1 /** 2 * This module provides API using Request structure. 3 * 4 * Structure Request provides configuration, connection pooling, cookie 5 * persistance. You can consider it as 'Session' and reuse it - all caches and settings will effective 6 * for next requests. 7 8 * Some most usefull settings: 9 10 $(TABLE 11 $(TR 12 $(TH name) 13 $(TH type) 14 $(TH meaning) 15 $(TH default) 16 ) 17 $(TR 18 $(TD keepAlive) 19 $(TD `bool`) 20 $(TD use keepalive connection) 21 $(TD `true`) 22 ) 23 $(TR 24 $(TD verbosity) 25 $(TD `uint`) 26 $(TD verbosity level $(LPAREN)0, 1, 2 or 3$(RPAREN)) 27 $(TD `16KB`) 28 ) 29 $(TR 30 $(TD maxRedirects) 31 $(TD `uint`) 32 $(TD maximum redirects allowed $(LPAREN)0 to disable redirects$(RPAREN)) 33 $(TD `true`) 34 ) 35 $(TR 36 $(TD maxHeadersLength) 37 $(TD `size_t`) 38 $(TD max. acceptable response headers length) 39 $(TD `32KB`) 40 ) 41 $(TR 42 $(TD maxContentLength) 43 $(TD `size_t`) 44 $(TD max. acceptable content length) 45 $(TD `0` - unlimited) 46 ) 47 $(TR 48 $(TD bufferSize) 49 $(TD `size_t`) 50 $(TD socket io buffer size) 51 $(TD `16KB`) 52 ) 53 $(TR 54 $(TD timeout) 55 $(TD `Duration`) 56 $(TD timeout on connect or data transfer) 57 $(TD `30.seconds`) 58 ) 59 $(TR 60 $(TD proxy) 61 $(TD `string`) 62 $(TD url of the HTTP/HTTPS/FTP proxy) 63 $(TD `null`) 64 ) 65 $(TR 66 $(TD bind) 67 $(TD `string`) 68 $(TD use local address for outgoing connections) 69 $(TD `null`) 70 ) 71 $(TR 72 $(TD useStreaming) 73 $(TD `bool`) 74 $(TD receive response data as `InputRange` in case it can't fit memory) 75 $(TD `false`) 76 ) 77 $(TR 78 $(TD addHeaders) 79 $(TD `string[string]`) 80 $(TD custom headers) 81 $(TD `null`) 82 ) 83 $(TR 84 $(TD cookie) 85 $(TD `Cookie[]`) 86 $(TD cookies you will send to server) 87 $(TD `null`) 88 ) 89 $(TR 90 $(TD authenticator) 91 $(TD `Auth`) 92 $(TD authenticator) 93 $(TD `null`) 94 ) 95 $(TR 96 $(TD socketFactory) 97 $(TD see doc for socketFactory) 98 $(TD user-provided connection factory) 99 $(TD `null`) 100 ) 101 ) 102 103 * Example: 104 * --- 105 * import requests; 106 * import std.datetime; 107 * 108 * void main() 109 * { 110 * Request rq = Request(); 111 * Response rs; 112 * rq.timeout = 10.seconds; 113 * rq.addHeaders(["User-Agent": "unknown"]); 114 * rs = rq.get("https://httpbin.org/get"); 115 * assert(rs.code==200); 116 * rs = rq.post("http://httpbin.org/post", "abc"); 117 * assert(rs.code==200); 118 * } 119 * --- 120 121 */ 122 module requests.request; 123 import requests.http; 124 import requests.ftp; 125 import requests.streams; 126 import requests.base; 127 import requests.uri; 128 129 import std.datetime; 130 import std.conv; 131 import std.range; 132 import std.experimental.logger; 133 import std.format; 134 import std.typecons; 135 import std.stdio; 136 import std.algorithm; 137 import std.uni; 138 139 import requests.utils; 140 import requests.connmanager; 141 import requests.rangeadapter; 142 143 alias NetStreamFactory = NetworkStream delegate(string, string, ushort); 144 145 /** 146 * 'Intercepror' intercepts Request. It can modify request, or log it, or cache it 147 * or do whatever you need. When done it can return response or 148 * pass it to next request handler. 149 * 150 * Example: 151 * --- 152 * class ChangePath : Interceptor 153 * { 154 * Response opCall(Request r, RequestHandler next) 155 * { 156 * r.path = r.path ~ "get"; 157 * auto rs = next.handle(r); 158 * return rs; 159 * } 160 * } 161 * --- 162 * Later in the code you can use this class: 163 * Example: 164 * --- 165 * Request rq; 166 * rq.addInterceptor(new ChangePath()); 167 * rq.get("http://example.com"); 168 * --- 169 * 170 */ 171 interface Interceptor { 172 Response opCall(Request r, RequestHandler next); 173 } 174 class RequestHandler { 175 private Interceptor[] _interceptors; 176 this(Interceptor[] i) 177 { 178 _interceptors = i; 179 } 180 Response handle(Request r) 181 { 182 auto i = _interceptors.front(); 183 _interceptors.popFront(); 184 return i(r, this); 185 } 186 } 187 188 private Interceptor[] _static_interceptors; 189 /** 190 * Add module-level interceptor. Each Request will include it in the processing. 191 * it is handy as you can change behaviour of your requests without any changes 192 * in your code, just install interceptor before any call to Request(). 193 */ 194 public void addInterceptor(Interceptor i) 195 { 196 _static_interceptors ~= i; 197 } 198 199 /** 200 * Structure Request provides configuration, connection pooling, cookie 201 * persistance. You can consider it as 'Session'. 202 */ 203 public struct Request { 204 private { 205 /// use streaming when receive response 206 bool _useStreaming; 207 208 /// limit redirect number 209 uint _maxRedirects = 10; 210 211 /// Auth provider 212 Auth _authenticator; 213 214 /// maximum headers length 215 size_t _maxHeadersLength = 32 * 1024; // 32 KB 216 217 /// maximum content length 218 size_t _maxContentLength; // 0 - Unlimited 219 220 /// logging verbosity 221 uint _verbosity = 0; 222 223 /// use keepalive requests 224 bool _keepAlive = true; 225 226 /// io buffer size 227 size_t _bufferSize = 16*1024; 228 229 /// http/https proxy 230 string _proxy; 231 232 /// content type for POST/PUT requests 233 string _contentType; // content type for POST/PUT payloads 234 235 /// timeout for connect/send/receive 236 Duration _timeout = 30.seconds; 237 238 /// verify peer when using ssl 239 bool _sslSetVerifyPeer = true; 240 SSLOptions _sslOptions; 241 242 /// bind outgoing connections to this addr (name or ip) 243 string _bind; 244 245 _UH _userHeaders; 246 247 /// user-provided headers(use addHeader to add) 248 string[string] _headers; 249 // 250 251 // instrumentation 252 /// user-provided socket factory 253 NetStreamFactory _socketFactory; 254 255 /// user-provided interceptors 256 Interceptor[] _interceptors; 257 // 258 259 // parameters for each request 260 /// uri for the request 261 URI _uri; 262 /// method (GET, POST, ...) 263 string _method; 264 /// request parameters 265 QueryParam[] _params; 266 /// multipart form (for multipart POST requests) 267 MultipartForm _multipartForm; 268 /// unified interface for post data 269 InputRangeAdapter _postData; 270 // 271 272 // can be changed during execution 273 /// connection cache 274 RefCounted!ConnManager _cm; // connection manager 275 /// cookie storage 276 RefCounted!Cookies _cookie; // cookies received 277 /// permanent redirect cache 278 string[URI] _permanent_redirects; // cache 301 redirects for GET requests 279 // 280 281 } 282 /** Get/Set timeout on connection and IO operation. 283 * **v** - timeout value 284 * If timeout expired Request operation will throw $(B TimeoutException). 285 */ 286 mixin(Getter_Setter!Duration("timeout")); 287 mixin(Getter_Setter!bool("keepAlive")); 288 mixin(Getter_Setter!uint("maxRedirects")); 289 mixin(Getter_Setter!size_t("maxContentLength")); 290 mixin(Getter_Setter!size_t("maxHeadersLength")); 291 mixin(Getter_Setter!size_t("bufferSize")); 292 mixin(Getter_Setter!uint("verbosity")); 293 mixin(Getter_Setter!Auth("authenticator")); 294 /// set proxy property. 295 /// $(B v) - full url to proxy. 296 /// 297 /// Note that we recognize proxy settings from process environment (see $(LINK https://github.com/ikod/dlang-requests/issues/46)): 298 /// you can use http_proxy, https_proxy, all_proxy (as well as uppercase names). 299 mixin(Getter_Setter!string("proxy")); 300 mixin(Getter_Setter!string("method")); 301 mixin(Getter("uri")); 302 mixin(Getter("params")); 303 mixin(Getter("contentType")); 304 mixin(Getter("postData")); 305 mixin(Getter("permanent_redirects")); 306 mixin(Getter("multipartForm")); 307 mixin(Getter_Setter!NetStreamFactory("socketFactory")); 308 mixin(Getter_Setter!bool("useStreaming")); 309 mixin(Getter_Setter!(RefCounted!Cookies)("cookie")); 310 mixin(Getter_Setter!string("bind")); 311 mixin(Getter_Setter!(string[string])("headers")); 312 313 /** 314 Set and Get uri for next request. 315 */ 316 @property void uri(string u) pure @safe 317 { 318 _uri = URI(u); 319 } 320 321 /** 322 Set/Get path for next request. 323 */ 324 @property void path(string p) pure @nogc 325 { 326 _uri.path = p; 327 } 328 /** 329 ditto 330 */ 331 @property string path() const @safe pure @nogc 332 { 333 return _uri.path; 334 } 335 336 mixin(Getter("sslOptions")); 337 /** 338 Enable/disable ssl peer verification.. 339 */ 340 @property void sslSetVerifyPeer(bool v) pure @safe nothrow @nogc { 341 _sslOptions.setVerifyPeer(v); 342 } 343 /** 344 Set path and format for ssl key file. 345 */ 346 @property void sslSetKeyFile(string p, SSLOptions.filetype t = SSLOptions.filetype.pem) pure @safe nothrow @nogc { 347 _sslOptions.setKeyFile(p, t); 348 } 349 /** 350 Set path and format for ssl certificate file. 351 */ 352 @property void sslSetCertFile(string p, SSLOptions.filetype t = SSLOptions.filetype.pem) pure @safe nothrow @nogc { 353 _sslOptions.setCertFile(p, t); 354 } 355 /** 356 Set path to certificate authority file. 357 */ 358 @property void sslSetCaCert(string path) pure @safe nothrow @nogc { 359 _sslOptions.setCaCert(path); 360 } 361 362 /* 363 userHeaders keep bitflags for user-setted important headers 364 */ 365 package @property auto userHeaders() pure @safe nothrow @nogc 366 { 367 return _userHeaders; 368 } 369 370 /++ 371 + Add headers to request 372 + Params: 373 + headers = headers to send. 374 + Example: 375 + --- 376 + rq = Request(); 377 + rq.keepAlive = true; 378 + rq.addHeaders(["X-Header": "test"]); 379 + --- 380 +/ 381 void addHeaders(in string[string] headers) { 382 foreach(pair; headers.byKeyValue) { 383 string _h = pair.key; 384 switch(toLower(_h)) { 385 case "host": 386 _userHeaders.Host = true; 387 break; 388 case "user-agent": 389 _userHeaders.UserAgent = true; 390 break; 391 case "content-length": 392 _userHeaders.ContentLength = true; 393 break; 394 case "content-type": 395 _userHeaders.ContentType = true; 396 break; 397 case "connection": 398 _userHeaders.Connection = true; 399 break; 400 case "cookie": 401 _userHeaders.Cookie = true; 402 break; 403 default: 404 break; 405 } 406 _headers[pair.key] = pair.value; 407 } 408 } 409 /// 410 /// Remove any previously added headers. 411 /// 412 void clearHeaders() { 413 _headers = null; 414 _userHeaders = _UH.init; 415 } 416 /// Execute GET for http and retrieve file for FTP. 417 /// You have to provide at least $(B uri). All other arguments should conform to HTTPRequest.get or FTPRequest.get depending on the URI scheme. 418 /// When arguments do not conform scheme (for example you try to call get("ftp://somehost.net/pub/README", {"a":"b"}) which doesn't make sense) 419 /// you will receive Exception("Operation not supported for ftp") 420 /// 421 422 /// Execute POST for http and STOR file for FTP. 423 /// You have to provide $(B uri) and data. Data should conform to HTTPRequest.post or FTPRequest.post depending on the URI scheme. 424 /// When arguments do not conform scheme you will receive Exception("Operation not supported for ftp") 425 /// 426 427 Response exec(string method="GET", A...)(string url, A args) 428 { 429 return execute(method, url, args); 430 } 431 Response exec(string method="GET")(string url, string[string] query) 432 { 433 return execute(method, url, aa2params(query)); 434 } 435 string toString() const { 436 return "Request(%s, %s)".format(_method, _uri.uri()); 437 } 438 string format(string fmt) const { 439 import std.array; 440 import std.stdio; 441 auto a = appender!string(); 442 auto f = FormatSpec!char(fmt); 443 while (f.writeUpToNextSpec(a)) { 444 switch(f.spec) { 445 case 'h': 446 // Remote hostname. 447 a.put(_uri.host); 448 break; 449 case 'm': 450 // method. 451 a.put(_method); 452 break; 453 case 'p': 454 // Remote port. 455 a.put("%d".format(_uri.port)); 456 break; 457 case 'P': 458 // Path 459 a.put(_uri.path); 460 break; 461 case 'q': 462 // query parameters supplied with url. 463 a.put(_uri.query); 464 break; 465 case 'U': 466 a.put(_uri.uri()); 467 break; 468 default: 469 throw new FormatException("Unknown Request format spec " ~ f.spec); 470 } 471 } 472 return a.data(); 473 } 474 @property auto cm() { 475 return _cm; 476 } 477 /// helper 478 @property bool hasMultipartForm() const 479 { 480 return !_multipartForm.empty; 481 } 482 /// Add interceptor to request. 483 void addInterceptor(Interceptor i) 484 { 485 _interceptors ~= i; 486 } 487 488 package class LastInterceptor : Interceptor 489 { 490 Response opCall(Request r, RequestHandler _) 491 { 492 switch (r._uri.scheme) 493 { 494 case "ftp": 495 FTPRequest ftp; 496 return ftp.execute(r); 497 case "https","http": 498 HTTPRequest http; 499 return http.execute(r); 500 default: 501 assert(0, "".format(r._uri.scheme)); 502 } 503 } 504 } 505 Response post(string uri, string[string] query) 506 { 507 return execute("POST", uri, aa2params(query)); 508 } 509 Response post(string uri, QueryParam[] query) 510 { 511 return execute("POST", uri, query); 512 } 513 Response post(string uri, MultipartForm form) 514 { 515 return execute("POST", uri, form); 516 } 517 Response post(R)(string uri, R content, string contentType="application/octet-stream") 518 { 519 return execute("POST", uri, content, contentType); 520 } 521 522 Response get(string uri, string[string] query) 523 { 524 return execute("GET", uri, aa2params(query)); 525 } 526 Response get(string uri, QueryParam[] params = null) 527 { 528 return execute("GET", uri, params); 529 } 530 Response execute(R)(string method, string url, R content, string ct = "application/octet-stream") 531 { 532 _method = method; 533 _uri = URI(url); 534 _uri.idn_encode(); 535 _params = null; 536 _multipartForm = MultipartForm.init; 537 _contentType = ct; 538 _postData = makeAdapter(content); 539 // https://issues.dlang.org/show_bug.cgi?id=10535 540 _permanent_redirects[URI.init] = ""; _permanent_redirects.remove(URI.init); 541 // 542 if ( _cm.refCountedStore().refCount() == 0) 543 { 544 _cm = RefCounted!ConnManager(10); 545 } 546 if ( _cookie.refCountedStore().refCount() == 0) 547 { 548 _cookie = RefCounted!Cookies(); 549 } 550 auto interceptors = _static_interceptors ~ _interceptors ~ new LastInterceptor(); 551 auto handler = new RequestHandler(interceptors); 552 return handler.handle(this); 553 554 } 555 Response execute(string method, string url, MultipartForm form) 556 { 557 _method = method; 558 _uri = URI(url); 559 _uri.idn_encode(); 560 _params = null; 561 _multipartForm = form; 562 563 // https://issues.dlang.org/show_bug.cgi?id=10535 564 _permanent_redirects[URI.init] = ""; _permanent_redirects.remove(URI.init); 565 // 566 567 if ( _cm.refCountedStore().refCount() == 0) 568 { 569 _cm = RefCounted!ConnManager(10); 570 } 571 if ( _cookie.refCountedStore().refCount() == 0) 572 { 573 _cookie = RefCounted!Cookies(); 574 } 575 auto interceptors = _static_interceptors ~ _interceptors ~ new LastInterceptor(); 576 auto handler = new RequestHandler(interceptors); 577 return handler.handle(this); 578 } 579 Response execute(string method, string url, QueryParam[] params = null) 580 { 581 _method = method; 582 _uri = URI(url); 583 _uri.idn_encode(); 584 _multipartForm = MultipartForm.init; 585 _params = params; 586 587 // https://issues.dlang.org/show_bug.cgi?id=10535 588 _permanent_redirects[URI.init] = ""; _permanent_redirects.remove(URI.init); 589 // 590 if ( _cm.refCountedStore().refCount() == 0) 591 { 592 _cm = RefCounted!ConnManager(10); 593 } 594 if ( _cookie.refCountedStore().refCount() == 0) 595 { 596 _cookie = RefCounted!Cookies(Cookies()); 597 } 598 auto interceptors = _static_interceptors ~ _interceptors ~ new LastInterceptor(); 599 auto handler = new RequestHandler(interceptors); 600 auto r = handler.handle(this); 601 return r; 602 } 603 } 604 605 unittest { 606 info("testing Request"); 607 int interceptorCalls; 608 609 class DummyInterceptor : Interceptor { 610 Response opCall(Request r, RequestHandler next) 611 { 612 interceptorCalls++; 613 auto rs = next.handle(r); 614 return rs; 615 } 616 } 617 globalLogLevel = LogLevel.info; 618 Request rq; 619 Response rs; 620 rq.addInterceptor(new DummyInterceptor()); 621 622 rs = rq.execute("GET", "http://httpbin.org/"); 623 rq.useStreaming = true; 624 rq.bufferSize = 128; 625 rs = rq.execute("GET", "http://httpbin.org/"); 626 auto s = rs.receiveAsRange; 627 while (!s.empty) 628 { 629 s.front; 630 s.popFront(); 631 } 632 assert(interceptorCalls == 2, "Expected interceptorCalls==2, got %d".format(interceptorCalls)); 633 634 // test global/static interceptors 635 // 636 // save and clear static_interceptors 637 Interceptor[] saved_interceptors = _static_interceptors; 638 scope(exit) 639 { 640 _static_interceptors = saved_interceptors; 641 } 642 _static_interceptors.length = 0; 643 644 // add RqModifier to static_interceptors 645 class RqModifier : Interceptor { 646 Response opCall(Request r, RequestHandler next) 647 { 648 r.path = "/get"; 649 r.method = "GET"; 650 auto rs = next.handle(r); 651 return rs; 652 } 653 } 654 addInterceptor(new RqModifier()); 655 // from now any request will pass through Pathmodifier without changing your code: 656 rq = Request(); 657 rs = rq.get("http://httpbin.org/"); 658 assert(rs.uri.path == "/get", "Expected /get, but got %s".format(rs.uri.path)); 659 660 rs = rq.post("http://httpbin.org/post", "abc"); 661 assert(rs.code == 200); 662 assert(rs.uri().path() == "/get"); 663 }