1 module requests.http; 2 3 private: 4 import std.algorithm; 5 import std.array; 6 import std.ascii; 7 import std.conv; 8 import std.datetime; 9 import std.exception; 10 import std.format; 11 import std.stdio; 12 import std.range; 13 import std.string; 14 import std.traits; 15 import std.typecons; 16 import std.bitmanip; 17 import std.experimental.logger; 18 import core.thread; 19 20 import requests.streams; 21 import requests.uri; 22 import requests.utils; 23 import requests.base; 24 25 static immutable ushort[] redirectCodes = [301, 302, 303, 307, 308]; 26 static immutable uint defaultBufferSize = 12*1024; 27 28 static immutable string[string] proxies; 29 static this() { 30 import std.process; 31 proxies["http"] = environment.get("http_proxy", environment.get("HTTP_PROXY")); 32 proxies["https"] = environment.get("https_proxy", environment.get("HTTPS_PROXY")); 33 proxies["all"] = environment.get("all_proxy", environment.get("ALL_PROXY")); 34 foreach(p; proxies.byKey()) { 35 if (proxies[p] is null) { 36 continue; 37 } 38 URI u = URI(proxies[p]); 39 } 40 } 41 42 public class MaxRedirectsException: Exception { 43 this(string message, string file = __FILE__, size_t line = __LINE__, Throwable next = null) @safe pure nothrow { 44 super(message, file, line, next); 45 } 46 } 47 48 /** 49 * Basic authentication. 50 * Adds $(B Authorization: Basic) header to request. 51 */ 52 public class BasicAuthentication: Auth { 53 private { 54 string _username, _password; 55 string[] _domains; 56 } 57 /// Constructor. 58 /// Params: 59 /// username = username 60 /// password = password 61 /// domains = not used now 62 /// 63 this(string username, string password, string[] domains = []) { 64 _username = username; 65 _password = password; 66 _domains = domains; 67 } 68 override string[string] authHeaders(string domain) { 69 import std.base64; 70 string[string] auth; 71 auth["Authorization"] = "Basic " ~ to!string(Base64.encode(cast(ubyte[])"%s:%s".format(_username, _password))); 72 return auth; 73 } 74 override string userName() { 75 return _username; 76 } 77 override string password() { 78 return _password; 79 } 80 } 81 /// 82 /// 83 /// 84 public auto queryParams(T...)(T params) pure nothrow @safe 85 { 86 static assert (T.length % 2 == 0, "wrong args count"); 87 88 QueryParam[] output; 89 output.reserve = T.length / 2; 90 91 void queryParamsHelper(T...)(T params, ref QueryParam[] output) 92 { 93 static if (T.length > 0) 94 { 95 output ~= QueryParam(params[0].to!string, params[1].to!string); 96 queryParamsHelper(params[2..$], output); 97 } 98 } 99 100 queryParamsHelper(params, output); 101 return output; 102 } 103 104 /// 105 /// Response - result of request execution. 106 /// 107 /// Response.code - response HTTP code. 108 /// Response.status_line - received HTTP status line. 109 /// Response.responseHeaders - received headers. 110 /// Response.responseBody - container for received body 111 /// Response.history - for redirected responses contain all history 112 /// 113 public class HTTPResponse : Response { 114 private { 115 string _status_line; 116 117 HTTPResponse[] _history; // redirects history 118 119 mixin(Setter!string("status_line")); 120 } 121 122 ~this() { 123 _responseHeaders = null; 124 _history.length = 0; 125 } 126 127 mixin(Getter!string("status_line")); 128 @property final string[string] responseHeaders() @safe @nogc nothrow { 129 return _responseHeaders; 130 } 131 @property final HTTPResponse[] history() @safe @nogc nothrow { 132 return _history; 133 } 134 135 @property auto getStats() const pure @safe { 136 alias statTuple = Tuple!(Duration, "connectTime", 137 Duration, "sendTime", 138 Duration, "recvTime"); 139 statTuple stat; 140 stat.connectTime = _connectedAt - _startedAt; 141 stat.sendTime = _requestSentAt - _connectedAt; 142 stat.recvTime = _finishedAt - _requestSentAt; 143 return stat; 144 } 145 } 146 /** 147 * Struct to send multiple files in POST request. 148 */ 149 public struct PostFile { 150 /// Path to the file to send. 151 string fileName; 152 /// Name of the field (if empty - send file base name) 153 string fieldName; 154 /// contentType of the file if not empty 155 string contentType; 156 } 157 /// 158 /// This is File-like interface for sending data to multipart fotms 159 /// 160 public interface FiniteReadable { 161 /// size of the content 162 abstract ulong getSize(); 163 /// file-like read() 164 abstract ubyte[] read(); 165 } 166 /// 167 /// Helper to create form elements from File. 168 /// Params: 169 /// name = name of the field in form 170 /// f = opened std.stio.File to send to server 171 /// parameters = optional parameters (most important are "filename" and "Content-Type") 172 /// 173 public auto formData(string name, File f, string[string] parameters = null) { 174 return MultipartForm.FormData(name, new FormDataFile(f), parameters); 175 } 176 /// 177 /// Helper to create form elements from ubyte[]. 178 /// Params: 179 /// name = name of the field in form 180 /// b = data to send to server 181 /// parameters = optional parameters (can be "filename" and "Content-Type") 182 /// 183 public auto formData(string name, ubyte[] b, string[string] parameters = null) { 184 return MultipartForm.FormData(name, new FormDataBytes(b), parameters); 185 } 186 public auto formData(string name, string b, string[string] parameters = null) { 187 return MultipartForm.FormData(name, new FormDataBytes(b.dup.representation), parameters); 188 } 189 public class FormDataBytes : FiniteReadable { 190 private { 191 ulong _size; 192 ubyte[] _data; 193 size_t _offset; 194 bool _exhausted; 195 } 196 this(ubyte[] data) { 197 _data = data; 198 _size = data.length; 199 } 200 final override ulong getSize() { 201 return _size; 202 } 203 final override ubyte[] read() { 204 enforce( !_exhausted, "You can't read froum exhausted source" ); 205 size_t toRead = min(defaultBufferSize, _size - _offset); 206 auto result = _data[_offset.._offset+toRead]; 207 _offset += toRead; 208 if ( toRead == 0 ) { 209 _exhausted = true; 210 } 211 return result; 212 } 213 } 214 public class FormDataFile : FiniteReadable { 215 import std.file; 216 private { 217 File _fileHandle; 218 ulong _fileSize; 219 size_t _processed; 220 bool _exhausted; 221 } 222 this(File file) { 223 import std.file; 224 _fileHandle = file; 225 _fileSize = std.file.getSize(file.name); 226 } 227 final override ulong getSize() pure nothrow @safe { 228 return _fileSize; 229 } 230 final override ubyte[] read() { 231 enforce( !_exhausted, "You can't read froum exhausted source" ); 232 auto b = new ubyte[defaultBufferSize]; 233 auto r = _fileHandle.rawRead(b); 234 auto toRead = min(r.length, _fileSize - _processed); 235 if ( toRead == 0 ) { 236 _exhausted = true; 237 } 238 _processed += toRead; 239 return r[0..toRead]; 240 } 241 } 242 /// 243 /// This struct used to bulld POST's to forms. 244 /// Each part have name and data. data is something that can be read-ed and have size. 245 /// For example this can be string-like object (wrapped for reading) or opened File. 246 /// 247 public struct MultipartForm { 248 package struct FormData { 249 FiniteReadable input; 250 string name; 251 string[string] parameters; 252 this(string name, FiniteReadable i, string[string] parameters = null) { 253 this.input = i; 254 this.name = name; 255 this.parameters = parameters; 256 } 257 } 258 259 private FormData[] _sources; 260 auto add(FormData d) { 261 _sources ~= d; 262 return this; 263 } 264 auto add(string name, FiniteReadable i, string[string]parameters = null) { 265 _sources ~= FormData(name, i, parameters); 266 return this; 267 } 268 } 269 /// 270 271 /// 272 /// Request. 273 /// Configurable parameters: 274 /// $(B method) - string, method to use (GET, POST, ...) 275 /// $(B headers) - string[string], add any additional headers you'd like to send. 276 /// $(B authenticator) - class Auth, class to send auth headers. 277 /// $(B keepAlive) - bool, set true for keepAlive requests. default true. 278 /// $(B maxRedirects) - uint, maximum number of redirects. default 10. 279 /// $(B maxHeadersLength) - size_t, maximum length of server response headers. default = 32KB. 280 /// $(B maxContentLength) - size_t, maximun content length. delault - 0 = unlimited. 281 /// $(B bufferSize) - size_t, send and receive buffer size. default = 16KB. 282 /// $(B verbosity) - uint, level of verbosity(0 - nothing, 1 - headers, 2 - headers and body progress). default = 0. 283 /// $(B proxy) - string, set proxy url if needed. default - null. 284 /// $(B cookie) - Tuple Cookie, Read/Write cookie You can get cookie setted by server, or set cookies before doing request. 285 /// $(B timeout) - Duration, Set timeout value for connect/receive/send. 286 /// 287 public struct HTTPRequest { 288 private { 289 struct _UH { 290 // flags for each important header, added by user using addHeaders 291 mixin(bitfields!( 292 bool, "Host", 1, 293 bool, "UserAgent", 1, 294 bool, "ContentLength", 1, 295 bool, "Connection", 1, 296 bool, "AcceptEncoding", 1, 297 bool, "ContentType", 1, 298 bool, "Cookie", 1, 299 uint, "", 1 300 )); 301 } 302 string _method = "GET"; 303 URI _uri; 304 string[string] _headers; 305 string[] _filteredHeaders; 306 Auth _authenticator; 307 bool _keepAlive = true; 308 uint _maxRedirects = 10; 309 size_t _maxHeadersLength = 32 * 1024; // 32 KB 310 size_t _maxContentLength; // 0 - Unlimited 311 string _proxy; 312 uint _verbosity = 0; // 0 - no output, 1 - headers, 2 - headers+body info 313 Duration _timeout = 30.seconds; 314 size_t _bufferSize = defaultBufferSize; // 16k 315 bool _useStreaming; // return iterator instead of completed request 316 317 NetworkStream _stream; 318 HTTPResponse[] _history; // redirects history 319 DataPipe!ubyte _bodyDecoder; 320 DecodeChunked _unChunker; 321 long _contentLength; 322 long _contentReceived; 323 Cookie[] _cookie; 324 SSLOptions _sslOptions; 325 string _bind; 326 _UH _userHeaders; 327 } 328 package HTTPResponse _response; 329 330 mixin(Getter_Setter!string ("method")); 331 mixin(Getter_Setter!bool ("keepAlive")); 332 mixin(Getter_Setter!size_t ("maxContentLength")); 333 mixin(Getter_Setter!size_t ("maxHeadersLength")); 334 mixin(Getter_Setter!size_t ("bufferSize")); 335 mixin(Getter_Setter!uint ("maxRedirects")); 336 mixin(Getter_Setter!uint ("verbosity")); 337 mixin(Getter!string ("proxy")); 338 mixin(Getter_Setter!Duration ("timeout")); 339 mixin(Setter!Auth ("authenticator")); 340 mixin(Getter_Setter!bool ("useStreaming")); 341 mixin(Getter!long ("contentLength")); 342 mixin(Getter!long ("contentReceived")); 343 mixin(Getter_Setter!SSLOptions ("sslOptions")); 344 mixin(Getter_Setter!string ("bind")); 345 346 @property void sslSetVerifyPeer(bool v) pure @safe nothrow @nogc { 347 _sslOptions.setVerifyPeer(v); 348 } 349 @property void sslSetKeyFile(string p, SSLOptions.filetype t = SSLOptions.filetype.pem) pure @safe nothrow @nogc { 350 _sslOptions.setKeyFile(p, t); 351 } 352 @property void sslSetCertFile(string p, SSLOptions.filetype t = SSLOptions.filetype.pem) pure @safe nothrow @nogc { 353 _sslOptions.setCertFile(p, t); 354 } 355 @property void sslSetCaCert(string path) pure @safe nothrow @nogc { 356 _sslOptions.setCaCert(path); 357 } 358 @property final void cookie(Cookie[] s) pure @safe @nogc nothrow { 359 _cookie = s; 360 } 361 @property final void proxy(string v) { 362 if ( v != _proxy ) { 363 if ( _stream && _stream.isOpen() ) { 364 debug(requests) tracef("Close connection because we reset proxy"); 365 _stream.close(); 366 } 367 } 368 _proxy = v; 369 } 370 @property final Cookie[] cookie() pure @safe @nogc nothrow { 371 return _cookie; 372 } 373 374 this(string uri) { 375 _uri = URI(uri); 376 } 377 ~this() { 378 if ( _stream && _stream.isConnected) { 379 _stream.close(); 380 } 381 _stream = null; 382 _headers = null; 383 _authenticator = null; 384 _history = null; 385 _bodyDecoder = null; 386 _unChunker = null; 387 } 388 string toString() const { 389 return "HTTPRequest(%s, %s)".format(_method, _uri.uri()); 390 } 391 string format(string fmt) const { 392 import std.array; 393 import std.stdio; 394 auto a = appender!string(); 395 auto f = FormatSpec!char(fmt); 396 while (f.writeUpToNextSpec(a)) { 397 switch(f.spec) { 398 case 'h': 399 // Remote hostname. 400 a.put(_uri.host); 401 break; 402 case 'm': 403 // method. 404 a.put(_method); 405 break; 406 case 'p': 407 // Remote port. 408 a.put("%d".format(_uri.port)); 409 break; 410 case 'P': 411 // Path 412 a.put(_uri.path); 413 break; 414 case 'q': 415 // query parameters supplied with url. 416 a.put(_uri.query); 417 break; 418 case 'U': 419 a.put(_uri.uri()); 420 break; 421 default: 422 throw new FormatException("Unknown Request format spec " ~ f.spec); 423 } 424 } 425 return a.data(); 426 } 427 string select_proxy(string scheme) { 428 if ( _proxy is null && proxies.length == 0 ) { 429 debug(requests) tracef("proxy=null"); 430 return null; 431 } 432 if ( _proxy ) { 433 debug(requests) tracef("proxy=%s", _proxy); 434 return _proxy; 435 } 436 auto p = scheme in proxies; 437 if ( p !is null && *p != "") { 438 debug(requests) tracef("proxy=%s", *p); 439 return *p; 440 } 441 p = "all" in proxies; 442 if ( p !is null && *p != "") { 443 debug(requests) tracef("proxy=%s", *p); 444 return *p; 445 } 446 debug(requests) tracef("proxy=null"); 447 return null; 448 } 449 void clearHeaders() { 450 _headers = null; 451 } 452 @property void uri(in URI newURI) { 453 handleURLChange(_uri, newURI); 454 _uri = newURI; 455 } 456 /// Add headers to request 457 /// Params: 458 /// headers = headers to send. 459 void addHeaders(in string[string] headers) { 460 foreach(pair; headers.byKeyValue) { 461 string _h = pair.key; 462 switch(toLower(_h)) { 463 case "host": 464 _userHeaders.Host = true; 465 break; 466 case "user-agent": 467 _userHeaders.UserAgent = true; 468 break; 469 case "content-length": 470 _userHeaders.ContentLength = true; 471 break; 472 case "content-type": 473 _userHeaders.ContentType = true; 474 break; 475 case "connection": 476 _userHeaders.Connection = true; 477 break; 478 case "cookie": 479 _userHeaders.Cookie = true; 480 break; 481 default: 482 break; 483 } 484 _headers[pair.key] = pair.value; 485 } 486 } 487 private void safeSetHeader(ref string[string] headers, bool userAdded, string h, string v) pure @safe { 488 if ( !userAdded ) { 489 headers[h] = v; 490 } 491 } 492 /// Remove headers from request 493 /// Params: 494 /// headers = headers to remove. 495 void removeHeaders(in string[] headers) pure { 496 _filteredHeaders ~= headers; 497 } 498 /// 499 /// compose headers to send 500 /// 501 private string[string] requestHeaders() { 502 503 string[string] generatedHeaders; 504 505 if ( _authenticator ) { 506 _authenticator. 507 authHeaders(_uri.host). 508 byKeyValue. 509 each!(pair => generatedHeaders[pair.key] = pair.value); 510 } 511 512 _headers.byKey.each!(h => generatedHeaders[h] = _headers[h]); 513 514 safeSetHeader(generatedHeaders, _userHeaders.AcceptEncoding, "Accept-Encoding", "gzip,deflate"); 515 safeSetHeader(generatedHeaders, _userHeaders.UserAgent, "User-Agent", "dlang-requests"); 516 safeSetHeader(generatedHeaders, _userHeaders.Connection, "Connection", _keepAlive?"Keep-Alive":"Close"); 517 518 if ( !_userHeaders.Host ) 519 { 520 generatedHeaders["Host"] = _uri.host; 521 if ( _uri.scheme !in standard_ports || _uri.port != standard_ports[_uri.scheme] ) { 522 generatedHeaders["Host"] ~= ":%d".format(_uri.port); 523 } 524 } 525 526 if ( _cookie.length && !_userHeaders.Cookie ) { 527 auto cs = _cookie. 528 filter!(c => _uri.path.pathMatches(c.path) && _uri.host.domainMatches(c.domain)). 529 map!(c => "%s=%s".format(c.attr, c.value)). 530 joiner(";"); 531 if ( ! cs.empty ) 532 { 533 generatedHeaders["Cookie"] = to!string(cs); 534 } 535 } 536 537 _filteredHeaders.each!(h => generatedHeaders.remove(h)); 538 539 return generatedHeaders; 540 } 541 /// 542 /// Build request string. 543 /// Handle proxy and query parameters. 544 /// 545 private @property string requestString(QueryParam[] params = null) { 546 string actual_proxy = select_proxy(_uri.scheme); 547 if ( actual_proxy && _uri.scheme != "https" ) { 548 return "%s %s HTTP/1.1\r\n".format(_method, _uri.uri); 549 } 550 auto query = _uri.query.dup; 551 if ( params ) { 552 query ~= "&" ~ params2query(params); 553 if ( query[0] != '?' ) { 554 query = "?" ~ query; 555 } 556 } 557 return "%s %s%s HTTP/1.1\r\n".format(_method, _uri.path, query); 558 } 559 /// 560 /// encode parameters and build query part of the url 561 /// 562 private static string params2query(in QueryParam[] params) pure @safe { 563 return params. 564 map!(a => "%s=%s".format(a.key.urlEncoded, a.value.urlEncoded)). 565 join("&"); 566 } 567 // 568 package unittest { 569 assert(params2query(queryParams("a","b", "c", " d "))=="a=b&c=%20d%20"); 570 } 571 /// 572 /// Analyze received headers, take appropriate actions: 573 /// check content length, attach unchunk and uncompress 574 /// 575 private void analyzeHeaders(in string[string] headers) { 576 577 _contentLength = -1; 578 _unChunker = null; 579 auto contentLength = "content-length" in headers; 580 if ( contentLength ) { 581 try { 582 _contentLength = to!long(*contentLength); 583 if ( _maxContentLength && _contentLength > _maxContentLength) { 584 throw new RequestException("ContentLength > maxContentLength (%d>%d)". 585 format(_contentLength, _maxContentLength)); 586 } 587 } catch (ConvException e) { 588 throw new RequestException("Can't convert Content-Length from %s".format(*contentLength)); 589 } 590 } 591 auto transferEncoding = "transfer-encoding" in headers; 592 if ( transferEncoding ) { 593 debug(requests) tracef("transferEncoding: %s", *transferEncoding); 594 if ( *transferEncoding == "chunked") { 595 _unChunker = new DecodeChunked(); 596 _bodyDecoder.insert(_unChunker); 597 } 598 } 599 auto contentEncoding = "content-encoding" in headers; 600 if ( contentEncoding ) switch (*contentEncoding) { 601 default: 602 throw new RequestException("Unknown content-encoding " ~ *contentEncoding); 603 case "gzip": 604 case "deflate": 605 _bodyDecoder.insert(new Decompressor!ubyte); 606 } 607 608 } 609 /// 610 /// Called when we know that all headers already received in buffer. 611 /// This routine does not interpret headers content (see analyzeHeaders). 612 /// 1. Split headers on lines 613 /// 2. store status line, store response code 614 /// 3. unfold headers if needed 615 /// 4. store headers 616 /// 617 private void parseResponseHeaders(in ubyte[] input) { 618 string lastHeader; 619 auto buffer = cast(string)input; 620 621 foreach(line; buffer.split("\n").map!(l => l.stripRight)) { 622 if ( ! _response.status_line.length ) { 623 debug (requests) tracef("statusLine: %s", line); 624 _response.status_line = line; 625 if ( _verbosity >= 1 ) { 626 writefln("< %s", line); 627 } 628 auto parsed = line.split(" "); 629 if ( parsed.length >= 2 ) { 630 _response.code = parsed[1].to!ushort; 631 } 632 continue; 633 } 634 if ( line[0] == ' ' || line[0] == '\t' ) { 635 // unfolding https://tools.ietf.org/html/rfc822#section-3.1 636 if ( auto stored = lastHeader in _response._responseHeaders) { 637 *stored ~= line; 638 } 639 continue; 640 } 641 auto parsed = line.findSplit(":"); 642 auto header = parsed[0].toLower; 643 auto value = parsed[2].strip; 644 645 if ( _verbosity >= 1 ) { 646 writefln("< %s: %s", header, value); 647 } 648 649 lastHeader = header; 650 debug (requests) tracef("Header %s = %s", header, value); 651 652 if ( header != "set-cookie" ) { 653 auto stored = _response.responseHeaders.get(header, null); 654 if ( stored ) { 655 value = stored ~ ", " ~ value; 656 } 657 _response._responseHeaders[header] = value; 658 continue; 659 } 660 _cookie ~= processCookie(value); 661 } 662 } 663 664 /// 665 /// Process Set-Cookie header from server response 666 /// 667 private Cookie[] processCookie(string value ) pure { 668 // cookie processing 669 // 670 // as we can't join several set-cookie lines in single line 671 // < Set-Cookie: idx=7f2800f63c112a65ef5082957bcca24b; expires=Mon, 29-May-2017 00:31:25 GMT; path=/; domain=example.com 672 // < Set-Cookie: idx=7f2800f63c112a65ef5082957bcca24b; expires=Mon, 29-May-2017 00:31:25 GMT; path=/; domain=example.com, cs=ip764-RgKqc-HvSkxRxdQQAKW8LA; path=/; domain=.example.com; HttpOnly 673 // 674 Cookie[] res; 675 string[string] kv; 676 auto fields = value.split(";").map!strip; 677 while(!fields.empty) { 678 auto s = fields.front.findSplit("="); 679 fields.popFront; 680 if ( s[1] != "=" ) { 681 continue; 682 } 683 auto k = s[0]; 684 auto v = s[2]; 685 switch(k.toLower()) { 686 case "domain": 687 k = "domain"; 688 break; 689 case "path": 690 k = "path"; 691 break; 692 case "expires": 693 continue; 694 default: 695 break; 696 } 697 kv[k] = v; 698 } 699 if ( "domain" !in kv ) { 700 kv["domain"] = _uri.host; 701 } 702 if ( "path" !in kv ) { 703 kv["path"] = _uri.path; 704 } 705 auto domain = kv["domain"]; kv.remove("domain"); 706 auto path = kv["path"]; kv.remove("path"); 707 foreach(pair; kv.byKeyValue) { 708 auto _attr = pair.key; 709 auto _value = pair.value; 710 auto cookie = Cookie(path, domain, _attr, _value); 711 res ~= cookie; 712 } 713 return res; 714 } 715 /// 716 /// Do we received \r\n\r\n? 717 /// 718 private bool headersHaveBeenReceived(in ubyte[] data, ref Buffer!ubyte buffer, out string separator) const @safe { 719 foreach(s; ["\r\n\r\n", "\n\n"]) { 720 if ( data.canFind(s) || buffer.canFind(s) ) { 721 separator = s; 722 return true; 723 } 724 } 725 return false; 726 } 727 728 private bool followRedirectResponse() { 729 if ( _history.length >= _maxRedirects ) { 730 throw new MaxRedirectsException("%d redirects reached maxRedirects %d.".format(_history.length, _maxRedirects)); 731 } 732 auto location = "location" in _response.responseHeaders; 733 if ( !location ) { 734 return false; 735 } 736 _history ~= _response; 737 auto connection = "connection" in _response._responseHeaders; 738 if ( !connection || *connection == "close" ) { 739 debug(requests) tracef("Closing connection because of 'Connection: close' or no 'Connection' header"); 740 _stream.close(); 741 } 742 URI oldURI = _uri; 743 URI newURI = oldURI; 744 try { 745 newURI = URI(*location); 746 } catch (UriException e) { 747 debug(requests) trace("Can't parse Location:, try relative uri"); 748 newURI.path = *location; 749 newURI.uri = newURI.recalc_uri; 750 } 751 handleURLChange(oldURI, newURI); 752 oldURI = _response.uri; 753 _uri = newURI; 754 _response = new HTTPResponse; 755 _response.uri = oldURI; 756 _response.finalURI = newURI; 757 return true; 758 } 759 /// 760 /// If uri changed so that we have to change host, port or proxy, then we have to close socket stream 761 /// 762 private void handleURLChange(in URI from, in URI to) { 763 if ( _stream is null || !_stream.isConnected ) { 764 return; 765 } 766 string proxy_from = select_proxy(from.scheme); 767 string proxy_to = select_proxy(to.scheme); 768 if ( proxy_from != proxy_to ) { 769 // we are switching proxies 770 _stream.close(); 771 return; 772 } 773 if ( proxy_to !is null ) { 774 // we do not have to close proxy connection if we will not change proxy 775 if ( (from.scheme=="https" || to.scheme=="https") 776 && (from.scheme != to.scheme) ) { 777 _stream.close(); 778 } 779 return; 780 } 781 if ( from.scheme != to.scheme || from.host != to.host || from.port != to.port ) { 782 debug tracef("Have to reopen stream, because of URI change"); 783 _stream.close(); 784 } 785 } 786 /// 787 /// if we have new uri, then we need to check if we have to reopen existent connection 788 /// 789 private void checkURL(string url, string file=__FILE__, size_t line=__LINE__) { 790 if (url is null && _uri.uri == "" ) { 791 throw new RequestException("No url configured", file, line); 792 } 793 794 if ( url !is null ) { 795 URI newURI = URI(url); 796 handleURLChange(_uri, newURI); 797 _uri = newURI; 798 } 799 } 800 /// 801 /// Setup connection. Handle proxy and https case 802 /// 803 private void setupConnection() { 804 if ( _stream && _stream.isConnected ) { 805 debug(requests) tracef("Use old connection"); 806 return; 807 } 808 809 debug(requests) tracef("Set up new connection"); 810 811 URI uri; // this URI will be used temporarry if we need proxy 812 string actual_proxy = select_proxy(_uri.scheme); 813 final switch (_uri.scheme) { 814 case "http": 815 if ( actual_proxy ) { 816 uri.uri_parse(actual_proxy); 817 uri.idn_encode(); 818 } else { 819 // use original uri 820 uri = _uri; 821 } 822 _stream = new TCPStream(); 823 _stream.bind(_bind); 824 _stream.connect(uri.host, uri.port, _timeout); 825 break; 826 case "https": 827 if ( actual_proxy ) { 828 uri.uri_parse(actual_proxy); 829 uri.idn_encode(); 830 _stream = new TCPStream(); 831 _stream.bind(_bind); 832 _stream.connect(uri.host, uri.port, _timeout); 833 if ( verbosity>=1 ) { 834 writeln("> CONNECT %s:%d HTTP/1.1".format(_uri.host, _uri.port)); 835 } 836 _stream.send("CONNECT %s:%d HTTP/1.1\r\n\r\n".format(_uri.host, _uri.port)); 837 while ( _stream.isConnected ) { 838 ubyte[1024] b; 839 auto read = _stream.receive(b); 840 if ( verbosity>=1) { 841 writefln("< %s", cast(string)b[0..read]); 842 } 843 debug(requests) tracef("read: %d", read); 844 if ( b[0..read].canFind("\r\n\r\n") || b[0..read].canFind("\n\n") ) { 845 debug(requests) tracef("proxy connection ready"); 846 // convert connection to ssl 847 _stream = new SSLStream(_stream, _sslOptions, _uri.host); 848 break; 849 } else { 850 debug(requests) tracef("still wait for proxy connection"); 851 } 852 } 853 } else { 854 uri = _uri; 855 _stream = new SSLStream(_sslOptions); 856 _stream.bind(_bind); 857 _stream.connect(uri.host, uri.port, _timeout); 858 debug(requests) tracef("ssl connection to origin server ready"); 859 } 860 break; 861 } 862 } 863 /// 864 /// Request sent, now receive response. 865 /// Find headers, split on headers and body, continue to receive body 866 /// 867 private void receiveResponse() { 868 869 _stream.readTimeout = timeout; 870 scope(exit) { 871 if ( _stream && _stream.isOpen ) { 872 _stream.readTimeout = 0.seconds; 873 } 874 } 875 876 _bodyDecoder = new DataPipe!ubyte(); 877 scope(exit) { 878 if ( !_useStreaming ) { 879 _bodyDecoder = null; 880 _unChunker = null; 881 } 882 } 883 884 auto buffer = Buffer!ubyte(); 885 Buffer!ubyte partialBody; 886 ptrdiff_t read; 887 string separator; 888 889 while(true) { 890 891 auto b = new ubyte[_bufferSize]; 892 read = _stream.receive(b); 893 894 debug(requests) tracef("read: %d", read); 895 if ( read == 0 ) { 896 break; 897 } 898 auto data = b[0..read]; 899 buffer.putNoCopy(data); 900 if ( verbosity>=3 ) { 901 writeln(data.dump.join("\n")); 902 } 903 904 if ( buffer.length > maxHeadersLength ) { 905 throw new RequestException("Headers length > maxHeadersLength (%d > %d)".format(buffer.length, maxHeadersLength)); 906 } 907 if ( headersHaveBeenReceived(data, buffer, separator) ) { 908 auto s = buffer.data.findSplit(separator); 909 auto ResponseHeaders = s[0]; 910 partialBody = Buffer!ubyte(s[2]); 911 _contentReceived += partialBody.length; 912 parseResponseHeaders(ResponseHeaders); 913 break; 914 } 915 } 916 917 analyzeHeaders(_response._responseHeaders); 918 919 _bodyDecoder.putNoCopy(partialBody.data); 920 921 if ( _verbosity >= 2 ) writefln("< %d bytes of body received", partialBody.length); 922 923 if ( _method == "HEAD" ) { 924 // HEAD response have ContentLength, but have no body 925 return; 926 } 927 928 while( true ) { 929 if ( _contentLength >= 0 && _contentReceived >= _contentLength ) { 930 debug(requests) trace("Body received."); 931 break; 932 } 933 if ( _unChunker && _unChunker.done ) { 934 break; 935 } 936 if ( _useStreaming && _response._responseBody.length && !redirectCodes.canFind(_response.code) ) { 937 debug(requests) trace("streaming requested"); 938 _response.receiveAsRange.activated = true; 939 _response.receiveAsRange.data = _response._responseBody.data; 940 _response.receiveAsRange.read = delegate ubyte[] () { 941 while(true) { 942 // check if we received everything we need 943 if ( ( _unChunker && _unChunker.done ) 944 || !_stream.isConnected() 945 || (_contentLength > 0 && _contentReceived >= _contentLength) ) 946 { 947 debug(requests) trace("streaming_in receive completed"); 948 _bodyDecoder.flush(); 949 return _bodyDecoder.get(); 950 } 951 // have to continue 952 auto b = new ubyte[_bufferSize]; 953 try { 954 read = _stream.receive(b); 955 } 956 catch (Exception e) { 957 throw new RequestException("streaming_in error reading from socket", __FILE__, __LINE__, e); 958 } 959 debug(requests) tracef("streaming_in received %d bytes", read); 960 961 if ( read == 0 ) { 962 debug(requests) tracef("streaming_in: server closed connection"); 963 _bodyDecoder.flush(); 964 return _bodyDecoder.get(); 965 } 966 967 if ( verbosity>=3 ) { 968 writeln(b[0..read].dump.join("\n")); 969 } 970 971 _contentReceived += read; 972 _bodyDecoder.putNoCopy(b[0..read]); 973 auto res = _bodyDecoder.getNoCopy(); 974 if ( res.length == 0 ) { 975 // there were nothing to produce (beginning of the chunk or no decompressed data) 976 continue; 977 } 978 if (res.length == 1) { 979 return res[0]; 980 } 981 // 982 // I'd like to "return _bodyDecoder.getNoCopy().join;" but if is slower 983 // 984 auto total = res.map!(b=>b.length).sum; 985 // create buffer for joined bytes 986 ubyte[] joined = new ubyte[total]; 987 size_t p; 988 // memcopy 989 foreach(ref _; res) { 990 joined[p .. p + _.length] = _; 991 p += _.length; 992 } 993 return joined; 994 } 995 assert(0); 996 }; 997 // we prepared for streaming 998 return; 999 } 1000 1001 auto b = new ubyte[_bufferSize]; 1002 read = _stream.receive(b); 1003 1004 if ( read == 0 ) { 1005 debug(requests) trace("read done"); 1006 break; 1007 } 1008 if ( _verbosity >= 2 ) { 1009 writefln("< %d bytes of body received", read); 1010 } 1011 1012 if ( verbosity>=3 ) { 1013 writeln(b[0..read].dump.join("\n")); 1014 } 1015 1016 debug(requests) tracef("read: %d", read); 1017 _contentReceived += read; 1018 if ( _maxContentLength && _contentReceived > _maxContentLength ) { 1019 throw new RequestException("ContentLength > maxContentLength (%d>%d)". 1020 format(_contentLength, _maxContentLength)); 1021 } 1022 1023 _bodyDecoder.putNoCopy(b[0..read]); // send buffer to all decoders 1024 1025 _bodyDecoder.getNoCopy. // fetch result and place to body 1026 each!(b => _response._responseBody.putNoCopy(b)); 1027 1028 debug(requests) tracef("receivedTotal: %d, contentLength: %d, bodyLength: %d", _contentReceived, _contentLength, _response._responseBody.length); 1029 1030 } 1031 _bodyDecoder.flush(); 1032 _response._responseBody.putNoCopy(_bodyDecoder.get()); 1033 } 1034 private bool serverClosedKeepAliveConnection() pure @safe nothrow { 1035 return _response._responseHeaders.length == 0 && _keepAlive; 1036 } 1037 private bool isIdempotent(in string method) pure @safe nothrow { 1038 return ["GET", "HEAD"].canFind(method); 1039 } 1040 /// 1041 /// Send multipart for request. 1042 /// You would like to use this method for sending large portions of mixed data or uploading files to forms. 1043 /// Content of the posted form consist of sources. Each source have at least name and value (can be string-like object or opened file, see more docs for MultipartForm struct) 1044 /// Params: 1045 /// url = url 1046 /// sources = array of sources. 1047 HTTPResponse exec(string method="POST")(string url, MultipartForm sources) { 1048 import std.uuid; 1049 import std.file; 1050 1051 if ( _response && _response._receiveAsRange.activated && _stream && _stream.isConnected ) { 1052 _stream.close(); 1053 } 1054 // 1055 // application/json 1056 // 1057 bool restartedRequest = false; 1058 1059 _method = method; 1060 1061 _response = new HTTPResponse; 1062 checkURL(url); 1063 _response.uri = _uri; 1064 _response.finalURI = _uri; 1065 1066 connect: 1067 _contentReceived = 0; 1068 _response._startedAt = Clock.currTime; 1069 setupConnection(); 1070 1071 if ( !_stream.isConnected() ) { 1072 return _response; 1073 } 1074 _response._connectedAt = Clock.currTime; 1075 1076 Appender!string req; 1077 req.put(requestString()); 1078 1079 string boundary = randomUUID().toString; 1080 string[] partHeaders; 1081 size_t contentLength; 1082 1083 foreach(ref part; sources._sources) { 1084 string h = "--" ~ boundary ~ "\r\n"; 1085 string disposition = `form-data; name="%s"`.format(part.name); 1086 string optionals = part. 1087 parameters.byKeyValue(). 1088 filter!(p => p.key!="Content-Type"). 1089 map! (p => "%s=%s".format(p.key, p.value)). 1090 join("; "); 1091 1092 h ~= `Content-Disposition: ` ~ [disposition, optionals].join("; ") ~ "\r\n"; 1093 1094 auto contentType = "Content-Type" in part.parameters; 1095 if ( contentType ) { 1096 h ~= "Content-Type: " ~ *contentType ~ "\r\n"; 1097 } 1098 1099 h ~= "\r\n"; 1100 partHeaders ~= h; 1101 contentLength += h.length + part.input.getSize() + "\r\n".length; 1102 } 1103 contentLength += "--".length + boundary.length + "--\r\n".length; 1104 1105 auto h = requestHeaders(); 1106 safeSetHeader(h, _userHeaders.ContentType, "Content-Type", "multipart/form-data; boundary=" ~ boundary); 1107 safeSetHeader(h, _userHeaders.ContentLength, "Content-Length", to!string(contentLength)); 1108 1109 h.byKeyValue. 1110 map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n"). 1111 each!(h => req.put(h)); 1112 req.put("\r\n"); 1113 1114 debug(requests) trace(req.data); 1115 if ( _verbosity >= 1 ) req.data.splitLines.each!(a => writeln("> " ~ a)); 1116 1117 try { 1118 _stream.send(req.data()); 1119 foreach(ref source; sources._sources) { 1120 debug(requests) tracef("sending part headers <%s>", partHeaders.front); 1121 _stream.send(partHeaders.front); 1122 partHeaders.popFront; 1123 while (true) { 1124 auto chunk = source.input.read(); 1125 if ( chunk.length <= 0 ) { 1126 break; 1127 } 1128 _stream.send(chunk); 1129 } 1130 _stream.send("\r\n"); 1131 } 1132 _stream.send("--" ~ boundary ~ "--\r\n"); 1133 _response._requestSentAt = Clock.currTime; 1134 receiveResponse(); 1135 _response._finishedAt = Clock.currTime; 1136 } 1137 catch (NetworkException e) { 1138 errorf("Error sending request: ", e.msg); 1139 return _response; 1140 } 1141 if ( _useStreaming ) { 1142 if ( _response._receiveAsRange.activated ) { 1143 debug(requests) trace("streaming_in activated"); 1144 return _response; 1145 } else { 1146 _response._receiveAsRange.data = _response.responseBody.data; 1147 } 1148 } 1149 auto connection = "connection" in _response._responseHeaders; 1150 if ( !connection || *connection == "close" ) { 1151 debug(requests) tracef("Closing connection because of 'Connection: close' or no 'Connection' header"); 1152 _stream.close(); 1153 } 1154 if ( canFind(redirectCodes, _response.code) && followRedirectResponse() ) { 1155 if ( _method != "GET" && _response.code != 307 && _response.code != 308 ) { 1156 // 307 and 308 do not change method 1157 return this.get(); 1158 } 1159 goto connect; 1160 } 1161 _response._history = _history; 1162 /// 1163 return _response; 1164 } 1165 1166 // we use this if we send from ubyte[][] and user provided Content-Length 1167 private void sendFlattenContent(T)(NetworkStream _stream, T content) { 1168 while ( !content.empty ) { 1169 auto chunk = content.front; 1170 _stream.send(chunk); 1171 content.popFront; 1172 } 1173 debug(requests) tracef("sent"); 1174 } 1175 // we use this if we send from ubyte[][] as chunked content 1176 private void sendChunkedContent(T)(NetworkStream _stream, T content) { 1177 while ( !content.empty ) { 1178 auto chunk = content.front; 1179 auto chunkHeader = "%x\r\n".format(chunk.length); 1180 debug(requests) tracef("sending %s%s", chunkHeader, chunk); 1181 _stream.send(chunkHeader); 1182 _stream.send(chunk); 1183 _stream.send("\r\n"); 1184 content.popFront; 1185 } 1186 debug(requests) tracef("sent"); 1187 _stream.send("0\r\n\r\n"); 1188 } 1189 /// 1190 /// POST/PUT/... data from some string(with Content-Length), or from range of strings/bytes (use Transfer-Encoding: chunked). 1191 /// When rank 1 (flat array) used as content it must have length. In that case "content" will be sent directly to network, and Content-Length headers will be added. 1192 /// If you are goung to send some range and do not know length at the moment when you start to send request, then you can send chunks of chars or ubyte. 1193 /// Try not to send too short chunks as this will put additional load on client and server. Chunks of length 2048 or 4096 are ok. 1194 /// 1195 /// Parameters: 1196 /// url = url 1197 /// content = string or input range 1198 /// contentType = content type 1199 /// Returns: 1200 /// Response 1201 /// Examples: 1202 /// --------------------------------------------------------------------------------------------------------- 1203 /// rs = rq.exec!"POST"("http://httpbin.org/post", "привiт, свiт!", "application/octet-stream"); 1204 /// 1205 /// auto s = lineSplitter("one,\ntwo,\nthree."); 1206 /// rs = rq.exec!"POST"("http://httpbin.org/post", s, "application/octet-stream"); 1207 /// 1208 /// auto s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; 1209 /// rs = rq.exec!"POST"("http://httpbin.org/post", s.representation.chunks(10), "application/octet-stream"); 1210 /// 1211 /// auto f = File("tests/test.txt", "rb"); 1212 /// rs = rq.exec!"POST"("http://httpbin.org/post", f.byChunk(3), "application/octet-stream"); 1213 /// -------------------------------------------------------------------------------------------------------- 1214 HTTPResponse exec(string method="POST", R)(string url, R content, string contentType="application/octet-stream") 1215 if ( (rank!R == 1) 1216 || (rank!R == 2 && isSomeChar!(Unqual!(typeof(content.front.front)))) 1217 || (rank!R == 2 && (is(Unqual!(typeof(content.front.front)) == ubyte))) 1218 ) { 1219 if ( _response && _response._receiveAsRange.activated && _stream && _stream.isConnected ) { 1220 _stream.close(); 1221 } 1222 // 1223 // application/json 1224 // 1225 bool restartedRequest = false; 1226 bool send_flat; 1227 1228 _method = method; 1229 1230 _response = new HTTPResponse; 1231 checkURL(url); 1232 _response.uri = _uri; 1233 _response.finalURI = _uri; 1234 1235 connect: 1236 _contentReceived = 0; 1237 _response._startedAt = Clock.currTime; 1238 setupConnection(); 1239 1240 if ( !_stream.isConnected() ) { 1241 return _response; 1242 } 1243 _response._connectedAt = Clock.currTime; 1244 1245 Appender!string req; 1246 req.put(requestString()); 1247 1248 auto h = requestHeaders; 1249 if ( contentType ) { 1250 safeSetHeader(h, _userHeaders.ContentType, "Content-Type", contentType); 1251 } 1252 static if ( rank!R == 1 ) { 1253 safeSetHeader(h, _userHeaders.ContentLength, "Content-Length", to!string(content.length)); 1254 } else { 1255 if ( _userHeaders.ContentLength ) { 1256 debug(requests) tracef("User provided content-length for chunked content"); 1257 send_flat = true; 1258 } else { 1259 h["Transfer-Encoding"] = "chunked"; 1260 send_flat = false; 1261 } 1262 } 1263 h.byKeyValue. 1264 map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n"). 1265 each!(h => req.put(h)); 1266 req.put("\r\n"); 1267 1268 debug(requests) trace(req.data); 1269 if ( _verbosity >= 1 ) { 1270 req.data.splitLines.each!(a => writeln("> " ~ a)); 1271 } 1272 1273 try { 1274 // send headers 1275 _stream.send(req.data()); 1276 // send body 1277 static if ( rank!R == 1) { 1278 _stream.send(content); 1279 } else { 1280 if ( send_flat ) { 1281 sendFlattenContent(_stream, content); 1282 } else { 1283 sendChunkedContent(_stream, content); 1284 } 1285 } 1286 _response._requestSentAt = Clock.currTime; 1287 receiveResponse(); 1288 _response._finishedAt = Clock.currTime; 1289 } catch (NetworkException e) { 1290 _stream.close(); 1291 throw new RequestException("Network error during data exchange"); 1292 } 1293 1294 if ( _useStreaming ) { 1295 if ( _response._receiveAsRange.activated ) { 1296 debug(requests) trace("streaming_in activated"); 1297 return _response; 1298 } else { 1299 _response._receiveAsRange.data = _response.responseBody.data; 1300 } 1301 } 1302 auto connection = "connection" in _response._responseHeaders; 1303 if ( !connection || *connection == "close" ) { 1304 debug(requests) tracef("Closing connection because of 'Connection: close' or no 'Connection' header"); 1305 _stream.close(); 1306 } 1307 if ( canFind(redirectCodes, _response.code) && followRedirectResponse() ) { 1308 if ( _method != "GET" && _response.code != 307 && _response.code != 308 ) { 1309 // 307 and 308 do not change method 1310 return this.get(); 1311 } 1312 goto connect; 1313 } 1314 /// 1315 _response._history = _history; 1316 return _response; 1317 } 1318 /// 1319 /// Send request with pameters. 1320 /// If used for POST or PUT requests then application/x-www-form-urlencoded used. 1321 /// Request parameters will be encoded into request string or placed in request body for POST/PUT 1322 /// requests. 1323 /// Parameters: 1324 /// url = url 1325 /// params = request parameters 1326 /// Returns: 1327 /// Response 1328 /// Examples: 1329 /// --------------------------------------------------------------------------------- 1330 /// rs = Request().exec!"GET"("http://httpbin.org/get", ["c":"d", "a":"b"]); 1331 /// --------------------------------------------------------------------------------- 1332 /// 1333 HTTPResponse exec(string method="GET")(string url = null, QueryParam[] params = null) { 1334 1335 if ( _response && _response._receiveAsRange.activated && _stream && _stream.isConnected ) { 1336 _stream.close(); 1337 } 1338 _method = method; 1339 _response = new HTTPResponse; 1340 _history.length = 0; 1341 bool restartedRequest = false; // True if this is restarted keepAlive request 1342 string encoded; 1343 1344 checkURL(url); 1345 _response.uri = _uri; 1346 _response.finalURI = _uri; 1347 1348 connect: 1349 _contentReceived = 0; 1350 _response._startedAt = Clock.currTime; 1351 setupConnection(); 1352 1353 if ( !_stream.isConnected() ) { 1354 return _response; 1355 } 1356 _response._connectedAt = Clock.currTime; 1357 1358 auto h = requestHeaders(); 1359 1360 Appender!string req; 1361 1362 switch (_method) { 1363 case "POST","PUT": 1364 encoded = params2query(params); 1365 safeSetHeader(h, _userHeaders.ContentType, "Content-Type", "application/x-www-form-urlencoded"); 1366 if ( encoded.length > 0) { 1367 safeSetHeader(h, _userHeaders.ContentLength, "Content-Length", to!string(encoded.length)); 1368 } 1369 req.put(requestString()); 1370 break; 1371 default: 1372 req.put(requestString(params)); 1373 } 1374 1375 h.byKeyValue. 1376 map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n"). 1377 each!(h => req.put(h)); 1378 req.put("\r\n"); 1379 if ( encoded ) { 1380 req.put(encoded); 1381 } 1382 1383 debug(requests) trace(req.data); 1384 1385 if ( _verbosity >= 1 ) req.data.splitLines.each!(a => writeln("> " ~ a)); 1386 // 1387 // Now send request and receive response 1388 // 1389 try { 1390 _stream.send(req.data()); 1391 _response._requestSentAt = Clock.currTime; 1392 receiveResponse(); 1393 _response._finishedAt = Clock.currTime; 1394 } 1395 catch (NetworkException e) { 1396 // On SEND this can means: 1397 // we started to send request to the server, but it closed connection because of keepalive timeout. 1398 // We have to restart request if possible. 1399 1400 // On RECEIVE - if we received something - then this exception is real and unexpected error. 1401 // If we didn't receive anything - we can restart request again as it can be 1402 if ( _response._responseHeaders.length != 0 ) { 1403 _stream.close(); 1404 throw new RequestException("Unexpected network error"); 1405 } 1406 } 1407 1408 if ( serverClosedKeepAliveConnection() 1409 && !restartedRequest 1410 && isIdempotent(_method) 1411 ) { 1412 /// 1413 /// We didn't receive any data (keepalive connectioin closed?) 1414 /// and we can restart this request. 1415 /// Go ahead. 1416 /// 1417 debug(requests) tracef("Server closed keepalive connection"); 1418 _stream.close(); 1419 restartedRequest = true; 1420 goto connect; 1421 } 1422 1423 if ( _useStreaming ) { 1424 if ( _response._receiveAsRange.activated ) { 1425 debug(requests) trace("streaming_in activated"); 1426 return _response; 1427 } else { 1428 // this can happen if whole response body received together with headers 1429 _response._receiveAsRange.data = _response.responseBody.data; 1430 } 1431 } 1432 1433 auto connection = "connection" in _response._responseHeaders; 1434 if ( !connection || *connection == "close" ) { 1435 debug(requests) tracef("Closing connection because of 'Connection: close' or no 'Connection' header"); 1436 _stream.close(); 1437 } 1438 if ( _verbosity >= 1 ) { 1439 writeln(">> Connect time: ", _response._connectedAt - _response._startedAt); 1440 writeln(">> Request send time: ", _response._requestSentAt - _response._connectedAt); 1441 writeln(">> Response recv time: ", _response._finishedAt - _response._requestSentAt); 1442 } 1443 if ( canFind(redirectCodes, _response.code) && followRedirectResponse() ) { 1444 if ( _method != "GET" && _response.code != 307 && _response.code != 308 ) { 1445 // 307 and 308 do not change method 1446 return this.get(); 1447 } 1448 goto connect; 1449 } 1450 /// 1451 _response._history = _history; 1452 return _response; 1453 } 1454 1455 /// WRAPPERS 1456 /// 1457 /// send file(s) using POST and multipart form. 1458 /// This wrapper will be deprecated, use post with MultipartForm - it is more general and clear. 1459 /// Parameters: 1460 /// url = url 1461 /// files = array of PostFile structures 1462 /// Returns: 1463 /// Response 1464 /// Each PostFile structure contain path to file, and optional field name and content type. 1465 /// If no field name provided, then basename of the file will be used. 1466 /// application/octet-stream is default when no content type provided. 1467 /// Example: 1468 /// --------------------------------------------------------------- 1469 /// PostFile[] files = [ 1470 /// {fileName:"tests/abc.txt", fieldName:"abc", contentType:"application/octet-stream"}, 1471 /// {fileName:"tests/test.txt"} 1472 /// ]; 1473 /// rs = rq.exec!"POST"("http://httpbin.org/post", files); 1474 /// --------------------------------------------------------------- 1475 /// 1476 HTTPResponse exec(string method="POST")(string url, PostFile[] files) if (method=="POST") { 1477 MultipartForm multipart; 1478 File[] toClose; 1479 foreach(ref f; files) { 1480 File file = File(f.fileName, "rb"); 1481 toClose ~= file; 1482 string fileName = f.fileName ? f.fileName : f.fieldName; 1483 string contentType = f.contentType ? f.contentType : "application/octetstream"; 1484 multipart.add(f.fieldName, new FormDataFile(file), ["filename":fileName, "Content-Type": contentType]); 1485 } 1486 auto res = exec!"POST"(url, multipart); 1487 toClose.each!"a.close"; 1488 return res; 1489 } 1490 /// 1491 /// exec request with parameters when you can use dictionary (when you have no duplicates in parameter names) 1492 /// Consider switch to exec(url, QueryParams) as it more generic and clear. 1493 /// Parameters: 1494 /// url = url 1495 /// params = dictionary with field names as keys and field values as values. 1496 /// Returns: 1497 /// Response 1498 HTTPResponse exec(string method="GET")(string url, string[string] params) { 1499 return exec!method(url, params.byKeyValue.map!(p => QueryParam(p.key, p.value)).array); 1500 } 1501 /// 1502 /// GET request. Simple wrapper over exec!"GET" 1503 /// Params: 1504 /// args = request parameters. see exec docs. 1505 /// 1506 HTTPResponse get(A...)(A args) { 1507 return exec!"GET"(args); 1508 } 1509 /// 1510 /// POST request. Simple wrapper over exec!"POST" 1511 /// Params: 1512 /// uri = endpoint uri 1513 /// args = request parameters. see exec docs. 1514 /// 1515 HTTPResponse post(A...)(string uri, A args) { 1516 return exec!"POST"(uri, args); 1517 } 1518 } 1519 1520 version(vibeD) { 1521 import std.json; 1522 package string httpTestServer() { 1523 return "http://httpbin.org/"; 1524 } 1525 package string fromJsonArrayToStr(JSONValue v) { 1526 return v.str; 1527 } 1528 } 1529 else { 1530 import std.json; 1531 package string httpTestServer() { 1532 return "http://127.0.0.1:8081/"; 1533 } 1534 package string fromJsonArrayToStr(JSONValue v) { 1535 return cast(string)(v.array.map!"cast(ubyte)a.integer".array); 1536 } 1537 } 1538 1539 1540 package unittest { 1541 import std.json; 1542 import std.array; 1543 1544 globalLogLevel(LogLevel.info); 1545 1546 string httpbinUrl = httpTestServer(); 1547 version(vibeD) { 1548 } 1549 else { 1550 import httpbin; 1551 auto server = httpbinApp(); 1552 server.start(); 1553 scope(exit) { 1554 server.stop(); 1555 } 1556 } 1557 HTTPRequest rq; 1558 HTTPResponse rs; 1559 info("Check GET"); 1560 URI uri = URI(httpbinUrl); 1561 rs = rq.get(httpbinUrl); 1562 assert(rs.code==200); 1563 assert(rs.responseBody.length > 0); 1564 assert(rq.format("%m|%h|%p|%P|%q|%U") == 1565 "GET|%s|%d|%s||%s" 1566 .format(uri.host, uri.port, uri.path, httpbinUrl)); 1567 info("Check GET with AA params"); 1568 { 1569 rs = HTTPRequest().get(httpbinUrl ~ "get", ["c":" d", "a":"b"]); 1570 assert(rs.code == 200); 1571 auto json = parseJSON(cast(string)rs.responseBody.data).object["args"].object; 1572 assert(json["c"].str == " d"); 1573 assert(json["a"].str == "b"); 1574 } 1575 info("Check POST files"); 1576 { 1577 import std.file; 1578 import std.path; 1579 auto tmpd = tempDir(); 1580 auto tmpfname = tmpd ~ dirSeparator ~ "request_test.txt"; 1581 auto f = File(tmpfname, "wb"); 1582 f.rawWrite("abcdefgh\n12345678\n"); 1583 f.close(); 1584 // files 1585 PostFile[] files = [ 1586 {fileName: tmpfname, fieldName:"abc", contentType:"application/octet-stream"}, 1587 {fileName: tmpfname} 1588 ]; 1589 rs = rq.post(httpbinUrl ~ "post", files); 1590 assert(rs.code==200); 1591 } 1592 info("Check POST chunked from file.byChunk"); 1593 { 1594 import std.file; 1595 import std.path; 1596 auto tmpd = tempDir(); 1597 auto tmpfname = tmpd ~ dirSeparator ~ "request_test.txt"; 1598 auto f = File(tmpfname, "wb"); 1599 f.rawWrite("abcdefgh\n12345678\n"); 1600 f.close(); 1601 f = File(tmpfname, "rb"); 1602 rs = rq.post(httpbinUrl ~ "post", f.byChunk(3), "application/octet-stream"); 1603 if (httpbinUrl != "http://httpbin.org/") { 1604 assert(rs.code==200); 1605 auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]); 1606 assert(data=="abcdefgh\n12345678\n"); 1607 } 1608 f.close(); 1609 } 1610 info("Check POST chunked from lineSplitter"); 1611 { 1612 auto s = lineSplitter("one,\ntwo,\nthree."); 1613 rs = rq.exec!"POST"(httpbinUrl ~ "post", s, "application/octet-stream"); 1614 if (httpbinUrl != "http://httpbin.org/") { 1615 assert(rs.code==200); 1616 auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]); 1617 assert(data=="one,two,three."); 1618 } 1619 } 1620 info("Check POST chunked from array"); 1621 { 1622 auto s = ["one,", "two,", "three."]; 1623 rs = rq.post(httpbinUrl ~ "post", s, "application/octet-stream"); 1624 if (httpbinUrl != "http://httpbin.org/") { 1625 assert(rs.code==200); 1626 auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]); 1627 assert(data=="one,two,three."); 1628 } 1629 } 1630 info("Check POST chunked using std.range.chunks()"); 1631 { 1632 auto s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; 1633 rs = rq.post(httpbinUrl ~ "post", s.representation.chunks(10), "application/octet-stream"); 1634 if (httpbinUrl != "http://httpbin.org/") { 1635 assert(rs.code==200); 1636 auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody.data).object["data"]); 1637 assert(data==s); 1638 } 1639 } 1640 info("Check POST from QueryParams"); 1641 { 1642 rs = rq.post(httpbinUrl ~ "post", queryParams("name[]", "first", "name[]", 2)); 1643 assert(rs.code==200); 1644 auto data = parseJSON(cast(string)rs.responseBody).object["form"].object; 1645 string[] a; 1646 try { 1647 a = to!(string[])(data["name[]"].str); 1648 } 1649 catch (JSONException e) { 1650 a = data["name[]"].array.map!"a.str".array; 1651 } 1652 assert(equal(["first", "2"], a)); 1653 } 1654 info("Check POST from AA"); 1655 { 1656 rs = rq.post(httpbinUrl ~ "post", ["a":"b ", "c":"d"]); 1657 assert(rs.code==200); 1658 auto form = parseJSON(cast(string)rs.responseBody.data).object["form"].object; 1659 assert(form["a"].str == "b "); 1660 assert(form["c"].str == "d"); 1661 } 1662 info("Check POST json"); 1663 { 1664 rs = rq.post(httpbinUrl ~ "post?b=x", `{"a":"a b", "c":[1,2,3]}`, "application/json"); 1665 assert(rs.code==200); 1666 auto json = parseJSON(cast(string)rs.responseBody).object["args"].object; 1667 assert(json["b"].str == "x"); 1668 json = parseJSON(cast(string)rs.responseBody).object["json"].object; 1669 assert(json["a"].str == "a b"); 1670 assert(json["c"].array.map!(a=>a.integer).array == [1,2,3]); 1671 } 1672 info("Check HEAD"); 1673 rs = rq.exec!"HEAD"(httpbinUrl); 1674 assert(rs.code==200); 1675 info("Check DELETE"); 1676 rs = rq.exec!"DELETE"(httpbinUrl ~ "delete"); 1677 assert(rs.code==200); 1678 info("Check PUT"); 1679 rs = rq.exec!"PUT"(httpbinUrl ~ "put", `{"a":"b", "c":[1,2,3]}`, "application/json"); 1680 assert(rs.code==200); 1681 assert(parseJSON(cast(string)rs.responseBody).object["json"].object["a"].str=="b"); 1682 info("Check PATCH"); 1683 rs = rq.exec!"PATCH"(httpbinUrl ~ "patch", "привiт, свiт!", "application/octet-stream"); 1684 assert(rs.code==200); 1685 info("Check compressed content"); 1686 rs = rq.get(httpbinUrl ~ "gzip"); 1687 assert(rs.code==200); 1688 bool gzipped = parseJSON(cast(string)rs.responseBody).object["gzipped"].type == JSON_TYPE.TRUE; 1689 assert(gzipped); 1690 info("gzip - ok"); 1691 rs = rq.get(httpbinUrl ~ "deflate"); 1692 assert(rs.code==200); 1693 bool deflated = parseJSON(cast(string)rs.responseBody).object["deflated"].type == JSON_TYPE.TRUE; 1694 assert(deflated); 1695 info("deflate - ok"); 1696 1697 info("Check redirects"); 1698 rs = rq.get(httpbinUrl ~ "relative-redirect/2"); 1699 assert(rs.history.length == 2); 1700 assert(rs.code==200); 1701 rs = rq.get(httpbinUrl ~ "absolute-redirect/2"); 1702 assert(rs.history.length == 2); 1703 assert(rs.code==200); 1704 1705 rq.maxRedirects = 2; 1706 assertThrown!MaxRedirectsException(rq.get(httpbinUrl ~ "absolute-redirect/3")); 1707 1708 info("Check cookie"); 1709 { 1710 rs = rq.get(httpbinUrl ~ "cookies/set?A=abcd&b=cdef"); 1711 assert(rs.code == 200); 1712 auto json = parseJSON(cast(string)rs.responseBody.data).object["cookies"].object; 1713 assert(json["A"].str == "abcd"); 1714 assert(json["b"].str == "cdef"); 1715 foreach(c; rq.cookie) { 1716 final switch(c.attr) { 1717 case "A": 1718 assert(c.value == "abcd"); 1719 break; 1720 case "b": 1721 assert(c.value == "cdef"); 1722 break; 1723 } 1724 } 1725 } 1726 info("Check chunked content"); 1727 rs = rq.get(httpbinUrl ~ "range/1024"); 1728 assert(rs.code==200); 1729 assert(rs.responseBody.length==1024); 1730 1731 info("Check basic auth"); 1732 rq.authenticator = new BasicAuthentication("user", "passwd"); 1733 rs = rq.get(httpbinUrl ~ "basic-auth/user/passwd"); 1734 assert(rs.code==200); 1735 1736 info("Check limits"); 1737 rq = HTTPRequest(); 1738 rq.maxContentLength = 1; 1739 assertThrown!RequestException(rq.get(httpbinUrl)); 1740 rq = HTTPRequest(); 1741 rq.maxHeadersLength = 1; 1742 assertThrown!RequestException(rq.get(httpbinUrl)); 1743 rq = HTTPRequest(); 1744 info("Check POST multiPartForm"); 1745 { 1746 /// This is example on usage files with MultipartForm data. 1747 /// For this example we have to create files which will be sent. 1748 import std.file; 1749 import std.path; 1750 /// preapare files 1751 auto tmpd = tempDir(); 1752 auto tmpfname1 = tmpd ~ dirSeparator ~ "request_test1.txt"; 1753 auto f = File(tmpfname1, "wb"); 1754 f.rawWrite("file1 content\n"); 1755 f.close(); 1756 auto tmpfname2 = tmpd ~ dirSeparator ~ "request_test2.txt"; 1757 f = File(tmpfname2, "wb"); 1758 f.rawWrite("file2 content\n"); 1759 f.close(); 1760 /// 1761 /// Ok, files ready. 1762 /// Now we will prepare Form data 1763 /// 1764 File f1 = File(tmpfname1, "rb"); 1765 File f2 = File(tmpfname2, "rb"); 1766 scope(exit) { 1767 f1.close(); 1768 f2.close(); 1769 } 1770 /// 1771 /// for each part we have to set field name, source (ubyte array or opened file) and optional filename and content-type 1772 /// 1773 MultipartForm mForm = MultipartForm(). 1774 add(formData("Field1", cast(ubyte[])"form field from memory")). 1775 add(formData("Field2", cast(ubyte[])"file field from memory", ["filename":"data2"])). 1776 add(formData("File1", f1, ["filename":"file1", "Content-Type": "application/octet-stream"])). 1777 add(formData("File2", f2, ["filename":"file2", "Content-Type": "application/octet-stream"])); 1778 /// everything ready, send request 1779 rs = rq.post(httpbinUrl ~ "post", mForm); 1780 } 1781 info("Check exception handling, error messages and timeous are OK"); 1782 rq.timeout = 1.seconds; 1783 assertThrown!TimeoutException(rq.get(httpbinUrl ~ "delay/3")); 1784 // assertThrown!ConnectError(rq.get("http://0.0.0.0:65000/")); 1785 // assertThrown!ConnectError(rq.get("http://1.1.1.1/")); 1786 // assertThrown!ConnectError(rq.get("http://gkhgkhgkjhgjhgfjhgfjhgf/")); 1787 }