1 module requests.http; 2 3 private: 4 import std.algorithm; 5 import std.array; 6 import std.ascii; 7 import std.conv; 8 import std.datetime; 9 import std.exception; 10 import std.format; 11 import std.stdio; 12 import std.range; 13 import std..string; 14 import std.traits; 15 import std.typecons; 16 import std.bitmanip; 17 import std.experimental.logger; 18 import core.thread; 19 20 import requests.streams; 21 import requests.uri; 22 import requests.utils; 23 import requests.base; 24 25 static immutable ushort[] redirectCodes = [301, 302, 303, 307, 308]; 26 static immutable uint defaultBufferSize = 12*1024; 27 28 static immutable string[string] proxies; 29 static this() { 30 import std.process; 31 proxies["http"] = environment.get("http_proxy", environment.get("HTTP_PROXY")); 32 proxies["https"] = environment.get("https_proxy", environment.get("HTTPS_PROXY")); 33 proxies["all"] = environment.get("all_proxy", environment.get("ALL_PROXY")); 34 foreach(p; proxies.byKey()) { 35 if (proxies[p] is null) { 36 continue; 37 } 38 URI u = URI(proxies[p]); 39 } 40 } 41 42 public class MaxRedirectsException: Exception { 43 this(string message, string file = __FILE__, size_t line = __LINE__, Throwable next = null) @safe pure nothrow { 44 super(message, file, line, next); 45 } 46 } 47 48 /** 49 * Basic authentication. 50 * Adds $(B Authorization: Basic) header to request. 51 */ 52 public class BasicAuthentication: Auth { 53 private { 54 string _username, _password; 55 string[] _domains; 56 } 57 /// Constructor. 58 /// Params: 59 /// username = username 60 /// password = password 61 /// domains = not used now 62 /// 63 this(string username, string password, string[] domains = []) { 64 _username = username; 65 _password = password; 66 _domains = domains; 67 } 68 override string[string] authHeaders(string domain) { 69 import std.base64; 70 string[string] auth; 71 auth["Authorization"] = "Basic " ~ to!string(Base64.encode(cast(ubyte[])"%s:%s".format(_username, _password))); 72 return auth; 73 } 74 override string userName() { 75 return _username; 76 } 77 override string password() { 78 return _password; 79 } 80 } 81 /// 82 /// 83 /// 84 public auto queryParams(T...)(T params) pure nothrow @safe 85 { 86 static assert (T.length % 2 == 0, "wrong args count"); 87 88 QueryParam[] output; 89 output.reserve = T.length / 2; 90 91 void queryParamsHelper(T...)(T params, ref QueryParam[] output) 92 { 93 static if (T.length > 0) 94 { 95 output ~= QueryParam(params[0].to!string, params[1].to!string); 96 queryParamsHelper(params[2..$], output); 97 } 98 } 99 100 queryParamsHelper(params, output); 101 return output; 102 } 103 104 /// 105 /// Response - result of request execution. 106 /// 107 /// Response.code - response HTTP code. 108 /// Response.status_line - received HTTP status line. 109 /// Response.responseHeaders - received headers. 110 /// Response.responseBody - container for received body 111 /// Response.history - for redirected responses contain all history 112 /// 113 public class HTTPResponse : Response { 114 private { 115 string _status_line; 116 117 HTTPResponse[] _history; // redirects history 118 119 mixin(Setter!string("status_line")); 120 } 121 122 ~this() { 123 _responseHeaders = null; 124 _history.length = 0; 125 } 126 127 mixin(Getter!string("status_line")); 128 @property final string[string] responseHeaders() @safe @nogc nothrow { 129 return _responseHeaders; 130 } 131 @property final HTTPResponse[] history() @safe @nogc nothrow { 132 return _history; 133 } 134 135 @property auto getStats() const pure @safe { 136 alias statTuple = Tuple!(Duration, "connectTime", 137 Duration, "sendTime", 138 Duration, "recvTime"); 139 statTuple stat; 140 stat.connectTime = _connectedAt - _startedAt; 141 stat.sendTime = _requestSentAt - _connectedAt; 142 stat.recvTime = _finishedAt - _requestSentAt; 143 return stat; 144 } 145 } 146 /** 147 * Struct to send multiple files in POST request. 148 */ 149 public struct PostFile { 150 /// Path to the file to send. 151 string fileName; 152 /// Name of the field (if empty - send file base name) 153 string fieldName; 154 /// contentType of the file if not empty 155 string contentType; 156 } 157 /// 158 /// This is File-like interface for sending data to multipart fotms 159 /// 160 public interface FiniteReadable { 161 /// size of the content 162 abstract ulong getSize(); 163 /// file-like read() 164 abstract ubyte[] read(); 165 } 166 /// 167 /// Helper to create form elements from File. 168 /// Params: 169 /// name = name of the field in form 170 /// f = opened std.stio.File to send to server 171 /// parameters = optional parameters (most important are "filename" and "Content-Type") 172 /// 173 public auto formData(string name, File f, string[string] parameters = null) { 174 return MultipartForm.FormData(name, new FormDataFile(f), parameters); 175 } 176 /// 177 /// Helper to create form elements from ubyte[]. 178 /// Params: 179 /// name = name of the field in form 180 /// b = data to send to server 181 /// parameters = optional parameters (can be "filename" and "Content-Type") 182 /// 183 public auto formData(string name, ubyte[] b, string[string] parameters = null) { 184 return MultipartForm.FormData(name, new FormDataBytes(b), parameters); 185 } 186 public auto formData(string name, string b, string[string] parameters = null) { 187 return MultipartForm.FormData(name, new FormDataBytes(b.dup.representation), parameters); 188 } 189 public class FormDataBytes : FiniteReadable { 190 private { 191 ulong _size; 192 ubyte[] _data; 193 size_t _offset; 194 bool _exhausted; 195 } 196 this(ubyte[] data) { 197 _data = data; 198 _size = data.length; 199 } 200 final override ulong getSize() { 201 return _size; 202 } 203 final override ubyte[] read() { 204 enforce( !_exhausted, "You can't read froum exhausted source" ); 205 size_t toRead = min(defaultBufferSize, _size - _offset); 206 auto result = _data[_offset.._offset+toRead]; 207 _offset += toRead; 208 if ( toRead == 0 ) { 209 _exhausted = true; 210 } 211 return result; 212 } 213 } 214 public class FormDataFile : FiniteReadable { 215 import std.file; 216 private { 217 File _fileHandle; 218 ulong _fileSize; 219 size_t _processed; 220 bool _exhausted; 221 } 222 this(File file) { 223 import std.file; 224 _fileHandle = file; 225 _fileSize = std.file.getSize(file.name); 226 } 227 final override ulong getSize() pure nothrow @safe { 228 return _fileSize; 229 } 230 final override ubyte[] read() { 231 enforce( !_exhausted, "You can't read froum exhausted source" ); 232 auto b = new ubyte[defaultBufferSize]; 233 auto r = _fileHandle.rawRead(b); 234 auto toRead = min(r.length, _fileSize - _processed); 235 if ( toRead == 0 ) { 236 _exhausted = true; 237 } 238 _processed += toRead; 239 return r[0..toRead]; 240 } 241 } 242 /// 243 /// This struct used to bulld POST's to forms. 244 /// Each part have name and data. data is something that can be read-ed and have size. 245 /// For example this can be string-like object (wrapped for reading) or opened File. 246 /// 247 public struct MultipartForm { 248 package struct FormData { 249 FiniteReadable input; 250 string name; 251 string[string] parameters; 252 this(string name, FiniteReadable i, string[string] parameters = null) { 253 this.input = i; 254 this.name = name; 255 this.parameters = parameters; 256 } 257 } 258 259 private FormData[] _sources; 260 auto add(FormData d) { 261 _sources ~= d; 262 return this; 263 } 264 auto add(string name, FiniteReadable i, string[string]parameters = null) { 265 _sources ~= FormData(name, i, parameters); 266 return this; 267 } 268 } 269 /// 270 271 /// 272 /// Request. 273 /// Configurable parameters: 274 /// $(B method) - string, method to use (GET, POST, ...) 275 /// $(B headers) - string[string], add any additional headers you'd like to send. 276 /// $(B authenticator) - class Auth, class to send auth headers. 277 /// $(B keepAlive) - bool, set true for keepAlive requests. default true. 278 /// $(B maxRedirects) - uint, maximum number of redirects. default 10. 279 /// $(B maxHeadersLength) - size_t, maximum length of server response headers. default = 32KB. 280 /// $(B maxContentLength) - size_t, maximun content length. delault - 0 = unlimited. 281 /// $(B bufferSize) - size_t, send and receive buffer size. default = 16KB. 282 /// $(B verbosity) - uint, level of verbosity(0 - nothing, 1 - headers, 2 - headers and body progress). default = 0. 283 /// $(B proxy) - string, set proxy url if needed. default - null. 284 /// $(B cookie) - Tuple Cookie, Read/Write cookie You can get cookie setted by server, or set cookies before doing request. 285 /// $(B timeout) - Duration, Set timeout value for connect/receive/send. 286 /// 287 public struct HTTPRequest { 288 private { 289 struct _UH { 290 // flags for each important header, added by user using addHeaders 291 mixin(bitfields!( 292 bool, "Host", 1, 293 bool, "UserAgent", 1, 294 bool, "ContentLength", 1, 295 bool, "Connection", 1, 296 bool, "AcceptEncoding", 1, 297 bool, "ContentType", 1, 298 bool, "Cookie", 1, 299 uint, "", 1 300 )); 301 } 302 string _method = "GET"; 303 URI _uri; 304 string[string] _headers; 305 string[] _filteredHeaders; 306 Auth _authenticator; 307 bool _keepAlive = true; 308 uint _maxRedirects = 10; 309 size_t _maxHeadersLength = 32 * 1024; // 32 KB 310 size_t _maxContentLength; // 0 - Unlimited 311 string _proxy; 312 uint _verbosity = 0; // 0 - no output, 1 - headers, 2 - headers+body info 313 Duration _timeout = 30.seconds; 314 size_t _bufferSize = defaultBufferSize; // 16k 315 bool _useStreaming; // return iterator instead of completed request 316 317 NetworkStream _stream; 318 HTTPResponse[] _history; // redirects history 319 DataPipe!ubyte _bodyDecoder; 320 DecodeChunked _unChunker; 321 long _contentLength; 322 long _contentReceived; 323 Cookie[] _cookie; 324 SSLOptions _sslOptions; 325 string _bind; 326 _UH _userHeaders; 327 } 328 package HTTPResponse _response; 329 330 mixin(Getter_Setter!string ("method")); 331 mixin(Getter_Setter!bool ("keepAlive")); 332 mixin(Getter_Setter!size_t ("maxContentLength")); 333 mixin(Getter_Setter!size_t ("maxHeadersLength")); 334 mixin(Getter_Setter!size_t ("bufferSize")); 335 mixin(Getter_Setter!uint ("maxRedirects")); 336 mixin(Getter_Setter!uint ("verbosity")); 337 mixin(Getter!string ("proxy")); 338 mixin(Getter_Setter!Duration ("timeout")); 339 mixin(Setter!Auth ("authenticator")); 340 mixin(Getter_Setter!bool ("useStreaming")); 341 mixin(Getter!long ("contentLength")); 342 mixin(Getter!long ("contentReceived")); 343 mixin(Getter_Setter!SSLOptions ("sslOptions")); 344 mixin(Getter_Setter!string ("bind")); 345 346 @property void sslSetVerifyPeer(bool v) pure @safe nothrow @nogc { 347 _sslOptions.setVerifyPeer(v); 348 } 349 @property void sslSetKeyFile(string p, SSLOptions.filetype t = SSLOptions.filetype.pem) pure @safe nothrow @nogc { 350 _sslOptions.setKeyFile(p, t); 351 } 352 @property void sslSetCertFile(string p, SSLOptions.filetype t = SSLOptions.filetype.pem) pure @safe nothrow @nogc { 353 _sslOptions.setCertFile(p, t); 354 } 355 @property void sslSetCaCert(string path) pure @safe nothrow @nogc { 356 _sslOptions.setCaCert(path); 357 } 358 @property final void cookie(Cookie[] s) pure @safe @nogc nothrow { 359 _cookie = s; 360 } 361 @property final void proxy(string v) { 362 if ( v != _proxy ) { 363 if ( _stream && _stream.isOpen() ) { 364 debug(requests) tracef("Close connection because we reset proxy"); 365 _stream.close(); 366 } 367 } 368 _proxy = v; 369 } 370 @property final Cookie[] cookie() pure @safe @nogc nothrow { 371 return _cookie; 372 } 373 374 this(string uri) { 375 _uri = URI(uri); 376 } 377 ~this() { 378 if ( _stream && _stream.isConnected) { 379 _stream.close(); 380 } 381 _stream = null; 382 _headers = null; 383 _authenticator = null; 384 _history = null; 385 _bodyDecoder = null; 386 _unChunker = null; 387 } 388 string toString() const { 389 return "HTTPRequest(%s, %s)".format(_method, _uri.uri()); 390 } 391 string format(string fmt) const { 392 import std.array; 393 import std.stdio; 394 auto a = appender!string(); 395 auto f = FormatSpec!char(fmt); 396 while (f.writeUpToNextSpec(a)) { 397 switch(f.spec) { 398 case 'h': 399 // Remote hostname. 400 a.put(_uri.host); 401 break; 402 case 'm': 403 // method. 404 a.put(_method); 405 break; 406 case 'p': 407 // Remote port. 408 a.put("%d".format(_uri.port)); 409 break; 410 case 'P': 411 // Path 412 a.put(_uri.path); 413 break; 414 case 'q': 415 // query parameters supplied with url. 416 a.put(_uri.query); 417 break; 418 case 'U': 419 a.put(_uri.uri()); 420 break; 421 default: 422 throw new FormatException("Unknown Request format spec " ~ f.spec); 423 } 424 } 425 return a.data(); 426 } 427 string select_proxy(string scheme) { 428 if ( _proxy is null && proxies.length == 0 ) { 429 debug(requests) tracef("proxy=null"); 430 return null; 431 } 432 if ( _proxy ) { 433 debug(requests) tracef("proxy=%s", _proxy); 434 return _proxy; 435 } 436 auto p = scheme in proxies; 437 if ( p !is null && *p != "") { 438 debug(requests) tracef("proxy=%s", *p); 439 return *p; 440 } 441 p = "all" in proxies; 442 if ( p !is null && *p != "") { 443 debug(requests) tracef("proxy=%s", *p); 444 return *p; 445 } 446 debug(requests) tracef("proxy=null"); 447 return null; 448 } 449 void clearHeaders() { 450 _headers = null; 451 } 452 @property void uri(in URI newURI) { 453 handleURLChange(_uri, newURI); 454 _uri = newURI; 455 } 456 /// Add headers to request 457 /// Params: 458 /// headers = headers to send. 459 void addHeaders(in string[string] headers) { 460 foreach(pair; headers.byKeyValue) { 461 string _h = pair.key; 462 switch(toLower(_h)) { 463 case "host": 464 _userHeaders.Host = true; 465 break; 466 case "user-agent": 467 _userHeaders.UserAgent = true; 468 break; 469 case "content-length": 470 _userHeaders.ContentLength = true; 471 break; 472 case "content-type": 473 _userHeaders.ContentType = true; 474 break; 475 case "connection": 476 _userHeaders.Connection = true; 477 break; 478 case "cookie": 479 _userHeaders.Cookie = true; 480 break; 481 default: 482 break; 483 } 484 _headers[pair.key] = pair.value; 485 } 486 } 487 private void safeSetHeader(ref string[string] headers, bool userAdded, string h, string v) pure @safe { 488 if ( !userAdded ) { 489 headers[h] = v; 490 } 491 } 492 /// Remove headers from request 493 /// Params: 494 /// headers = headers to remove. 495 void removeHeaders(in string[] headers) pure { 496 _filteredHeaders ~= headers; 497 } 498 /// 499 /// compose headers to send 500 /// 501 private string[string] requestHeaders() { 502 503 string[string] generatedHeaders; 504 505 if ( _authenticator ) { 506 _authenticator. 507 authHeaders(_uri.host). 508 byKeyValue. 509 each!(pair => generatedHeaders[pair.key] = pair.value); 510 } 511 512 _headers.byKey.each!(h => generatedHeaders[h] = _headers[h]); 513 514 safeSetHeader(generatedHeaders, _userHeaders.AcceptEncoding, "Accept-Encoding", "gzip,deflate"); 515 safeSetHeader(generatedHeaders, _userHeaders.UserAgent, "User-Agent", "dlang-requests"); 516 safeSetHeader(generatedHeaders, _userHeaders.Connection, "Connection", _keepAlive?"Keep-Alive":"Close"); 517 518 if ( !_userHeaders.Host ) 519 { 520 generatedHeaders["Host"] = _uri.host; 521 if ( _uri.scheme !in standard_ports || _uri.port != standard_ports[_uri.scheme] ) { 522 generatedHeaders["Host"] ~= ":%d".format(_uri.port); 523 } 524 } 525 526 if ( _cookie.length && !_userHeaders.Cookie ) { 527 auto cs = _cookie. 528 filter!(c => _uri.path.pathMatches(c.path) && _uri.host.domainMatches(c.domain)). 529 map!(c => "%s=%s".format(c.attr, c.value)). 530 joiner(";"); 531 if ( ! cs.empty ) 532 { 533 generatedHeaders["Cookie"] = to!string(cs); 534 } 535 } 536 537 _filteredHeaders.each!(h => generatedHeaders.remove(h)); 538 539 return generatedHeaders; 540 } 541 /// 542 /// Build request string. 543 /// Handle proxy and query parameters. 544 /// 545 private @property string requestString(QueryParam[] params = null) { 546 string actual_proxy = select_proxy(_uri.scheme); 547 if ( actual_proxy && _uri.scheme != "https" ) { 548 return "%s %s HTTP/1.1\r\n".format(_method, _uri.uri); 549 } 550 auto query = _uri.query.dup; 551 if ( params ) { 552 query ~= "&" ~ params2query(params); 553 if ( query[0] != '?' ) { 554 query = "?" ~ query; 555 } 556 } 557 return "%s %s%s HTTP/1.1\r\n".format(_method, _uri.path, query); 558 } 559 /// 560 /// encode parameters and build query part of the url 561 /// 562 private static string params2query(in QueryParam[] params) pure @safe { 563 return params. 564 map!(a => "%s=%s".format(a.key.urlEncoded, a.value.urlEncoded)). 565 join("&"); 566 } 567 // 568 package unittest { 569 assert(params2query(queryParams("a","b", "c", " d "))=="a=b&c=%20d%20"); 570 } 571 /// 572 /// Analyze received headers, take appropriate actions: 573 /// check content length, attach unchunk and uncompress 574 /// 575 private void analyzeHeaders(in string[string] headers) { 576 577 _contentLength = -1; 578 _unChunker = null; 579 auto contentLength = "content-length" in headers; 580 if ( contentLength ) { 581 try { 582 _contentLength = to!long(*contentLength); 583 if ( _maxContentLength && _contentLength > _maxContentLength) { 584 throw new RequestException("ContentLength > maxContentLength (%d>%d)". 585 format(_contentLength, _maxContentLength)); 586 } 587 } catch (ConvException e) { 588 throw new RequestException("Can't convert Content-Length from %s".format(*contentLength)); 589 } 590 } 591 auto transferEncoding = "transfer-encoding" in headers; 592 if ( transferEncoding ) { 593 debug(requests) tracef("transferEncoding: %s", *transferEncoding); 594 if ( (*transferEncoding).toLower == "chunked") { 595 _unChunker = new DecodeChunked(); 596 _bodyDecoder.insert(_unChunker); 597 } 598 } 599 auto contentEncoding = "content-encoding" in headers; 600 if ( contentEncoding ) switch (*contentEncoding) { 601 default: 602 throw new RequestException("Unknown content-encoding " ~ *contentEncoding); 603 case "gzip": 604 case "deflate": 605 _bodyDecoder.insert(new Decompressor!ubyte); 606 } 607 608 } 609 /// 610 /// Called when we know that all headers already received in buffer. 611 /// This routine does not interpret headers content (see analyzeHeaders). 612 /// 1. Split headers on lines 613 /// 2. store status line, store response code 614 /// 3. unfold headers if needed 615 /// 4. store headers 616 /// 617 private void parseResponseHeaders(in ubyte[] input) { 618 string lastHeader; 619 auto buffer = cast(string)input; 620 621 foreach(line; buffer.split("\n").map!(l => l.stripRight)) { 622 if ( ! _response.status_line.length ) { 623 debug (requests) tracef("statusLine: %s", line); 624 _response.status_line = line; 625 if ( _verbosity >= 1 ) { 626 writefln("< %s", line); 627 } 628 auto parsed = line.split(" "); 629 if ( parsed.length >= 2 ) { 630 _response.code = parsed[1].to!ushort; 631 } 632 continue; 633 } 634 if ( line[0] == ' ' || line[0] == '\t' ) { 635 // unfolding https://tools.ietf.org/html/rfc822#section-3.1 636 if ( auto stored = lastHeader in _response._responseHeaders) { 637 *stored ~= line; 638 } 639 continue; 640 } 641 auto parsed = line.findSplit(":"); 642 auto header = parsed[0].toLower; 643 auto value = parsed[2].strip; 644 645 if ( _verbosity >= 1 ) { 646 writefln("< %s: %s", header, value); 647 } 648 649 lastHeader = header; 650 debug (requests) tracef("Header %s = %s", header, value); 651 652 if ( header != "set-cookie" ) { 653 auto stored = _response.responseHeaders.get(header, null); 654 if ( stored ) { 655 value = stored ~ ", " ~ value; 656 } 657 _response._responseHeaders[header] = value; 658 continue; 659 } 660 _cookie ~= processCookie(value); 661 } 662 } 663 664 /// 665 /// Process Set-Cookie header from server response 666 /// 667 private Cookie[] processCookie(string value ) pure { 668 // cookie processing 669 // 670 // as we can't join several set-cookie lines in single line 671 // < Set-Cookie: idx=7f2800f63c112a65ef5082957bcca24b; expires=Mon, 29-May-2017 00:31:25 GMT; path=/; domain=example.com 672 // < Set-Cookie: idx=7f2800f63c112a65ef5082957bcca24b; expires=Mon, 29-May-2017 00:31:25 GMT; path=/; domain=example.com, cs=ip764-RgKqc-HvSkxRxdQQAKW8LA; path=/; domain=.example.com; HttpOnly 673 // 674 Cookie[] res; 675 string[string] kv; 676 auto fields = value.split(";").map!strip; 677 while(!fields.empty) { 678 auto s = fields.front.findSplit("="); 679 fields.popFront; 680 if ( s[1] != "=" ) { 681 continue; 682 } 683 auto k = s[0]; 684 auto v = s[2]; 685 switch(k.toLower()) { 686 case "domain": 687 k = "domain"; 688 break; 689 case "path": 690 k = "path"; 691 break; 692 case "expires": 693 continue; 694 default: 695 break; 696 } 697 kv[k] = v; 698 } 699 if ( "domain" !in kv ) { 700 kv["domain"] = _uri.host; 701 } 702 if ( "path" !in kv ) { 703 kv["path"] = _uri.path; 704 } 705 auto domain = kv["domain"]; kv.remove("domain"); 706 auto path = kv["path"]; kv.remove("path"); 707 foreach(pair; kv.byKeyValue) { 708 auto _attr = pair.key; 709 auto _value = pair.value; 710 auto cookie = Cookie(path, domain, _attr, _value); 711 res ~= cookie; 712 } 713 return res; 714 } 715 /// 716 /// Do we received \r\n\r\n? 717 /// 718 private bool headersHaveBeenReceived(in ubyte[] data, ref Buffer!ubyte buffer, out string separator) const @safe { 719 foreach(s; ["\r\n\r\n", "\n\n"]) { 720 if ( data.canFind(s) || buffer.canFind(s) ) { 721 separator = s; 722 return true; 723 } 724 } 725 return false; 726 } 727 728 private bool followRedirectResponse() { 729 if ( _history.length >= _maxRedirects ) { 730 throw new MaxRedirectsException("%d redirects reached maxRedirects %d.".format(_history.length, _maxRedirects)); 731 } 732 auto location = "location" in _response.responseHeaders; 733 if ( !location ) { 734 return false; 735 } 736 _history ~= _response; 737 auto connection = "connection" in _response._responseHeaders; 738 if ( !connection || *connection == "close" ) { 739 debug(requests) tracef("Closing connection because of 'Connection: close' or no 'Connection' header"); 740 _stream.close(); 741 } 742 URI oldURI = _uri; 743 URI newURI = oldURI; 744 try { 745 newURI = URI(*location); 746 } catch (UriException e) { 747 debug(requests) trace("Can't parse Location:, try relative uri"); 748 newURI.path = *location; 749 newURI.uri = newURI.recalc_uri; 750 } 751 handleURLChange(oldURI, newURI); 752 oldURI = _response.uri; 753 _uri = newURI; 754 _response = new HTTPResponse; 755 _response.uri = oldURI; 756 _response.finalURI = newURI; 757 return true; 758 } 759 /// 760 /// If uri changed so that we have to change host, port or proxy, then we have to close socket stream 761 /// 762 private void handleURLChange(in URI from, in URI to) { 763 if ( _stream is null || !_stream.isConnected ) { 764 return; 765 } 766 string proxy_from = select_proxy(from.scheme); 767 string proxy_to = select_proxy(to.scheme); 768 if ( proxy_from != proxy_to ) { 769 // we are switching proxies 770 _stream.close(); 771 return; 772 } 773 if ( proxy_to !is null ) { 774 // we do not have to close proxy connection if we will not change proxy 775 if ( (from.scheme=="https" || to.scheme=="https") 776 && (from.scheme != to.scheme) ) { 777 _stream.close(); 778 } 779 return; 780 } 781 if ( from.scheme != to.scheme || from.host != to.host || from.port != to.port ) { 782 debug tracef("Have to reopen stream, because of URI change"); 783 _stream.close(); 784 } 785 } 786 /// 787 /// if we have new uri, then we need to check if we have to reopen existent connection 788 /// 789 private void checkURL(string url, string file=__FILE__, size_t line=__LINE__) { 790 if (url is null && _uri.uri == "" ) { 791 throw new RequestException("No url configured", file, line); 792 } 793 794 if ( url !is null ) { 795 URI newURI = URI(url); 796 handleURLChange(_uri, newURI); 797 _uri = newURI; 798 } 799 } 800 /// 801 /// Setup connection. Handle proxy and https case 802 /// 803 private void setupConnection() { 804 if ( _stream && _stream.isConnected ) { 805 debug(requests) tracef("Use old connection"); 806 return; 807 } 808 809 debug(requests) tracef("Set up new connection"); 810 811 URI uri; // this URI will be used temporarry if we need proxy 812 string actual_proxy = select_proxy(_uri.scheme); 813 final switch (_uri.scheme) { 814 case "http": 815 if ( actual_proxy ) { 816 uri.uri_parse(actual_proxy); 817 uri.idn_encode(); 818 } else { 819 // use original uri 820 uri = _uri; 821 } 822 _stream = new TCPStream(); 823 _stream.bind(_bind); 824 _stream.connect(uri.host, uri.port, _timeout); 825 break; 826 case "https": 827 if ( actual_proxy ) { 828 uri.uri_parse(actual_proxy); 829 uri.idn_encode(); 830 _stream = new TCPStream(); 831 _stream.bind(_bind); 832 _stream.connect(uri.host, uri.port, _timeout); 833 if ( verbosity>=1 ) { 834 writeln("> CONNECT %s:%d HTTP/1.1".format(_uri.host, _uri.port)); 835 } 836 _stream.send("CONNECT %s:%d HTTP/1.1\r\n\r\n".format(_uri.host, _uri.port)); 837 while ( _stream.isConnected ) { 838 ubyte[1024] b; 839 auto read = _stream.receive(b); 840 if ( verbosity>=1) { 841 writefln("< %s", cast(string)b[0..read]); 842 } 843 debug(requests) tracef("read: %d", read); 844 if ( b[0..read].canFind("\r\n\r\n") || b[0..read].canFind("\n\n") ) { 845 debug(requests) tracef("proxy connection ready"); 846 // convert connection to ssl 847 _stream = new SSLStream(_stream, _sslOptions, _uri.host); 848 break; 849 } else { 850 debug(requests) tracef("still wait for proxy connection"); 851 } 852 } 853 } else { 854 uri = _uri; 855 _stream = new SSLStream(_sslOptions); 856 _stream.bind(_bind); 857 _stream.connect(uri.host, uri.port, _timeout); 858 debug(requests) tracef("ssl connection to origin server ready"); 859 } 860 break; 861 } 862 } 863 /// 864 /// Request sent, now receive response. 865 /// Find headers, split on headers and body, continue to receive body 866 /// 867 private void receiveResponse() { 868 869 _stream.readTimeout = timeout; 870 scope(exit) { 871 if ( _stream && _stream.isOpen ) { 872 _stream.readTimeout = 0.seconds; 873 } 874 } 875 876 _bodyDecoder = new DataPipe!ubyte(); 877 scope(exit) { 878 if ( !_useStreaming ) { 879 _bodyDecoder = null; 880 _unChunker = null; 881 } 882 } 883 884 auto buffer = Buffer!ubyte(); 885 Buffer!ubyte partialBody; 886 ptrdiff_t read; 887 string separator; 888 889 while(true) { 890 891 auto b = new ubyte[_bufferSize]; 892 read = _stream.receive(b); 893 894 debug(requests) tracef("read: %d", read); 895 if ( read == 0 ) { 896 break; 897 } 898 auto data = b[0..read]; 899 buffer.putNoCopy(data); 900 if ( verbosity>=3 ) { 901 writeln(data.dump.join("\n")); 902 } 903 904 if ( buffer.length > maxHeadersLength ) { 905 throw new RequestException("Headers length > maxHeadersLength (%d > %d)".format(buffer.length, maxHeadersLength)); 906 } 907 if ( headersHaveBeenReceived(data, buffer, separator) ) { 908 auto s = buffer.data.findSplit(separator); 909 auto ResponseHeaders = s[0]; 910 partialBody = Buffer!ubyte(s[2]); 911 _contentReceived += partialBody.length; 912 parseResponseHeaders(ResponseHeaders); 913 break; 914 } 915 } 916 917 analyzeHeaders(_response._responseHeaders); 918 919 _bodyDecoder.putNoCopy(partialBody.data); 920 921 auto v = _bodyDecoder.get(); 922 _response._responseBody.putNoCopy(v); 923 924 if ( _verbosity >= 2 ) writefln("< %d bytes of body received", partialBody.length); 925 926 if ( _method == "HEAD" ) { 927 // HEAD response have ContentLength, but have no body 928 return; 929 } 930 931 while( true ) { 932 if ( _contentLength >= 0 && _contentReceived >= _contentLength ) { 933 debug(requests) trace("Body received."); 934 break; 935 } 936 if ( _unChunker && _unChunker.done ) { 937 break; 938 } 939 940 if ( _useStreaming && _response._responseBody.length && !redirectCodes.canFind(_response.code) ) { 941 debug(requests) trace("streaming requested"); 942 _response.receiveAsRange.activated = true; 943 _response.receiveAsRange.data = _response._responseBody.data; 944 _response.receiveAsRange.read = delegate ubyte[] () { 945 while(true) { 946 // check if we received everything we need 947 if ( ( _unChunker && _unChunker.done ) 948 || !_stream.isConnected() 949 || (_contentLength > 0 && _contentReceived >= _contentLength) ) 950 { 951 debug(requests) trace("streaming_in receive completed"); 952 _bodyDecoder.flush(); 953 return _bodyDecoder.get(); 954 } 955 // have to continue 956 auto b = new ubyte[_bufferSize]; 957 try { 958 read = _stream.receive(b); 959 } 960 catch (Exception e) { 961 throw new RequestException("streaming_in error reading from socket", __FILE__, __LINE__, e); 962 } 963 debug(requests) tracef("streaming_in received %d bytes", read); 964 965 if ( read == 0 ) { 966 debug(requests) tracef("streaming_in: server closed connection"); 967 _bodyDecoder.flush(); 968 return _bodyDecoder.get(); 969 } 970 971 if ( verbosity>=3 ) { 972 writeln(b[0..read].dump.join("\n")); 973 } 974 975 _contentReceived += read; 976 _bodyDecoder.putNoCopy(b[0..read]); 977 auto res = _bodyDecoder.getNoCopy(); 978 if ( res.length == 0 ) { 979 // there were nothing to produce (beginning of the chunk or no decompressed data) 980 continue; 981 } 982 if (res.length == 1) { 983 return res[0]; 984 } 985 // 986 // I'd like to "return _bodyDecoder.getNoCopy().join;" but it is slower 987 // 988 auto total = res.map!(b=>b.length).sum; 989 // create buffer for joined bytes 990 ubyte[] joined = new ubyte[total]; 991 size_t p; 992 // memcopy 993 foreach(ref _; res) { 994 joined[p .. p + _.length] = _; 995 p += _.length; 996 } 997 return joined; 998 } 999 assert(0); 1000 }; 1001 // we prepared for streaming 1002 return; 1003 } 1004 1005 auto b = new ubyte[_bufferSize]; 1006 read = _stream.receive(b); 1007 1008 if ( read == 0 ) { 1009 debug(requests) trace("read done"); 1010 break; 1011 } 1012 if ( _verbosity >= 2 ) { 1013 writefln("< %d bytes of body received", read); 1014 } 1015 1016 if ( verbosity>=3 ) { 1017 writeln(b[0..read].dump.join("\n")); 1018 } 1019 1020 debug(requests) tracef("read: %d", read); 1021 _contentReceived += read; 1022 if ( _maxContentLength && _contentReceived > _maxContentLength ) { 1023 throw new RequestException("ContentLength > maxContentLength (%d>%d)". 1024 format(_contentLength, _maxContentLength)); 1025 } 1026 1027 _bodyDecoder.putNoCopy(b[0..read]); // send buffer to all decoders 1028 1029 _bodyDecoder.getNoCopy. // fetch result and place to body 1030 each!(b => _response._responseBody.putNoCopy(b)); 1031 1032 debug(requests) tracef("receivedTotal: %d, contentLength: %d, bodyLength: %d", _contentReceived, _contentLength, _response._responseBody.length); 1033 1034 } 1035 _bodyDecoder.flush(); 1036 _response._responseBody.putNoCopy(_bodyDecoder.get()); 1037 } 1038 private bool serverClosedKeepAliveConnection() pure @safe nothrow { 1039 return _response._responseHeaders.length == 0 && _response._status_line.length == 0 && _keepAlive; 1040 } 1041 private bool isIdempotent(in string method) pure @safe nothrow { 1042 return ["GET", "HEAD"].canFind(method); 1043 } 1044 /// 1045 /// Send multipart for request. 1046 /// You would like to use this method for sending large portions of mixed data or uploading files to forms. 1047 /// Content of the posted form consist of sources. Each source have at least name and value (can be string-like object or opened file, see more docs for MultipartForm struct) 1048 /// Params: 1049 /// url = url 1050 /// sources = array of sources. 1051 HTTPResponse exec(string method="POST")(string url, MultipartForm sources) { 1052 import std.uuid; 1053 import std.file; 1054 1055 if ( _response && _response._receiveAsRange.activated && _stream && _stream.isConnected ) { 1056 _stream.close(); 1057 } 1058 // 1059 // application/json 1060 // 1061 bool restartedRequest = false; 1062 1063 _method = method; 1064 1065 _response = new HTTPResponse; 1066 checkURL(url); 1067 _response.uri = _uri; 1068 _response.finalURI = _uri; 1069 1070 connect: 1071 _contentReceived = 0; 1072 _response._startedAt = Clock.currTime; 1073 setupConnection(); 1074 1075 if ( !_stream.isConnected() ) { 1076 return _response; 1077 } 1078 _response._connectedAt = Clock.currTime; 1079 1080 Appender!string req; 1081 req.put(requestString()); 1082 1083 string boundary = randomUUID().toString; 1084 string[] partHeaders; 1085 size_t contentLength; 1086 1087 foreach(ref part; sources._sources) { 1088 string h = "--" ~ boundary ~ "\r\n"; 1089 string disposition = `form-data; name="%s"`.format(part.name); 1090 string optionals = part. 1091 parameters.byKeyValue(). 1092 filter!(p => p.key!="Content-Type"). 1093 map! (p => "%s=%s".format(p.key, p.value)). 1094 join("; "); 1095 1096 h ~= `Content-Disposition: ` ~ [disposition, optionals].join("; ") ~ "\r\n"; 1097 1098 auto contentType = "Content-Type" in part.parameters; 1099 if ( contentType ) { 1100 h ~= "Content-Type: " ~ *contentType ~ "\r\n"; 1101 } 1102 1103 h ~= "\r\n"; 1104 partHeaders ~= h; 1105 contentLength += h.length + part.input.getSize() + "\r\n".length; 1106 } 1107 contentLength += "--".length + boundary.length + "--\r\n".length; 1108 1109 auto h = requestHeaders(); 1110 safeSetHeader(h, _userHeaders.ContentType, "Content-Type", "multipart/form-data; boundary=" ~ boundary); 1111 safeSetHeader(h, _userHeaders.ContentLength, "Content-Length", to!string(contentLength)); 1112 1113 h.byKeyValue. 1114 map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n"). 1115 each!(h => req.put(h)); 1116 req.put("\r\n"); 1117 1118 debug(requests) trace(req.data); 1119 if ( _verbosity >= 1 ) req.data.splitLines.each!(a => writeln("> " ~ a)); 1120 1121 try { 1122 _stream.send(req.data()); 1123 foreach(ref source; sources._sources) { 1124 debug(requests) tracef("sending part headers <%s>", partHeaders.front); 1125 _stream.send(partHeaders.front); 1126 partHeaders.popFront; 1127 while (true) { 1128 auto chunk = source.input.read(); 1129 if ( chunk.length <= 0 ) { 1130 break; 1131 } 1132 _stream.send(chunk); 1133 } 1134 _stream.send("\r\n"); 1135 } 1136 _stream.send("--" ~ boundary ~ "--\r\n"); 1137 _response._requestSentAt = Clock.currTime; 1138 receiveResponse(); 1139 _response._finishedAt = Clock.currTime; 1140 } 1141 catch (NetworkException e) { 1142 errorf("Error sending request: ", e.msg); 1143 return _response; 1144 } 1145 if ( _useStreaming ) { 1146 if ( _response._receiveAsRange.activated ) { 1147 debug(requests) trace("streaming_in activated"); 1148 return _response; 1149 } else { 1150 _response._receiveAsRange.data = _response.responseBody.data; 1151 } 1152 } 1153 auto connection = "connection" in _response._responseHeaders; 1154 if ( !connection || *connection == "close" ) { 1155 debug(requests) tracef("Closing connection because of 'Connection: close' or no 'Connection' header"); 1156 _stream.close(); 1157 } 1158 if ( canFind(redirectCodes, _response.code) && followRedirectResponse() ) { 1159 if ( _method != "GET" && _response.code != 307 && _response.code != 308 ) { 1160 // 307 and 308 do not change method 1161 return this.get(); 1162 } 1163 goto connect; 1164 } 1165 _response._history = _history; 1166 /// 1167 return _response; 1168 } 1169 1170 // we use this if we send from ubyte[][] and user provided Content-Length 1171 private void sendFlattenContent(T)(NetworkStream _stream, T content) { 1172 while ( !content.empty ) { 1173 auto chunk = content.front; 1174 _stream.send(chunk); 1175 content.popFront; 1176 } 1177 debug(requests) tracef("sent"); 1178 } 1179 // we use this if we send from ubyte[][] as chunked content 1180 private void sendChunkedContent(T)(NetworkStream _stream, T content) { 1181 while ( !content.empty ) { 1182 auto chunk = content.front; 1183 auto chunkHeader = "%x\r\n".format(chunk.length); 1184 debug(requests) tracef("sending %s%s", chunkHeader, chunk); 1185 _stream.send(chunkHeader); 1186 _stream.send(chunk); 1187 _stream.send("\r\n"); 1188 content.popFront; 1189 } 1190 debug(requests) tracef("sent"); 1191 _stream.send("0\r\n\r\n"); 1192 } 1193 /// 1194 /// POST/PUT/... data from some string(with Content-Length), or from range of strings/bytes (use Transfer-Encoding: chunked). 1195 /// When rank 1 (flat array) used as content it must have length. In that case "content" will be sent directly to network, and Content-Length headers will be added. 1196 /// If you are goung to send some range and do not know length at the moment when you start to send request, then you can send chunks of chars or ubyte. 1197 /// Try not to send too short chunks as this will put additional load on client and server. Chunks of length 2048 or 4096 are ok. 1198 /// 1199 /// Parameters: 1200 /// url = url 1201 /// content = string or input range 1202 /// contentType = content type 1203 /// Returns: 1204 /// Response 1205 /// Examples: 1206 /// --------------------------------------------------------------------------------------------------------- 1207 /// rs = rq.exec!"POST"("http://httpbin.org/post", "привiт, свiт!", "application/octet-stream"); 1208 /// 1209 /// auto s = lineSplitter("one,\ntwo,\nthree."); 1210 /// rs = rq.exec!"POST"("http://httpbin.org/post", s, "application/octet-stream"); 1211 /// 1212 /// auto s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; 1213 /// rs = rq.exec!"POST"("http://httpbin.org/post", s.representation.chunks(10), "application/octet-stream"); 1214 /// 1215 /// auto f = File("tests/test.txt", "rb"); 1216 /// rs = rq.exec!"POST"("http://httpbin.org/post", f.byChunk(3), "application/octet-stream"); 1217 /// -------------------------------------------------------------------------------------------------------- 1218 HTTPResponse exec(string method="POST", R)(string url, R content, string contentType="application/octet-stream") 1219 if ( (rank!R == 1) 1220 || (rank!R == 2 && isSomeChar!(Unqual!(typeof(content.front.front)))) 1221 || (rank!R == 2 && (is(Unqual!(typeof(content.front.front)) == ubyte))) 1222 ) { 1223 if ( _response && _response._receiveAsRange.activated && _stream && _stream.isConnected ) { 1224 _stream.close(); 1225 } 1226 // 1227 // application/json 1228 // 1229 bool restartedRequest = false; 1230 bool send_flat; 1231 1232 _method = method; 1233 1234 _response = new HTTPResponse; 1235 checkURL(url); 1236 _response.uri = _uri; 1237 _response.finalURI = _uri; 1238 1239 connect: 1240 _contentReceived = 0; 1241 _response._startedAt = Clock.currTime; 1242 setupConnection(); 1243 1244 if ( !_stream.isConnected() ) { 1245 return _response; 1246 } 1247 _response._connectedAt = Clock.currTime; 1248 1249 Appender!string req; 1250 req.put(requestString()); 1251 1252 auto h = requestHeaders; 1253 if ( contentType ) { 1254 safeSetHeader(h, _userHeaders.ContentType, "Content-Type", contentType); 1255 } 1256 static if ( rank!R == 1 ) { 1257 safeSetHeader(h, _userHeaders.ContentLength, "Content-Length", to!string(content.length)); 1258 } else { 1259 if ( _userHeaders.ContentLength ) { 1260 debug(requests) tracef("User provided content-length for chunked content"); 1261 send_flat = true; 1262 } else { 1263 h["Transfer-Encoding"] = "chunked"; 1264 send_flat = false; 1265 } 1266 } 1267 h.byKeyValue. 1268 map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n"). 1269 each!(h => req.put(h)); 1270 req.put("\r\n"); 1271 1272 debug(requests) trace(req.data); 1273 if ( _verbosity >= 1 ) { 1274 req.data.splitLines.each!(a => writeln("> " ~ a)); 1275 } 1276 1277 try { 1278 // send headers 1279 _stream.send(req.data()); 1280 // send body 1281 static if ( rank!R == 1) { 1282 _stream.send(content); 1283 } else { 1284 if ( send_flat ) { 1285 sendFlattenContent(_stream, content); 1286 } else { 1287 sendChunkedContent(_stream, content); 1288 } 1289 } 1290 _response._requestSentAt = Clock.currTime; 1291 receiveResponse(); 1292 _response._finishedAt = Clock.currTime; 1293 } catch (NetworkException e) { 1294 _stream.close(); 1295 throw new RequestException("Network error during data exchange"); 1296 } 1297 1298 if ( _useStreaming ) { 1299 if ( _response._receiveAsRange.activated ) { 1300 debug(requests) trace("streaming_in activated"); 1301 return _response; 1302 } else { 1303 _response._receiveAsRange.data = _response.responseBody.data; 1304 } 1305 } 1306 auto connection = "connection" in _response._responseHeaders; 1307 if ( !connection || *connection == "close" ) { 1308 debug(requests) tracef("Closing connection because of 'Connection: close' or no 'Connection' header"); 1309 _stream.close(); 1310 } 1311 if ( canFind(redirectCodes, _response.code) && followRedirectResponse() ) { 1312 if ( _method != "GET" && _response.code != 307 && _response.code != 308 ) { 1313 // 307 and 308 do not change method 1314 return this.get(); 1315 } 1316 goto connect; 1317 } 1318 /// 1319 _response._history = _history; 1320 return _response; 1321 } 1322 /// 1323 /// Send request with pameters. 1324 /// If used for POST or PUT requests then application/x-www-form-urlencoded used. 1325 /// Request parameters will be encoded into request string or placed in request body for POST/PUT 1326 /// requests. 1327 /// Parameters: 1328 /// url = url 1329 /// params = request parameters 1330 /// Returns: 1331 /// Response 1332 /// Examples: 1333 /// --------------------------------------------------------------------------------- 1334 /// rs = Request().exec!"GET"("http://httpbin.org/get", ["c":"d", "a":"b"]); 1335 /// --------------------------------------------------------------------------------- 1336 /// 1337 HTTPResponse exec(string method="GET")(string url = null, QueryParam[] params = null) { 1338 1339 if ( _response && _response._receiveAsRange.activated && _stream && _stream.isConnected ) { 1340 _stream.close(); 1341 } 1342 _method = method; 1343 _response = new HTTPResponse; 1344 _history.length = 0; 1345 bool restartedRequest = false; // True if this is restarted keepAlive request 1346 string encoded; 1347 1348 checkURL(url); 1349 _response.uri = _uri; 1350 _response.finalURI = _uri; 1351 1352 connect: 1353 _contentReceived = 0; 1354 _response._startedAt = Clock.currTime; 1355 setupConnection(); 1356 1357 if ( !_stream.isConnected() ) { 1358 return _response; 1359 } 1360 _response._connectedAt = Clock.currTime; 1361 1362 auto h = requestHeaders(); 1363 1364 Appender!string req; 1365 1366 switch (_method) { 1367 case "POST","PUT": 1368 encoded = params2query(params); 1369 safeSetHeader(h, _userHeaders.ContentType, "Content-Type", "application/x-www-form-urlencoded"); 1370 if ( encoded.length > 0) { 1371 safeSetHeader(h, _userHeaders.ContentLength, "Content-Length", to!string(encoded.length)); 1372 } 1373 req.put(requestString()); 1374 break; 1375 default: 1376 req.put(requestString(params)); 1377 } 1378 1379 h.byKeyValue. 1380 map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n"). 1381 each!(h => req.put(h)); 1382 req.put("\r\n"); 1383 if ( encoded ) { 1384 req.put(encoded); 1385 } 1386 1387 debug(requests) trace(req.data); 1388 1389 if ( _verbosity >= 1 ) req.data.splitLines.each!(a => writeln("> " ~ a)); 1390 // 1391 // Now send request and receive response 1392 // 1393 try { 1394 _stream.send(req.data()); 1395 _response._requestSentAt = Clock.currTime; 1396 receiveResponse(); 1397 _response._finishedAt = Clock.currTime; 1398 } 1399 catch (NetworkException e) { 1400 // On SEND this can means: 1401 // we started to send request to the server, but it closed connection because of keepalive timeout. 1402 // We have to restart request if possible. 1403 1404 // On RECEIVE - if we received something - then this exception is real and unexpected error. 1405 // If we didn't receive anything - we can restart request again as it can be 1406 if ( _response._responseHeaders.length != 0 ) { 1407 _stream.close(); 1408 throw new RequestException("Unexpected network error"); 1409 } 1410 } 1411 1412 if ( serverClosedKeepAliveConnection() 1413 && !restartedRequest 1414 && isIdempotent(_method) 1415 ) { 1416 /// 1417 /// We didn't receive any data (keepalive connectioin closed?) 1418 /// and we can restart this request. 1419 /// Go ahead. 1420 /// 1421 debug(requests) tracef("Server closed keepalive connection"); 1422 _stream.close(); 1423 restartedRequest = true; 1424 goto connect; 1425 } 1426 1427 if ( _useStreaming ) { 1428 if ( _response._receiveAsRange.activated ) { 1429 debug(requests) trace("streaming_in activated"); 1430 return _response; 1431 } else { 1432 // this can happen if whole response body received together with headers 1433 _response._receiveAsRange.data = _response.responseBody.data; 1434 } 1435 } 1436 1437 auto connection = "connection" in _response._responseHeaders; 1438 if ( !connection || *connection == "close" ) { 1439 debug(requests) tracef("Closing connection because of 'Connection: close' or no 'Connection' header"); 1440 _stream.close(); 1441 } 1442 if ( _verbosity >= 1 ) { 1443 writeln(">> Connect time: ", _response._connectedAt - _response._startedAt); 1444 writeln(">> Request send time: ", _response._requestSentAt - _response._connectedAt); 1445 writeln(">> Response recv time: ", _response._finishedAt - _response._requestSentAt); 1446 } 1447 if ( canFind(redirectCodes, _response.code) && followRedirectResponse() ) { 1448 if ( _method != "GET" && _response.code != 307 && _response.code != 308 ) { 1449 // 307 and 308 do not change method 1450 return this.get(); 1451 } 1452 goto connect; 1453 } 1454 /// 1455 _response._history = _history; 1456 return _response; 1457 } 1458 1459 /// WRAPPERS 1460 /// 1461 /// send file(s) using POST and multipart form. 1462 /// This wrapper will be deprecated, use post with MultipartForm - it is more general and clear. 1463 /// Parameters: 1464 /// url = url 1465 /// files = array of PostFile structures 1466 /// Returns: 1467 /// Response 1468 /// Each PostFile structure contain path to file, and optional field name and content type. 1469 /// If no field name provided, then basename of the file will be used. 1470 /// application/octet-stream is default when no content type provided. 1471 /// Example: 1472 /// --------------------------------------------------------------- 1473 /// PostFile[] files = [ 1474 /// {fileName:"tests/abc.txt", fieldName:"abc", contentType:"application/octet-stream"}, 1475 /// {fileName:"tests/test.txt"} 1476 /// ]; 1477 /// rs = rq.exec!"POST"("http://httpbin.org/post", files); 1478 /// --------------------------------------------------------------- 1479 /// 1480 HTTPResponse exec(string method="POST")(string url, PostFile[] files) if (method=="POST") { 1481 MultipartForm multipart; 1482 File[] toClose; 1483 foreach(ref f; files) { 1484 File file = File(f.fileName, "rb"); 1485 toClose ~= file; 1486 string fileName = f.fileName ? f.fileName : f.fieldName; 1487 string contentType = f.contentType ? f.contentType : "application/octetstream"; 1488 multipart.add(f.fieldName, new FormDataFile(file), ["filename":fileName, "Content-Type": contentType]); 1489 } 1490 auto res = exec!"POST"(url, multipart); 1491 toClose.each!"a.close"; 1492 return res; 1493 } 1494 /// 1495 /// exec request with parameters when you can use dictionary (when you have no duplicates in parameter names) 1496 /// Consider switch to exec(url, QueryParams) as it more generic and clear. 1497 /// Parameters: 1498 /// url = url 1499 /// params = dictionary with field names as keys and field values as values. 1500 /// Returns: 1501 /// Response 1502 HTTPResponse exec(string method="GET")(string url, string[string] params) { 1503 return exec!method(url, params.byKeyValue.map!(p => QueryParam(p.key, p.value)).array); 1504 } 1505 /// 1506 /// GET request. Simple wrapper over exec!"GET" 1507 /// Params: 1508 /// args = request parameters. see exec docs. 1509 /// 1510 HTTPResponse get(A...)(A args) { 1511 return exec!"GET"(args); 1512 } 1513 /// 1514 /// POST request. Simple wrapper over exec!"POST" 1515 /// Params: 1516 /// uri = endpoint uri 1517 /// args = request parameters. see exec docs. 1518 /// 1519 HTTPResponse post(A...)(string uri, A args) { 1520 return exec!"POST"(uri, args); 1521 } 1522 } 1523 1524 version(vibeD) { 1525 import std.json; 1526 package string httpTestServer() { 1527 return "http://httpbin.org/"; 1528 } 1529 package string fromJsonArrayToStr(JSONValue v) { 1530 return v.str; 1531 } 1532 } 1533 else { 1534 import std.json; 1535 package string httpTestServer() { 1536 return "http://127.0.0.1:8081/"; 1537 } 1538 package string fromJsonArrayToStr(JSONValue v) { 1539 return cast(string)(v.array.map!"cast(ubyte)a.integer".array); 1540 } 1541 } 1542 1543 1544 package unittest { 1545 import std.json; 1546 import std.array; 1547 1548 globalLogLevel(LogLevel.info); 1549 1550 string httpbinUrl = httpTestServer(); 1551 version(vibeD) { 1552 } 1553 else { 1554 import httpbin; 1555 auto server = httpbinApp(); 1556 server.start(); 1557 scope(exit) { 1558 server.stop(); 1559 } 1560 } 1561 HTTPRequest rq; 1562 HTTPResponse rs; 1563 info("Check GET"); 1564 URI uri = URI(httpbinUrl); 1565 rs = rq.get(httpbinUrl); 1566 assert(rs.code==200); 1567 assert(rs.responseBody.length > 0); 1568 assert(rq.format("%m|%h|%p|%P|%q|%U") == 1569 "GET|%s|%d|%s||%s" 1570 .format(uri.host, uri.port, uri.path, httpbinUrl)); 1571 info("Check GET with AA params"); 1572 { 1573 rs = HTTPRequest().get(httpbinUrl ~ "get", ["c":" d", "a":"b"]); 1574 assert(rs.code == 200); 1575 auto json = parseJSON(cast(string)rs.responseBody.data).object["args"].object; 1576 assert(json["c"].str == " d"); 1577 assert(json["a"].str == "b"); 1578 } 1579 info("Check POST files"); 1580 { 1581 import std.file; 1582 import std.path; 1583 auto tmpd = tempDir(); 1584 auto tmpfname = tmpd ~ dirSeparator ~ "request_test.txt"; 1585 auto f = File(tmpfname, "wb"); 1586 f.rawWrite("abcdefgh\n12345678\n"); 1587 f.close(); 1588 // files 1589 PostFile[] files = [ 1590 {fileName: tmpfname, fieldName:"abc", contentType:"application/octet-stream"}, 1591 {fileName: tmpfname} 1592 ]; 1593 rs = rq.post(httpbinUrl ~ "post", files); 1594 assert(rs.code==200); 1595 } 1596 info("Check POST chunked from file.byChunk"); 1597 { 1598 import std.file; 1599 import std.path; 1600 auto tmpd = tempDir(); 1601 auto tmpfname = tmpd ~ dirSeparator ~ "request_test.txt"; 1602 auto f = File(tmpfname, "wb"); 1603 f.rawWrite("abcdefgh\n12345678\n"); 1604 f.close(); 1605 f = File(tmpfname, "rb"); 1606 rs = rq.post(httpbinUrl ~ "post", f.byChunk(3), "application/octet-stream"); 1607 if (httpbinUrl != "http://httpbin.org/") { 1608 assert(rs.code==200); 1609 auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]); 1610 assert(data=="abcdefgh\n12345678\n"); 1611 } 1612 f.close(); 1613 } 1614 info("Check POST chunked from lineSplitter"); 1615 { 1616 auto s = lineSplitter("one,\ntwo,\nthree."); 1617 rs = rq.exec!"POST"(httpbinUrl ~ "post", s, "application/octet-stream"); 1618 if (httpbinUrl != "http://httpbin.org/") { 1619 assert(rs.code==200); 1620 auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]); 1621 assert(data=="one,two,three."); 1622 } 1623 } 1624 info("Check POST chunked from array"); 1625 { 1626 auto s = ["one,", "two,", "three."]; 1627 rs = rq.post(httpbinUrl ~ "post", s, "application/octet-stream"); 1628 if (httpbinUrl != "http://httpbin.org/") { 1629 assert(rs.code==200); 1630 auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]); 1631 assert(data=="one,two,three."); 1632 } 1633 } 1634 info("Check POST chunked using std.range.chunks()"); 1635 { 1636 auto s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; 1637 rs = rq.post(httpbinUrl ~ "post", s.representation.chunks(10), "application/octet-stream"); 1638 if (httpbinUrl != "http://httpbin.org/") { 1639 assert(rs.code==200); 1640 auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody.data).object["data"]); 1641 assert(data==s); 1642 } 1643 } 1644 info("Check POST from QueryParams"); 1645 { 1646 rs = rq.post(httpbinUrl ~ "post", queryParams("name[]", "first", "name[]", 2)); 1647 assert(rs.code==200); 1648 auto data = parseJSON(cast(string)rs.responseBody).object["form"].object; 1649 string[] a; 1650 try { 1651 a = to!(string[])(data["name[]"].str); 1652 } 1653 catch (JSONException e) { 1654 a = data["name[]"].array.map!"a.str".array; 1655 } 1656 assert(equal(["first", "2"], a)); 1657 } 1658 info("Check POST from AA"); 1659 { 1660 rs = rq.post(httpbinUrl ~ "post", ["a":"b ", "c":"d"]); 1661 assert(rs.code==200); 1662 auto form = parseJSON(cast(string)rs.responseBody.data).object["form"].object; 1663 assert(form["a"].str == "b "); 1664 assert(form["c"].str == "d"); 1665 } 1666 info("Check POST json"); 1667 { 1668 rs = rq.post(httpbinUrl ~ "post?b=x", `{"a":"a b", "c":[1,2,3]}`, "application/json"); 1669 assert(rs.code==200); 1670 auto json = parseJSON(cast(string)rs.responseBody).object["args"].object; 1671 assert(json["b"].str == "x"); 1672 json = parseJSON(cast(string)rs.responseBody).object["json"].object; 1673 assert(json["a"].str == "a b"); 1674 assert(json["c"].array.map!(a=>a.integer).array == [1,2,3]); 1675 } 1676 info("Check HEAD"); 1677 rs = rq.exec!"HEAD"(httpbinUrl); 1678 assert(rs.code==200); 1679 info("Check DELETE"); 1680 rs = rq.exec!"DELETE"(httpbinUrl ~ "delete"); 1681 assert(rs.code==200); 1682 info("Check PUT"); 1683 rs = rq.exec!"PUT"(httpbinUrl ~ "put", `{"a":"b", "c":[1,2,3]}`, "application/json"); 1684 assert(rs.code==200); 1685 assert(parseJSON(cast(string)rs.responseBody).object["json"].object["a"].str=="b"); 1686 info("Check PATCH"); 1687 rs = rq.exec!"PATCH"(httpbinUrl ~ "patch", "привiт, свiт!", "application/octet-stream"); 1688 assert(rs.code==200); 1689 info("Check compressed content"); 1690 rs = rq.get(httpbinUrl ~ "gzip"); 1691 assert(rs.code==200); 1692 bool gzipped = parseJSON(cast(string)rs.responseBody).object["gzipped"].type == JSON_TYPE.TRUE; 1693 assert(gzipped); 1694 info("gzip - ok"); 1695 rs = rq.get(httpbinUrl ~ "deflate"); 1696 assert(rs.code==200); 1697 bool deflated = parseJSON(cast(string)rs.responseBody).object["deflated"].type == JSON_TYPE.TRUE; 1698 assert(deflated); 1699 info("deflate - ok"); 1700 1701 info("Check redirects"); 1702 rs = rq.get(httpbinUrl ~ "relative-redirect/2"); 1703 assert(rs.history.length == 2); 1704 assert(rs.code==200); 1705 rs = rq.get(httpbinUrl ~ "absolute-redirect/2"); 1706 assert(rs.history.length == 2); 1707 assert(rs.code==200); 1708 1709 rq.maxRedirects = 2; 1710 assertThrown!MaxRedirectsException(rq.get(httpbinUrl ~ "absolute-redirect/3")); 1711 1712 info("Check cookie"); 1713 { 1714 rs = rq.get(httpbinUrl ~ "cookies/set?A=abcd&b=cdef"); 1715 assert(rs.code == 200); 1716 auto json = parseJSON(cast(string)rs.responseBody.data).object["cookies"].object; 1717 assert(json["A"].str == "abcd"); 1718 assert(json["b"].str == "cdef"); 1719 foreach(c; rq.cookie) { 1720 final switch(c.attr) { 1721 case "A": 1722 assert(c.value == "abcd"); 1723 break; 1724 case "b": 1725 assert(c.value == "cdef"); 1726 break; 1727 } 1728 } 1729 } 1730 info("Check chunked content"); 1731 rs = rq.get(httpbinUrl ~ "range/1024"); 1732 assert(rs.code==200); 1733 assert(rs.responseBody.length==1024); 1734 1735 info("Check basic auth"); 1736 rq.authenticator = new BasicAuthentication("user", "passwd"); 1737 rs = rq.get(httpbinUrl ~ "basic-auth/user/passwd"); 1738 assert(rs.code==200); 1739 1740 info("Check limits"); 1741 rq = HTTPRequest(); 1742 rq.maxContentLength = 1; 1743 assertThrown!RequestException(rq.get(httpbinUrl)); 1744 rq = HTTPRequest(); 1745 rq.maxHeadersLength = 1; 1746 assertThrown!RequestException(rq.get(httpbinUrl)); 1747 rq = HTTPRequest(); 1748 info("Check POST multiPartForm"); 1749 { 1750 /// This is example on usage files with MultipartForm data. 1751 /// For this example we have to create files which will be sent. 1752 import std.file; 1753 import std.path; 1754 /// preapare files 1755 auto tmpd = tempDir(); 1756 auto tmpfname1 = tmpd ~ dirSeparator ~ "request_test1.txt"; 1757 auto f = File(tmpfname1, "wb"); 1758 f.rawWrite("file1 content\n"); 1759 f.close(); 1760 auto tmpfname2 = tmpd ~ dirSeparator ~ "request_test2.txt"; 1761 f = File(tmpfname2, "wb"); 1762 f.rawWrite("file2 content\n"); 1763 f.close(); 1764 /// 1765 /// Ok, files ready. 1766 /// Now we will prepare Form data 1767 /// 1768 File f1 = File(tmpfname1, "rb"); 1769 File f2 = File(tmpfname2, "rb"); 1770 scope(exit) { 1771 f1.close(); 1772 f2.close(); 1773 } 1774 /// 1775 /// for each part we have to set field name, source (ubyte array or opened file) and optional filename and content-type 1776 /// 1777 MultipartForm mForm = MultipartForm(). 1778 add(formData("Field1", cast(ubyte[])"form field from memory")). 1779 add(formData("Field2", cast(ubyte[])"file field from memory", ["filename":"data2"])). 1780 add(formData("File1", f1, ["filename":"file1", "Content-Type": "application/octet-stream"])). 1781 add(formData("File2", f2, ["filename":"file2", "Content-Type": "application/octet-stream"])); 1782 /// everything ready, send request 1783 rs = rq.post(httpbinUrl ~ "post", mForm); 1784 } 1785 info("Check exception handling, error messages and timeous are OK"); 1786 rq.timeout = 1.seconds; 1787 assertThrown!TimeoutException(rq.get(httpbinUrl ~ "delay/3")); 1788 // assertThrown!ConnectError(rq.get("http://0.0.0.0:65000/")); 1789 // assertThrown!ConnectError(rq.get("http://1.1.1.1/")); 1790 // assertThrown!ConnectError(rq.get("http://gkhgkhgkjhgjhgfjhgfjhgf/")); 1791 }