1 module requests.http; 2 3 private: 4 import std.algorithm; 5 import std.array; 6 import std.ascii; 7 import std.conv; 8 import std.datetime; 9 import std.exception; 10 import std.format; 11 import std.stdio; 12 import std.range; 13 import std.string; 14 import std.traits; 15 import std.typecons; 16 import std.experimental.logger; 17 import core.thread; 18 19 import requests.streams; 20 import requests.uri; 21 import requests.utils; 22 import requests.base; 23 import requests.connmanager; 24 import requests.rangeadapter; 25 26 static immutable ushort[] redirectCodes = [301, 302, 303, 307, 308]; 27 28 enum HTTP11 = 101; 29 enum HTTP10 = 100; 30 31 static immutable string[string] proxies; 32 shared static this() { 33 import std.process; 34 import std.string; 35 foreach(p; ["http", "https", "all"]) 36 { 37 auto v = environment.get(p ~ "_proxy", environment.get(p.toUpper() ~ "_PROXY")); 38 if ( v !is null && v.length > 0 ) 39 { 40 debug(requests) tracef("will use %s for %s as proxy", v, p); 41 proxies[p] = v; 42 } 43 } 44 } 45 46 public class MaxRedirectsException: Exception { 47 this(string message, string file = __FILE__, size_t line = __LINE__, Throwable next = null) @safe pure nothrow { 48 super(message, file, line, next); 49 } 50 } 51 52 /// 53 /// 54 /// 55 //public auto queryParams(T...)(T params) pure nothrow @safe 56 //{ 57 // static assert (T.length % 2 == 0, "wrong args count"); 58 // 59 // QueryParam[] output; 60 // output.reserve = T.length / 2; 61 // 62 // void queryParamsHelper(T...)(T params, ref QueryParam[] output) 63 // { 64 // static if (T.length > 0) 65 // { 66 // output ~= QueryParam(params[0].to!string, params[1].to!string); 67 // queryParamsHelper(params[2..$], output); 68 // } 69 // } 70 // 71 // queryParamsHelper(params, output); 72 // return output; 73 //} 74 75 /// 76 /// Response - result of request execution. 77 /// 78 /// Response.code - response HTTP code. 79 /// Response.status_line - received HTTP status line. 80 /// Response.responseHeaders - received headers. 81 /// Response.responseBody - container for received body 82 /// Response.history - for redirected responses contain all history 83 /// 84 public class HTTPResponse : Response { 85 private { 86 string _status_line; 87 88 HTTPResponse[] _history; // redirects history 89 90 mixin(Setter!string("status_line")); 91 92 int _version; 93 } 94 95 ~this() { 96 _responseHeaders = null; 97 _history.length = 0; 98 } 99 100 mixin(Getter("status_line")); 101 102 @property final string[string] responseHeaders() @safe @nogc nothrow { 103 return _responseHeaders; 104 } 105 @property final HTTPResponse[] history() @safe @nogc nothrow { 106 return _history; 107 } 108 109 private int parse_version(in string v) pure const nothrow @safe { 110 // try to parse HTTP/1.x to version 111 try if ( v.length > 5 ) { 112 return (v[5..$].split(".").map!"to!int(a)".array[0..2].reduce!((a,b) => a*100 + b)); 113 } catch (Exception e) { 114 } 115 return 0; 116 } 117 unittest { 118 auto r = new HTTPResponse(); 119 assert(r.parse_version("HTTP/1.1") == 101); 120 assert(r.parse_version("HTTP/1.0") == 100); 121 assert(r.parse_version("HTTP/0.9") == 9); 122 assert(r.parse_version("HTTP/xxx") == 0); 123 } 124 } 125 126 /// 127 /// Request. 128 /// Configurable parameters: 129 /// $(B method) - string, method to use (GET, POST, ...) 130 /// $(B headers) - string[string], add any additional headers you'd like to send. 131 /// $(B authenticator) - class Auth, class to send auth headers. 132 /// $(B keepAlive) - bool, set true for keepAlive requests. default true. 133 /// $(B maxRedirects) - uint, maximum number of redirects. default 10. 134 /// $(B maxHeadersLength) - size_t, maximum length of server response headers. default = 32KB. 135 /// $(B maxContentLength) - size_t, maximun content length. delault - 0 = unlimited. 136 /// $(B bufferSize) - size_t, send and receive buffer size. default = 16KB. 137 /// $(B verbosity) - uint, level of verbosity(0 - nothing, 1 - headers, 2 - headers and body progress). default = 0. 138 /// $(B proxy) - string, set proxy url if needed. default - null. 139 /// $(B cookie) - Tuple Cookie, Read/Write cookie You can get cookie setted by server, or set cookies before doing request. 140 /// $(B timeout) - Duration, Set timeout value for connect/receive/send. 141 /// 142 public struct HTTPRequest { 143 private { 144 string _method = "GET"; 145 URI _uri; 146 string[string] _headers; 147 string[] _filteredHeaders; 148 Auth _authenticator; 149 bool _keepAlive = true; 150 uint _maxRedirects = 10; 151 size_t _maxHeadersLength = 32 * 1024; // 32 KB 152 size_t _maxContentLength; // 0 - Unlimited 153 string _proxy; 154 uint _verbosity = 0; // 0 - no output, 1 - headers, 2 - headers+body info 155 Duration _timeout = 30.seconds; 156 size_t _bufferSize = 16*1024; // 16k 157 bool _useStreaming; // return iterator instead of completed request 158 159 HTTPResponse[] _history; // redirects history 160 DataPipe!ubyte _bodyDecoder; 161 DecodeChunked _unChunker; 162 long _contentLength; 163 long _contentReceived; 164 SSLOptions _sslOptions; 165 string _bind; 166 _UH _userHeaders; 167 168 RefCounted!ConnManager _cm; 169 RefCounted!Cookies _cookie; 170 string[URI] _permanent_redirects; // cache 301 redirects for GET requests 171 MultipartForm _multipartForm; 172 173 NetStreamFactory _socketFactory; 174 175 QueryParam[] _params; 176 string _contentType; 177 InputRangeAdapter _postData; 178 } 179 package HTTPResponse _response; 180 181 mixin(Getter_Setter!string ("method")); 182 mixin(Getter_Setter!bool ("keepAlive")); 183 mixin(Getter_Setter!size_t ("maxContentLength")); 184 mixin(Getter_Setter!size_t ("maxHeadersLength")); 185 mixin(Getter_Setter!size_t ("bufferSize")); 186 mixin(Getter_Setter!uint ("maxRedirects")); 187 mixin(Getter_Setter!uint ("verbosity")); 188 mixin(Getter ("proxy")); 189 mixin(Getter_Setter!Duration ("timeout")); 190 mixin(Setter!Auth ("authenticator")); 191 mixin(Getter_Setter!bool ("useStreaming")); 192 mixin(Getter ("contentLength")); 193 mixin(Getter ("contentReceived")); 194 mixin(Getter_Setter!SSLOptions ("sslOptions")); 195 mixin(Getter_Setter!string ("bind")); 196 mixin(Setter!NetStreamFactory ("socketFactory")); 197 198 @property void sslSetVerifyPeer(bool v) pure @safe nothrow @nogc { 199 _sslOptions.setVerifyPeer(v); 200 } 201 @property void sslSetKeyFile(string p, SSLOptions.filetype t = SSLOptions.filetype.pem) pure @safe nothrow @nogc { 202 _sslOptions.setKeyFile(p, t); 203 } 204 @property void sslSetCertFile(string p, SSLOptions.filetype t = SSLOptions.filetype.pem) pure @safe nothrow @nogc { 205 _sslOptions.setCertFile(p, t); 206 } 207 @property void sslSetCaCert(string path) pure @safe nothrow @nogc { 208 _sslOptions.setCaCert(path); 209 } 210 //@property final void cookie(Cookie[] s) pure @safe @nogc nothrow { 211 // _cookie = s; 212 //} 213 @property final void proxy(string v) { 214 if ( v != _proxy ) { 215 _cm.clear(); 216 } 217 _proxy = v; 218 } 219 //@property final Cookie[] cookie() pure @safe @nogc nothrow { 220 // return _cookie; 221 //} 222 223 this(string uri) { 224 _uri = URI(uri); 225 _cm = ConnManager(10); 226 } 227 ~this() { 228 _headers = null; 229 _authenticator = null; 230 _history = null; 231 _bodyDecoder = null; 232 _unChunker = null; 233 //if ( _cm ) { 234 // _cm.clear(); 235 //} 236 } 237 string toString() const { 238 return "HTTPRequest(%s, %s)".format(_method, _uri.uri()); 239 } 240 string format(string fmt) const { 241 import std.array; 242 import std.stdio; 243 auto a = appender!string(); 244 auto f = FormatSpec!char(fmt); 245 while (f.writeUpToNextSpec(a)) { 246 switch(f.spec) { 247 case 'h': 248 // Remote hostname. 249 a.put(_uri.host); 250 break; 251 case 'm': 252 // method. 253 a.put(_method); 254 break; 255 case 'p': 256 // Remote port. 257 a.put("%d".format(_uri.port)); 258 break; 259 case 'P': 260 // Path 261 a.put(_uri.path); 262 break; 263 case 'q': 264 // query parameters supplied with url. 265 a.put(_uri.query); 266 break; 267 case 'U': 268 a.put(_uri.uri()); 269 break; 270 default: 271 throw new FormatException("Unknown Request format spec " ~ f.spec); 272 } 273 } 274 return a.data(); 275 } 276 string select_proxy(string scheme) { 277 if ( _proxy is null && proxies.length == 0 ) { 278 debug(requests) tracef("proxy=null"); 279 return null; 280 } 281 if ( _proxy ) { 282 debug(requests) tracef("proxy=%s", _proxy); 283 return _proxy; 284 } 285 auto p = scheme in proxies; 286 if ( p !is null && *p != "") { 287 debug(requests) tracef("proxy=%s", *p); 288 return *p; 289 } 290 p = "all" in proxies; 291 if ( p !is null && *p != "") { 292 debug(requests) tracef("proxy=%s", *p); 293 return *p; 294 } 295 debug(requests) tracef("proxy=null"); 296 return null; 297 } 298 void clearHeaders() { 299 _headers = null; 300 } 301 @property void uri(in URI newURI) { 302 //handleURLChange(_uri, newURI); 303 _uri = newURI; 304 } 305 /// Add headers to request 306 /// Params: 307 /// headers = headers to send. 308 void addHeaders(in string[string] headers) { 309 foreach(pair; headers.byKeyValue) { 310 string _h = pair.key; 311 switch(toLower(_h)) { 312 case "host": 313 _userHeaders.Host = true; 314 break; 315 case "user-agent": 316 _userHeaders.UserAgent = true; 317 break; 318 case "content-length": 319 _userHeaders.ContentLength = true; 320 break; 321 case "content-type": 322 _userHeaders.ContentType = true; 323 break; 324 case "connection": 325 _userHeaders.Connection = true; 326 break; 327 case "cookie": 328 _userHeaders.Cookie = true; 329 break; 330 default: 331 break; 332 } 333 _headers[pair.key] = pair.value; 334 } 335 } 336 private void safeSetHeader(ref string[string] headers, bool userAdded, string h, string v) pure @safe { 337 if ( !userAdded ) { 338 headers[h] = v; 339 } 340 } 341 /// Remove headers from request 342 /// Params: 343 /// headers = headers to remove. 344 void removeHeaders(in string[] headers) pure { 345 _filteredHeaders ~= headers; 346 } 347 /// 348 /// compose headers to send 349 /// 350 private string[string] requestHeaders() { 351 352 string[string] generatedHeaders; 353 354 if ( _authenticator ) { 355 _authenticator. 356 authHeaders(_uri.host). 357 byKeyValue. 358 each!(pair => generatedHeaders[pair.key] = pair.value); 359 } 360 361 _headers.byKey.each!(h => generatedHeaders[h] = _headers[h]); 362 363 safeSetHeader(generatedHeaders, _userHeaders.AcceptEncoding, "Accept-Encoding", "gzip,deflate"); 364 safeSetHeader(generatedHeaders, _userHeaders.UserAgent, "User-Agent", "dlang-requests"); 365 safeSetHeader(generatedHeaders, _userHeaders.Connection, "Connection", _keepAlive?"Keep-Alive":"Close"); 366 367 if ( !_userHeaders.Host ) 368 { 369 generatedHeaders["Host"] = _uri.host; 370 if ( _uri.scheme !in standard_ports || _uri.port != standard_ports[_uri.scheme] ) { 371 generatedHeaders["Host"] ~= ":%d".format(_uri.port); 372 } 373 } 374 375 if ( _cookie._array.length && !_userHeaders.Cookie ) { 376 auto cs = _cookie._array. 377 filter!(c => _uri.path.pathMatches(c.path) && _uri.host.domainMatches(c.domain)). 378 map!(c => "%s=%s".format(c.attr, c.value)). 379 joiner(";"); 380 if ( ! cs.empty ) 381 { 382 generatedHeaders["Cookie"] = to!string(cs); 383 } 384 } 385 386 _filteredHeaders.each!(h => generatedHeaders.remove(h)); 387 388 return generatedHeaders; 389 } 390 /// 391 /// Build request string. 392 /// Handle proxy and query parameters. 393 /// 394 private @property string requestString(QueryParam[] params = null) { 395 auto query = _uri.query.dup; 396 if ( params ) { 397 query ~= "&" ~ params2query(params); 398 if ( query[0] != '?' ) { 399 query = "?" ~ query; 400 } 401 } 402 string actual_proxy = select_proxy(_uri.scheme); 403 if ( actual_proxy && _uri.scheme != "https" ) { 404 return "%s %s%s HTTP/1.1\r\n".format(_method, _uri.uri(No.params), query); 405 } 406 return "%s %s%s HTTP/1.1\r\n".format(_method, _uri.path, query); 407 } 408 /// 409 /// encode parameters and build query part of the url 410 /// 411 private static string params2query(in QueryParam[] params) pure @safe { 412 return params. 413 map!(a => "%s=%s".format(a.key.urlEncoded, a.value.urlEncoded)). 414 join("&"); 415 } 416 // 417 package unittest { 418 assert(params2query(queryParams("a","b", "c", " d "))=="a=b&c=%20d%20"); 419 } 420 /// 421 /// Analyze received headers, take appropriate actions: 422 /// check content length, attach unchunk and uncompress 423 /// 424 private void analyzeHeaders(in string[string] headers) { 425 426 _contentLength = -1; 427 _unChunker = null; 428 auto contentLength = "content-length" in headers; 429 if ( contentLength ) { 430 try { 431 string l = *contentLength; 432 _contentLength = parse!long(l); 433 // TODO: maybe add a strict mode that checks if l was parsed completely 434 if ( _maxContentLength && _contentLength > _maxContentLength) { 435 throw new RequestException("ContentLength > maxContentLength (%d>%d)". 436 format(_contentLength, _maxContentLength)); 437 } 438 } catch (ConvException e) { 439 throw new RequestException("Can't convert Content-Length from %s".format(*contentLength)); 440 } 441 } 442 auto transferEncoding = "transfer-encoding" in headers; 443 if ( transferEncoding ) { 444 debug(requests) tracef("transferEncoding: %s", *transferEncoding); 445 if ( (*transferEncoding).toLower == "chunked") { 446 _unChunker = new DecodeChunked(); 447 _bodyDecoder.insert(_unChunker); 448 } 449 } 450 auto contentEncoding = "content-encoding" in headers; 451 if ( contentEncoding ) switch (*contentEncoding) { 452 default: 453 throw new RequestException("Unknown content-encoding " ~ *contentEncoding); 454 case "gzip": 455 case "deflate": 456 _bodyDecoder.insert(new Decompressor!ubyte); 457 } 458 459 } 460 /// 461 /// Called when we know that all headers already received in buffer. 462 /// This routine does not interpret headers content (see analyzeHeaders). 463 /// 1. Split headers on lines 464 /// 2. store status line, store response code 465 /// 3. unfold headers if needed 466 /// 4. store headers 467 /// 468 private void parseResponseHeaders(in ubyte[] input, string lineSep) { 469 string lastHeader; 470 auto buffer = cast(string)input; 471 472 foreach(line; buffer.split(lineSep).map!(l => l.stripRight)) { 473 if ( ! _response.status_line.length ) { 474 debug (requests) tracef("statusLine: %s", line); 475 _response.status_line = line; 476 if ( _verbosity >= 1 ) { 477 writefln("< %s", line); 478 } 479 auto parsed = line.split(" "); 480 if ( parsed.length >= 2 ) { 481 _response.code = parsed[1].to!ushort; 482 _response._version = _response.parse_version(parsed[0]); 483 } 484 continue; 485 } 486 if ( line[0] == ' ' || line[0] == '\t' ) { 487 // unfolding https://tools.ietf.org/html/rfc822#section-3.1 488 if ( auto stored = lastHeader in _response._responseHeaders) { 489 *stored ~= line; 490 } 491 continue; 492 } 493 auto parsed = line.findSplit(":"); 494 auto header = parsed[0].toLower; 495 auto value = parsed[2].strip; 496 497 if ( _verbosity >= 1 ) { 498 writefln("< %s: %s", header, value); 499 } 500 501 lastHeader = header; 502 debug (requests) tracef("Header %s = %s", header, value); 503 504 if ( header != "set-cookie" ) { 505 auto stored = _response.responseHeaders.get(header, null); 506 if ( stored ) { 507 value = stored ~ "," ~ value; 508 } 509 _response._responseHeaders[header] = value; 510 continue; 511 } 512 _cookie._array ~= processCookie(value); 513 } 514 } 515 516 /// 517 /// Process Set-Cookie header from server response 518 /// 519 private Cookie[] processCookie(string value ) pure { 520 // cookie processing 521 // 522 // as we can't join several set-cookie lines in single line 523 // < Set-Cookie: idx=7f2800f63c112a65ef5082957bcca24b; expires=Mon, 29-May-2017 00:31:25 GMT; path=/; domain=example.com 524 // < Set-Cookie: idx=7f2800f63c112a65ef5082957bcca24b; expires=Mon, 29-May-2017 00:31:25 GMT; path=/; domain=example.com, cs=ip764-RgKqc-HvSkxRxdQQAKW8LA; path=/; domain=.example.com; HttpOnly 525 // 526 Cookie[] res; 527 string[string] kv; 528 auto fields = value.split(";").map!strip; 529 while(!fields.empty) { 530 auto s = fields.front.findSplit("="); 531 fields.popFront; 532 if ( s[1] != "=" ) { 533 continue; 534 } 535 auto k = s[0]; 536 auto v = s[2]; 537 switch(k.toLower()) { 538 case "domain": 539 k = "domain"; 540 break; 541 case "path": 542 k = "path"; 543 break; 544 case "expires": 545 continue; 546 case "max-age": 547 continue; 548 default: 549 break; 550 } 551 kv[k] = v; 552 } 553 if ( "domain" !in kv ) { 554 kv["domain"] = _uri.host; 555 } 556 if ( "path" !in kv ) { 557 kv["path"] = _uri.path; 558 } 559 auto domain = kv["domain"]; kv.remove("domain"); 560 auto path = kv["path"]; kv.remove("path"); 561 foreach(pair; kv.byKeyValue) { 562 auto _attr = pair.key; 563 auto _value = pair.value; 564 auto cookie = Cookie(path, domain, _attr, _value); 565 res ~= cookie; 566 } 567 return res; 568 } 569 570 private bool willFollowRedirect() { 571 if ( !canFind(redirectCodes, _response.code) ) { 572 return false; 573 } 574 if ( !_maxRedirects ) { 575 return false; 576 } 577 if ( "location" !in _response.responseHeaders ) { 578 return false; 579 } 580 return true; 581 } 582 private URI uriFromLocation(const ref URI uri, in string location) { 583 URI newURI = uri; 584 try { 585 newURI = URI(location); 586 } catch (UriException e) { 587 debug(requests) trace("Can't parse Location:, try relative uri"); 588 newURI.path = location; 589 newURI.uri = newURI.recalc_uri; 590 } 591 return newURI; 592 } 593 /// 594 /// if we have new uri, then we need to check if we have to reopen existent connection 595 /// 596 private void checkURL(string url, string file=__FILE__, size_t line=__LINE__) { 597 if (url is null && _uri.uri == "" ) { 598 throw new RequestException("No url configured", file, line); 599 } 600 601 if ( url !is null ) { 602 URI newURI = URI(url); 603 //handleURLChange(_uri, newURI); 604 _uri = newURI; 605 } 606 } 607 /// 608 /// Setup connection. Handle proxy and https case 609 /// 610 /// Place new connection in ConnManager cache 611 /// 612 private NetworkStream setupConnection() 613 do { 614 615 debug(requests) tracef("Set up new connection"); 616 NetworkStream stream; 617 618 // on exit 619 // place created connection to conn. manager 620 // close connection purged from manager (if any) 621 // 622 scope(exit) { 623 if ( stream ) 624 { 625 if ( auto purged_connection = _cm.put(_uri.scheme, _uri.host, _uri.port, stream) ) 626 { 627 debug(requests) tracef("closing purged connection %s", purged_connection); 628 purged_connection.close(); 629 } 630 } 631 } 632 633 if ( _socketFactory ) 634 { 635 debug(requests) tracef("use socketFactory"); 636 stream = _socketFactory(_uri.scheme, _uri.host, _uri.port); 637 } 638 639 if ( stream ) // socket factory created connection 640 { 641 return stream; 642 } 643 644 URI uri; // this URI will be used temporarry if we need proxy 645 string actual_proxy = select_proxy(_uri.scheme); 646 final switch (_uri.scheme) { 647 case"http": 648 if ( actual_proxy ) { 649 uri.uri_parse(actual_proxy); 650 uri.idn_encode(); 651 } else { 652 // use original uri 653 uri = _uri; 654 } 655 stream = new TCPStream(); 656 stream.bind(_bind); 657 stream.connect(uri.host, uri.port, _timeout); 658 break ; 659 case"https": 660 if ( actual_proxy ) { 661 uri.uri_parse(actual_proxy); 662 uri.idn_encode(); 663 stream = new TCPStream(); 664 stream.bind(_bind); 665 stream.connect(uri.host, uri.port, _timeout); 666 if ( verbosity>=1 ) { 667 writeln("> CONNECT %s:%d HTTP/1.1".format(_uri.host, _uri.port)); 668 } 669 stream.send("CONNECT %s:%d HTTP/1.1\r\n".format(_uri.host, _uri.port)); 670 if (uri.username) 671 { 672 debug(requests) tracef("Add Proxy-Authorization header"); 673 auto auth = new BasicAuthentication(uri.username, uri.password); 674 auto header = auth.authHeaders(""); 675 stream.send("Proxy-Authorization: %s\r\n".format(header["Authorization"])); 676 } 677 stream.send("\r\n"); 678 while ( stream.isConnected ) { 679 ubyte[1024] b; 680 auto read = stream.receive(b); 681 if ( verbosity>=1) { 682 writefln("< %s", cast(string)b[0..read]); 683 } 684 debug(requests) tracef("read: %d", read); 685 if ( b[0..read].canFind("\r\n\r\n") || b[0..read].canFind("\n\n") ) { 686 debug(requests) tracef("proxy connection ready"); 687 // convert connection to ssl 688 stream = new SSLStream(stream, _sslOptions, _uri.host); 689 break ; 690 } else { 691 debug(requests) tracef("still wait for proxy connection"); 692 } 693 } 694 } else { 695 uri = _uri; 696 stream = new SSLStream(_sslOptions); 697 stream.bind(_bind); 698 stream.connect(uri.host, uri.port, _timeout); 699 debug(requests) tracef("ssl connection to origin server ready"); 700 } 701 break ; 702 } 703 704 return stream; 705 } 706 /// 707 /// Request sent, now receive response. 708 /// Find headers, split on headers and body, continue to receive body 709 /// 710 private void receiveResponse(NetworkStream _stream) { 711 712 try { 713 _stream.readTimeout = timeout; 714 } catch (Exception e) { 715 debug(requests) tracef("Failed to set read timeout for stream: %s", e.msg); 716 return; 717 } 718 // Commented this out as at exit we can have alreade closed socket 719 // scope(exit) { 720 // if ( _stream && _stream.isOpen ) { 721 // _stream.readTimeout = 0.seconds; 722 // } 723 // } 724 725 _bodyDecoder = new DataPipe!ubyte(); 726 scope(exit) { 727 if ( !_useStreaming ) { 728 _bodyDecoder = null; 729 _unChunker = null; 730 } 731 } 732 733 auto buffer = Buffer!ubyte(); 734 Buffer!ubyte partialBody; 735 ptrdiff_t read; 736 string lineSep = null, headersEnd = null; 737 bool headersHaveBeenReceived; 738 739 while( !headersHaveBeenReceived ) { 740 741 auto b = new ubyte[_bufferSize]; 742 read = _stream.receive(b); 743 744 debug(requests) tracef("read: %d", read); 745 if ( read == 0 ) { 746 break; 747 } 748 auto data = b[0..read]; 749 buffer.putNoCopy(data); 750 if ( verbosity>=3 ) { 751 writeln(data.dump.join("\n")); 752 } 753 754 if ( buffer.length > maxHeadersLength ) { 755 throw new RequestException("Headers length > maxHeadersLength (%d > %d)".format(buffer.length, maxHeadersLength)); 756 } 757 758 // Proper HTTP uses "\r\n" as a line separator, but broken servers sometimes use "\n". 759 // Servers that use "\r\n" might have "\n" inside a header. 760 // For any half-sane server, the first '\n' should be at the end of the status line, so this can be used to detect the line separator. 761 // In any case, all the interesting points in the header for now are at '\n' characters, so scan the newly read data for them. 762 foreach (idx; buffer.length-read..buffer.length) 763 { 764 if ( buffer[idx] == '\n' ) 765 { 766 if ( lineSep is null ) 767 { 768 // First '\n'. Detect line/header endings. 769 // HTTP header sections end with a double line separator 770 lineSep = "\n"; 771 headersEnd = "\n\n"; 772 if ( idx > 0 && buffer[idx-1] == '\r' ) 773 { 774 lineSep = "\r\n"; 775 headersEnd = "\r\n\r\n"; 776 } 777 } 778 else 779 { 780 // Potential header ending. 781 if ( buffer.data[0..idx+1].endsWith(headersEnd) ) 782 { 783 auto ResponseHeaders = buffer.data[0..idx+1-headersEnd.length]; 784 partialBody = buffer[idx+1..$]; 785 _contentReceived += partialBody.length; 786 parseResponseHeaders(ResponseHeaders, lineSep); 787 headersHaveBeenReceived = true; 788 break; 789 } 790 } 791 } 792 } 793 } 794 795 analyzeHeaders(_response._responseHeaders); 796 797 _bodyDecoder.putNoCopy(partialBody.data); 798 799 auto v = _bodyDecoder.get(); 800 _response._responseBody.putNoCopy(v); 801 802 // https://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.4 803 if ( (_method == "HEAD") || responseMustNotIncludeBody(_response.code) || (_contentLength < 0 && _unChunker is null) ) 804 { 805 debug(requests) tracef("response without body"); 806 return; 807 } 808 809 _response._contentLength = _contentLength; 810 _response._contentReceived = _contentReceived; 811 812 if ( _verbosity >= 2 ) writefln("< %d bytes of body received", partialBody.length); 813 814 while( true ) { 815 if ( _contentLength >= 0 && _contentReceived >= _contentLength ) { 816 debug(requests) trace("Body received."); 817 break; 818 } 819 if ( _unChunker && _unChunker.done ) { 820 break; 821 } 822 823 if ( _useStreaming && _response._responseBody.length && !redirectCodes.canFind(_response.code) ) { 824 debug(requests) trace("streaming requested"); 825 // save _stream in closure 826 auto __stream = _stream; 827 auto __bodyDecoder = _bodyDecoder; 828 auto __unChunker = _unChunker; 829 auto __contentReceived = _contentReceived; 830 auto __contentLength = _contentLength; 831 auto __bufferSize = _bufferSize; 832 auto __response = _response; 833 auto __verbosity = _verbosity; 834 835 // set up response 836 _response._contentLength = _contentLength; 837 _response.receiveAsRange.activated = true; 838 _response.receiveAsRange.data = _response._responseBody.data; 839 _response.receiveAsRange.read = delegate ubyte[] () { 840 841 while(true) { 842 // check if we received everything we need 843 if ( ( __unChunker && __unChunker.done ) 844 || !__stream.isConnected() 845 || (__contentLength > 0 && __contentReceived >= __contentLength) ) 846 { 847 debug(requests) trace("streaming_in receive completed"); 848 __bodyDecoder.flush(); 849 return __bodyDecoder.get(); 850 } 851 // have to continue 852 auto b = new ubyte[__bufferSize]; 853 try { 854 read = __stream.receive(b); 855 } 856 catch (Exception e) { 857 throw new RequestException("streaming_in error reading from socket", __FILE__, __LINE__, e); 858 } 859 debug(requests) tracef("streaming_in received %d bytes", read); 860 861 if ( read == 0 ) { 862 debug(requests) tracef("streaming_in: server closed connection"); 863 __bodyDecoder.flush(); 864 return __bodyDecoder.get(); 865 } 866 867 if ( __verbosity>=3 ) { 868 writeln(b[0..read].dump.join("\n")); 869 } 870 __response._contentReceived += read; 871 __contentReceived += read; 872 __bodyDecoder.putNoCopy(b[0..read]); 873 auto res = __bodyDecoder.getNoCopy(); 874 if ( res.length == 0 ) { 875 // there were nothing to produce (beginning of the chunk or no decompressed data) 876 continue; 877 } 878 if (res.length == 1) { 879 return res[0]; 880 } 881 // 882 // I'd like to "return _bodyDecoder.getNoCopy().join;" but it is slower 883 // 884 auto total = res.map!(b=>b.length).sum; 885 // create buffer for joined bytes 886 ubyte[] joined = new ubyte[total]; 887 size_t p; 888 // memcopy 889 foreach(ref _; res) { 890 joined[p .. p + _.length] = _; 891 p += _.length; 892 } 893 return joined; 894 } 895 assert(0); 896 }; 897 // we prepared for streaming 898 return; 899 } 900 901 auto b = new ubyte[_bufferSize]; 902 read = _stream.receive(b); 903 904 if ( read == 0 ) { 905 debug(requests) trace("read done"); 906 break; 907 } 908 if ( _verbosity >= 2 ) { 909 writefln("< %d bytes of body received", read); 910 } 911 912 if ( verbosity>=3 ) { 913 writeln(b[0..read].dump.join("\n")); 914 } 915 916 debug(requests) tracef("read: %d", read); 917 _contentReceived += read; 918 if ( _maxContentLength && _contentReceived > _maxContentLength ) { 919 throw new RequestException("ContentLength > maxContentLength (%d>%d)". 920 format(_contentLength, _maxContentLength)); 921 } 922 923 _bodyDecoder.putNoCopy(b[0..read]); // send buffer to all decoders 924 925 _bodyDecoder.getNoCopy. // fetch result and place to body 926 each!(b => _response._responseBody.putNoCopy(b)); 927 928 debug(requests) tracef("receivedTotal: %d, contentLength: %d, bodyLength: %d", _contentReceived, _contentLength, _response._responseBody.length); 929 930 } 931 _bodyDecoder.flush(); 932 _response._responseBody.putNoCopy(_bodyDecoder.get()); 933 _response._contentReceived = _contentReceived; 934 } 935 /// 936 /// Check that we received anything. 937 /// Server can close previous connection (keepalive or not) 938 /// 939 private bool serverPrematurelyClosedConnection() pure @safe { 940 immutable server_closed_connection = _response._responseHeaders.length == 0 && _response._status_line.length == 0; 941 // debug(requests) tracef("server closed connection = %s (headers.length=%s, status_line.length=%s)", 942 // server_closed_connection, _response._responseHeaders.length, _response._status_line.length); 943 return server_closed_connection; 944 } 945 private bool isIdempotent(in string method) pure @safe nothrow { 946 return ["GET", "HEAD"].canFind(method); 947 } 948 /// 949 /// If we do not want keepalive request, 950 /// or server signalled to close connection, 951 /// then close it 952 /// 953 void close_connection_if_not_keepalive(NetworkStream _stream) { 954 auto connection = "connection" in _response._responseHeaders; 955 if ( !_keepAlive ) { 956 _stream.close(); 957 } else switch(_response._version) { 958 case HTTP11: 959 // HTTP/1.1 defines the "close" connection option for the sender to signal that the connection 960 // will be closed after completion of the response. For example, 961 // Connection: close 962 // in either the request or the response header fields indicates that the connection 963 // SHOULD NOT be considered `persistent' (section 8.1) after the current request/response is complete. 964 // HTTP/1.1 applications that do not support persistent connections MUST include the "close" connection 965 // option in every message. 966 if ( connection && (*connection).toLower.split(",").canFind("close") ) { 967 _stream.close(); 968 } 969 break; 970 default: 971 // for anything else close connection if there is no keep-alive in Connection 972 if ( connection && !(*connection).toLower.split(",").canFind("keep-alive") ) { 973 _stream.close(); 974 } 975 break; 976 } 977 } 978 /// 979 /// Send multipart for request. 980 /// You would like to use this method for sending large portions of mixed data or uploading files to forms. 981 /// Content of the posted form consist of sources. Each source have at least name and value (can be string-like object or opened file, see more docs for MultipartForm struct) 982 /// Params: 983 /// url = url 984 /// sources = array of sources. 985 deprecated("Use Request() instead of HTTPRequest(); will be removed 2019-07") 986 HTTPResponse exec(string method="POST")(string url, MultipartForm sources) { 987 import std.uuid; 988 import std.file; 989 990 checkURL(url); 991 //if ( _cm is null ) { 992 // _cm = new ConnManager(); 993 //} 994 995 NetworkStream _stream; 996 _method = method; 997 _response = new HTTPResponse; 998 _response.uri = _uri; 999 _response.finalURI = _uri; 1000 bool restartedRequest = false; 1001 1002 connect: 1003 _contentReceived = 0; 1004 _response._startedAt = Clock.currTime; 1005 1006 assert(_stream is null); 1007 1008 _stream = _cm.get(_uri.scheme, _uri.host, _uri.port); 1009 1010 if ( _stream is null ) { 1011 debug(requests) trace("create new connection"); 1012 _stream = setupConnection(); 1013 } else { 1014 debug(requests) trace("reuse old connection"); 1015 } 1016 1017 assert(_stream !is null); 1018 1019 if ( !_stream.isConnected ) { 1020 debug(requests) trace("disconnected stream on enter"); 1021 if ( !restartedRequest ) { 1022 debug(requests) trace("disconnected stream on enter: retry"); 1023 assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream); 1024 1025 _cm.del(_uri.scheme, _uri.host, _uri.port); 1026 _stream.close(); 1027 _stream = null; 1028 1029 restartedRequest = true; 1030 goto connect; 1031 } 1032 debug(requests) trace("disconnected stream on enter: return response"); 1033 //_stream = null; 1034 return _response; 1035 } 1036 _response._connectedAt = Clock.currTime; 1037 1038 Appender!string req; 1039 req.put(requestString()); 1040 1041 string boundary = randomUUID().toString; 1042 string[] partHeaders; 1043 size_t contentLength; 1044 1045 foreach(ref part; sources._sources) { 1046 string h = "--" ~ boundary ~ "\r\n"; 1047 string disposition = `form-data; name="%s"`.format(part.name); 1048 string optionals = part. 1049 parameters.byKeyValue(). 1050 filter!(p => p.key!="Content-Type"). 1051 map! (p => "%s=%s".format(p.key, p.value)). 1052 join("; "); 1053 1054 h ~= `Content-Disposition: ` ~ [disposition, optionals].join("; ") ~ "\r\n"; 1055 1056 auto contentType = "Content-Type" in part.parameters; 1057 if ( contentType ) { 1058 h ~= "Content-Type: " ~ *contentType ~ "\r\n"; 1059 } 1060 1061 h ~= "\r\n"; 1062 partHeaders ~= h; 1063 contentLength += h.length + part.input.getSize() + "\r\n".length; 1064 } 1065 contentLength += "--".length + boundary.length + "--\r\n".length; 1066 1067 auto h = requestHeaders(); 1068 safeSetHeader(h, _userHeaders.ContentType, "Content-Type", "multipart/form-data; boundary=" ~ boundary); 1069 safeSetHeader(h, _userHeaders.ContentLength, "Content-Length", to!string(contentLength)); 1070 1071 h.byKeyValue. 1072 map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n"). 1073 each!(h => req.put(h)); 1074 req.put("\r\n"); 1075 1076 debug(requests) trace(req.data); 1077 if ( _verbosity >= 1 ) req.data.splitLines.each!(a => writeln("> " ~ a)); 1078 1079 try { 1080 _stream.send(req.data()); 1081 foreach(ref source; sources._sources) { 1082 debug(requests) tracef("sending part headers <%s>", partHeaders.front); 1083 _stream.send(partHeaders.front); 1084 partHeaders.popFront; 1085 while (true) { 1086 auto chunk = source.input.read(); 1087 if ( chunk.length <= 0 ) { 1088 break; 1089 } 1090 _stream.send(chunk); 1091 } 1092 _stream.send("\r\n"); 1093 } 1094 _stream.send("--" ~ boundary ~ "--\r\n"); 1095 _response._requestSentAt = Clock.currTime; 1096 receiveResponse(_stream); 1097 _response._finishedAt = Clock.currTime; 1098 } 1099 catch (NetworkException e) { 1100 errorf("Error sending request: ", e.msg); 1101 _stream.close(); 1102 return _response; 1103 } 1104 1105 if ( serverPrematurelyClosedConnection() 1106 && !restartedRequest 1107 && isIdempotent(_method) 1108 ) { 1109 /// 1110 /// We didn't receive any data (keepalive connectioin closed?) 1111 /// and we can restart this request. 1112 /// Go ahead. 1113 /// 1114 debug(requests) tracef("Server closed keepalive connection"); 1115 1116 assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream); 1117 1118 _cm.del(_uri.scheme, _uri.host, _uri.port); 1119 _stream.close(); 1120 _stream = null; 1121 1122 restartedRequest = true; 1123 goto connect; 1124 } 1125 1126 if ( _useStreaming ) { 1127 if ( _response._receiveAsRange.activated ) { 1128 debug(requests) trace("streaming_in activated"); 1129 return _response; 1130 } else { 1131 // this can happen if whole response body received together with headers 1132 _response._receiveAsRange.data = _response.responseBody.data; 1133 } 1134 } 1135 1136 close_connection_if_not_keepalive(_stream); 1137 1138 if ( _verbosity >= 1 ) { 1139 writeln(">> Connect time: ", _response._connectedAt - _response._startedAt); 1140 writeln(">> Request send time: ", _response._requestSentAt - _response._connectedAt); 1141 writeln(">> Response recv time: ", _response._finishedAt - _response._requestSentAt); 1142 } 1143 1144 if ( willFollowRedirect ) { 1145 if ( _history.length >= _maxRedirects ) { 1146 _stream = null; 1147 throw new MaxRedirectsException("%d redirects reached maxRedirects %d.".format(_history.length, _maxRedirects)); 1148 } 1149 // "location" in response already checked in canFollowRedirect 1150 immutable new_location = *("location" in _response.responseHeaders); 1151 immutable current_uri = _uri, next_uri = uriFromLocation(_uri, new_location); 1152 1153 // save current response for history 1154 _history ~= _response; 1155 1156 // prepare new response (for redirected request) 1157 _response = new HTTPResponse; 1158 _response.uri = current_uri; 1159 _response.finalURI = next_uri; 1160 _stream = null; 1161 1162 // set new uri 1163 this._uri = next_uri; 1164 debug(requests) tracef("Redirected to %s", next_uri); 1165 if ( _method != "GET" && _response.code != 307 && _response.code != 308 ) { 1166 // 307 and 308 do not change method 1167 return this.get(); 1168 } 1169 if ( restartedRequest ) { 1170 debug(requests) trace("Rare event: clearing 'restartedRequest' on redirect"); 1171 restartedRequest = false; 1172 } 1173 goto connect; 1174 } 1175 1176 _response._history = _history; 1177 return _response; 1178 } 1179 1180 // we use this if we send from ubyte[][] and user provided Content-Length 1181 private void sendFlattenContent(T)(NetworkStream _stream, T content) { 1182 while ( !content.empty ) { 1183 auto chunk = content.front; 1184 _stream.send(chunk); 1185 content.popFront; 1186 } 1187 debug(requests) tracef("sent"); 1188 } 1189 // we use this if we send from ubyte[][] as chunked content 1190 private void sendChunkedContent(T)(NetworkStream _stream, T content) { 1191 while ( !content.empty ) { 1192 auto chunk = content.front; 1193 auto chunkHeader = "%x\r\n".format(chunk.length); 1194 debug(requests) tracef("sending %s%s", chunkHeader, chunk); 1195 _stream.send(chunkHeader); 1196 _stream.send(chunk); 1197 _stream.send("\r\n"); 1198 content.popFront; 1199 } 1200 debug(requests) tracef("sent"); 1201 _stream.send("0\r\n\r\n"); 1202 } 1203 /// 1204 /// POST/PUT/... data from some string(with Content-Length), or from range of strings/bytes (use Transfer-Encoding: chunked). 1205 /// When rank 1 (flat array) used as content it must have length. In that case "content" will be sent directly to network, and Content-Length headers will be added. 1206 /// If you are goung to send some range and do not know length at the moment when you start to send request, then you can send chunks of chars or ubyte. 1207 /// Try not to send too short chunks as this will put additional load on client and server. Chunks of length 2048 or 4096 are ok. 1208 /// 1209 /// Parameters: 1210 /// url = url 1211 /// content = string or input range 1212 /// contentType = content type 1213 /// Returns: 1214 /// Response 1215 /// Examples: 1216 /// --------------------------------------------------------------------------------------------------------- 1217 /// rs = rq.exec!"POST"("http://httpbin.org/post", "привiт, свiт!", "application/octet-stream"); 1218 /// 1219 /// auto s = lineSplitter("one,\ntwo,\nthree."); 1220 /// rs = rq.exec!"POST"("http://httpbin.org/post", s, "application/octet-stream"); 1221 /// 1222 /// auto s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; 1223 /// rs = rq.exec!"POST"("http://httpbin.org/post", s.representation.chunks(10), "application/octet-stream"); 1224 /// 1225 /// auto f = File("tests/test.txt", "rb"); 1226 /// rs = rq.exec!"POST"("http://httpbin.org/post", f.byChunk(3), "application/octet-stream"); 1227 /// -------------------------------------------------------------------------------------------------------- 1228 deprecated("Use Request() instead of HTTPRequest(); will be removed 2019-07") 1229 HTTPResponse exec(string method="POST", R)(string url, R content, string contentType="application/octet-stream") 1230 if ( (rank!R == 1) 1231 || (rank!R == 2 && isSomeChar!(Unqual!(typeof(content.front.front)))) 1232 || (rank!R == 2 && (is(Unqual!(typeof(content.front.front)) == ubyte))) 1233 ) 1234 do { 1235 debug(requests) tracef("started url=%s, this._uri=%s", url, _uri); 1236 1237 checkURL(url); 1238 //if ( _cm is null ) { 1239 // _cm = new ConnManager(); 1240 //} 1241 1242 NetworkStream _stream; 1243 _method = method; 1244 _response = new HTTPResponse; 1245 _history.length = 0; 1246 _response.uri = _uri; 1247 _response.finalURI = _uri; 1248 bool restartedRequest = false; 1249 bool send_flat; 1250 1251 connect: 1252 _contentReceived = 0; 1253 _response._startedAt = Clock.currTime; 1254 1255 assert(_stream is null); 1256 1257 _stream = _cm.get(_uri.scheme, _uri.host, _uri.port); 1258 1259 if ( _stream is null ) { 1260 debug(requests) trace("create new connection"); 1261 _stream = setupConnection(); 1262 } else { 1263 debug(requests) trace("reuse old connection"); 1264 } 1265 1266 assert(_stream !is null); 1267 1268 if ( !_stream.isConnected ) { 1269 debug(requests) trace("disconnected stream on enter"); 1270 if ( !restartedRequest ) { 1271 debug(requests) trace("disconnected stream on enter: retry"); 1272 assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream); 1273 1274 _cm.del(_uri.scheme, _uri.host, _uri.port); 1275 _stream.close(); 1276 _stream = null; 1277 1278 restartedRequest = true; 1279 goto connect; 1280 } 1281 debug(requests) trace("disconnected stream on enter: return response"); 1282 //_stream = null; 1283 return _response; 1284 } 1285 _response._connectedAt = Clock.currTime; 1286 1287 Appender!string req; 1288 req.put(requestString()); 1289 1290 auto h = requestHeaders; 1291 if ( contentType ) { 1292 safeSetHeader(h, _userHeaders.ContentType, "Content-Type", contentType); 1293 } 1294 static if ( rank!R == 1 ) { 1295 safeSetHeader(h, _userHeaders.ContentLength, "Content-Length", to!string(content.length)); 1296 } else { 1297 if ( _userHeaders.ContentLength ) { 1298 debug(requests) tracef("User provided content-length for chunked content"); 1299 send_flat = true; 1300 } else { 1301 h["Transfer-Encoding"] = "chunked"; 1302 send_flat = false; 1303 } 1304 } 1305 h.byKeyValue. 1306 map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n"). 1307 each!(h => req.put(h)); 1308 req.put("\r\n"); 1309 1310 debug(requests) trace(req.data); 1311 if ( _verbosity >= 1 ) { 1312 req.data.splitLines.each!(a => writeln("> " ~ a)); 1313 } 1314 1315 try { 1316 // send headers 1317 _stream.send(req.data()); 1318 // send body 1319 static if ( rank!R == 1) { 1320 _stream.send(content); 1321 } else { 1322 if ( send_flat ) { 1323 sendFlattenContent(_stream, content); 1324 } else { 1325 sendChunkedContent(_stream, content); 1326 } 1327 } 1328 _response._requestSentAt = Clock.currTime; 1329 debug(requests) trace("starting receive response"); 1330 receiveResponse(_stream); 1331 debug(requests) trace("finished receive response"); 1332 _response._finishedAt = Clock.currTime; 1333 } catch (NetworkException e) { 1334 _stream.close(); 1335 throw new RequestException("Network error during data exchange"); 1336 } 1337 1338 if ( serverPrematurelyClosedConnection() 1339 && !restartedRequest 1340 && isIdempotent(_method)) 1341 { 1342 /// 1343 /// We didn't receive any data (keepalive connectioin closed?) 1344 /// and we can restart this request. 1345 /// Go ahead. 1346 /// 1347 debug(requests) tracef("Server closed keepalive connection"); 1348 1349 assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream); 1350 1351 _cm.del(_uri.scheme, _uri.host, _uri.port); 1352 _stream.close(); 1353 _stream = null; 1354 1355 restartedRequest = true; 1356 goto connect; 1357 } 1358 1359 if ( _useStreaming ) { 1360 if ( _response._receiveAsRange.activated ) { 1361 debug(requests) trace("streaming_in activated"); 1362 return _response; 1363 } else { 1364 // this can happen if whole response body received together with headers 1365 _response._receiveAsRange.data = _response.responseBody.data; 1366 } 1367 } 1368 1369 close_connection_if_not_keepalive(_stream); 1370 1371 if ( _verbosity >= 1 ) { 1372 writeln(">> Connect time: ", _response._connectedAt - _response._startedAt); 1373 writeln(">> Request send time: ", _response._requestSentAt - _response._connectedAt); 1374 writeln(">> Response recv time: ", _response._finishedAt - _response._requestSentAt); 1375 } 1376 1377 1378 if ( willFollowRedirect ) { 1379 if ( _history.length >= _maxRedirects ) { 1380 _stream = null; 1381 throw new MaxRedirectsException("%d redirects reached maxRedirects %d.".format(_history.length, _maxRedirects)); 1382 } 1383 // "location" in response already checked in canFollowRedirect 1384 immutable new_location = *("location" in _response.responseHeaders); 1385 immutable current_uri = _uri, next_uri = uriFromLocation(_uri, new_location); 1386 1387 // save current response for history 1388 _history ~= _response; 1389 1390 // prepare new response (for redirected request) 1391 _response = new HTTPResponse; 1392 _response.uri = current_uri; 1393 _response.finalURI = next_uri; 1394 1395 _stream = null; 1396 1397 // set new uri 1398 this._uri = next_uri; 1399 debug(requests) tracef("Redirected to %s", next_uri); 1400 if ( _method != "GET" && _response.code != 307 && _response.code != 308 ) { 1401 // 307 and 308 do not change method 1402 return this.get(); 1403 } 1404 if ( restartedRequest ) { 1405 debug(requests) trace("Rare event: clearing 'restartedRequest' on redirect"); 1406 restartedRequest = false; 1407 } 1408 goto connect; 1409 } 1410 1411 _response._history = _history; 1412 return _response; 1413 } 1414 /// 1415 /// Send request with parameters. 1416 /// If used for POST or PUT requests then application/x-www-form-urlencoded used. 1417 /// Request parameters will be encoded into request string or placed in request body for POST/PUT 1418 /// requests. 1419 /// Parameters: 1420 /// url = url 1421 /// params = request parameters 1422 /// Returns: 1423 /// Response 1424 /// Examples: 1425 /// --------------------------------------------------------------------------------- 1426 /// rs = Request().exec!"GET"("http://httpbin.org/get", ["c":"d", "a":"b"]); 1427 /// --------------------------------------------------------------------------------- 1428 /// 1429 deprecated("Use Request() instead of HTTPRequest; will be removed 2019-07") 1430 HTTPResponse exec(string method="GET")(string url = null, QueryParam[] params = null) 1431 do { 1432 debug(requests) tracef("started url=%s, this._uri=%s", url, _uri); 1433 1434 checkURL(url); 1435 //if ( _cm is null ) { 1436 // _cm = new ConnManager(); 1437 //} 1438 1439 NetworkStream _stream; 1440 _method = method; 1441 _response = new HTTPResponse; 1442 _history.length = 0; 1443 _response.uri = _uri; 1444 _response.finalURI = _uri; 1445 bool restartedRequest = false; // True if this is restarted keepAlive request 1446 1447 connect: 1448 if ( _method == "GET" && _uri in _permanent_redirects ) { 1449 debug(requests) trace("use parmanent redirects cache"); 1450 _uri = uriFromLocation(_uri, _permanent_redirects[_uri]); 1451 _response._finalURI = _uri; 1452 } 1453 _contentReceived = 0; 1454 _response._startedAt = Clock.currTime; 1455 1456 assert(_stream is null); 1457 1458 _stream = _cm.get(_uri.scheme, _uri.host, _uri.port); 1459 1460 if ( _stream is null ) { 1461 debug(requests) trace("create new connection"); 1462 _stream = setupConnection(); 1463 } else { 1464 debug(requests) trace("reuse old connection"); 1465 } 1466 1467 assert(_stream !is null); 1468 1469 if ( !_stream.isConnected ) { 1470 debug(requests) trace("disconnected stream on enter"); 1471 if ( !restartedRequest ) { 1472 debug(requests) trace("disconnected stream on enter: retry"); 1473 assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream); 1474 1475 _cm.del(_uri.scheme, _uri.host, _uri.port); 1476 _stream.close(); 1477 _stream = null; 1478 1479 restartedRequest = true; 1480 goto connect; 1481 } 1482 debug(requests) trace("disconnected stream on enter: return response"); 1483 //_stream = null; 1484 return _response; 1485 } 1486 _response._connectedAt = Clock.currTime; 1487 1488 auto h = requestHeaders(); 1489 1490 Appender!string req; 1491 1492 string encoded; 1493 1494 switch (_method) { 1495 case "POST","PUT","PATCH": 1496 encoded = params2query(params); 1497 safeSetHeader(h, _userHeaders.ContentType, "Content-Type", "application/x-www-form-urlencoded"); 1498 if ( encoded.length > 0) { 1499 safeSetHeader(h, _userHeaders.ContentLength, "Content-Length", to!string(encoded.length)); 1500 } 1501 req.put(requestString()); 1502 break; 1503 default: 1504 req.put(requestString(params)); 1505 } 1506 1507 h.byKeyValue. 1508 map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n"). 1509 each!(h => req.put(h)); 1510 req.put("\r\n"); 1511 if ( encoded ) { 1512 req.put(encoded); 1513 } 1514 1515 debug(requests) trace(req.data); 1516 if ( _verbosity >= 1 ) { 1517 req.data.splitLines.each!(a => writeln("> " ~ a)); 1518 } 1519 // 1520 // Now send request and receive response 1521 // 1522 try { 1523 _stream.send(req.data()); 1524 _response._requestSentAt = Clock.currTime; 1525 debug(requests) trace("starting receive response"); 1526 receiveResponse(_stream); 1527 debug(requests) trace("done receive response"); 1528 _response._finishedAt = Clock.currTime; 1529 } 1530 catch (NetworkException e) { 1531 // On SEND this can means: 1532 // we started to send request to the server, but it closed connection because of keepalive timeout. 1533 // We have to restart request if possible. 1534 1535 // On RECEIVE - if we received something - then this exception is real and unexpected error. 1536 // If we didn't receive anything - we can restart request again as it can be 1537 debug(requests) tracef("Exception on receive response: %s", e.msg); 1538 if ( _response._responseHeaders.length != 0 ) 1539 { 1540 _stream.close(); 1541 throw new RequestException("Unexpected network error"); 1542 } 1543 } 1544 1545 if ( serverPrematurelyClosedConnection() 1546 && !restartedRequest 1547 && isIdempotent(_method) 1548 ) { 1549 /// 1550 /// We didn't receive any data (keepalive connectioin closed?) 1551 /// and we can restart this request. 1552 /// Go ahead. 1553 /// 1554 debug(requests) tracef("Server closed keepalive connection"); 1555 1556 assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream); 1557 1558 _cm.del(_uri.scheme, _uri.host, _uri.port); 1559 _stream.close(); 1560 _stream = null; 1561 1562 restartedRequest = true; 1563 goto connect; 1564 } 1565 1566 if ( _useStreaming ) { 1567 if ( _response._receiveAsRange.activated ) { 1568 debug(requests) trace("streaming_in activated"); 1569 return _response; 1570 } else { 1571 // this can happen if whole response body received together with headers 1572 _response._receiveAsRange.data = _response.responseBody.data; 1573 } 1574 } 1575 1576 close_connection_if_not_keepalive(_stream); 1577 1578 if ( _verbosity >= 1 ) { 1579 writeln(">> Connect time: ", _response._connectedAt - _response._startedAt); 1580 writeln(">> Request send time: ", _response._requestSentAt - _response._connectedAt); 1581 writeln(">> Response recv time: ", _response._finishedAt - _response._requestSentAt); 1582 } 1583 1584 if ( willFollowRedirect ) { 1585 debug(requests) trace("going to follow redirect"); 1586 if ( _history.length >= _maxRedirects ) { 1587 _stream = null; 1588 throw new MaxRedirectsException("%d redirects reached maxRedirects %d.".format(_history.length, _maxRedirects)); 1589 } 1590 // "location" in response already checked in canFollowRedirect 1591 immutable new_location = *("location" in _response.responseHeaders); 1592 immutable current_uri = _uri, next_uri = uriFromLocation(_uri, new_location); 1593 1594 if ( _method == "GET" && _response.code == 301 ) { 1595 _permanent_redirects[_uri] = new_location; 1596 } 1597 1598 // save current response for history 1599 _history ~= _response; 1600 1601 // prepare new response (for redirected request) 1602 _response = new HTTPResponse; 1603 _response.uri = current_uri; 1604 _response.finalURI = next_uri; 1605 _stream = null; 1606 1607 // set new uri 1608 _uri = next_uri; 1609 debug(requests) tracef("Redirected to %s", next_uri); 1610 if ( _method != "GET" && _response.code != 307 && _response.code != 308 ) { 1611 // 307 and 308 do not change method 1612 return this.get(); 1613 } 1614 if ( restartedRequest ) { 1615 debug(requests) trace("Rare event: clearing 'restartedRequest' on redirect"); 1616 restartedRequest = false; 1617 } 1618 goto connect; 1619 } 1620 1621 _response._history = _history; 1622 return _response; 1623 } 1624 1625 /// WRAPPERS 1626 /// 1627 /// send file(s) using POST and multipart form. 1628 /// This wrapper will be deprecated, use post with MultipartForm - it is more general and clear. 1629 /// Parameters: 1630 /// url = url 1631 /// files = array of PostFile structures 1632 /// Returns: 1633 /// Response 1634 /// Each PostFile structure contain path to file, and optional field name and content type. 1635 /// If no field name provided, then basename of the file will be used. 1636 /// application/octet-stream is default when no content type provided. 1637 /// Example: 1638 /// --------------------------------------------------------------- 1639 /// PostFile[] files = [ 1640 /// {fileName:"tests/abc.txt", fieldName:"abc", contentType:"application/octet-stream"}, 1641 /// {fileName:"tests/test.txt"} 1642 /// ]; 1643 /// rs = rq.exec!"POST"("http://httpbin.org/post", files); 1644 /// --------------------------------------------------------------- 1645 /// 1646 deprecated("Use Request() instead of HTTPRequest(); will be removed 2019-07") 1647 HTTPResponse exec(string method="POST")(string url, PostFile[] files) if (method=="POST") { 1648 MultipartForm multipart; 1649 File[] toClose; 1650 foreach(ref f; files) { 1651 File file = File(f.fileName, "rb"); 1652 toClose ~= file; 1653 string fileName = f.fileName ? f.fileName : f.fieldName; 1654 string contentType = f.contentType ? f.contentType : "application/octetstream"; 1655 multipart.add(f.fieldName, new FormDataFile(file), ["filename":fileName, "Content-Type": contentType]); 1656 } 1657 auto res = exec!"POST"(url, multipart); 1658 toClose.each!"a.close"; 1659 return res; 1660 } 1661 /// 1662 /// exec request with parameters when you can use dictionary (when you have no duplicates in parameter names) 1663 /// Consider switch to exec(url, QueryParams) as it more generic and clear. 1664 /// Parameters: 1665 /// url = url 1666 /// params = dictionary with field names as keys and field values as values. 1667 /// Returns: 1668 /// Response 1669 deprecated("Use Request() instead of HTTPRequest(); will be removed 2019-07") 1670 HTTPResponse exec(string method="GET")(string url, string[string] params) { 1671 return exec!method(url, params.byKeyValue.map!(p => QueryParam(p.key, p.value)).array); 1672 } 1673 /// 1674 /// GET request. Simple wrapper over exec!"GET" 1675 /// Params: 1676 /// args = request parameters. see exec docs. 1677 /// 1678 deprecated("Use Request() instead of HTTPRequest; will be removed 2019-07") 1679 HTTPResponse get(A...)(A args) { 1680 return exec!"GET"(args); 1681 } 1682 /// 1683 /// POST request. Simple wrapper over exec!"POST" 1684 /// Params: 1685 /// uri = endpoint uri 1686 /// args = request parameters. see exec docs. 1687 /// 1688 deprecated("Use Request() instead of HTTPRequest; will be removed 2019-07") 1689 HTTPResponse post(A...)(string uri, A args) { 1690 return exec!"POST"(uri, args); 1691 } 1692 1693 import requests.request; 1694 1695 // we use this if we send from ubyte[][] and user provided Content-Length 1696 private void sendFlattenContent(NetworkStream _stream) { 1697 while ( !_postData.empty ) { 1698 auto chunk = _postData.front; 1699 _stream.send(chunk); 1700 _postData.popFront; 1701 } 1702 debug(requests) tracef("sent"); 1703 } 1704 // we use this if we send from ubyte[][] as chunked content 1705 private void sendChunkedContent(NetworkStream _stream) { 1706 while ( !_postData.empty ) { 1707 auto chunk = _postData.front; 1708 auto chunkHeader = "%x\r\n".format(chunk.length); 1709 debug(requests) tracef("sending %s%s", chunkHeader, cast(string)chunk); 1710 _stream.send(chunkHeader); 1711 _stream.send(chunk); 1712 _stream.send("\r\n"); 1713 debug(requests) tracef("chunk sent"); 1714 _postData.popFront; 1715 } 1716 debug(requests) tracef("sent"); 1717 _stream.send("0\r\n\r\n"); 1718 } 1719 1720 HTTPResponse exec_from_range(InputRangeAdapter postData) 1721 do { 1722 1723 _postData = postData; 1724 1725 debug(requests) tracef("exec from range"); 1726 1727 NetworkStream _stream; 1728 _response = new HTTPResponse; 1729 _history.length = 0; 1730 _response.uri = _uri; 1731 _response.finalURI = _uri; 1732 bool restartedRequest = false; 1733 bool send_flat; 1734 1735 connect: 1736 _contentReceived = 0; 1737 _response._startedAt = Clock.currTime; 1738 1739 assert(_stream is null); 1740 1741 _stream = _cm.get(_uri.scheme, _uri.host, _uri.port); 1742 1743 if ( _stream is null ) { 1744 debug(requests) trace("create new connection"); 1745 _stream = setupConnection(); 1746 } else { 1747 debug(requests) trace("reuse old connection"); 1748 } 1749 1750 assert(_stream !is null); 1751 1752 if ( !_stream.isConnected ) { 1753 debug(requests) trace("disconnected stream on enter"); 1754 if ( !restartedRequest ) { 1755 debug(requests) trace("disconnected stream on enter: retry"); 1756 assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream); 1757 1758 _cm.del(_uri.scheme, _uri.host, _uri.port); 1759 _stream.close(); 1760 _stream = null; 1761 1762 restartedRequest = true; 1763 goto connect; 1764 } 1765 debug(requests) trace("disconnected stream on enter: return response"); 1766 //_stream = null; 1767 return _response; 1768 } 1769 _response._connectedAt = Clock.currTime; 1770 1771 Appender!string req; 1772 req.put(requestString()); 1773 1774 auto h = requestHeaders; 1775 if ( _contentType ) { 1776 safeSetHeader(h, _userHeaders.ContentType, "Content-Type", _contentType); 1777 } 1778 1779 if ( _postData.length >= 0 ) 1780 { 1781 // we know t 1782 safeSetHeader(h, _userHeaders.ContentLength, "Content-Length", to!string(_postData.length)); 1783 } 1784 1785 if ( _userHeaders.ContentLength || "Content-Length" in h ) 1786 { 1787 debug(requests) tracef("User provided content-length for chunked content"); 1788 send_flat = true; 1789 } 1790 else 1791 { 1792 h["Transfer-Encoding"] = "chunked"; 1793 send_flat = false; 1794 } 1795 h.byKeyValue. 1796 map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n"). 1797 each!(h => req.put(h)); 1798 req.put("\r\n"); 1799 1800 debug(requests) tracef("send <%s>", req.data); 1801 if ( _verbosity >= 1 ) { 1802 req.data.splitLines.each!(a => writeln("> " ~ a)); 1803 } 1804 1805 try { 1806 // send headers 1807 _stream.send(req.data()); 1808 // send body 1809 if ( send_flat ) { 1810 sendFlattenContent(_stream); 1811 } else { 1812 sendChunkedContent(_stream); 1813 } 1814 _response._requestSentAt = Clock.currTime; 1815 debug(requests) trace("starting receive response"); 1816 receiveResponse(_stream); 1817 debug(requests) trace("finished receive response"); 1818 _response._finishedAt = Clock.currTime; 1819 } 1820 catch (NetworkException e) 1821 { 1822 _stream.close(); 1823 throw new RequestException("Network error during data exchange"); 1824 } 1825 if ( serverPrematurelyClosedConnection() 1826 && !restartedRequest 1827 && isIdempotent(_method) 1828 ) { 1829 /// 1830 /// We didn't receive any data (keepalive connectioin closed?) 1831 /// and we can restart this request. 1832 /// Go ahead. 1833 /// 1834 debug(requests) tracef("Server closed keepalive connection"); 1835 1836 assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream); 1837 1838 _cm.del(_uri.scheme, _uri.host, _uri.port); 1839 _stream.close(); 1840 _stream = null; 1841 1842 restartedRequest = true; 1843 goto connect; 1844 } 1845 1846 if ( _useStreaming ) { 1847 if ( _response._receiveAsRange.activated ) { 1848 debug(requests) trace("streaming_in activated"); 1849 return _response; 1850 } else { 1851 // this can happen if whole response body received together with headers 1852 _response._receiveAsRange.data = _response.responseBody.data; 1853 } 1854 } 1855 1856 close_connection_if_not_keepalive(_stream); 1857 1858 if ( _verbosity >= 1 ) { 1859 writeln(">> Connect time: ", _response._connectedAt - _response._startedAt); 1860 writeln(">> Request send time: ", _response._requestSentAt - _response._connectedAt); 1861 writeln(">> Response recv time: ", _response._finishedAt - _response._requestSentAt); 1862 } 1863 1864 1865 if ( willFollowRedirect ) { 1866 if ( _history.length >= _maxRedirects ) { 1867 _stream = null; 1868 throw new MaxRedirectsException("%d redirects reached maxRedirects %d.".format(_history.length, _maxRedirects)); 1869 } 1870 // "location" in response already checked in canFollowRedirect 1871 immutable new_location = *("location" in _response.responseHeaders); 1872 immutable current_uri = _uri, next_uri = uriFromLocation(_uri, new_location); 1873 1874 immutable get_or_head = _method == "GET" || _method == "HEAD"; 1875 immutable code = _response.code; 1876 1877 // save current response for history 1878 _history ~= _response; 1879 1880 if ( code == 301 ) 1881 { 1882 // permanent redirect and change method 1883 _permanent_redirects[_uri] = new_location; 1884 if ( !get_or_head ) 1885 { 1886 _method = "GET"; 1887 } 1888 } 1889 if ( (code == 302 || code == 303) && !get_or_head) 1890 { 1891 // only change method 1892 _method = "GET"; 1893 } 1894 if ( code == 307 ) 1895 { 1896 // no change method, no permanent 1897 } 1898 if ( code == 308 ) 1899 { 1900 // permanent redirection and do not change method 1901 _permanent_redirects[_uri] = new_location; 1902 } 1903 1904 // prepare new response (for redirected request) 1905 _response = new HTTPResponse; 1906 _response.uri = current_uri; 1907 _response.finalURI = next_uri; 1908 1909 _stream = null; 1910 1911 // set new uri 1912 this._uri = next_uri; 1913 debug(requests) tracef("Redirected to %s", next_uri); 1914 if ( restartedRequest ) { 1915 debug(requests) trace("Rare event: clearing 'restartedRequest' on redirect"); 1916 restartedRequest = false; 1917 } 1918 if ( _method == "GET") 1919 { 1920 return exec_from_parameters(); 1921 } 1922 goto connect; 1923 } 1924 1925 _response._history = _history; 1926 return _response; 1927 } 1928 1929 HTTPResponse exec_from_multipart_form(MultipartForm form) { 1930 import std.uuid; 1931 import std.file; 1932 1933 _multipartForm = form; 1934 1935 debug(requests) tracef("exec from multipart form"); 1936 1937 NetworkStream _stream; 1938 _response = new HTTPResponse; 1939 _response.uri = _uri; 1940 _response.finalURI = _uri; 1941 bool restartedRequest = false; 1942 1943 connect: 1944 _contentReceived = 0; 1945 _response._startedAt = Clock.currTime; 1946 1947 assert(_stream is null); 1948 1949 _stream = _cm.get(_uri.scheme, _uri.host, _uri.port); 1950 1951 if ( _stream is null ) { 1952 debug(requests) trace("create new connection"); 1953 _stream = setupConnection(); 1954 } else { 1955 debug(requests) trace("reuse old connection"); 1956 } 1957 1958 assert(_stream !is null); 1959 1960 if ( !_stream.isConnected ) { 1961 debug(requests) trace("disconnected stream on enter"); 1962 if ( !restartedRequest ) { 1963 debug(requests) trace("disconnected stream on enter: retry"); 1964 assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream); 1965 1966 _cm.del(_uri.scheme, _uri.host, _uri.port); 1967 _stream.close(); 1968 _stream = null; 1969 1970 restartedRequest = true; 1971 goto connect; 1972 } 1973 debug(requests) trace("disconnected stream on enter: return response"); 1974 //_stream = null; 1975 return _response; 1976 } 1977 _response._connectedAt = Clock.currTime; 1978 1979 Appender!string req; 1980 req.put(requestString()); 1981 1982 string boundary = randomUUID().toString; 1983 string[] partHeaders; 1984 size_t contentLength; 1985 1986 foreach(ref part; _multipartForm._sources) { 1987 string h = "--" ~ boundary ~ "\r\n"; 1988 string disposition = `form-data; name="%s"`.format(part.name); 1989 string optionals = part. 1990 parameters.byKeyValue(). 1991 filter!(p => p.key!="Content-Type"). 1992 map! (p => "%s=%s".format(p.key, p.value)). 1993 join("; "); 1994 1995 h ~= `Content-Disposition: ` ~ [disposition, optionals].join("; ") ~ "\r\n"; 1996 1997 auto contentType = "Content-Type" in part.parameters; 1998 if ( contentType ) { 1999 h ~= "Content-Type: " ~ *contentType ~ "\r\n"; 2000 } 2001 2002 h ~= "\r\n"; 2003 partHeaders ~= h; 2004 contentLength += h.length + part.input.getSize() + "\r\n".length; 2005 } 2006 contentLength += "--".length + boundary.length + "--\r\n".length; 2007 2008 auto h = requestHeaders(); 2009 safeSetHeader(h, _userHeaders.ContentType, "Content-Type", "multipart/form-data; boundary=" ~ boundary); 2010 safeSetHeader(h, _userHeaders.ContentLength, "Content-Length", to!string(contentLength)); 2011 2012 h.byKeyValue. 2013 map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n"). 2014 each!(h => req.put(h)); 2015 req.put("\r\n"); 2016 2017 debug(requests) trace(req.data); 2018 if ( _verbosity >= 1 ) req.data.splitLines.each!(a => writeln("> " ~ a)); 2019 2020 try { 2021 _stream.send(req.data()); 2022 foreach(ref source; _multipartForm._sources) { 2023 debug(requests) tracef("sending part headers <%s>", partHeaders.front); 2024 _stream.send(partHeaders.front); 2025 partHeaders.popFront; 2026 while (true) { 2027 auto chunk = source.input.read(); 2028 if ( chunk.length <= 0 ) { 2029 break; 2030 } 2031 _stream.send(chunk); 2032 } 2033 _stream.send("\r\n"); 2034 } 2035 _stream.send("--" ~ boundary ~ "--\r\n"); 2036 _response._requestSentAt = Clock.currTime; 2037 receiveResponse(_stream); 2038 _response._finishedAt = Clock.currTime; 2039 } 2040 catch (NetworkException e) { 2041 errorf("Error sending request: ", e.msg); 2042 _stream.close(); 2043 return _response; 2044 } 2045 2046 if ( serverPrematurelyClosedConnection() 2047 && !restartedRequest 2048 && isIdempotent(_method) 2049 ) { 2050 /// 2051 /// We didn't receive any data (keepalive connectioin closed?) 2052 /// and we can restart this request. 2053 /// Go ahead. 2054 /// 2055 debug(requests) tracef("Server closed keepalive connection"); 2056 2057 assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream); 2058 2059 _cm.del(_uri.scheme, _uri.host, _uri.port); 2060 _stream.close(); 2061 _stream = null; 2062 2063 restartedRequest = true; 2064 goto connect; 2065 } 2066 2067 if ( _useStreaming ) { 2068 if ( _response._receiveAsRange.activated ) { 2069 debug(requests) trace("streaming_in activated"); 2070 return _response; 2071 } else { 2072 // this can happen if whole response body received together with headers 2073 _response._receiveAsRange.data = _response.responseBody.data; 2074 } 2075 } 2076 2077 close_connection_if_not_keepalive(_stream); 2078 2079 if ( _verbosity >= 1 ) { 2080 writeln(">> Connect time: ", _response._connectedAt - _response._startedAt); 2081 writeln(">> Request send time: ", _response._requestSentAt - _response._connectedAt); 2082 writeln(">> Response recv time: ", _response._finishedAt - _response._requestSentAt); 2083 } 2084 2085 if ( willFollowRedirect ) { 2086 if ( _history.length >= _maxRedirects ) { 2087 _stream = null; 2088 throw new MaxRedirectsException("%d redirects reached maxRedirects %d.".format(_history.length, _maxRedirects)); 2089 } 2090 // "location" in response already checked in canFollowRedirect 2091 immutable new_location = *("location" in _response.responseHeaders); 2092 immutable current_uri = _uri; 2093 immutable next_uri = uriFromLocation(_uri, new_location); 2094 2095 immutable get_or_head = _method == "GET" || _method == "HEAD"; 2096 immutable code = _response.code; 2097 2098 // save current response for history 2099 _history ~= _response; 2100 2101 if ( code == 301 ) 2102 { 2103 // permanent redirect and change method 2104 _permanent_redirects[_uri] = new_location; 2105 if ( !get_or_head ) 2106 { 2107 _method = "GET"; 2108 } 2109 } 2110 if ( (code == 302 || code == 303) && !get_or_head) 2111 { 2112 // only change method 2113 _method = "GET"; 2114 } 2115 if ( code == 307 ) 2116 { 2117 // no change method, no permanent 2118 } 2119 if ( code == 308 ) 2120 { 2121 // permanent redirection and do not change method 2122 _permanent_redirects[_uri] = new_location; 2123 } 2124 2125 // prepare new response (for redirected request) 2126 _response = new HTTPResponse; 2127 _response.uri = current_uri; 2128 _response.finalURI = next_uri; 2129 _stream = null; 2130 2131 // set new uri 2132 this._uri = next_uri; 2133 debug(requests) tracef("Redirected to %s", next_uri); 2134 if ( restartedRequest ) { 2135 debug(requests) trace("Rare event: clearing 'restartedRequest' on redirect"); 2136 restartedRequest = false; 2137 } 2138 if ( _method == "GET") 2139 { 2140 return exec_from_parameters(); 2141 } 2142 goto connect; 2143 } 2144 2145 _response._history = _history; 2146 return _response; 2147 } 2148 2149 HTTPResponse exec_from_parameters() { 2150 2151 debug(requests) tracef("exec from parameters request"); 2152 2153 assert(_uri != URI.init); 2154 NetworkStream _stream; 2155 _response = new HTTPResponse; 2156 _history.length = 0; 2157 _response.uri = _uri; 2158 _response.finalURI = _uri; 2159 bool restartedRequest = false; // True if this is restarted keepAlive request 2160 2161 connect: 2162 if ( _method == "GET" && _uri in _permanent_redirects ) { 2163 debug(requests) trace("use parmanent redirects cache"); 2164 _uri = uriFromLocation(_uri, _permanent_redirects[_uri]); 2165 _response._finalURI = _uri; 2166 } 2167 _contentReceived = 0; 2168 _response._startedAt = Clock.currTime; 2169 2170 assert(_stream is null); 2171 2172 _stream = _cm.get(_uri.scheme, _uri.host, _uri.port); 2173 2174 if ( _stream is null ) { 2175 debug(requests) trace("create new connection"); 2176 _stream = setupConnection(); 2177 } else { 2178 debug(requests) trace("reuse old connection"); 2179 } 2180 2181 assert(_stream !is null); 2182 2183 if ( !_stream.isConnected ) { 2184 debug(requests) trace("disconnected stream on enter"); 2185 if ( !restartedRequest ) { 2186 debug(requests) trace("disconnected stream on enter: retry"); 2187 assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream); 2188 2189 _cm.del(_uri.scheme, _uri.host, _uri.port); 2190 _stream.close(); 2191 _stream = null; 2192 2193 restartedRequest = true; 2194 goto connect; 2195 } 2196 debug(requests) trace("disconnected stream on enter: return response"); 2197 //_stream = null; 2198 return _response; 2199 } 2200 _response._connectedAt = Clock.currTime; 2201 2202 auto h = requestHeaders(); 2203 2204 Appender!string req; 2205 2206 string encoded; 2207 2208 switch (_method) { 2209 case "POST","PUT","PATCH": 2210 encoded = params2query(_params); 2211 safeSetHeader(h, _userHeaders.ContentType, "Content-Type", "application/x-www-form-urlencoded"); 2212 safeSetHeader(h, _userHeaders.ContentLength, "Content-Length", to!string(encoded.length)); 2213 req.put(requestString()); 2214 break; 2215 default: 2216 req.put(requestString(_params)); 2217 } 2218 2219 h.byKeyValue. 2220 map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n"). 2221 each!(h => req.put(h)); 2222 req.put("\r\n"); 2223 if ( encoded ) { 2224 req.put(encoded); 2225 } 2226 2227 debug(requests) trace(req.data); 2228 if ( _verbosity >= 1 ) { 2229 req.data.splitLines.each!(a => writeln("> " ~ a)); 2230 } 2231 // 2232 // Now send request and receive response 2233 // 2234 try { 2235 _stream.send(req.data()); 2236 _response._requestSentAt = Clock.currTime; 2237 debug(requests) trace("starting receive response"); 2238 receiveResponse(_stream); 2239 debug(requests) tracef("done receive response"); 2240 _response._finishedAt = Clock.currTime; 2241 } 2242 catch (NetworkException e) { 2243 // On SEND this can means: 2244 // we started to send request to the server, but it closed connection because of keepalive timeout. 2245 // We have to restart request if possible. 2246 2247 // On RECEIVE - if we received something - then this exception is real and unexpected error. 2248 // If we didn't receive anything - we can restart request again as it can be 2249 debug(requests) tracef("Exception on receive response: %s", e.msg); 2250 if ( _response._responseHeaders.length != 0 ) 2251 { 2252 _stream.close(); 2253 throw new RequestException("Unexpected network error"); 2254 } 2255 } 2256 2257 if ( serverPrematurelyClosedConnection() 2258 && !restartedRequest 2259 && isIdempotent(_method) 2260 ) { 2261 /// 2262 /// We didn't receive any data (keepalive connectioin closed?) 2263 /// and we can restart this request. 2264 /// Go ahead. 2265 /// 2266 debug(requests) tracef("Server closed keepalive connection"); 2267 2268 assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream); 2269 2270 _cm.del(_uri.scheme, _uri.host, _uri.port); 2271 _stream.close(); 2272 _stream = null; 2273 2274 restartedRequest = true; 2275 goto connect; 2276 } 2277 2278 if ( _useStreaming ) { 2279 if ( _response._receiveAsRange.activated ) { 2280 debug(requests) trace("streaming_in activated"); 2281 return _response; 2282 } else { 2283 // this can happen if whole response body received together with headers 2284 _response._receiveAsRange.data = _response.responseBody.data; 2285 } 2286 } 2287 2288 close_connection_if_not_keepalive(_stream); 2289 2290 if ( _verbosity >= 1 ) { 2291 writeln(">> Connect time: ", _response._connectedAt - _response._startedAt); 2292 writeln(">> Request send time: ", _response._requestSentAt - _response._connectedAt); 2293 writeln(">> Response recv time: ", _response._finishedAt - _response._requestSentAt); 2294 } 2295 2296 if ( willFollowRedirect ) { 2297 debug(requests) trace("going to follow redirect"); 2298 if ( _history.length >= _maxRedirects ) { 2299 _stream = null; 2300 throw new MaxRedirectsException("%d redirects reached maxRedirects %d.".format(_history.length, _maxRedirects)); 2301 } 2302 // "location" in response already checked in canFollowRedirect 2303 immutable new_location = *("location" in _response.responseHeaders); 2304 immutable current_uri = _uri; 2305 immutable next_uri = uriFromLocation(_uri, new_location); 2306 2307 immutable get_or_head = _method == "GET" || _method == "HEAD"; 2308 immutable code = _response.code; 2309 2310 // save current response for history 2311 _history ~= _response; 2312 2313 if ( code == 301 ) 2314 { 2315 // permanent redirect and change method 2316 _permanent_redirects[_uri] = new_location; 2317 if ( !get_or_head ) 2318 { 2319 _method = "GET"; 2320 } 2321 } 2322 if ( (code == 302 || code == 303) && !get_or_head) 2323 { 2324 // only change method 2325 _method = "GET"; 2326 } 2327 if ( code == 307 ) 2328 { 2329 // no change method, no permanent 2330 } 2331 if ( code == 308 ) 2332 { 2333 // permanent redirection and do not change method 2334 _permanent_redirects[_uri] = new_location; 2335 } 2336 2337 // prepare new response (for redirected request) 2338 _response = new HTTPResponse; 2339 _response.uri = current_uri; 2340 _response.finalURI = next_uri; 2341 _stream = null; 2342 2343 // set new uri 2344 _uri = next_uri; 2345 debug(requests) tracef("Redirected to %s", next_uri); 2346 //if ( _method != "GET" && _response.code != 307 && _response.code != 308 ) { 2347 // // 307 and 308 do not change method 2348 // return exec_from_parameters(r); 2349 //} 2350 if ( restartedRequest ) { 2351 debug(requests) trace("Rare event: clearing 'restartedRequest' on redirect"); 2352 restartedRequest = false; 2353 } 2354 goto connect; 2355 } 2356 2357 _response._history = _history; 2358 return _response; 2359 } 2360 HTTPResponse execute(Request r) 2361 { 2362 _method = r.method; 2363 _uri = r.uri; 2364 _useStreaming = r.useStreaming; 2365 _permanent_redirects = r.permanent_redirects; 2366 _maxRedirects = r.maxRedirects; 2367 _authenticator = r.authenticator; 2368 _maxHeadersLength = r.maxHeadersLength; 2369 _maxContentLength = r.maxContentLength; 2370 _verbosity = r.verbosity; 2371 _keepAlive = r.keepAlive; 2372 _bufferSize = r.bufferSize; 2373 _proxy = r.proxy; 2374 _timeout = r.timeout; 2375 _contentType = r.contentType; 2376 _socketFactory = r.socketFactory; 2377 _sslOptions = r.sslOptions; 2378 _bind = r.bind; 2379 _headers = r.headers; 2380 _userHeaders = r.userHeaders; 2381 2382 _params = r.params; 2383 2384 // this assignments increments refCounts, so we can't use const Request 2385 // but Request is anyway struct and called by-value 2386 _cm = r.cm; 2387 _cookie = r.cookie; 2388 2389 debug(requests) trace("serving %s".format(r)); 2390 if ( !r.postData.empty) 2391 { 2392 return exec_from_range(r.postData); 2393 } 2394 if ( r.hasMultipartForm ) 2395 { 2396 return exec_from_multipart_form(r.multipartForm); 2397 } 2398 auto rs = exec_from_parameters(); 2399 return rs; 2400 } 2401 } 2402 2403 version(vibeD) { 2404 import std.json; 2405 package string httpTestServer() { 2406 return "http://httpbin.org/"; 2407 } 2408 package string fromJsonArrayToStr(JSONValue v) { 2409 return v.str; 2410 } 2411 } 2412 else { 2413 import std.json; 2414 package string httpTestServer() { 2415 return "http://127.0.0.1:8081/"; 2416 } 2417 package string fromJsonArrayToStr(JSONValue v) { 2418 return cast(string)(v.array.map!"cast(ubyte)a.integer".array); 2419 } 2420 }