Unofficial Dub Package Documentation 
1 module requests.http;
2 
3 private:
4 import std.algorithm;
5 import std.array;
6 import std.ascii;
7 import std.conv;
8 import std.datetime;
9 import std.exception;
10 import std.format;
11 import std.stdio;
12 import std.range;
13 import std.string;
14 import std.traits;
15 import std.typecons;
16 import std.experimental.logger;
17 import core.thread;
18 
19 import requests.streams;
20 import requests.uri;
21 import requests.utils;
22 import requests.base;
23 import requests.connmanager;
24 import requests.rangeadapter;
25 
26 static immutable ushort[] redirectCodes = [301, 302, 303, 307, 308];
27 
28 enum   HTTP11 = 101;
29 enum   HTTP10 = 100;
30 
31 static immutable string[string] proxies;
32 shared static this() {
33     import std.process;
34     import std.string;
35     foreach(p; ["http", "https", "all"])
36     {
37         auto v = environment.get(p ~ "_proxy", environment.get(p.toUpper() ~ "_PROXY"));
38         if ( v !is null && v.length > 0 )
39         {
40             debug(requests) tracef("will use %s for %s as proxy", v, p);
41             proxies[p] = v;
42         }
43     }
44 }
45 
46 public class MaxRedirectsException: Exception {
47     this(string message, string file = __FILE__, size_t line = __LINE__, Throwable next = null) @safe pure nothrow {
48         super(message, file, line, next);
49     }
50 }
51 
52 ///
53 ///
54 ///
55 //public auto queryParams(T...)(T params) pure nothrow @safe
56 //{
57 //    static assert (T.length % 2 == 0, "wrong args count");
58 //
59 //    QueryParam[] output;
60 //    output.reserve = T.length / 2;
61 //
62 //    void queryParamsHelper(T...)(T params, ref QueryParam[] output)
63 //    {
64 //        static if (T.length > 0)
65 //        {
66 //            output ~= QueryParam(params[0].to!string, params[1].to!string);
67 //            queryParamsHelper(params[2..$], output);
68 //        }
69 //    }
70 //
71 //    queryParamsHelper(params, output);
72 //    return output;
73 //}
74 
75 ///
76 /// Response - result of request execution.
77 ///
78 /// Response.code - response HTTP code.
79 /// Response.status_line - received HTTP status line.
80 /// Response.responseHeaders - received headers.
81 /// Response.responseBody - container for received body
82 /// Response.history - for redirected responses contain all history
83 ///
84 public class HTTPResponse : Response {
85     private {
86         string              _status_line;
87 
88         HTTPResponse[]      _history; // redirects history
89 
90         mixin(Setter!string("status_line"));
91 
92         int                 _version;
93     }
94 
95     ~this() {
96         _responseHeaders = null;
97         _history.length = 0;
98     }
99 
100     mixin(Getter("status_line"));
101 
102     @property final string[string] responseHeaders() @safe @nogc nothrow {
103         return _responseHeaders;
104     }
105     @property final HTTPResponse[] history() @safe @nogc nothrow {
106         return _history;
107     }
108 
109     private int parse_version(in string v) pure const nothrow @safe {
110         // try to parse HTTP/1.x to version
111         try if ( v.length > 5 ) {
112             return (v[5..$].split(".").map!"to!int(a)".array[0..2].reduce!((a,b) => a*100 + b));
113         } catch (Exception e) {
114         }
115         return 0;
116     }
117     unittest {
118         auto r = new HTTPResponse();
119         assert(r.parse_version("HTTP/1.1") == 101);
120         assert(r.parse_version("HTTP/1.0") == 100);
121         assert(r.parse_version("HTTP/0.9") == 9);
122         assert(r.parse_version("HTTP/xxx") == 0);
123     }
124 }
125 
126 ///
127 /// Request.
128 /// Configurable parameters:
129 /// $(B method) - string, method to use (GET, POST, ...)
130 /// $(B headers) - string[string], add any additional headers you'd like to send.
131 /// $(B authenticator) - class Auth, class to send auth headers.
132 /// $(B keepAlive) - bool, set true for keepAlive requests. default true.
133 /// $(B maxRedirects) - uint, maximum number of redirects. default 10.
134 /// $(B maxHeadersLength) - size_t, maximum length of server response headers. default = 32KB.
135 /// $(B maxContentLength) - size_t, maximun content length. delault - 0 = unlimited.
136 /// $(B bufferSize) - size_t, send and receive buffer size. default = 16KB.
137 /// $(B verbosity) - uint, level of verbosity(0 - nothing, 1 - headers, 2 - headers and body progress). default = 0.
138 /// $(B proxy) - string, set proxy url if needed. default - null.
139 /// $(B cookie) - Tuple Cookie, Read/Write cookie You can get cookie setted by server, or set cookies before doing request.
140 /// $(B timeout) - Duration, Set timeout value for connect/receive/send.
141 ///
142 public struct HTTPRequest {
143     private {
144         string         _method = "GET";
145         URI            _uri;
146         string[string] _headers;
147         string[]       _filteredHeaders;
148         Auth           _authenticator;
149         bool           _keepAlive = true;
150         uint           _maxRedirects = 10;
151         size_t         _maxHeadersLength = 32 * 1024;   // 32 KB
152         size_t         _maxContentLength;               // 0 - Unlimited
153         string         _proxy;
154         uint           _verbosity = 0;                  // 0 - no output, 1 - headers, 2 - headers+body info
155         Duration       _timeout = 30.seconds;
156         size_t         _bufferSize = 16*1024; // 16k
157         bool           _useStreaming;                   // return iterator instead of completed request
158 
159         HTTPResponse[] _history;                        // redirects history
160         DataPipe!ubyte _bodyDecoder;
161         DecodeChunked  _unChunker;
162         long           _contentLength;
163         long           _contentReceived;
164         SSLOptions     _sslOptions;
165         string         _bind;
166         _UH            _userHeaders;
167 
168         RefCounted!ConnManager      _cm;
169         RefCounted!Cookies          _cookie;
170         string[URI]                 _permanent_redirects;            // cache 301 redirects for GET requests
171         MultipartForm               _multipartForm;
172 
173         NetStreamFactory  _socketFactory;
174 
175         QueryParam[]        _params;
176         string              _contentType;
177         InputRangeAdapter   _postData;
178     }
179     package HTTPResponse   _response;
180 
181     mixin(Getter_Setter!string     ("method"));
182     mixin(Getter_Setter!bool       ("keepAlive"));
183     mixin(Getter_Setter!size_t     ("maxContentLength"));
184     mixin(Getter_Setter!size_t     ("maxHeadersLength"));
185     mixin(Getter_Setter!size_t     ("bufferSize"));
186     mixin(Getter_Setter!uint       ("maxRedirects"));
187     mixin(Getter_Setter!uint       ("verbosity"));
188     mixin(Getter                   ("proxy"));
189     mixin(Getter_Setter!Duration   ("timeout"));
190     mixin(Setter!Auth              ("authenticator"));
191     mixin(Getter_Setter!bool       ("useStreaming"));
192     mixin(Getter                   ("contentLength"));
193     mixin(Getter                   ("contentReceived"));
194     mixin(Getter_Setter!SSLOptions ("sslOptions"));
195     mixin(Getter_Setter!string     ("bind"));
196     mixin(Setter!NetStreamFactory  ("socketFactory"));
197 
198     @property void sslSetVerifyPeer(bool v) pure @safe nothrow @nogc {
199         _sslOptions.setVerifyPeer(v);
200     }
201     @property void sslSetKeyFile(string p, SSLOptions.filetype t = SSLOptions.filetype.pem) pure @safe nothrow @nogc {
202         _sslOptions.setKeyFile(p, t);
203     }
204     @property void sslSetCertFile(string p, SSLOptions.filetype t = SSLOptions.filetype.pem) pure @safe nothrow @nogc {
205         _sslOptions.setCertFile(p, t);
206     }
207     @property void sslSetCaCert(string path) pure @safe nothrow @nogc {
208         _sslOptions.setCaCert(path);
209     }
210     //@property final void cookie(Cookie[] s) pure @safe @nogc nothrow {
211     //    _cookie = s;
212     //}
213     @property final void proxy(string v) {
214         if ( v != _proxy ) {
215             _cm.clear();
216         }
217         _proxy = v;
218     }
219     //@property final Cookie[] cookie() pure @safe @nogc nothrow {
220     //    return _cookie;
221     //}
222 
223     this(string uri) {
224         _uri = URI(uri);
225         _cm = ConnManager(10);
226     }
227     ~this() {
228         _headers = null;
229         _authenticator = null;
230         _history = null;
231         _bodyDecoder = null;
232         _unChunker = null;
233         //if ( _cm ) {
234         //    _cm.clear();
235         //}
236     }
237     string toString() const {
238         return "HTTPRequest(%s, %s)".format(_method, _uri.uri());
239     }
240     string format(string fmt) const {
241         import std.array;
242         import std.stdio;
243         auto a = appender!string();
244         auto f = FormatSpec!char(fmt);
245         while (f.writeUpToNextSpec(a)) {
246             switch(f.spec) {
247                 case 'h':
248                     // Remote hostname.
249                     a.put(_uri.host);
250                     break;
251                 case 'm':
252                     // method.
253                     a.put(_method);
254                     break;
255                 case 'p':
256                     // Remote port.
257                     a.put("%d".format(_uri.port));
258                     break;
259                 case 'P':
260                     // Path
261                     a.put(_uri.path);
262                     break;
263                 case 'q':
264                     // query parameters supplied with url.
265                     a.put(_uri.query);
266                     break;
267                 case 'U':
268                     a.put(_uri.uri());
269                     break;
270                 default:
271                     throw new FormatException("Unknown Request format spec " ~ f.spec);
272             }
273         }
274         return a.data();
275     }
276     string select_proxy(string scheme) {
277         if ( _proxy is null && proxies.length == 0 ) {
278             debug(requests) tracef("proxy=null");
279             return null;
280         }
281         if ( _proxy ) {
282             debug(requests) tracef("proxy=%s", _proxy);
283             return _proxy;
284         }
285         auto p = scheme in proxies;
286         if ( p !is null && *p != "") {
287             debug(requests) tracef("proxy=%s", *p);
288             return *p;
289         }
290         p = "all" in proxies;
291         if ( p !is null && *p != "") {
292             debug(requests) tracef("proxy=%s", *p);
293             return *p;
294         }
295         debug(requests) tracef("proxy=null");
296         return null;
297     }
298     void clearHeaders() {
299         _headers = null;
300     }
301     @property void uri(in URI newURI) {
302         //handleURLChange(_uri, newURI);
303         _uri = newURI;
304     }
305     /// Add headers to request
306     /// Params:
307     /// headers = headers to send.
308     void addHeaders(in string[string] headers) {
309         foreach(pair; headers.byKeyValue) {
310             string _h = pair.key;
311             switch(toLower(_h)) {
312             case "host":
313                 _userHeaders.Host = true;
314                 break;
315             case "user-agent":
316                 _userHeaders.UserAgent = true;
317                 break;
318             case "content-length":
319                 _userHeaders.ContentLength = true;
320                 break;
321             case "content-type":
322                 _userHeaders.ContentType = true;
323                 break;
324             case "connection":
325                 _userHeaders.Connection = true;
326                 break;
327             case "cookie":
328                 _userHeaders.Cookie = true;
329                 break;
330             default:
331                 break;
332             }
333             _headers[pair.key] = pair.value;
334         }
335     }
336     private void safeSetHeader(ref string[string] headers, bool userAdded, string h, string v) pure @safe {
337         if ( !userAdded ) {
338             headers[h] = v;
339         }
340     }
341     /// Remove headers from request
342     /// Params:
343     /// headers = headers to remove.
344     void removeHeaders(in string[] headers) pure {
345         _filteredHeaders ~= headers;
346     }
347     ///
348     /// compose headers to send
349     ///
350     private string[string] requestHeaders() {
351 
352         string[string] generatedHeaders;
353 
354         if ( _authenticator ) {
355             _authenticator.
356                 authHeaders(_uri.host).
357                 byKeyValue.
358                 each!(pair => generatedHeaders[pair.key] = pair.value);
359         }
360 
361         _headers.byKey.each!(h => generatedHeaders[h] = _headers[h]);
362 
363         safeSetHeader(generatedHeaders, _userHeaders.AcceptEncoding, "Accept-Encoding", "gzip,deflate");
364         safeSetHeader(generatedHeaders, _userHeaders.UserAgent, "User-Agent", "dlang-requests");
365         safeSetHeader(generatedHeaders, _userHeaders.Connection, "Connection", _keepAlive?"Keep-Alive":"Close");
366 
367         if ( !_userHeaders.Host )
368         {
369             generatedHeaders["Host"] = _uri.host;
370             if ( _uri.scheme !in standard_ports || _uri.port != standard_ports[_uri.scheme] ) {
371                 generatedHeaders["Host"] ~= ":%d".format(_uri.port);
372             }
373         }
374 
375         if ( _cookie._array.length && !_userHeaders.Cookie ) {
376             auto cs = _cookie._array.
377                 filter!(c => _uri.path.pathMatches(c.path) && _uri.host.domainMatches(c.domain)).
378                 map!(c => "%s=%s".format(c.attr, c.value)).
379                 joiner(";");
380             if ( ! cs.empty )
381             {
382                 generatedHeaders["Cookie"] = to!string(cs);
383             }
384         }
385 
386         _filteredHeaders.each!(h => generatedHeaders.remove(h));
387 
388         return generatedHeaders;
389     }
390     ///
391     /// Build request string.
392     /// Handle proxy and query parameters.
393     ///
394     private @property string requestString(QueryParam[] params = null) {
395         auto query = _uri.query.dup;
396         if ( params ) {
397             query ~= "&" ~ params2query(params);
398             if ( query[0] != '?' ) {
399                 query = "?" ~ query;
400             }
401         }
402         string actual_proxy = select_proxy(_uri.scheme);
403         if ( actual_proxy && _uri.scheme != "https" ) {
404             return "%s %s%s HTTP/1.1\r\n".format(_method, _uri.uri(No.params), query);
405         }
406         return "%s %s%s HTTP/1.1\r\n".format(_method, _uri.path, query);
407     }
408     ///
409     /// encode parameters and build query part of the url
410     ///
411     private static string params2query(in QueryParam[] params) pure @safe {
412         return params.
413                 map!(a => "%s=%s".format(a.key.urlEncoded, a.value.urlEncoded)).
414                 join("&");
415     }
416     //
417     package unittest {
418         assert(params2query(queryParams("a","b", "c", " d "))=="a=b&c=%20d%20");
419     }
420     ///
421     /// Analyze received headers, take appropriate actions:
422     /// check content length, attach unchunk and uncompress
423     ///
424     private void analyzeHeaders(in string[string] headers) {
425 
426         _contentLength = -1;
427         _unChunker = null;
428         auto contentLength = "content-length" in headers;
429         if ( contentLength ) {
430             try {
431                 string l = *contentLength;
432                 _contentLength = parse!long(l);
433                 // TODO: maybe add a strict mode that checks if l was parsed completely
434                 if ( _maxContentLength && _contentLength > _maxContentLength) {
435                     throw new RequestException("ContentLength > maxContentLength (%d>%d)".
436                                 format(_contentLength, _maxContentLength));
437                 }
438             } catch (ConvException e) {
439                 throw new RequestException("Can't convert Content-Length from %s".format(*contentLength));
440             }
441         }
442         auto transferEncoding = "transfer-encoding" in headers;
443         if ( transferEncoding ) {
444             debug(requests) tracef("transferEncoding: %s", *transferEncoding);
445             if ( (*transferEncoding).toLower == "chunked") {
446                 _unChunker = new DecodeChunked();
447                 _bodyDecoder.insert(_unChunker);
448             }
449         }
450         auto contentEncoding = "content-encoding" in headers;
451         if ( contentEncoding ) switch (*contentEncoding) {
452             default:
453                 throw new RequestException("Unknown content-encoding " ~ *contentEncoding);
454             case "gzip":
455             case "deflate":
456                 _bodyDecoder.insert(new Decompressor!ubyte);
457         }
458 
459     }
460     ///
461     /// Called when we know that all headers already received in buffer.
462     /// This routine does not interpret headers content (see analyzeHeaders).
463     /// 1. Split headers on lines
464     /// 2. store status line, store response code
465     /// 3. unfold headers if needed
466     /// 4. store headers
467     ///
468     private void parseResponseHeaders(in ubyte[] input, string lineSep) {
469         string lastHeader;
470         auto buffer = cast(string)input;
471 
472         foreach(line; buffer.split(lineSep).map!(l => l.stripRight)) {
473             if ( ! _response.status_line.length ) {
474                 debug (requests) tracef("statusLine: %s", line);
475                 _response.status_line = line;
476                 if ( _verbosity >= 1 ) {
477                     writefln("< %s", line);
478                 }
479                 auto parsed = line.split(" ");
480                 if ( parsed.length >= 2 ) {
481                     _response.code = parsed[1].to!ushort;
482                     _response._version = _response.parse_version(parsed[0]);
483                 }
484                 continue;
485             }
486             if ( line[0] == ' ' || line[0] == '\t' ) {
487                 // unfolding https://tools.ietf.org/html/rfc822#section-3.1
488                 if ( auto stored = lastHeader in _response._responseHeaders) {
489                     *stored ~= line;
490                 }
491                 continue;
492             }
493             auto parsed = line.findSplit(":");
494             auto header = parsed[0].toLower;
495             auto value = parsed[2].strip;
496 
497             if ( _verbosity >= 1 ) {
498                 writefln("< %s: %s", header, value);
499             }
500 
501             lastHeader = header;
502             debug (requests) tracef("Header %s = %s", header, value);
503 
504             if ( header != "set-cookie" ) {
505                 auto stored = _response.responseHeaders.get(header, null);
506                 if ( stored ) {
507                     value = stored ~ "," ~ value;
508                 }
509                 _response._responseHeaders[header] = value;
510                 continue;
511             }
512             _cookie._array ~= processCookie(value);
513         }
514     }
515 
516     ///
517     /// Process Set-Cookie header from server response
518     ///
519     private Cookie[] processCookie(string value ) pure {
520         // cookie processing
521         //
522         // as we can't join several set-cookie lines in single line
523         // < Set-Cookie: idx=7f2800f63c112a65ef5082957bcca24b; expires=Mon, 29-May-2017 00:31:25 GMT; path=/; domain=example.com
524         // < Set-Cookie: idx=7f2800f63c112a65ef5082957bcca24b; expires=Mon, 29-May-2017 00:31:25 GMT; path=/; domain=example.com, cs=ip764-RgKqc-HvSkxRxdQQAKW8LA; path=/; domain=.example.com; HttpOnly
525         //
526         Cookie[] res;
527         string[string] kv;
528         auto fields = value.split(";").map!strip;
529         while(!fields.empty) {
530             auto s = fields.front.findSplit("=");
531             fields.popFront;
532             if ( s[1] != "=" ) {
533                 continue;
534             }
535             auto k = s[0];
536             auto v = s[2];
537             switch(k.toLower()) {
538                 case "domain":
539                     k = "domain";
540                     break;
541                 case "path":
542                     k = "path";
543                     break;
544                 case "expires":
545                     continue;
546                 case "max-age":
547                     continue;
548                 default:
549                     break;
550             }
551             kv[k] = v;
552         }
553         if ( "domain" !in kv ) {
554             kv["domain"] = _uri.host;
555         }
556         if ( "path" !in kv ) {
557             kv["path"] = _uri.path;
558         }
559         auto domain = kv["domain"]; kv.remove("domain");
560         auto path   = kv["path"];   kv.remove("path");
561         foreach(pair; kv.byKeyValue) {
562             auto _attr = pair.key;
563             auto _value = pair.value;
564             auto cookie = Cookie(path, domain, _attr, _value);
565             res ~= cookie;
566         }
567         return res;
568     }
569 
570     private bool willFollowRedirect() {
571         if ( !canFind(redirectCodes, _response.code) ) {
572             return false;
573         }
574         if ( !_maxRedirects ) {
575             return false;
576         }
577         if ( "location" !in _response.responseHeaders ) {
578             return false;
579         }
580         return true;
581     }
582     private URI uriFromLocation(const ref URI uri, in string location) {
583         URI newURI = uri;
584         try {
585             newURI = URI(location);
586         } catch (UriException e) {
587             debug(requests) trace("Can't parse Location:, try relative uri");
588             newURI.path = location;
589             newURI.uri = newURI.recalc_uri;
590         }
591         return newURI;
592     }
593     ///
594     /// if we have new uri, then we need to check if we have to reopen existent connection
595     ///
596     private void checkURL(string url, string file=__FILE__, size_t line=__LINE__) {
597         if (url is null && _uri.uri == "" ) {
598             throw new RequestException("No url configured", file, line);
599         }
600 
601         if ( url !is null ) {
602             URI newURI = URI(url);
603             //handleURLChange(_uri, newURI);
604             _uri = newURI;
605         }
606     }
607     ///
608     /// Setup connection. Handle proxy and https case
609     ///
610     /// Place new connection in ConnManager cache
611     ///
612     private NetworkStream setupConnection()
613     do {
614 
615         debug(requests) tracef("Set up new connection");
616         NetworkStream stream;
617 
618         // on exit
619         // place created connection to conn. manager
620         // close connection purged from manager (if any)
621         //
622         scope(exit) {
623             if ( stream )
624             {
625                 if ( auto purged_connection = _cm.put(_uri.scheme, _uri.host, _uri.port, stream) )
626                 {
627                     debug(requests) tracef("closing purged connection %s", purged_connection);
628                     purged_connection.close();
629                 }
630             }
631         }
632 
633         if ( _socketFactory )
634         {
635             debug(requests) tracef("use socketFactory");
636             stream = _socketFactory(_uri.scheme, _uri.host, _uri.port);
637         }
638 
639         if ( stream ) // socket factory created connection
640         {
641             return stream;
642         }
643 
644         URI   uri; // this URI will be used temporarry if we need proxy
645         string actual_proxy = select_proxy(_uri.scheme);
646         final switch (_uri.scheme) {
647             case"http":
648                 if ( actual_proxy ) {
649                     uri.uri_parse(actual_proxy);
650                     uri.idn_encode();
651                 } else {
652                     // use original uri
653                     uri = _uri;
654                 }
655                 stream = new TCPStream();
656                 stream.bind(_bind);
657                 stream.connect(uri.host, uri.port, _timeout);
658                 break ;
659             case"https":
660                 if ( actual_proxy ) {
661                     uri.uri_parse(actual_proxy);
662                     uri.idn_encode();
663                     stream = new TCPStream();
664                     stream.bind(_bind);
665                     stream.connect(uri.host, uri.port, _timeout);
666                     if ( verbosity>=1 ) {
667                         writeln("> CONNECT %s:%d HTTP/1.1".format(_uri.host, _uri.port));
668                     }
669                     stream.send("CONNECT %s:%d HTTP/1.1\r\n\r\n".format(_uri.host, _uri.port));
670                     while ( stream.isConnected ) {
671                         ubyte[1024] b;
672                         auto read = stream.receive(b);
673                         if ( verbosity>=1) {
674                             writefln("< %s", cast(string)b[0..read]);
675                         }
676                         debug(requests) tracef("read: %d", read);
677                         if ( b[0..read].canFind("\r\n\r\n") || b[0..read].canFind("\n\n") ) {
678                             debug(requests) tracef("proxy connection ready");
679                             // convert connection to ssl
680                             stream = new SSLStream(stream, _sslOptions, _uri.host);
681                             break ;
682                         } else {
683                             debug(requests) tracef("still wait for proxy connection");
684                         }
685                     }
686                 } else {
687                     uri = _uri;
688                     stream = new SSLStream(_sslOptions);
689                     stream.bind(_bind);
690                     stream.connect(uri.host, uri.port, _timeout);
691                     debug(requests) tracef("ssl connection to origin server ready");
692                 }
693                 break ;
694         }
695 
696         return stream;
697     }
698     ///
699     /// Request sent, now receive response.
700     /// Find headers, split on headers and body, continue to receive body
701     ///
702     private void receiveResponse(NetworkStream _stream) {
703 
704         try {
705             _stream.readTimeout = timeout;
706         } catch (Exception e) {
707             debug(requests) tracef("Failed to set read timeout for stream: %s", e.msg);
708             return;
709         }
710         // Commented this out as at exit we can have alreade closed socket
711         // scope(exit) {
712         //     if ( _stream && _stream.isOpen ) {
713         //         _stream.readTimeout = 0.seconds;
714         //     }
715         // }
716 
717         _bodyDecoder = new DataPipe!ubyte();
718         scope(exit) {
719             if ( !_useStreaming ) {
720                 _bodyDecoder = null;
721                 _unChunker = null;
722             }
723         }
724 
725         auto buffer = Buffer!ubyte();
726         Buffer!ubyte partialBody;
727         ptrdiff_t read;
728         string lineSep = null, headersEnd = null;
729         bool headersHaveBeenReceived;
730 
731         while( !headersHaveBeenReceived ) {
732 
733             auto b = new ubyte[_bufferSize];
734             read = _stream.receive(b);
735 
736             debug(requests) tracef("read: %d", read);
737             if ( read == 0 ) {
738                 break;
739             }
740             auto data = b[0..read];
741             buffer.putNoCopy(data);
742             if ( verbosity>=3 ) {
743                 writeln(data.dump.join("\n"));
744             }
745 
746             if ( buffer.length > maxHeadersLength ) {
747                 throw new RequestException("Headers length > maxHeadersLength (%d > %d)".format(buffer.length, maxHeadersLength));
748             }
749 
750             // Proper HTTP uses "\r\n" as a line separator, but broken servers sometimes use "\n".
751             // Servers that use "\r\n" might have "\n" inside a header.
752             // For any half-sane server, the first '\n' should be at the end of the status line, so this can be used to detect the line separator.
753             // In any case, all the interesting points in the header for now are at '\n' characters, so scan the newly read data for them.
754             foreach (idx; buffer.length-read..buffer.length)
755             {
756                 if ( buffer[idx] == '\n' )
757                 {
758                     if ( lineSep is null )
759                     {
760                         // First '\n'. Detect line/header endings.
761                         // HTTP header sections end with a double line separator
762                         lineSep = "\n";
763                         headersEnd = "\n\n";
764                         if ( idx > 0 && buffer[idx-1] == '\r' )
765                         {
766                             lineSep = "\r\n";
767                             headersEnd = "\r\n\r\n";
768                         }
769                     }
770                     else
771                     {
772                         // Potential header ending.
773                         if ( buffer.data[0..idx+1].endsWith(headersEnd) )
774                         {
775                             auto ResponseHeaders = buffer.data[0..idx+1-headersEnd.length];
776                             partialBody = buffer[idx+1..$];
777                             _contentReceived += partialBody.length;
778                             parseResponseHeaders(ResponseHeaders, lineSep);
779                             headersHaveBeenReceived = true;
780                             break;
781                         }
782                     }
783                 }
784             }
785         }
786 
787         analyzeHeaders(_response._responseHeaders);
788 
789         _bodyDecoder.putNoCopy(partialBody.data);
790 
791         auto v = _bodyDecoder.get();
792         _response._responseBody.putNoCopy(v);
793 
794         // https://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.4
795         if ( (_method == "HEAD") || responseMustNotIncludeBody(_response.code) || (_contentLength < 0 && _unChunker is null) )
796         {
797             debug(requests) tracef("response without body");
798             return;
799         }
800 
801         _response._contentLength = _contentLength;
802         _response._contentReceived = _contentReceived;
803 
804         if ( _verbosity >= 2 ) writefln("< %d bytes of body received", partialBody.length);
805 
806         while( true ) {
807             if ( _contentLength >= 0 && _contentReceived >= _contentLength ) {
808                 debug(requests) trace("Body received.");
809                 break;
810             }
811             if ( _unChunker && _unChunker.done ) {
812                 break;
813             }
814 
815             if ( _useStreaming && _response._responseBody.length && !redirectCodes.canFind(_response.code) ) {
816                 debug(requests) trace("streaming requested");
817                 // save _stream in closure
818                 auto __stream = _stream;
819                 auto __bodyDecoder = _bodyDecoder;
820                 auto __unChunker = _unChunker;
821                 auto __contentReceived = _contentReceived;
822                 auto __contentLength = _contentLength;
823                 auto __bufferSize = _bufferSize;
824                 auto __response = _response;
825                 auto __verbosity = _verbosity;
826 
827                 // set up response
828                 _response._contentLength = _contentLength;
829                 _response.receiveAsRange.activated = true;
830                 _response.receiveAsRange.data = _response._responseBody.data;
831                 _response.receiveAsRange.read = delegate ubyte[] () {
832 
833                     while(true) {
834                         // check if we received everything we need
835                         if ( ( __unChunker && __unChunker.done )
836                             || !__stream.isConnected()
837                             || (__contentLength > 0 && __contentReceived >= __contentLength) )
838                         {
839                             debug(requests) trace("streaming_in receive completed");
840                             __bodyDecoder.flush();
841                             return __bodyDecoder.get();
842                         }
843                         // have to continue
844                         auto b = new ubyte[__bufferSize];
845                         try {
846                             read = __stream.receive(b);
847                         }
848                         catch (Exception e) {
849                             throw new RequestException("streaming_in error reading from socket", __FILE__, __LINE__, e);
850                         }
851                         debug(requests) tracef("streaming_in received %d bytes", read);
852 
853                         if ( read == 0 ) {
854                             debug(requests) tracef("streaming_in: server closed connection");
855                             __bodyDecoder.flush();
856                             return __bodyDecoder.get();
857                         }
858 
859                         if ( __verbosity>=3 ) {
860                             writeln(b[0..read].dump.join("\n"));
861                         }
862                         __response._contentReceived += read;
863                         __contentReceived += read;
864                         __bodyDecoder.putNoCopy(b[0..read]);
865                         auto res = __bodyDecoder.getNoCopy();
866                         if ( res.length == 0 ) {
867                             // there were nothing to produce (beginning of the chunk or no decompressed data)
868                             continue;
869                         }
870                         if (res.length == 1) {
871                             return res[0];
872                         }
873                         //
874                         // I'd like to "return _bodyDecoder.getNoCopy().join;" but it is slower
875                         //
876                         auto total = res.map!(b=>b.length).sum;
877                         // create buffer for joined bytes
878                         ubyte[] joined = new ubyte[total];
879                         size_t p;
880                         // memcopy
881                         foreach(ref _; res) {
882                             joined[p .. p + _.length] = _;
883                             p += _.length;
884                         }
885                         return joined;
886                     }
887                     assert(0);
888                 };
889                 // we prepared for streaming
890                 return;
891             }
892 
893             auto b = new ubyte[_bufferSize];
894             read = _stream.receive(b);
895 
896             if ( read == 0 ) {
897                 debug(requests) trace("read done");
898                 break;
899             }
900             if ( _verbosity >= 2 ) {
901                 writefln("< %d bytes of body received", read);
902             }
903 
904             if ( verbosity>=3 ) {
905                 writeln(b[0..read].dump.join("\n"));
906             }
907 
908             debug(requests) tracef("read: %d", read);
909             _contentReceived += read;
910             if ( _maxContentLength && _contentReceived > _maxContentLength ) {
911                 throw new RequestException("ContentLength > maxContentLength (%d>%d)".
912                     format(_contentLength, _maxContentLength));
913             }
914 
915             _bodyDecoder.putNoCopy(b[0..read]); // send buffer to all decoders
916 
917             _bodyDecoder.getNoCopy.             // fetch result and place to body
918                 each!(b => _response._responseBody.putNoCopy(b));
919 
920             debug(requests) tracef("receivedTotal: %d, contentLength: %d, bodyLength: %d", _contentReceived, _contentLength, _response._responseBody.length);
921 
922         }
923         _bodyDecoder.flush();
924         _response._responseBody.putNoCopy(_bodyDecoder.get());
925         _response._contentReceived = _contentReceived;
926     }
927     ///
928     /// Check that we received anything.
929     /// Server can close previous connection (keepalive or not)
930     ///
931     private bool serverPrematurelyClosedConnection() pure @safe {
932         immutable server_closed_connection = _response._responseHeaders.length == 0 && _response._status_line.length == 0;
933         // debug(requests) tracef("server closed connection = %s (headers.length=%s, status_line.length=%s)",
934         //     server_closed_connection, _response._responseHeaders.length,  _response._status_line.length);
935         return server_closed_connection;
936     }
937     private bool isIdempotent(in string method) pure @safe nothrow {
938         return ["GET", "HEAD"].canFind(method);
939     }
940     ///
941     /// If we do not want keepalive request,
942     /// or server signalled to close connection,
943     /// then close it
944     ///
945     void close_connection_if_not_keepalive(NetworkStream _stream) {
946         auto connection = "connection" in _response._responseHeaders;
947         if ( !_keepAlive ) {
948             _stream.close();
949         } else switch(_response._version) {
950             case HTTP11:
951                 // HTTP/1.1 defines the "close" connection option for the sender to signal that the connection
952                 // will be closed after completion of the response. For example,
953                 //        Connection: close
954                 // in either the request or the response header fields indicates that the connection
955                 // SHOULD NOT be considered `persistent' (section 8.1) after the current request/response is complete.
956                 // HTTP/1.1 applications that do not support persistent connections MUST include the "close" connection
957                 // option in every message.
958                 if ( connection && (*connection).toLower.split(",").canFind("close") ) {
959                     _stream.close();
960                 }
961                 break;
962             default:
963                 // for anything else close connection if there is no keep-alive in Connection
964                 if ( connection && !(*connection).toLower.split(",").canFind("keep-alive") ) {
965                     _stream.close();
966                 }
967                 break;
968         }
969     }
970     ///
971     /// Send multipart for request.
972     /// You would like to use this method for sending large portions of mixed data or uploading files to forms.
973     /// Content of the posted form consist of sources. Each source have at least name and value (can be string-like object or opened file, see more docs for MultipartForm struct)
974     /// Params:
975     ///     url = url
976     ///     sources = array of sources.
977     deprecated("Use Request() instead of HTTPRequest(); will be removed 2019-07")
978     HTTPResponse exec(string method="POST")(string url, MultipartForm sources) {
979         import std.uuid;
980         import std.file;
981 
982         checkURL(url);
983         //if ( _cm is null ) {
984         //    _cm = new ConnManager();
985         //}
986 
987         NetworkStream _stream;
988         _method = method;
989         _response = new HTTPResponse;
990         _response.uri = _uri;
991         _response.finalURI = _uri;
992         bool restartedRequest = false;
993 
994     connect:
995         _contentReceived = 0;
996         _response._startedAt = Clock.currTime;
997 
998         assert(_stream is null);
999 
1000         _stream = _cm.get(_uri.scheme, _uri.host, _uri.port);
1001 
1002         if ( _stream is null ) {
1003             debug(requests) trace("create new connection");
1004             _stream = setupConnection();
1005         } else {
1006             debug(requests) trace("reuse old connection");
1007         }
1008 
1009         assert(_stream !is null);
1010 
1011         if ( !_stream.isConnected ) {
1012             debug(requests) trace("disconnected stream on enter");
1013             if ( !restartedRequest ) {
1014                 debug(requests) trace("disconnected stream on enter: retry");
1015                 assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
1016 
1017                 _cm.del(_uri.scheme, _uri.host, _uri.port);
1018                 _stream.close();
1019                 _stream = null;
1020 
1021                 restartedRequest = true;
1022                 goto connect;
1023             }
1024             debug(requests) trace("disconnected stream on enter: return response");
1025             //_stream = null;
1026             return _response;
1027         }
1028         _response._connectedAt = Clock.currTime;
1029 
1030         Appender!string req;
1031         req.put(requestString());
1032 
1033         string   boundary = randomUUID().toString;
1034         string[] partHeaders;
1035         size_t   contentLength;
1036 
1037         foreach(ref part; sources._sources) {
1038             string h = "--" ~ boundary ~ "\r\n";
1039             string disposition = `form-data; name="%s"`.format(part.name);
1040             string optionals = part.
1041                 parameters.byKeyValue().
1042                 filter!(p => p.key!="Content-Type").
1043                 map!   (p => "%s=%s".format(p.key, p.value)).
1044                 join("; ");
1045 
1046             h ~= `Content-Disposition: ` ~ [disposition, optionals].join("; ") ~ "\r\n";
1047 
1048             auto contentType = "Content-Type" in part.parameters;
1049             if ( contentType ) {
1050                 h ~= "Content-Type: " ~ *contentType ~ "\r\n";
1051             }
1052 
1053             h ~= "\r\n";
1054             partHeaders ~= h;
1055             contentLength += h.length + part.input.getSize() + "\r\n".length;
1056         }
1057         contentLength += "--".length + boundary.length + "--\r\n".length;
1058 
1059         auto h = requestHeaders();
1060         safeSetHeader(h, _userHeaders.ContentType, "Content-Type", "multipart/form-data; boundary=" ~ boundary);
1061         safeSetHeader(h, _userHeaders.ContentLength, "Content-Length", to!string(contentLength));
1062 
1063         h.byKeyValue.
1064             map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n").
1065                 each!(h => req.put(h));
1066         req.put("\r\n");
1067 
1068         debug(requests) trace(req.data);
1069         if ( _verbosity >= 1 ) req.data.splitLines.each!(a => writeln("> " ~ a));
1070 
1071         try {
1072             _stream.send(req.data());
1073             foreach(ref source; sources._sources) {
1074                 debug(requests) tracef("sending part headers <%s>", partHeaders.front);
1075                 _stream.send(partHeaders.front);
1076                 partHeaders.popFront;
1077                 while (true) {
1078                     auto chunk = source.input.read();
1079                     if ( chunk.length <= 0 ) {
1080                         break;
1081                     }
1082                     _stream.send(chunk);
1083                 }
1084                 _stream.send("\r\n");
1085             }
1086             _stream.send("--" ~ boundary ~ "--\r\n");
1087             _response._requestSentAt = Clock.currTime;
1088             receiveResponse(_stream);
1089             _response._finishedAt = Clock.currTime;
1090         }
1091         catch (NetworkException e) {
1092             errorf("Error sending request: ", e.msg);
1093             _stream.close();
1094             return _response;
1095         }
1096 
1097         if ( serverPrematurelyClosedConnection()
1098         && !restartedRequest
1099         && isIdempotent(_method)
1100         ) {
1101             ///
1102             /// We didn't receive any data (keepalive connectioin closed?)
1103             /// and we can restart this request.
1104             /// Go ahead.
1105             ///
1106             debug(requests) tracef("Server closed keepalive connection");
1107 
1108             assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
1109 
1110             _cm.del(_uri.scheme, _uri.host, _uri.port);
1111             _stream.close();
1112             _stream = null;
1113 
1114             restartedRequest = true;
1115             goto connect;
1116         }
1117 
1118         if ( _useStreaming ) {
1119             if ( _response._receiveAsRange.activated ) {
1120                 debug(requests) trace("streaming_in activated");
1121                 return _response;
1122             } else {
1123                 // this can happen if whole response body received together with headers
1124                 _response._receiveAsRange.data = _response.responseBody.data;
1125             }
1126         }
1127 
1128         close_connection_if_not_keepalive(_stream);
1129 
1130         if ( _verbosity >= 1 ) {
1131             writeln(">> Connect time: ", _response._connectedAt - _response._startedAt);
1132             writeln(">> Request send time: ", _response._requestSentAt - _response._connectedAt);
1133             writeln(">> Response recv time: ", _response._finishedAt - _response._requestSentAt);
1134         }
1135 
1136         if ( willFollowRedirect ) {
1137             if ( _history.length >= _maxRedirects ) {
1138                 _stream = null;
1139                 throw new MaxRedirectsException("%d redirects reached maxRedirects %d.".format(_history.length, _maxRedirects));
1140             }
1141             // "location" in response already checked in canFollowRedirect
1142             immutable new_location = *("location" in _response.responseHeaders);
1143             immutable current_uri = _uri, next_uri = uriFromLocation(_uri, new_location);
1144 
1145             // save current response for history
1146             _history ~= _response;
1147 
1148             // prepare new response (for redirected request)
1149             _response = new HTTPResponse;
1150             _response.uri = current_uri;
1151             _response.finalURI = next_uri;
1152             _stream = null;
1153 
1154             // set new uri
1155             this._uri = next_uri;
1156             debug(requests) tracef("Redirected to %s", next_uri);
1157             if ( _method != "GET" && _response.code != 307 && _response.code != 308 ) {
1158                 // 307 and 308 do not change method
1159                 return this.get();
1160             }
1161             if ( restartedRequest ) {
1162                 debug(requests) trace("Rare event: clearing 'restartedRequest' on redirect");
1163                 restartedRequest = false;
1164             }
1165             goto connect;
1166         }
1167 
1168         _response._history = _history;
1169         return _response;
1170     }
1171 
1172     // we use this if we send from ubyte[][] and user provided Content-Length
1173     private void sendFlattenContent(T)(NetworkStream _stream, T content) {
1174         while ( !content.empty ) {
1175             auto chunk = content.front;
1176             _stream.send(chunk);
1177             content.popFront;
1178         }
1179         debug(requests) tracef("sent");
1180     }
1181     // we use this if we send from ubyte[][] as chunked content
1182     private void sendChunkedContent(T)(NetworkStream _stream, T content) {
1183         while ( !content.empty ) {
1184             auto chunk = content.front;
1185             auto chunkHeader = "%x\r\n".format(chunk.length);
1186             debug(requests) tracef("sending %s%s", chunkHeader, chunk);
1187             _stream.send(chunkHeader);
1188             _stream.send(chunk);
1189             _stream.send("\r\n");
1190             content.popFront;
1191         }
1192         debug(requests) tracef("sent");
1193         _stream.send("0\r\n\r\n");
1194     }
1195     ///
1196     /// POST/PUT/... data from some string(with Content-Length), or from range of strings/bytes (use Transfer-Encoding: chunked).
1197     /// When rank 1 (flat array) used as content it must have length. In that case "content" will be sent directly to network, and Content-Length headers will be added.
1198     /// If you are goung to send some range and do not know length at the moment when you start to send request, then you can send chunks of chars or ubyte.
1199     /// Try not to send too short chunks as this will put additional load on client and server. Chunks of length 2048 or 4096 are ok.
1200     ///
1201     /// Parameters:
1202     ///    url = url
1203     ///    content = string or input range
1204     ///    contentType = content type
1205     ///  Returns:
1206     ///     Response
1207     ///  Examples:
1208     ///  ---------------------------------------------------------------------------------------------------------
1209     ///      rs = rq.exec!"POST"("http://httpbin.org/post", "привiт, свiт!", "application/octet-stream");
1210     ///
1211     ///      auto s = lineSplitter("one,\ntwo,\nthree.");
1212     ///      rs = rq.exec!"POST"("http://httpbin.org/post", s, "application/octet-stream");
1213     ///
1214     ///      auto s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
1215     ///      rs = rq.exec!"POST"("http://httpbin.org/post", s.representation.chunks(10), "application/octet-stream");
1216     ///
1217     ///      auto f = File("tests/test.txt", "rb");
1218     ///      rs = rq.exec!"POST"("http://httpbin.org/post", f.byChunk(3), "application/octet-stream");
1219     ///  --------------------------------------------------------------------------------------------------------
1220     deprecated("Use Request() instead of HTTPRequest(); will be removed 2019-07")
1221     HTTPResponse exec(string method="POST", R)(string url, R content, string contentType="application/octet-stream")
1222         if ( (rank!R == 1)
1223             || (rank!R == 2 && isSomeChar!(Unqual!(typeof(content.front.front))))
1224             || (rank!R == 2 && (is(Unqual!(typeof(content.front.front)) == ubyte)))
1225         )
1226     do {
1227         debug(requests) tracef("started url=%s, this._uri=%s", url, _uri);
1228 
1229         checkURL(url);
1230         //if ( _cm is null ) {
1231         //    _cm = new ConnManager();
1232         //}
1233 
1234         NetworkStream _stream;
1235         _method = method;
1236         _response = new HTTPResponse;
1237         _history.length = 0;
1238         _response.uri = _uri;
1239         _response.finalURI = _uri;
1240         bool restartedRequest = false;
1241         bool send_flat;
1242 
1243     connect:
1244         _contentReceived = 0;
1245         _response._startedAt = Clock.currTime;
1246 
1247         assert(_stream is null);
1248 
1249         _stream = _cm.get(_uri.scheme, _uri.host, _uri.port);
1250 
1251         if ( _stream is null ) {
1252             debug(requests) trace("create new connection");
1253             _stream = setupConnection();
1254         } else {
1255             debug(requests) trace("reuse old connection");
1256         }
1257 
1258         assert(_stream !is null);
1259 
1260         if ( !_stream.isConnected ) {
1261             debug(requests) trace("disconnected stream on enter");
1262             if ( !restartedRequest ) {
1263                 debug(requests) trace("disconnected stream on enter: retry");
1264                 assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
1265 
1266                 _cm.del(_uri.scheme, _uri.host, _uri.port);
1267                 _stream.close();
1268                 _stream = null;
1269 
1270                 restartedRequest = true;
1271                 goto connect;
1272             }
1273             debug(requests) trace("disconnected stream on enter: return response");
1274             //_stream = null;
1275             return _response;
1276         }
1277         _response._connectedAt = Clock.currTime;
1278 
1279         Appender!string req;
1280         req.put(requestString());
1281 
1282         auto h = requestHeaders;
1283         if ( contentType ) {
1284             safeSetHeader(h, _userHeaders.ContentType, "Content-Type", contentType);
1285         }
1286         static if ( rank!R == 1 ) {
1287             safeSetHeader(h, _userHeaders.ContentLength, "Content-Length", to!string(content.length));
1288         } else {
1289             if ( _userHeaders.ContentLength ) {
1290                 debug(requests) tracef("User provided content-length for chunked content");
1291                 send_flat = true;
1292             } else {
1293                 h["Transfer-Encoding"] = "chunked";
1294                 send_flat = false;
1295             }
1296         }
1297         h.byKeyValue.
1298             map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n").
1299             each!(h => req.put(h));
1300         req.put("\r\n");
1301 
1302         debug(requests) trace(req.data);
1303         if ( _verbosity >= 1 ) {
1304             req.data.splitLines.each!(a => writeln("> " ~ a));
1305         }
1306 
1307         try {
1308             // send headers
1309             _stream.send(req.data());
1310             // send body
1311             static if ( rank!R == 1) {
1312                 _stream.send(content);
1313             } else {
1314                 if ( send_flat ) {
1315                     sendFlattenContent(_stream, content);
1316                 } else {
1317                     sendChunkedContent(_stream, content);
1318                 }
1319             }
1320             _response._requestSentAt = Clock.currTime;
1321             debug(requests) trace("starting receive response");
1322             receiveResponse(_stream);
1323             debug(requests) trace("finished receive response");
1324             _response._finishedAt = Clock.currTime;
1325         } catch (NetworkException e) {
1326             _stream.close();
1327             throw new RequestException("Network error during data exchange");
1328         }
1329 
1330         if ( serverPrematurelyClosedConnection()
1331             && !restartedRequest
1332             && isIdempotent(_method))
1333         {
1334             ///
1335             /// We didn't receive any data (keepalive connectioin closed?)
1336             /// and we can restart this request.
1337             /// Go ahead.
1338             ///
1339             debug(requests) tracef("Server closed keepalive connection");
1340 
1341             assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
1342 
1343             _cm.del(_uri.scheme, _uri.host, _uri.port);
1344             _stream.close();
1345             _stream = null;
1346 
1347             restartedRequest = true;
1348             goto connect;
1349         }
1350 
1351         if ( _useStreaming ) {
1352             if ( _response._receiveAsRange.activated ) {
1353                 debug(requests) trace("streaming_in activated");
1354                 return _response;
1355             } else {
1356                 // this can happen if whole response body received together with headers
1357                 _response._receiveAsRange.data = _response.responseBody.data;
1358             }
1359         }
1360 
1361         close_connection_if_not_keepalive(_stream);
1362 
1363         if ( _verbosity >= 1 ) {
1364             writeln(">> Connect time: ", _response._connectedAt - _response._startedAt);
1365             writeln(">> Request send time: ", _response._requestSentAt - _response._connectedAt);
1366             writeln(">> Response recv time: ", _response._finishedAt - _response._requestSentAt);
1367         }
1368 
1369 
1370         if ( willFollowRedirect ) {
1371             if ( _history.length >= _maxRedirects ) {
1372                 _stream = null;
1373                 throw new MaxRedirectsException("%d redirects reached maxRedirects %d.".format(_history.length, _maxRedirects));
1374             }
1375             // "location" in response already checked in canFollowRedirect
1376             immutable new_location = *("location" in _response.responseHeaders);
1377             immutable current_uri = _uri, next_uri = uriFromLocation(_uri, new_location);
1378 
1379             // save current response for history
1380             _history ~= _response;
1381 
1382             // prepare new response (for redirected request)
1383             _response = new HTTPResponse;
1384             _response.uri = current_uri;
1385             _response.finalURI = next_uri;
1386 
1387             _stream = null;
1388 
1389             // set new uri
1390             this._uri = next_uri;
1391             debug(requests) tracef("Redirected to %s", next_uri);
1392             if ( _method != "GET" && _response.code != 307 && _response.code != 308 ) {
1393                 // 307 and 308 do not change method
1394                 return this.get();
1395             }
1396             if ( restartedRequest ) {
1397                 debug(requests) trace("Rare event: clearing 'restartedRequest' on redirect");
1398                 restartedRequest = false;
1399             }
1400             goto connect;
1401         }
1402 
1403         _response._history = _history;
1404         return _response;
1405     }
1406     ///
1407     /// Send request with parameters.
1408     /// If used for POST or PUT requests then application/x-www-form-urlencoded used.
1409     /// Request parameters will be encoded into request string or placed in request body for POST/PUT
1410     /// requests.
1411     /// Parameters:
1412     ///     url = url
1413     ///     params = request parameters
1414     ///  Returns:
1415     ///     Response
1416     ///  Examples:
1417     ///  ---------------------------------------------------------------------------------
1418     ///     rs = Request().exec!"GET"("http://httpbin.org/get", ["c":"d", "a":"b"]);
1419     ///  ---------------------------------------------------------------------------------
1420     ///
1421     deprecated("Use Request() instead of HTTPRequest; will be removed 2019-07")
1422     HTTPResponse exec(string method="GET")(string url = null, QueryParam[] params = null)
1423     do {
1424         debug(requests) tracef("started url=%s, this._uri=%s", url, _uri);
1425 
1426         checkURL(url);
1427         //if ( _cm is null ) {
1428         //    _cm = new ConnManager();
1429         //}
1430 
1431         NetworkStream _stream;
1432         _method = method;
1433         _response = new HTTPResponse;
1434         _history.length = 0;
1435         _response.uri = _uri;
1436         _response.finalURI = _uri;
1437         bool restartedRequest = false; // True if this is restarted keepAlive request
1438 
1439     connect:
1440         if ( _method == "GET" && _uri in _permanent_redirects ) {
1441             debug(requests) trace("use parmanent redirects cache");
1442             _uri = uriFromLocation(_uri, _permanent_redirects[_uri]);
1443             _response._finalURI = _uri;
1444         }
1445         _contentReceived = 0;
1446         _response._startedAt = Clock.currTime;
1447 
1448         assert(_stream is null);
1449 
1450         _stream = _cm.get(_uri.scheme, _uri.host, _uri.port);
1451 
1452         if ( _stream is null ) {
1453             debug(requests) trace("create new connection");
1454             _stream = setupConnection();
1455         } else {
1456             debug(requests) trace("reuse old connection");
1457         }
1458 
1459         assert(_stream !is null);
1460 
1461         if ( !_stream.isConnected ) {
1462             debug(requests) trace("disconnected stream on enter");
1463             if ( !restartedRequest ) {
1464                 debug(requests) trace("disconnected stream on enter: retry");
1465                 assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
1466 
1467                 _cm.del(_uri.scheme, _uri.host, _uri.port);
1468                 _stream.close();
1469                 _stream = null;
1470 
1471                 restartedRequest = true;
1472                 goto connect;
1473             }
1474             debug(requests) trace("disconnected stream on enter: return response");
1475             //_stream = null;
1476             return _response;
1477         }
1478         _response._connectedAt = Clock.currTime;
1479 
1480         auto h = requestHeaders();
1481 
1482         Appender!string req;
1483 
1484         string encoded;
1485 
1486         switch (_method) {
1487             case "POST","PUT","PATCH":
1488                 encoded = params2query(params);
1489                 safeSetHeader(h, _userHeaders.ContentType, "Content-Type", "application/x-www-form-urlencoded");
1490                 if ( encoded.length > 0) {
1491                     safeSetHeader(h, _userHeaders.ContentLength, "Content-Length", to!string(encoded.length));
1492                 }
1493                 req.put(requestString());
1494                 break;
1495             default:
1496                 req.put(requestString(params));
1497         }
1498 
1499         h.byKeyValue.
1500             map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n").
1501             each!(h => req.put(h));
1502         req.put("\r\n");
1503         if ( encoded ) {
1504             req.put(encoded);
1505         }
1506 
1507         debug(requests) trace(req.data);
1508         if ( _verbosity >= 1 ) {
1509             req.data.splitLines.each!(a => writeln("> " ~ a));
1510         }
1511         //
1512         // Now send request and receive response
1513         //
1514         try {
1515             _stream.send(req.data());
1516             _response._requestSentAt = Clock.currTime;
1517             debug(requests) trace("starting receive response");
1518             receiveResponse(_stream);
1519             debug(requests) trace("done receive response");
1520             _response._finishedAt = Clock.currTime;
1521         }
1522         catch (NetworkException e) {
1523             // On SEND this can means:
1524             // we started to send request to the server, but it closed connection because of keepalive timeout.
1525             // We have to restart request if possible.
1526 
1527             // On RECEIVE - if we received something - then this exception is real and unexpected error.
1528             // If we didn't receive anything - we can restart request again as it can be
1529             debug(requests) tracef("Exception on receive response: %s", e.msg);
1530             if ( _response._responseHeaders.length != 0 )
1531             {
1532                 _stream.close();
1533                 throw new RequestException("Unexpected network error");
1534             }
1535         }
1536 
1537         if ( serverPrematurelyClosedConnection()
1538             && !restartedRequest
1539             && isIdempotent(_method)
1540             ) {
1541             ///
1542             /// We didn't receive any data (keepalive connectioin closed?)
1543             /// and we can restart this request.
1544             /// Go ahead.
1545             ///
1546             debug(requests) tracef("Server closed keepalive connection");
1547 
1548             assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
1549 
1550             _cm.del(_uri.scheme, _uri.host, _uri.port);
1551             _stream.close();
1552             _stream = null;
1553 
1554             restartedRequest = true;
1555             goto connect;
1556         }
1557 
1558         if ( _useStreaming ) {
1559             if ( _response._receiveAsRange.activated ) {
1560                 debug(requests) trace("streaming_in activated");
1561                 return _response;
1562             } else {
1563                 // this can happen if whole response body received together with headers
1564                 _response._receiveAsRange.data = _response.responseBody.data;
1565             }
1566         }
1567 
1568         close_connection_if_not_keepalive(_stream);
1569 
1570         if ( _verbosity >= 1 ) {
1571             writeln(">> Connect time: ", _response._connectedAt - _response._startedAt);
1572             writeln(">> Request send time: ", _response._requestSentAt - _response._connectedAt);
1573             writeln(">> Response recv time: ", _response._finishedAt - _response._requestSentAt);
1574         }
1575 
1576         if ( willFollowRedirect ) {
1577             debug(requests) trace("going to follow redirect");
1578             if ( _history.length >= _maxRedirects ) {
1579                 _stream = null;
1580                 throw new MaxRedirectsException("%d redirects reached maxRedirects %d.".format(_history.length, _maxRedirects));
1581             }
1582             // "location" in response already checked in canFollowRedirect
1583             immutable new_location = *("location" in _response.responseHeaders);
1584             immutable current_uri = _uri, next_uri = uriFromLocation(_uri, new_location);
1585 
1586             if ( _method == "GET" && _response.code == 301 ) {
1587                 _permanent_redirects[_uri] = new_location;
1588             }
1589 
1590             // save current response for history
1591             _history ~= _response;
1592 
1593             // prepare new response (for redirected request)
1594             _response = new HTTPResponse;
1595             _response.uri = current_uri;
1596             _response.finalURI = next_uri;
1597             _stream = null;
1598 
1599             // set new uri
1600             _uri = next_uri;
1601             debug(requests) tracef("Redirected to %s", next_uri);
1602             if ( _method != "GET" && _response.code != 307 && _response.code != 308 ) {
1603                 // 307 and 308 do not change method
1604                 return this.get();
1605             }
1606             if ( restartedRequest ) {
1607                 debug(requests) trace("Rare event: clearing 'restartedRequest' on redirect");
1608                 restartedRequest = false;
1609             }
1610             goto connect;
1611         }
1612 
1613         _response._history = _history;
1614         return _response;
1615     }
1616 
1617     /// WRAPPERS
1618     ///
1619     /// send file(s) using POST and multipart form.
1620     /// This wrapper will be deprecated, use post with MultipartForm - it is more general and clear.
1621     /// Parameters:
1622     ///     url = url
1623     ///     files = array of PostFile structures
1624     /// Returns:
1625     ///     Response
1626     /// Each PostFile structure contain path to file, and optional field name and content type.
1627     /// If no field name provided, then basename of the file will be used.
1628     /// application/octet-stream is default when no content type provided.
1629     /// Example:
1630     /// ---------------------------------------------------------------
1631     ///    PostFile[] files = [
1632     ///                   {fileName:"tests/abc.txt", fieldName:"abc", contentType:"application/octet-stream"},
1633     ///                   {fileName:"tests/test.txt"}
1634     ///               ];
1635     ///    rs = rq.exec!"POST"("http://httpbin.org/post", files);
1636     /// ---------------------------------------------------------------
1637     ///
1638     deprecated("Use Request() instead of HTTPRequest(); will be removed 2019-07")
1639     HTTPResponse exec(string method="POST")(string url, PostFile[] files) if (method=="POST") {
1640         MultipartForm multipart;
1641         File[]        toClose;
1642         foreach(ref f; files) {
1643             File file = File(f.fileName, "rb");
1644             toClose ~= file;
1645             string fileName = f.fileName ? f.fileName : f.fieldName;
1646             string contentType = f.contentType ? f.contentType : "application/octetstream";
1647             multipart.add(f.fieldName, new FormDataFile(file), ["filename":fileName, "Content-Type": contentType]);
1648         }
1649         auto res = exec!"POST"(url, multipart);
1650         toClose.each!"a.close";
1651         return res;
1652     }
1653     ///
1654     /// exec request with parameters when you can use dictionary (when you have no duplicates in parameter names)
1655     /// Consider switch to exec(url, QueryParams) as it more generic and clear.
1656     /// Parameters:
1657     ///     url = url
1658     ///     params = dictionary with field names as keys and field values as values.
1659     /// Returns:
1660     ///     Response
1661     deprecated("Use Request() instead of HTTPRequest(); will be removed 2019-07")
1662     HTTPResponse exec(string method="GET")(string url, string[string] params) {
1663         return exec!method(url, params.byKeyValue.map!(p => QueryParam(p.key, p.value)).array);
1664     }
1665     ///
1666     /// GET request. Simple wrapper over exec!"GET"
1667     /// Params:
1668     /// args = request parameters. see exec docs.
1669     ///
1670     deprecated("Use Request() instead of HTTPRequest; will be removed 2019-07")
1671     HTTPResponse get(A...)(A args) {
1672         return exec!"GET"(args);
1673     }
1674     ///
1675     /// POST request. Simple wrapper over exec!"POST"
1676     /// Params:
1677     /// uri = endpoint uri
1678     /// args = request parameters. see exec docs.
1679     ///
1680     deprecated("Use Request() instead of HTTPRequest; will be removed 2019-07")
1681     HTTPResponse post(A...)(string uri, A args) {
1682         return exec!"POST"(uri, args);
1683     }
1684 
1685     import requests.request;
1686 
1687     // we use this if we send from ubyte[][] and user provided Content-Length
1688     private void sendFlattenContent(NetworkStream _stream) {
1689         while ( !_postData.empty ) {
1690             auto chunk = _postData.front;
1691             _stream.send(chunk);
1692             _postData.popFront;
1693         }
1694         debug(requests) tracef("sent");
1695     }
1696     // we use this if we send from ubyte[][] as chunked content
1697     private void sendChunkedContent(NetworkStream _stream) {
1698         while ( !_postData.empty ) {
1699             auto chunk = _postData.front;
1700             auto chunkHeader = "%x\r\n".format(chunk.length);
1701             debug(requests) tracef("sending %s%s", chunkHeader, cast(string)chunk);
1702             _stream.send(chunkHeader);
1703             _stream.send(chunk);
1704             _stream.send("\r\n");
1705             debug(requests) tracef("chunk sent");
1706             _postData.popFront;
1707         }
1708         debug(requests) tracef("sent");
1709         _stream.send("0\r\n\r\n");
1710     }
1711 
1712     HTTPResponse exec_from_range(InputRangeAdapter postData)
1713     do {
1714 
1715         _postData = postData;
1716 
1717         debug(requests) tracef("exec from range");
1718 
1719         NetworkStream _stream;
1720         _response = new HTTPResponse;
1721         _history.length = 0;
1722         _response.uri = _uri;
1723         _response.finalURI = _uri;
1724         bool restartedRequest = false;
1725         bool send_flat;
1726 
1727     connect:
1728         _contentReceived = 0;
1729         _response._startedAt = Clock.currTime;
1730 
1731         assert(_stream is null);
1732 
1733         _stream = _cm.get(_uri.scheme, _uri.host, _uri.port);
1734 
1735         if ( _stream is null ) {
1736             debug(requests) trace("create new connection");
1737             _stream = setupConnection();
1738         } else {
1739             debug(requests) trace("reuse old connection");
1740         }
1741 
1742         assert(_stream !is null);
1743 
1744         if ( !_stream.isConnected ) {
1745             debug(requests) trace("disconnected stream on enter");
1746             if ( !restartedRequest ) {
1747                 debug(requests) trace("disconnected stream on enter: retry");
1748                 assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
1749 
1750                 _cm.del(_uri.scheme, _uri.host, _uri.port);
1751                 _stream.close();
1752                 _stream = null;
1753 
1754                 restartedRequest = true;
1755                 goto connect;
1756             }
1757             debug(requests) trace("disconnected stream on enter: return response");
1758             //_stream = null;
1759             return _response;
1760         }
1761         _response._connectedAt = Clock.currTime;
1762 
1763         Appender!string req;
1764         req.put(requestString());
1765 
1766         auto h = requestHeaders;
1767         if ( _contentType ) {
1768             safeSetHeader(h, _userHeaders.ContentType, "Content-Type", _contentType);
1769         }
1770 
1771         if ( _postData.length >= 0 )
1772         {
1773             // we know t
1774             safeSetHeader(h, _userHeaders.ContentLength, "Content-Length", to!string(_postData.length));
1775         }
1776 
1777         if ( _userHeaders.ContentLength || "Content-Length" in h )
1778         {
1779             debug(requests) tracef("User provided content-length for chunked content");
1780             send_flat = true;
1781         }
1782         else
1783         {
1784             h["Transfer-Encoding"] = "chunked";
1785             send_flat = false;
1786         }
1787         h.byKeyValue.
1788             map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n").
1789             each!(h => req.put(h));
1790         req.put("\r\n");
1791 
1792         debug(requests) tracef("send <%s>", req.data);
1793         if ( _verbosity >= 1 ) {
1794             req.data.splitLines.each!(a => writeln("> " ~ a));
1795         }
1796 
1797         try {
1798             // send headers
1799             _stream.send(req.data());
1800             // send body
1801             if ( send_flat ) {
1802                 sendFlattenContent(_stream);
1803             } else {
1804                 sendChunkedContent(_stream);
1805             }
1806             _response._requestSentAt = Clock.currTime;
1807             debug(requests) trace("starting receive response");
1808             receiveResponse(_stream);
1809             debug(requests) trace("finished receive response");
1810             _response._finishedAt = Clock.currTime;
1811         }
1812         catch (NetworkException e)
1813         {
1814             _stream.close();
1815             throw new RequestException("Network error during data exchange");
1816         }
1817         if ( serverPrematurelyClosedConnection()
1818         && !restartedRequest
1819         && isIdempotent(_method)
1820         ) {
1821             ///
1822             /// We didn't receive any data (keepalive connectioin closed?)
1823             /// and we can restart this request.
1824             /// Go ahead.
1825             ///
1826             debug(requests) tracef("Server closed keepalive connection");
1827 
1828             assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
1829 
1830             _cm.del(_uri.scheme, _uri.host, _uri.port);
1831             _stream.close();
1832             _stream = null;
1833 
1834             restartedRequest = true;
1835             goto connect;
1836         }
1837 
1838         if ( _useStreaming ) {
1839             if ( _response._receiveAsRange.activated ) {
1840                 debug(requests) trace("streaming_in activated");
1841                 return _response;
1842             } else {
1843                 // this can happen if whole response body received together with headers
1844                 _response._receiveAsRange.data = _response.responseBody.data;
1845             }
1846         }
1847 
1848         close_connection_if_not_keepalive(_stream);
1849 
1850         if ( _verbosity >= 1 ) {
1851             writeln(">> Connect time: ", _response._connectedAt - _response._startedAt);
1852             writeln(">> Request send time: ", _response._requestSentAt - _response._connectedAt);
1853             writeln(">> Response recv time: ", _response._finishedAt - _response._requestSentAt);
1854         }
1855 
1856 
1857         if ( willFollowRedirect ) {
1858             if ( _history.length >= _maxRedirects ) {
1859                 _stream = null;
1860                 throw new MaxRedirectsException("%d redirects reached maxRedirects %d.".format(_history.length, _maxRedirects));
1861             }
1862             // "location" in response already checked in canFollowRedirect
1863             immutable new_location = *("location" in _response.responseHeaders);
1864             immutable current_uri = _uri, next_uri = uriFromLocation(_uri, new_location);
1865 
1866             immutable get_or_head = _method == "GET" || _method == "HEAD";
1867             immutable code = _response.code;
1868 
1869             // save current response for history
1870             _history ~= _response;
1871 
1872             if ( code == 301 )
1873             {
1874                 // permanent redirect and change method
1875                 _permanent_redirects[_uri] = new_location;
1876                 if ( !get_or_head )
1877                 {
1878                     _method = "GET";
1879                 }
1880             }
1881             if ( (code == 302 || code == 303) && !get_or_head)
1882             {
1883                 // only change method
1884                 _method = "GET";
1885             }
1886             if ( code == 307 )
1887             {
1888                 // no change method, no permanent
1889             }
1890             if ( code == 308 )
1891             {
1892                 // permanent redirection and do not change method
1893                 _permanent_redirects[_uri] = new_location;
1894             }
1895 
1896             // prepare new response (for redirected request)
1897             _response = new HTTPResponse;
1898             _response.uri = current_uri;
1899             _response.finalURI = next_uri;
1900 
1901             _stream = null;
1902 
1903             // set new uri
1904             this._uri = next_uri;
1905             debug(requests) tracef("Redirected to %s", next_uri);
1906             if ( restartedRequest ) {
1907                 debug(requests) trace("Rare event: clearing 'restartedRequest' on redirect");
1908                 restartedRequest = false;
1909             }
1910             if ( _method == "GET")
1911             {
1912                 return exec_from_parameters();
1913             }
1914             goto connect;
1915         }
1916 
1917         _response._history = _history;
1918         return _response;
1919     }
1920 
1921     HTTPResponse exec_from_multipart_form(MultipartForm form) {
1922         import std.uuid;
1923         import std.file;
1924 
1925         _multipartForm = form;
1926 
1927         debug(requests) tracef("exec from multipart form");
1928 
1929         NetworkStream _stream;
1930         _response = new HTTPResponse;
1931         _response.uri = _uri;
1932         _response.finalURI = _uri;
1933         bool restartedRequest = false;
1934 
1935     connect:
1936         _contentReceived = 0;
1937         _response._startedAt = Clock.currTime;
1938 
1939         assert(_stream is null);
1940 
1941         _stream = _cm.get(_uri.scheme, _uri.host, _uri.port);
1942 
1943         if ( _stream is null ) {
1944             debug(requests) trace("create new connection");
1945             _stream = setupConnection();
1946         } else {
1947             debug(requests) trace("reuse old connection");
1948         }
1949 
1950         assert(_stream !is null);
1951 
1952         if ( !_stream.isConnected ) {
1953             debug(requests) trace("disconnected stream on enter");
1954             if ( !restartedRequest ) {
1955                 debug(requests) trace("disconnected stream on enter: retry");
1956                 assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
1957 
1958                 _cm.del(_uri.scheme, _uri.host, _uri.port);
1959                 _stream.close();
1960                 _stream = null;
1961 
1962                 restartedRequest = true;
1963                 goto connect;
1964             }
1965             debug(requests) trace("disconnected stream on enter: return response");
1966             //_stream = null;
1967             return _response;
1968         }
1969         _response._connectedAt = Clock.currTime;
1970 
1971         Appender!string req;
1972         req.put(requestString());
1973 
1974         string   boundary = randomUUID().toString;
1975         string[] partHeaders;
1976         size_t   contentLength;
1977 
1978         foreach(ref part; _multipartForm._sources) {
1979             string h = "--" ~ boundary ~ "\r\n";
1980             string disposition = `form-data; name="%s"`.format(part.name);
1981             string optionals = part.
1982             parameters.byKeyValue().
1983             filter!(p => p.key!="Content-Type").
1984             map!   (p => "%s=%s".format(p.key, p.value)).
1985             join("; ");
1986 
1987             h ~= `Content-Disposition: ` ~ [disposition, optionals].join("; ") ~ "\r\n";
1988 
1989             auto contentType = "Content-Type" in part.parameters;
1990             if ( contentType ) {
1991                 h ~= "Content-Type: " ~ *contentType ~ "\r\n";
1992             }
1993 
1994             h ~= "\r\n";
1995             partHeaders ~= h;
1996             contentLength += h.length + part.input.getSize() + "\r\n".length;
1997         }
1998         contentLength += "--".length + boundary.length + "--\r\n".length;
1999 
2000         auto h = requestHeaders();
2001         safeSetHeader(h, _userHeaders.ContentType, "Content-Type", "multipart/form-data; boundary=" ~ boundary);
2002         safeSetHeader(h, _userHeaders.ContentLength, "Content-Length", to!string(contentLength));
2003 
2004         h.byKeyValue.
2005         map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n").
2006         each!(h => req.put(h));
2007         req.put("\r\n");
2008 
2009         debug(requests) trace(req.data);
2010         if ( _verbosity >= 1 ) req.data.splitLines.each!(a => writeln("> " ~ a));
2011 
2012         try {
2013             _stream.send(req.data());
2014             foreach(ref source; _multipartForm._sources) {
2015                 debug(requests) tracef("sending part headers <%s>", partHeaders.front);
2016                 _stream.send(partHeaders.front);
2017                 partHeaders.popFront;
2018                 while (true) {
2019                     auto chunk = source.input.read();
2020                     if ( chunk.length <= 0 ) {
2021                         break;
2022                     }
2023                     _stream.send(chunk);
2024                 }
2025                 _stream.send("\r\n");
2026             }
2027             _stream.send("--" ~ boundary ~ "--\r\n");
2028             _response._requestSentAt = Clock.currTime;
2029             receiveResponse(_stream);
2030             _response._finishedAt = Clock.currTime;
2031         }
2032         catch (NetworkException e) {
2033             errorf("Error sending request: ", e.msg);
2034             _stream.close();
2035             return _response;
2036         }
2037 
2038         if ( serverPrematurelyClosedConnection()
2039         && !restartedRequest
2040         && isIdempotent(_method)
2041         ) {
2042             ///
2043             /// We didn't receive any data (keepalive connectioin closed?)
2044             /// and we can restart this request.
2045             /// Go ahead.
2046             ///
2047             debug(requests) tracef("Server closed keepalive connection");
2048 
2049             assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
2050 
2051             _cm.del(_uri.scheme, _uri.host, _uri.port);
2052             _stream.close();
2053             _stream = null;
2054 
2055             restartedRequest = true;
2056             goto connect;
2057         }
2058 
2059         if ( _useStreaming ) {
2060             if ( _response._receiveAsRange.activated ) {
2061                 debug(requests) trace("streaming_in activated");
2062                 return _response;
2063             } else {
2064                 // this can happen if whole response body received together with headers
2065                 _response._receiveAsRange.data = _response.responseBody.data;
2066             }
2067         }
2068 
2069         close_connection_if_not_keepalive(_stream);
2070 
2071         if ( _verbosity >= 1 ) {
2072             writeln(">> Connect time: ", _response._connectedAt - _response._startedAt);
2073             writeln(">> Request send time: ", _response._requestSentAt - _response._connectedAt);
2074             writeln(">> Response recv time: ", _response._finishedAt - _response._requestSentAt);
2075         }
2076 
2077         if ( willFollowRedirect ) {
2078             if ( _history.length >= _maxRedirects ) {
2079                 _stream = null;
2080                 throw new MaxRedirectsException("%d redirects reached maxRedirects %d.".format(_history.length, _maxRedirects));
2081             }
2082             // "location" in response already checked in canFollowRedirect
2083             immutable new_location = *("location" in _response.responseHeaders);
2084             immutable current_uri = _uri;
2085             immutable next_uri = uriFromLocation(_uri, new_location);
2086 
2087             immutable get_or_head = _method == "GET" || _method == "HEAD";
2088             immutable code = _response.code;
2089 
2090             // save current response for history
2091             _history ~= _response;
2092 
2093             if ( code == 301 )
2094             {
2095                 // permanent redirect and change method
2096                 _permanent_redirects[_uri] = new_location;
2097                 if ( !get_or_head )
2098                 {
2099                     _method = "GET";
2100                 }
2101             }
2102             if ( (code == 302 || code == 303) && !get_or_head)
2103             {
2104                 // only change method
2105                 _method = "GET";
2106             }
2107             if ( code == 307 )
2108             {
2109                 // no change method, no permanent
2110             }
2111             if ( code == 308 )
2112             {
2113                 // permanent redirection and do not change method
2114                 _permanent_redirects[_uri] = new_location;
2115             }
2116 
2117             // prepare new response (for redirected request)
2118             _response = new HTTPResponse;
2119             _response.uri = current_uri;
2120             _response.finalURI = next_uri;
2121             _stream = null;
2122 
2123             // set new uri
2124             this._uri = next_uri;
2125             debug(requests) tracef("Redirected to %s", next_uri);
2126             if ( restartedRequest ) {
2127                 debug(requests) trace("Rare event: clearing 'restartedRequest' on redirect");
2128                 restartedRequest = false;
2129             }
2130             if ( _method == "GET")
2131             {
2132                 return exec_from_parameters();
2133             }
2134             goto connect;
2135         }
2136 
2137         _response._history = _history;
2138         return _response;
2139     }
2140 
2141     HTTPResponse exec_from_parameters() {
2142 
2143         debug(requests) tracef("exec from parameters request");
2144 
2145         assert(_uri != URI.init);
2146         NetworkStream _stream;
2147         _response = new HTTPResponse;
2148         _history.length = 0;
2149         _response.uri = _uri;
2150         _response.finalURI = _uri;
2151         bool restartedRequest = false; // True if this is restarted keepAlive request
2152 
2153     connect:
2154         if ( _method == "GET" && _uri in _permanent_redirects ) {
2155             debug(requests) trace("use parmanent redirects cache");
2156             _uri = uriFromLocation(_uri, _permanent_redirects[_uri]);
2157             _response._finalURI = _uri;
2158         }
2159         _contentReceived = 0;
2160         _response._startedAt = Clock.currTime;
2161 
2162         assert(_stream is null);
2163 
2164         _stream = _cm.get(_uri.scheme, _uri.host, _uri.port);
2165 
2166         if ( _stream is null ) {
2167             debug(requests) trace("create new connection");
2168             _stream = setupConnection();
2169         } else {
2170             debug(requests) trace("reuse old connection");
2171         }
2172 
2173         assert(_stream !is null);
2174 
2175         if ( !_stream.isConnected ) {
2176             debug(requests) trace("disconnected stream on enter");
2177             if ( !restartedRequest ) {
2178                 debug(requests) trace("disconnected stream on enter: retry");
2179                 assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
2180 
2181                 _cm.del(_uri.scheme, _uri.host, _uri.port);
2182                 _stream.close();
2183                 _stream = null;
2184 
2185                 restartedRequest = true;
2186                 goto connect;
2187             }
2188             debug(requests) trace("disconnected stream on enter: return response");
2189             //_stream = null;
2190             return _response;
2191         }
2192         _response._connectedAt = Clock.currTime;
2193 
2194         auto h = requestHeaders();
2195 
2196         Appender!string req;
2197 
2198         string encoded;
2199 
2200         switch (_method) {
2201             case "POST","PUT","PATCH":
2202                 encoded = params2query(_params);
2203                 safeSetHeader(h, _userHeaders.ContentType, "Content-Type", "application/x-www-form-urlencoded");
2204                 if ( encoded.length > 0) {
2205                     safeSetHeader(h, _userHeaders.ContentLength, "Content-Length", to!string(encoded.length));
2206                 }
2207                 req.put(requestString());
2208                 break;
2209             default:
2210                 req.put(requestString(_params));
2211         }
2212 
2213         h.byKeyValue.
2214         map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n").
2215         each!(h => req.put(h));
2216         req.put("\r\n");
2217         if ( encoded ) {
2218             req.put(encoded);
2219         }
2220 
2221         debug(requests) trace(req.data);
2222         if ( _verbosity >= 1 ) {
2223             req.data.splitLines.each!(a => writeln("> " ~ a));
2224         }
2225         //
2226         // Now send request and receive response
2227         //
2228         try {
2229             _stream.send(req.data());
2230             _response._requestSentAt = Clock.currTime;
2231             debug(requests) trace("starting receive response");
2232             receiveResponse(_stream);
2233             debug(requests) tracef("done receive response");
2234             _response._finishedAt = Clock.currTime;
2235         }
2236         catch (NetworkException e) {
2237             // On SEND this can means:
2238             // we started to send request to the server, but it closed connection because of keepalive timeout.
2239             // We have to restart request if possible.
2240 
2241             // On RECEIVE - if we received something - then this exception is real and unexpected error.
2242             // If we didn't receive anything - we can restart request again as it can be
2243             debug(requests) tracef("Exception on receive response: %s", e.msg);
2244             if ( _response._responseHeaders.length != 0 )
2245             {
2246                 _stream.close();
2247                 throw new RequestException("Unexpected network error");
2248             }
2249         }
2250 
2251         if ( serverPrematurelyClosedConnection()
2252             && !restartedRequest
2253             && isIdempotent(_method)
2254             ) {
2255             ///
2256             /// We didn't receive any data (keepalive connectioin closed?)
2257             /// and we can restart this request.
2258             /// Go ahead.
2259             ///
2260             debug(requests) tracef("Server closed keepalive connection");
2261 
2262             assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
2263 
2264             _cm.del(_uri.scheme, _uri.host, _uri.port);
2265             _stream.close();
2266             _stream = null;
2267 
2268             restartedRequest = true;
2269             goto connect;
2270         }
2271 
2272         if ( _useStreaming ) {
2273             if ( _response._receiveAsRange.activated ) {
2274                 debug(requests) trace("streaming_in activated");
2275                 return _response;
2276             } else {
2277                 // this can happen if whole response body received together with headers
2278                 _response._receiveAsRange.data = _response.responseBody.data;
2279             }
2280         }
2281 
2282         close_connection_if_not_keepalive(_stream);
2283 
2284         if ( _verbosity >= 1 ) {
2285             writeln(">> Connect time: ", _response._connectedAt - _response._startedAt);
2286             writeln(">> Request send time: ", _response._requestSentAt - _response._connectedAt);
2287             writeln(">> Response recv time: ", _response._finishedAt - _response._requestSentAt);
2288         }
2289 
2290         if ( willFollowRedirect ) {
2291             debug(requests) trace("going to follow redirect");
2292             if ( _history.length >= _maxRedirects ) {
2293                 _stream = null;
2294                 throw new MaxRedirectsException("%d redirects reached maxRedirects %d.".format(_history.length, _maxRedirects));
2295             }
2296             // "location" in response already checked in canFollowRedirect
2297             immutable new_location = *("location" in _response.responseHeaders);
2298             immutable current_uri = _uri;
2299             immutable next_uri = uriFromLocation(_uri, new_location);
2300 
2301             immutable get_or_head = _method == "GET" || _method == "HEAD";
2302             immutable code = _response.code;
2303 
2304             // save current response for history
2305             _history ~= _response;
2306 
2307             if ( code == 301 )
2308             {
2309                 // permanent redirect and change method
2310                 _permanent_redirects[_uri] = new_location;
2311                 if ( !get_or_head )
2312                 {
2313                     _method = "GET";
2314                 }
2315             }
2316             if ( (code == 302 || code == 303) && !get_or_head)
2317             {
2318                 // only change method
2319                 _method = "GET";
2320             }
2321             if ( code == 307 )
2322             {
2323                 // no change method, no permanent
2324             }
2325             if ( code == 308 )
2326             {
2327                 // permanent redirection and do not change method
2328                 _permanent_redirects[_uri] = new_location;
2329             }
2330 
2331             // prepare new response (for redirected request)
2332             _response = new HTTPResponse;
2333             _response.uri = current_uri;
2334             _response.finalURI = next_uri;
2335             _stream = null;
2336 
2337             // set new uri
2338             _uri = next_uri;
2339             debug(requests) tracef("Redirected to %s", next_uri);
2340             //if ( _method != "GET" && _response.code != 307 && _response.code != 308 ) {
2341             //    // 307 and 308 do not change method
2342             //    return exec_from_parameters(r);
2343             //}
2344             if ( restartedRequest ) {
2345                 debug(requests) trace("Rare event: clearing 'restartedRequest' on redirect");
2346                 restartedRequest = false;
2347             }
2348             goto connect;
2349         }
2350 
2351         _response._history = _history;
2352         return _response;
2353     }
2354     HTTPResponse execute(Request r)
2355     {
2356         _method = r.method;
2357         _uri = r.uri;
2358         _useStreaming = r.useStreaming;
2359         _permanent_redirects = r.permanent_redirects;
2360         _maxRedirects = r.maxRedirects;
2361         _authenticator = r.authenticator;
2362         _maxHeadersLength = r.maxHeadersLength;
2363         _maxContentLength = r.maxContentLength;
2364         _verbosity = r.verbosity;
2365         _keepAlive = r.keepAlive;
2366         _bufferSize = r.bufferSize;
2367         _proxy = r.proxy;
2368         _timeout = r.timeout;
2369         _contentType = r.contentType;
2370         _socketFactory = r.socketFactory;
2371         _sslOptions = r.sslOptions;
2372         _bind = r.bind;
2373         _headers = r.headers;
2374         _userHeaders = r.userHeaders;
2375 
2376         _params = r.params;
2377 
2378         // this assignments increments refCounts, so we can't use const Request
2379         // but Request is anyway struct and called by-value
2380         _cm = r.cm;
2381         _cookie = r.cookie;
2382 
2383         debug(requests) trace("serving %s".format(r));
2384         if ( !r.postData.empty)
2385         {
2386             return exec_from_range(r.postData);
2387         }
2388         if ( r.hasMultipartForm )
2389         {
2390             return exec_from_multipart_form(r.multipartForm);
2391         }
2392         auto rs = exec_from_parameters();
2393         return rs;
2394     }
2395 }
2396 
2397 version(vibeD) {
2398     import std.json;
2399     package string httpTestServer() {
2400         return "http://httpbin.org/";
2401     }
2402     package string fromJsonArrayToStr(JSONValue v) {
2403         return v.str;
2404     }
2405 }
2406 else {
2407     import std.json;
2408     package string httpTestServer() {
2409         return "http://127.0.0.1:8081/";
2410     }
2411     package string fromJsonArrayToStr(JSONValue v) {
2412         return cast(string)(v.array.map!"cast(ubyte)a.integer".array);
2413     }
2414 }

Page generated by adrdox