1 module requests.http;
2 
3 private:
4 import std.algorithm;
5 import std.array;
6 import std.ascii;
7 import std.conv;
8 import std.datetime;
9 import std.exception;
10 import std.format;
11 import std.stdio;
12 import std.range;
13 import std.string;
14 import std.traits;
15 import std.typecons;
16 import std.experimental.logger;
17 import core.thread;
18 
19 import requests.streams;
20 import requests.uri;
21 import requests.utils;
22 import requests.base;
23 import requests.connmanager;
24 import requests.rangeadapter;
25 
26 static immutable ushort[] redirectCodes = [301, 302, 303, 307, 308];
27 
28 enum   HTTP11 = 101;
29 enum   HTTP10 = 100;
30 
31 static immutable string[string] proxies;
32 shared static this() {
33     import std.process;
34     import std.string;
35     foreach(p; ["http", "https", "all"])
36     {
37         auto v = environment.get(p ~ "_proxy", environment.get(p.toUpper() ~ "_PROXY"));
38         if ( v !is null && v.length > 0 )
39         {
40             debug(requests) tracef("will use %s for %s as proxy", v, p);
41             proxies[p] = v;
42         }
43     }
44 }
45 
46 public class MaxRedirectsException: Exception {
47     this(string message, string file = __FILE__, size_t line = __LINE__, Throwable next = null) @safe pure nothrow {
48         super(message, file, line, next);
49     }
50 }
51 
52 ///
53 ///
54 ///
55 //public auto queryParams(T...)(T params) pure nothrow @safe
56 //{
57 //    static assert (T.length % 2 == 0, "wrong args count");
58 //
59 //    QueryParam[] output;
60 //    output.reserve = T.length / 2;
61 //
62 //    void queryParamsHelper(T...)(T params, ref QueryParam[] output)
63 //    {
64 //        static if (T.length > 0)
65 //        {
66 //            output ~= QueryParam(params[0].to!string, params[1].to!string);
67 //            queryParamsHelper(params[2..$], output);
68 //        }
69 //    }
70 //
71 //    queryParamsHelper(params, output);
72 //    return output;
73 //}
74 
75 ///
76 /// Response - result of request execution.
77 ///
78 /// Response.code - response HTTP code.
79 /// Response.status_line - received HTTP status line.
80 /// Response.responseHeaders - received headers.
81 /// Response.responseBody - container for received body
82 /// Response.history - for redirected responses contain all history
83 ///
84 public class HTTPResponse : Response {
85     private {
86         string              _status_line;
87 
88         HTTPResponse[]      _history; // redirects history
89 
90         mixin(Setter!string("status_line"));
91 
92         int                 _version;
93     }
94 
95     ~this() {
96         _responseHeaders = null;
97         _history.length = 0;
98     }
99 
100     mixin(Getter("status_line"));
101 
102     @property final string[string] responseHeaders() @safe @nogc nothrow {
103         return _responseHeaders;
104     }
105     @property final HTTPResponse[] history() @safe @nogc nothrow {
106         return _history;
107     }
108 
109     private int parse_version(in string v) pure const nothrow @safe {
110         // try to parse HTTP/1.x to version
111         try if ( v.length > 5 ) {
112             return (v[5..$].split(".").map!"to!int(a)".array[0..2].reduce!((a,b) => a*100 + b));
113         } catch (Exception e) {
114         }
115         return 0;
116     }
117     unittest {
118         auto r = new HTTPResponse();
119         assert(r.parse_version("HTTP/1.1") == 101);
120         assert(r.parse_version("HTTP/1.0") == 100);
121         assert(r.parse_version("HTTP/0.9") == 9);
122         assert(r.parse_version("HTTP/xxx") == 0);
123     }
124 }
125 
126 ///
127 /// Request.
128 /// Configurable parameters:
129 /// $(B method) - string, method to use (GET, POST, ...)
130 /// $(B headers) - string[string], add any additional headers you'd like to send.
131 /// $(B authenticator) - class Auth, class to send auth headers.
132 /// $(B keepAlive) - bool, set true for keepAlive requests. default true.
133 /// $(B maxRedirects) - uint, maximum number of redirects. default 10.
134 /// $(B maxHeadersLength) - size_t, maximum length of server response headers. default = 32KB.
135 /// $(B maxContentLength) - size_t, maximun content length. delault - 0 = unlimited.
136 /// $(B bufferSize) - size_t, send and receive buffer size. default = 16KB.
137 /// $(B verbosity) - uint, level of verbosity(0 - nothing, 1 - headers, 2 - headers and body progress). default = 0.
138 /// $(B proxy) - string, set proxy url if needed. default - null.
139 /// $(B cookie) - Tuple Cookie, Read/Write cookie You can get cookie setted by server, or set cookies before doing request.
140 /// $(B timeout) - Duration, Set timeout value for connect/receive/send.
141 ///
142 public struct HTTPRequest {
143     private {
144         string         _method = "GET";
145         URI            _uri;
146         string[string] _headers;
147         string[]       _filteredHeaders;
148         Auth           _authenticator;
149         bool           _keepAlive = true;
150         uint           _maxRedirects = 10;
151         size_t         _maxHeadersLength = 32 * 1024;   // 32 KB
152         size_t         _maxContentLength;               // 0 - Unlimited
153         string         _proxy;
154         uint           _verbosity = 0;                  // 0 - no output, 1 - headers, 2 - headers+body info
155         Duration       _timeout = 30.seconds;
156         size_t         _bufferSize = 16*1024; // 16k
157         bool           _useStreaming;                   // return iterator instead of completed request
158 
159         HTTPResponse[] _history;                        // redirects history
160         DataPipe!ubyte _bodyDecoder;
161         DecodeChunked  _unChunker;
162         long           _contentLength;
163         long           _contentReceived;
164         SSLOptions     _sslOptions;
165         string         _bind;
166         _UH            _userHeaders;
167 
168         RefCounted!ConnManager      _cm;
169         RefCounted!Cookies          _cookie;
170         string[URI]                 _permanent_redirects;            // cache 301 redirects for GET requests
171         MultipartForm               _multipartForm;
172 
173         NetStreamFactory  _socketFactory;
174 
175         QueryParam[]        _params;
176         string              _contentType;
177         InputRangeAdapter   _postData;
178     }
179     package HTTPResponse   _response;
180 
181     mixin(Getter_Setter!string     ("method"));
182     mixin(Getter_Setter!bool       ("keepAlive"));
183     mixin(Getter_Setter!size_t     ("maxContentLength"));
184     mixin(Getter_Setter!size_t     ("maxHeadersLength"));
185     mixin(Getter_Setter!size_t     ("bufferSize"));
186     mixin(Getter_Setter!uint       ("maxRedirects"));
187     mixin(Getter_Setter!uint       ("verbosity"));
188     mixin(Getter                   ("proxy"));
189     mixin(Getter_Setter!Duration   ("timeout"));
190     mixin(Setter!Auth              ("authenticator"));
191     mixin(Getter_Setter!bool       ("useStreaming"));
192     mixin(Getter                   ("contentLength"));
193     mixin(Getter                   ("contentReceived"));
194     mixin(Getter_Setter!SSLOptions ("sslOptions"));
195     mixin(Getter_Setter!string     ("bind"));
196     mixin(Setter!NetStreamFactory  ("socketFactory"));
197 
198     @property void sslSetVerifyPeer(bool v) pure @safe nothrow @nogc {
199         _sslOptions.setVerifyPeer(v);
200     }
201     @property void sslSetKeyFile(string p, SSLOptions.filetype t = SSLOptions.filetype.pem) pure @safe nothrow @nogc {
202         _sslOptions.setKeyFile(p, t);
203     }
204     @property void sslSetCertFile(string p, SSLOptions.filetype t = SSLOptions.filetype.pem) pure @safe nothrow @nogc {
205         _sslOptions.setCertFile(p, t);
206     }
207     @property void sslSetCaCert(string path) pure @safe nothrow @nogc {
208         _sslOptions.setCaCert(path);
209     }
210     //@property final void cookie(Cookie[] s) pure @safe @nogc nothrow {
211     //    _cookie = s;
212     //}
213     @property final void proxy(string v) {
214         if ( v != _proxy ) {
215             _cm.clear();
216         }
217         _proxy = v;
218     }
219     //@property final Cookie[] cookie() pure @safe @nogc nothrow {
220     //    return _cookie;
221     //}
222 
223     this(string uri) {
224         _uri = URI(uri);
225         _cm = ConnManager(10);
226     }
227     ~this() {
228         _headers = null;
229         _authenticator = null;
230         _history = null;
231         _bodyDecoder = null;
232         _unChunker = null;
233         //if ( _cm ) {
234         //    _cm.clear();
235         //}
236     }
237     string toString() const {
238         return "HTTPRequest(%s, %s)".format(_method, _uri.uri());
239     }
240     string format(string fmt) const {
241         import std.array;
242         import std.stdio;
243         auto a = appender!string();
244         auto f = FormatSpec!char(fmt);
245         while (f.writeUpToNextSpec(a)) {
246             switch(f.spec) {
247                 case 'h':
248                     // Remote hostname.
249                     a.put(_uri.host);
250                     break;
251                 case 'm':
252                     // method.
253                     a.put(_method);
254                     break;
255                 case 'p':
256                     // Remote port.
257                     a.put("%d".format(_uri.port));
258                     break;
259                 case 'P':
260                     // Path
261                     a.put(_uri.path);
262                     break;
263                 case 'q':
264                     // query parameters supplied with url.
265                     a.put(_uri.query);
266                     break;
267                 case 'U':
268                     a.put(_uri.uri());
269                     break;
270                 default:
271                     throw new FormatException("Unknown Request format spec " ~ f.spec);
272             }
273         }
274         return a.data();
275     }
276     string select_proxy(string scheme) {
277         if ( _proxy is null && proxies.length == 0 ) {
278             debug(requests) tracef("proxy=null");
279             return null;
280         }
281         if ( _proxy ) {
282             debug(requests) tracef("proxy=%s", _proxy);
283             return _proxy;
284         }
285         auto p = scheme in proxies;
286         if ( p !is null && *p != "") {
287             debug(requests) tracef("proxy=%s", *p);
288             return *p;
289         }
290         p = "all" in proxies;
291         if ( p !is null && *p != "") {
292             debug(requests) tracef("proxy=%s", *p);
293             return *p;
294         }
295         debug(requests) tracef("proxy=null");
296         return null;
297     }
298     void clearHeaders() {
299         _headers = null;
300     }
301     @property void uri(in URI newURI) {
302         //handleURLChange(_uri, newURI);
303         _uri = newURI;
304     }
305     /// Add headers to request
306     /// Params:
307     /// headers = headers to send.
308     void addHeaders(in string[string] headers) {
309         foreach(pair; headers.byKeyValue) {
310             string _h = pair.key;
311             switch(toLower(_h)) {
312             case "host":
313                 _userHeaders.Host = true;
314                 break;
315             case "user-agent":
316                 _userHeaders.UserAgent = true;
317                 break;
318             case "content-length":
319                 _userHeaders.ContentLength = true;
320                 break;
321             case "content-type":
322                 _userHeaders.ContentType = true;
323                 break;
324             case "connection":
325                 _userHeaders.Connection = true;
326                 break;
327             case "cookie":
328                 _userHeaders.Cookie = true;
329                 break;
330             default:
331                 break;
332             }
333             _headers[pair.key] = pair.value;
334         }
335     }
336     private void safeSetHeader(ref string[string] headers, bool userAdded, string h, string v) pure @safe {
337         if ( !userAdded ) {
338             headers[h] = v;
339         }
340     }
341     /// Remove headers from request
342     /// Params:
343     /// headers = headers to remove.
344     void removeHeaders(in string[] headers) pure {
345         _filteredHeaders ~= headers;
346     }
347     ///
348     /// compose headers to send
349     ///
350     private string[string] requestHeaders() {
351 
352         string[string] generatedHeaders;
353 
354         if ( _authenticator ) {
355             _authenticator.
356                 authHeaders(_uri.host).
357                 byKeyValue.
358                 each!(pair => generatedHeaders[pair.key] = pair.value);
359         }
360 
361         _headers.byKey.each!(h => generatedHeaders[h] = _headers[h]);
362 
363         safeSetHeader(generatedHeaders, _userHeaders.AcceptEncoding, "Accept-Encoding", "gzip,deflate");
364         safeSetHeader(generatedHeaders, _userHeaders.UserAgent, "User-Agent", "dlang-requests");
365         safeSetHeader(generatedHeaders, _userHeaders.Connection, "Connection", _keepAlive?"Keep-Alive":"Close");
366 
367         if ( !_userHeaders.Host )
368         {
369             generatedHeaders["Host"] = _uri.host;
370             if ( _uri.scheme !in standard_ports || _uri.port != standard_ports[_uri.scheme] ) {
371                 generatedHeaders["Host"] ~= ":%d".format(_uri.port);
372             }
373         }
374 
375         if ( _cookie._array.length && !_userHeaders.Cookie ) {
376             auto cs = _cookie._array.
377                 filter!(c => _uri.path.pathMatches(c.path) && _uri.host.domainMatches(c.domain)).
378                 map!(c => "%s=%s".format(c.attr, c.value)).
379                 joiner(";");
380             if ( ! cs.empty )
381             {
382                 generatedHeaders["Cookie"] = to!string(cs);
383             }
384         }
385 
386         _filteredHeaders.each!(h => generatedHeaders.remove(h));
387 
388         return generatedHeaders;
389     }
390     ///
391     /// Build request string.
392     /// Handle proxy and query parameters.
393     ///
394     private @property string requestString(QueryParam[] params = null) {
395         auto query = _uri.query.dup;
396         if ( params ) {
397             query ~= "&" ~ params2query(params);
398             if ( query[0] != '?' ) {
399                 query = "?" ~ query;
400             }
401         }
402         string actual_proxy = select_proxy(_uri.scheme);
403         if ( actual_proxy && _uri.scheme != "https" ) {
404             return "%s %s%s HTTP/1.1\r\n".format(_method, _uri.uri(No.params), query);
405         }
406         return "%s %s%s HTTP/1.1\r\n".format(_method, _uri.path, query);
407     }
408     ///
409     /// encode parameters and build query part of the url
410     ///
411     private static string params2query(in QueryParam[] params) pure @safe {
412         return params.
413                 map!(a => "%s=%s".format(a.key.urlEncoded, a.value.urlEncoded)).
414                 join("&");
415     }
416     //
417     package unittest {
418         assert(params2query(queryParams("a","b", "c", " d "))=="a=b&c=%20d%20");
419     }
420     ///
421     /// Analyze received headers, take appropriate actions:
422     /// check content length, attach unchunk and uncompress
423     ///
424     private void analyzeHeaders(in string[string] headers) {
425 
426         _contentLength = -1;
427         _unChunker = null;
428         auto contentLength = "content-length" in headers;
429         if ( contentLength ) {
430             try {
431                 string l = *contentLength;
432                 _contentLength = parse!long(l);
433                 // TODO: maybe add a strict mode that checks if l was parsed completely
434                 if ( _maxContentLength && _contentLength > _maxContentLength) {
435                     throw new RequestException("ContentLength > maxContentLength (%d>%d)".
436                                 format(_contentLength, _maxContentLength));
437                 }
438             } catch (ConvException e) {
439                 throw new RequestException("Can't convert Content-Length from %s".format(*contentLength));
440             }
441         }
442         auto transferEncoding = "transfer-encoding" in headers;
443         if ( transferEncoding ) {
444             debug(requests) tracef("transferEncoding: %s", *transferEncoding);
445             if ( (*transferEncoding).toLower == "chunked") {
446                 _unChunker = new DecodeChunked();
447                 _bodyDecoder.insert(_unChunker);
448             }
449         }
450         auto contentEncoding = "content-encoding" in headers;
451         if ( contentEncoding ) switch (*contentEncoding) {
452             default:
453                 throw new RequestException("Unknown content-encoding " ~ *contentEncoding);
454             case "gzip":
455             case "deflate":
456                 _bodyDecoder.insert(new Decompressor!ubyte);
457         }
458 
459     }
460     ///
461     /// Called when we know that all headers already received in buffer.
462     /// This routine does not interpret headers content (see analyzeHeaders).
463     /// 1. Split headers on lines
464     /// 2. store status line, store response code
465     /// 3. unfold headers if needed
466     /// 4. store headers
467     ///
468     private void parseResponseHeaders(in ubyte[] input, string lineSep) {
469         string lastHeader;
470         auto buffer = cast(string)input;
471 
472         foreach(line; buffer.split(lineSep).map!(l => l.stripRight)) {
473             if ( ! _response.status_line.length ) {
474                 debug (requests) tracef("statusLine: %s", line);
475                 _response.status_line = line;
476                 if ( _verbosity >= 1 ) {
477                     writefln("< %s", line);
478                 }
479                 auto parsed = line.split(" ");
480                 if ( parsed.length >= 2 ) {
481                     _response.code = parsed[1].to!ushort;
482                     _response._version = _response.parse_version(parsed[0]);
483                 }
484                 continue;
485             }
486             if ( line[0] == ' ' || line[0] == '\t' ) {
487                 // unfolding https://tools.ietf.org/html/rfc822#section-3.1
488                 if ( auto stored = lastHeader in _response._responseHeaders) {
489                     *stored ~= line;
490                 }
491                 continue;
492             }
493             auto parsed = line.findSplit(":");
494             auto header = parsed[0].toLower;
495             auto value = parsed[2].strip;
496 
497             if ( _verbosity >= 1 ) {
498                 writefln("< %s: %s", header, value);
499             }
500 
501             lastHeader = header;
502             debug (requests) tracef("Header %s = %s", header, value);
503 
504             if ( header != "set-cookie" ) {
505                 auto stored = _response.responseHeaders.get(header, null);
506                 if ( stored ) {
507                     value = stored ~ "," ~ value;
508                 }
509                 _response._responseHeaders[header] = value;
510                 continue;
511             }
512             _cookie._array ~= processCookie(value);
513         }
514     }
515 
516     ///
517     /// Process Set-Cookie header from server response
518     ///
519     private Cookie[] processCookie(string value ) pure {
520         // cookie processing
521         //
522         // as we can't join several set-cookie lines in single line
523         // < Set-Cookie: idx=7f2800f63c112a65ef5082957bcca24b; expires=Mon, 29-May-2017 00:31:25 GMT; path=/; domain=example.com
524         // < Set-Cookie: idx=7f2800f63c112a65ef5082957bcca24b; expires=Mon, 29-May-2017 00:31:25 GMT; path=/; domain=example.com, cs=ip764-RgKqc-HvSkxRxdQQAKW8LA; path=/; domain=.example.com; HttpOnly
525         //
526         Cookie[] res;
527         string[string] kv;
528         auto fields = value.split(";").map!strip;
529         while(!fields.empty) {
530             auto s = fields.front.findSplit("=");
531             fields.popFront;
532             if ( s[1] != "=" ) {
533                 continue;
534             }
535             auto k = s[0];
536             auto v = s[2];
537             switch(k.toLower()) {
538                 case "domain":
539                     k = "domain";
540                     break;
541                 case "path":
542                     k = "path";
543                     break;
544                 case "expires":
545                     continue;
546                 case "max-age":
547                     continue;
548                 default:
549                     break;
550             }
551             kv[k] = v;
552         }
553         if ( "domain" !in kv ) {
554             kv["domain"] = _uri.host;
555         }
556         if ( "path" !in kv ) {
557             kv["path"] = _uri.path;
558         }
559         auto domain = kv["domain"]; kv.remove("domain");
560         auto path   = kv["path"];   kv.remove("path");
561         foreach(pair; kv.byKeyValue) {
562             auto _attr = pair.key;
563             auto _value = pair.value;
564             auto cookie = Cookie(path, domain, _attr, _value);
565             res ~= cookie;
566         }
567         return res;
568     }
569 
570     private bool willFollowRedirect() {
571         if ( !canFind(redirectCodes, _response.code) ) {
572             return false;
573         }
574         if ( !_maxRedirects ) {
575             return false;
576         }
577         if ( "location" !in _response.responseHeaders ) {
578             return false;
579         }
580         return true;
581     }
582     private URI uriFromLocation(const ref URI uri, in string location) {
583         URI newURI = uri;
584         try {
585             newURI = URI(location);
586         } catch (UriException e) {
587             debug(requests) trace("Can't parse Location:, try relative uri");
588             newURI.path = location;
589             newURI.uri = newURI.recalc_uri;
590         }
591         return newURI;
592     }
593     ///
594     /// if we have new uri, then we need to check if we have to reopen existent connection
595     ///
596     private void checkURL(string url, string file=__FILE__, size_t line=__LINE__) {
597         if (url is null && _uri.uri == "" ) {
598             throw new RequestException("No url configured", file, line);
599         }
600 
601         if ( url !is null ) {
602             URI newURI = URI(url);
603             //handleURLChange(_uri, newURI);
604             _uri = newURI;
605         }
606     }
607     ///
608     /// Setup connection. Handle proxy and https case
609     ///
610     /// Place new connection in ConnManager cache
611     ///
612     private NetworkStream setupConnection()
613     do {
614 
615         debug(requests) tracef("Set up new connection");
616         NetworkStream stream;
617 
618         // on exit
619         // place created connection to conn. manager
620         // close connection purged from manager (if any)
621         //
622         scope(exit) {
623             if ( stream )
624             {
625                 if ( auto purged_connection = _cm.put(_uri.scheme, _uri.host, _uri.port, stream) )
626                 {
627                     debug(requests) tracef("closing purged connection %s", purged_connection);
628                     purged_connection.close();
629                 }
630             }
631         }
632 
633         if ( _socketFactory )
634         {
635             debug(requests) tracef("use socketFactory");
636             stream = _socketFactory(_uri.scheme, _uri.host, _uri.port);
637         }
638 
639         if ( stream ) // socket factory created connection
640         {
641             return stream;
642         }
643 
644         URI   uri; // this URI will be used temporarry if we need proxy
645         string actual_proxy = select_proxy(_uri.scheme);
646         final switch (_uri.scheme) {
647             case"http":
648                 if ( actual_proxy ) {
649                     uri.uri_parse(actual_proxy);
650                     uri.idn_encode();
651                 } else {
652                     // use original uri
653                     uri = _uri;
654                 }
655                 stream = new TCPStream();
656                 stream.bind(_bind);
657                 stream.connect(uri.host, uri.port, _timeout);
658                 break ;
659             case"https":
660                 if ( actual_proxy ) {
661                     uri.uri_parse(actual_proxy);
662                     uri.idn_encode();
663                     stream = new TCPStream();
664                     stream.bind(_bind);
665                     stream.connect(uri.host, uri.port, _timeout);
666                     if ( verbosity>=1 ) {
667                         writeln("> CONNECT %s:%d HTTP/1.1".format(_uri.host, _uri.port));
668                     }
669                     stream.send("CONNECT %s:%d HTTP/1.1\r\n".format(_uri.host, _uri.port));
670                     if (uri.username)
671                     {
672                         debug(requests) tracef("Add Proxy-Authorization header");
673                         auto auth = new BasicAuthentication(uri.username, uri.password);
674                         auto header = auth.authHeaders("");
675                         stream.send("Proxy-Authorization: %s\r\n".format(header["Authorization"]));
676                     }
677                     stream.send("\r\n");
678                     while ( stream.isConnected ) {
679                         ubyte[1024] b;
680                         auto read = stream.receive(b);
681                         if ( verbosity>=1) {
682                             writefln("< %s", cast(string)b[0..read]);
683                         }
684                         debug(requests) tracef("read: %d", read);
685                         if ( b[0..read].canFind("\r\n\r\n") || b[0..read].canFind("\n\n") ) {
686                             debug(requests) tracef("proxy connection ready");
687                             // convert connection to ssl
688                             stream = new SSLStream(stream, _sslOptions, _uri.host);
689                             break ;
690                         } else {
691                             debug(requests) tracef("still wait for proxy connection");
692                         }
693                     }
694                 } else {
695                     uri = _uri;
696                     stream = new SSLStream(_sslOptions);
697                     stream.bind(_bind);
698                     stream.connect(uri.host, uri.port, _timeout);
699                     debug(requests) tracef("ssl connection to origin server ready");
700                 }
701                 break ;
702         }
703 
704         return stream;
705     }
706     ///
707     /// Request sent, now receive response.
708     /// Find headers, split on headers and body, continue to receive body
709     ///
710     private void receiveResponse(NetworkStream _stream) {
711 
712         try {
713             _stream.readTimeout = timeout;
714         } catch (Exception e) {
715             debug(requests) tracef("Failed to set read timeout for stream: %s", e.msg);
716             return;
717         }
718         // Commented this out as at exit we can have alreade closed socket
719         // scope(exit) {
720         //     if ( _stream && _stream.isOpen ) {
721         //         _stream.readTimeout = 0.seconds;
722         //     }
723         // }
724 
725         _bodyDecoder = new DataPipe!ubyte();
726         scope(exit) {
727             if ( !_useStreaming ) {
728                 _bodyDecoder = null;
729                 _unChunker = null;
730             }
731         }
732 
733         auto buffer = Buffer!ubyte();
734         Buffer!ubyte partialBody;
735         ptrdiff_t read;
736         string lineSep = null, headersEnd = null;
737         bool headersHaveBeenReceived;
738 
739         while( !headersHaveBeenReceived ) {
740 
741             auto b = new ubyte[_bufferSize];
742             read = _stream.receive(b);
743 
744             debug(requests) tracef("read: %d", read);
745             if ( read == 0 ) {
746                 break;
747             }
748             auto data = b[0..read];
749             buffer.putNoCopy(data);
750             if ( verbosity>=3 ) {
751                 writeln(data.dump.join("\n"));
752             }
753 
754             if ( buffer.length > maxHeadersLength ) {
755                 throw new RequestException("Headers length > maxHeadersLength (%d > %d)".format(buffer.length, maxHeadersLength));
756             }
757 
758             // Proper HTTP uses "\r\n" as a line separator, but broken servers sometimes use "\n".
759             // Servers that use "\r\n" might have "\n" inside a header.
760             // For any half-sane server, the first '\n' should be at the end of the status line, so this can be used to detect the line separator.
761             // In any case, all the interesting points in the header for now are at '\n' characters, so scan the newly read data for them.
762             foreach (idx; buffer.length-read..buffer.length)
763             {
764                 if ( buffer[idx] == '\n' )
765                 {
766                     if ( lineSep is null )
767                     {
768                         // First '\n'. Detect line/header endings.
769                         // HTTP header sections end with a double line separator
770                         lineSep = "\n";
771                         headersEnd = "\n\n";
772                         if ( idx > 0 && buffer[idx-1] == '\r' )
773                         {
774                             lineSep = "\r\n";
775                             headersEnd = "\r\n\r\n";
776                         }
777                     }
778                     else
779                     {
780                         // Potential header ending.
781                         if ( buffer.data[0..idx+1].endsWith(headersEnd) )
782                         {
783                             auto ResponseHeaders = buffer.data[0..idx+1-headersEnd.length];
784                             partialBody = buffer[idx+1..$];
785                             _contentReceived += partialBody.length;
786                             parseResponseHeaders(ResponseHeaders, lineSep);
787                             headersHaveBeenReceived = true;
788                             break;
789                         }
790                     }
791                 }
792             }
793         }
794 
795         analyzeHeaders(_response._responseHeaders);
796 
797         _bodyDecoder.putNoCopy(partialBody.data);
798 
799         auto v = _bodyDecoder.get();
800         _response._responseBody.putNoCopy(v);
801 
802         // https://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.4
803         if ( (_method == "HEAD") || responseMustNotIncludeBody(_response.code) || (_contentLength < 0 && _unChunker is null) )
804         {
805             debug(requests) tracef("response without body");
806             return;
807         }
808 
809         _response._contentLength = _contentLength;
810         _response._contentReceived = _contentReceived;
811 
812         if ( _verbosity >= 2 ) writefln("< %d bytes of body received", partialBody.length);
813 
814         while( true ) {
815             if ( _contentLength >= 0 && _contentReceived >= _contentLength ) {
816                 debug(requests) trace("Body received.");
817                 break;
818             }
819             if ( _unChunker && _unChunker.done ) {
820                 break;
821             }
822 
823             if ( _useStreaming && _response._responseBody.length && !redirectCodes.canFind(_response.code) ) {
824                 debug(requests) trace("streaming requested");
825                 // save _stream in closure
826                 auto __stream = _stream;
827                 auto __bodyDecoder = _bodyDecoder;
828                 auto __unChunker = _unChunker;
829                 auto __contentReceived = _contentReceived;
830                 auto __contentLength = _contentLength;
831                 auto __bufferSize = _bufferSize;
832                 auto __response = _response;
833                 auto __verbosity = _verbosity;
834 
835                 // set up response
836                 _response._contentLength = _contentLength;
837                 _response.receiveAsRange.activated = true;
838                 _response.receiveAsRange.data = _response._responseBody.data;
839                 _response.receiveAsRange.read = delegate ubyte[] () {
840 
841                     while(true) {
842                         // check if we received everything we need
843                         if ( ( __unChunker && __unChunker.done )
844                             || !__stream.isConnected()
845                             || (__contentLength > 0 && __contentReceived >= __contentLength) )
846                         {
847                             debug(requests) trace("streaming_in receive completed");
848                             __bodyDecoder.flush();
849                             return __bodyDecoder.get();
850                         }
851                         // have to continue
852                         auto b = new ubyte[__bufferSize];
853                         try {
854                             read = __stream.receive(b);
855                         }
856                         catch (Exception e) {
857                             throw new RequestException("streaming_in error reading from socket", __FILE__, __LINE__, e);
858                         }
859                         debug(requests) tracef("streaming_in received %d bytes", read);
860 
861                         if ( read == 0 ) {
862                             debug(requests) tracef("streaming_in: server closed connection");
863                             __bodyDecoder.flush();
864                             return __bodyDecoder.get();
865                         }
866 
867                         if ( __verbosity>=3 ) {
868                             writeln(b[0..read].dump.join("\n"));
869                         }
870                         __response._contentReceived += read;
871                         __contentReceived += read;
872                         __bodyDecoder.putNoCopy(b[0..read]);
873                         auto res = __bodyDecoder.getNoCopy();
874                         if ( res.length == 0 ) {
875                             // there were nothing to produce (beginning of the chunk or no decompressed data)
876                             continue;
877                         }
878                         if (res.length == 1) {
879                             return res[0];
880                         }
881                         //
882                         // I'd like to "return _bodyDecoder.getNoCopy().join;" but it is slower
883                         //
884                         auto total = res.map!(b=>b.length).sum;
885                         // create buffer for joined bytes
886                         ubyte[] joined = new ubyte[total];
887                         size_t p;
888                         // memcopy
889                         foreach(ref _; res) {
890                             joined[p .. p + _.length] = _;
891                             p += _.length;
892                         }
893                         return joined;
894                     }
895                     assert(0);
896                 };
897                 // we prepared for streaming
898                 return;
899             }
900 
901             auto b = new ubyte[_bufferSize];
902             read = _stream.receive(b);
903 
904             if ( read == 0 ) {
905                 debug(requests) trace("read done");
906                 break;
907             }
908             if ( _verbosity >= 2 ) {
909                 writefln("< %d bytes of body received", read);
910             }
911 
912             if ( verbosity>=3 ) {
913                 writeln(b[0..read].dump.join("\n"));
914             }
915 
916             debug(requests) tracef("read: %d", read);
917             _contentReceived += read;
918             if ( _maxContentLength && _contentReceived > _maxContentLength ) {
919                 throw new RequestException("ContentLength > maxContentLength (%d>%d)".
920                     format(_contentLength, _maxContentLength));
921             }
922 
923             _bodyDecoder.putNoCopy(b[0..read]); // send buffer to all decoders
924 
925             _bodyDecoder.getNoCopy.             // fetch result and place to body
926                 each!(b => _response._responseBody.putNoCopy(b));
927 
928             debug(requests) tracef("receivedTotal: %d, contentLength: %d, bodyLength: %d", _contentReceived, _contentLength, _response._responseBody.length);
929 
930         }
931         _bodyDecoder.flush();
932         _response._responseBody.putNoCopy(_bodyDecoder.get());
933         _response._contentReceived = _contentReceived;
934     }
935     ///
936     /// Check that we received anything.
937     /// Server can close previous connection (keepalive or not)
938     ///
939     private bool serverPrematurelyClosedConnection() pure @safe {
940         immutable server_closed_connection = _response._responseHeaders.length == 0 && _response._status_line.length == 0;
941         // debug(requests) tracef("server closed connection = %s (headers.length=%s, status_line.length=%s)",
942         //     server_closed_connection, _response._responseHeaders.length,  _response._status_line.length);
943         return server_closed_connection;
944     }
945     private bool isIdempotent(in string method) pure @safe nothrow {
946         return ["GET", "HEAD"].canFind(method);
947     }
948     ///
949     /// If we do not want keepalive request,
950     /// or server signalled to close connection,
951     /// then close it
952     ///
953     void close_connection_if_not_keepalive(NetworkStream _stream) {
954         auto connection = "connection" in _response._responseHeaders;
955         if ( !_keepAlive ) {
956             _stream.close();
957         } else switch(_response._version) {
958             case HTTP11:
959                 // HTTP/1.1 defines the "close" connection option for the sender to signal that the connection
960                 // will be closed after completion of the response. For example,
961                 //        Connection: close
962                 // in either the request or the response header fields indicates that the connection
963                 // SHOULD NOT be considered `persistent' (section 8.1) after the current request/response is complete.
964                 // HTTP/1.1 applications that do not support persistent connections MUST include the "close" connection
965                 // option in every message.
966                 if ( connection && (*connection).toLower.split(",").canFind("close") ) {
967                     _stream.close();
968                 }
969                 break;
970             default:
971                 // for anything else close connection if there is no keep-alive in Connection
972                 if ( connection && !(*connection).toLower.split(",").canFind("keep-alive") ) {
973                     _stream.close();
974                 }
975                 break;
976         }
977     }
978     ///
979     /// Send multipart for request.
980     /// You would like to use this method for sending large portions of mixed data or uploading files to forms.
981     /// Content of the posted form consist of sources. Each source have at least name and value (can be string-like object or opened file, see more docs for MultipartForm struct)
982     /// Params:
983     ///     url = url
984     ///     sources = array of sources.
985     deprecated("Use Request() instead of HTTPRequest(); will be removed 2019-07")
986     HTTPResponse exec(string method="POST")(string url, MultipartForm sources) {
987         import std.uuid;
988         import std.file;
989 
990         checkURL(url);
991         //if ( _cm is null ) {
992         //    _cm = new ConnManager();
993         //}
994 
995         NetworkStream _stream;
996         _method = method;
997         _response = new HTTPResponse;
998         _response.uri = _uri;
999         _response.finalURI = _uri;
1000         bool restartedRequest = false;
1001 
1002     connect:
1003         _contentReceived = 0;
1004         _response._startedAt = Clock.currTime;
1005 
1006         assert(_stream is null);
1007 
1008         _stream = _cm.get(_uri.scheme, _uri.host, _uri.port);
1009 
1010         if ( _stream is null ) {
1011             debug(requests) trace("create new connection");
1012             _stream = setupConnection();
1013         } else {
1014             debug(requests) trace("reuse old connection");
1015         }
1016 
1017         assert(_stream !is null);
1018 
1019         if ( !_stream.isConnected ) {
1020             debug(requests) trace("disconnected stream on enter");
1021             if ( !restartedRequest ) {
1022                 debug(requests) trace("disconnected stream on enter: retry");
1023                 assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
1024 
1025                 _cm.del(_uri.scheme, _uri.host, _uri.port);
1026                 _stream.close();
1027                 _stream = null;
1028 
1029                 restartedRequest = true;
1030                 goto connect;
1031             }
1032             debug(requests) trace("disconnected stream on enter: return response");
1033             //_stream = null;
1034             return _response;
1035         }
1036         _response._connectedAt = Clock.currTime;
1037 
1038         Appender!string req;
1039         req.put(requestString());
1040 
1041         string   boundary = randomUUID().toString;
1042         string[] partHeaders;
1043         size_t   contentLength;
1044 
1045         foreach(ref part; sources._sources) {
1046             string h = "--" ~ boundary ~ "\r\n";
1047             string disposition = `form-data; name="%s"`.format(part.name);
1048             string optionals = part.
1049                 parameters.byKeyValue().
1050                 filter!(p => p.key!="Content-Type").
1051                 map!   (p => "%s=%s".format(p.key, p.value)).
1052                 join("; ");
1053 
1054             h ~= `Content-Disposition: ` ~ [disposition, optionals].join("; ") ~ "\r\n";
1055 
1056             auto contentType = "Content-Type" in part.parameters;
1057             if ( contentType ) {
1058                 h ~= "Content-Type: " ~ *contentType ~ "\r\n";
1059             }
1060 
1061             h ~= "\r\n";
1062             partHeaders ~= h;
1063             contentLength += h.length + part.input.getSize() + "\r\n".length;
1064         }
1065         contentLength += "--".length + boundary.length + "--\r\n".length;
1066 
1067         auto h = requestHeaders();
1068         safeSetHeader(h, _userHeaders.ContentType, "Content-Type", "multipart/form-data; boundary=" ~ boundary);
1069         safeSetHeader(h, _userHeaders.ContentLength, "Content-Length", to!string(contentLength));
1070 
1071         h.byKeyValue.
1072             map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n").
1073                 each!(h => req.put(h));
1074         req.put("\r\n");
1075 
1076         debug(requests) trace(req.data);
1077         if ( _verbosity >= 1 ) req.data.splitLines.each!(a => writeln("> " ~ a));
1078 
1079         try {
1080             _stream.send(req.data());
1081             foreach(ref source; sources._sources) {
1082                 debug(requests) tracef("sending part headers <%s>", partHeaders.front);
1083                 _stream.send(partHeaders.front);
1084                 partHeaders.popFront;
1085                 while (true) {
1086                     auto chunk = source.input.read();
1087                     if ( chunk.length <= 0 ) {
1088                         break;
1089                     }
1090                     _stream.send(chunk);
1091                 }
1092                 _stream.send("\r\n");
1093             }
1094             _stream.send("--" ~ boundary ~ "--\r\n");
1095             _response._requestSentAt = Clock.currTime;
1096             receiveResponse(_stream);
1097             _response._finishedAt = Clock.currTime;
1098         }
1099         catch (NetworkException e) {
1100             errorf("Error sending request: ", e.msg);
1101             _stream.close();
1102             return _response;
1103         }
1104 
1105         if ( serverPrematurelyClosedConnection()
1106         && !restartedRequest
1107         && isIdempotent(_method)
1108         ) {
1109             ///
1110             /// We didn't receive any data (keepalive connectioin closed?)
1111             /// and we can restart this request.
1112             /// Go ahead.
1113             ///
1114             debug(requests) tracef("Server closed keepalive connection");
1115 
1116             assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
1117 
1118             _cm.del(_uri.scheme, _uri.host, _uri.port);
1119             _stream.close();
1120             _stream = null;
1121 
1122             restartedRequest = true;
1123             goto connect;
1124         }
1125 
1126         if ( _useStreaming ) {
1127             if ( _response._receiveAsRange.activated ) {
1128                 debug(requests) trace("streaming_in activated");
1129                 return _response;
1130             } else {
1131                 // this can happen if whole response body received together with headers
1132                 _response._receiveAsRange.data = _response.responseBody.data;
1133             }
1134         }
1135 
1136         close_connection_if_not_keepalive(_stream);
1137 
1138         if ( _verbosity >= 1 ) {
1139             writeln(">> Connect time: ", _response._connectedAt - _response._startedAt);
1140             writeln(">> Request send time: ", _response._requestSentAt - _response._connectedAt);
1141             writeln(">> Response recv time: ", _response._finishedAt - _response._requestSentAt);
1142         }
1143 
1144         if ( willFollowRedirect ) {
1145             if ( _history.length >= _maxRedirects ) {
1146                 _stream = null;
1147                 throw new MaxRedirectsException("%d redirects reached maxRedirects %d.".format(_history.length, _maxRedirects));
1148             }
1149             // "location" in response already checked in canFollowRedirect
1150             immutable new_location = *("location" in _response.responseHeaders);
1151             immutable current_uri = _uri, next_uri = uriFromLocation(_uri, new_location);
1152 
1153             // save current response for history
1154             _history ~= _response;
1155 
1156             // prepare new response (for redirected request)
1157             _response = new HTTPResponse;
1158             _response.uri = current_uri;
1159             _response.finalURI = next_uri;
1160             _stream = null;
1161 
1162             // set new uri
1163             this._uri = next_uri;
1164             debug(requests) tracef("Redirected to %s", next_uri);
1165             if ( _method != "GET" && _response.code != 307 && _response.code != 308 ) {
1166                 // 307 and 308 do not change method
1167                 return this.get();
1168             }
1169             if ( restartedRequest ) {
1170                 debug(requests) trace("Rare event: clearing 'restartedRequest' on redirect");
1171                 restartedRequest = false;
1172             }
1173             goto connect;
1174         }
1175 
1176         _response._history = _history;
1177         return _response;
1178     }
1179 
1180     // we use this if we send from ubyte[][] and user provided Content-Length
1181     private void sendFlattenContent(T)(NetworkStream _stream, T content) {
1182         while ( !content.empty ) {
1183             auto chunk = content.front;
1184             _stream.send(chunk);
1185             content.popFront;
1186         }
1187         debug(requests) tracef("sent");
1188     }
1189     // we use this if we send from ubyte[][] as chunked content
1190     private void sendChunkedContent(T)(NetworkStream _stream, T content) {
1191         while ( !content.empty ) {
1192             auto chunk = content.front;
1193             auto chunkHeader = "%x\r\n".format(chunk.length);
1194             debug(requests) tracef("sending %s%s", chunkHeader, chunk);
1195             _stream.send(chunkHeader);
1196             _stream.send(chunk);
1197             _stream.send("\r\n");
1198             content.popFront;
1199         }
1200         debug(requests) tracef("sent");
1201         _stream.send("0\r\n\r\n");
1202     }
1203     ///
1204     /// POST/PUT/... data from some string(with Content-Length), or from range of strings/bytes (use Transfer-Encoding: chunked).
1205     /// When rank 1 (flat array) used as content it must have length. In that case "content" will be sent directly to network, and Content-Length headers will be added.
1206     /// If you are goung to send some range and do not know length at the moment when you start to send request, then you can send chunks of chars or ubyte.
1207     /// Try not to send too short chunks as this will put additional load on client and server. Chunks of length 2048 or 4096 are ok.
1208     ///
1209     /// Parameters:
1210     ///    url = url
1211     ///    content = string or input range
1212     ///    contentType = content type
1213     ///  Returns:
1214     ///     Response
1215     ///  Examples:
1216     ///  ---------------------------------------------------------------------------------------------------------
1217     ///      rs = rq.exec!"POST"("http://httpbin.org/post", "привiт, свiт!", "application/octet-stream");
1218     ///
1219     ///      auto s = lineSplitter("one,\ntwo,\nthree.");
1220     ///      rs = rq.exec!"POST"("http://httpbin.org/post", s, "application/octet-stream");
1221     ///
1222     ///      auto s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
1223     ///      rs = rq.exec!"POST"("http://httpbin.org/post", s.representation.chunks(10), "application/octet-stream");
1224     ///
1225     ///      auto f = File("tests/test.txt", "rb");
1226     ///      rs = rq.exec!"POST"("http://httpbin.org/post", f.byChunk(3), "application/octet-stream");
1227     ///  --------------------------------------------------------------------------------------------------------
1228     deprecated("Use Request() instead of HTTPRequest(); will be removed 2019-07")
1229     HTTPResponse exec(string method="POST", R)(string url, R content, string contentType="application/octet-stream")
1230         if ( (rank!R == 1)
1231             || (rank!R == 2 && isSomeChar!(Unqual!(typeof(content.front.front))))
1232             || (rank!R == 2 && (is(Unqual!(typeof(content.front.front)) == ubyte)))
1233         )
1234     do {
1235         debug(requests) tracef("started url=%s, this._uri=%s", url, _uri);
1236 
1237         checkURL(url);
1238         //if ( _cm is null ) {
1239         //    _cm = new ConnManager();
1240         //}
1241 
1242         NetworkStream _stream;
1243         _method = method;
1244         _response = new HTTPResponse;
1245         _history.length = 0;
1246         _response.uri = _uri;
1247         _response.finalURI = _uri;
1248         bool restartedRequest = false;
1249         bool send_flat;
1250 
1251     connect:
1252         _contentReceived = 0;
1253         _response._startedAt = Clock.currTime;
1254 
1255         assert(_stream is null);
1256 
1257         _stream = _cm.get(_uri.scheme, _uri.host, _uri.port);
1258 
1259         if ( _stream is null ) {
1260             debug(requests) trace("create new connection");
1261             _stream = setupConnection();
1262         } else {
1263             debug(requests) trace("reuse old connection");
1264         }
1265 
1266         assert(_stream !is null);
1267 
1268         if ( !_stream.isConnected ) {
1269             debug(requests) trace("disconnected stream on enter");
1270             if ( !restartedRequest ) {
1271                 debug(requests) trace("disconnected stream on enter: retry");
1272                 assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
1273 
1274                 _cm.del(_uri.scheme, _uri.host, _uri.port);
1275                 _stream.close();
1276                 _stream = null;
1277 
1278                 restartedRequest = true;
1279                 goto connect;
1280             }
1281             debug(requests) trace("disconnected stream on enter: return response");
1282             //_stream = null;
1283             return _response;
1284         }
1285         _response._connectedAt = Clock.currTime;
1286 
1287         Appender!string req;
1288         req.put(requestString());
1289 
1290         auto h = requestHeaders;
1291         if ( contentType ) {
1292             safeSetHeader(h, _userHeaders.ContentType, "Content-Type", contentType);
1293         }
1294         static if ( rank!R == 1 ) {
1295             safeSetHeader(h, _userHeaders.ContentLength, "Content-Length", to!string(content.length));
1296         } else {
1297             if ( _userHeaders.ContentLength ) {
1298                 debug(requests) tracef("User provided content-length for chunked content");
1299                 send_flat = true;
1300             } else {
1301                 h["Transfer-Encoding"] = "chunked";
1302                 send_flat = false;
1303             }
1304         }
1305         h.byKeyValue.
1306             map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n").
1307             each!(h => req.put(h));
1308         req.put("\r\n");
1309 
1310         debug(requests) trace(req.data);
1311         if ( _verbosity >= 1 ) {
1312             req.data.splitLines.each!(a => writeln("> " ~ a));
1313         }
1314 
1315         try {
1316             // send headers
1317             _stream.send(req.data());
1318             // send body
1319             static if ( rank!R == 1) {
1320                 _stream.send(content);
1321             } else {
1322                 if ( send_flat ) {
1323                     sendFlattenContent(_stream, content);
1324                 } else {
1325                     sendChunkedContent(_stream, content);
1326                 }
1327             }
1328             _response._requestSentAt = Clock.currTime;
1329             debug(requests) trace("starting receive response");
1330             receiveResponse(_stream);
1331             debug(requests) trace("finished receive response");
1332             _response._finishedAt = Clock.currTime;
1333         } catch (NetworkException e) {
1334             _stream.close();
1335             throw new RequestException("Network error during data exchange");
1336         }
1337 
1338         if ( serverPrematurelyClosedConnection()
1339             && !restartedRequest
1340             && isIdempotent(_method))
1341         {
1342             ///
1343             /// We didn't receive any data (keepalive connectioin closed?)
1344             /// and we can restart this request.
1345             /// Go ahead.
1346             ///
1347             debug(requests) tracef("Server closed keepalive connection");
1348 
1349             assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
1350 
1351             _cm.del(_uri.scheme, _uri.host, _uri.port);
1352             _stream.close();
1353             _stream = null;
1354 
1355             restartedRequest = true;
1356             goto connect;
1357         }
1358 
1359         if ( _useStreaming ) {
1360             if ( _response._receiveAsRange.activated ) {
1361                 debug(requests) trace("streaming_in activated");
1362                 return _response;
1363             } else {
1364                 // this can happen if whole response body received together with headers
1365                 _response._receiveAsRange.data = _response.responseBody.data;
1366             }
1367         }
1368 
1369         close_connection_if_not_keepalive(_stream);
1370 
1371         if ( _verbosity >= 1 ) {
1372             writeln(">> Connect time: ", _response._connectedAt - _response._startedAt);
1373             writeln(">> Request send time: ", _response._requestSentAt - _response._connectedAt);
1374             writeln(">> Response recv time: ", _response._finishedAt - _response._requestSentAt);
1375         }
1376 
1377 
1378         if ( willFollowRedirect ) {
1379             if ( _history.length >= _maxRedirects ) {
1380                 _stream = null;
1381                 throw new MaxRedirectsException("%d redirects reached maxRedirects %d.".format(_history.length, _maxRedirects));
1382             }
1383             // "location" in response already checked in canFollowRedirect
1384             immutable new_location = *("location" in _response.responseHeaders);
1385             immutable current_uri = _uri, next_uri = uriFromLocation(_uri, new_location);
1386 
1387             // save current response for history
1388             _history ~= _response;
1389 
1390             // prepare new response (for redirected request)
1391             _response = new HTTPResponse;
1392             _response.uri = current_uri;
1393             _response.finalURI = next_uri;
1394 
1395             _stream = null;
1396 
1397             // set new uri
1398             this._uri = next_uri;
1399             debug(requests) tracef("Redirected to %s", next_uri);
1400             if ( _method != "GET" && _response.code != 307 && _response.code != 308 ) {
1401                 // 307 and 308 do not change method
1402                 return this.get();
1403             }
1404             if ( restartedRequest ) {
1405                 debug(requests) trace("Rare event: clearing 'restartedRequest' on redirect");
1406                 restartedRequest = false;
1407             }
1408             goto connect;
1409         }
1410 
1411         _response._history = _history;
1412         return _response;
1413     }
1414     ///
1415     /// Send request with parameters.
1416     /// If used for POST or PUT requests then application/x-www-form-urlencoded used.
1417     /// Request parameters will be encoded into request string or placed in request body for POST/PUT
1418     /// requests.
1419     /// Parameters:
1420     ///     url = url
1421     ///     params = request parameters
1422     ///  Returns:
1423     ///     Response
1424     ///  Examples:
1425     ///  ---------------------------------------------------------------------------------
1426     ///     rs = Request().exec!"GET"("http://httpbin.org/get", ["c":"d", "a":"b"]);
1427     ///  ---------------------------------------------------------------------------------
1428     ///
1429     deprecated("Use Request() instead of HTTPRequest; will be removed 2019-07")
1430     HTTPResponse exec(string method="GET")(string url = null, QueryParam[] params = null)
1431     do {
1432         debug(requests) tracef("started url=%s, this._uri=%s", url, _uri);
1433 
1434         checkURL(url);
1435         //if ( _cm is null ) {
1436         //    _cm = new ConnManager();
1437         //}
1438 
1439         NetworkStream _stream;
1440         _method = method;
1441         _response = new HTTPResponse;
1442         _history.length = 0;
1443         _response.uri = _uri;
1444         _response.finalURI = _uri;
1445         bool restartedRequest = false; // True if this is restarted keepAlive request
1446 
1447     connect:
1448         if ( _method == "GET" && _uri in _permanent_redirects ) {
1449             debug(requests) trace("use parmanent redirects cache");
1450             _uri = uriFromLocation(_uri, _permanent_redirects[_uri]);
1451             _response._finalURI = _uri;
1452         }
1453         _contentReceived = 0;
1454         _response._startedAt = Clock.currTime;
1455 
1456         assert(_stream is null);
1457 
1458         _stream = _cm.get(_uri.scheme, _uri.host, _uri.port);
1459 
1460         if ( _stream is null ) {
1461             debug(requests) trace("create new connection");
1462             _stream = setupConnection();
1463         } else {
1464             debug(requests) trace("reuse old connection");
1465         }
1466 
1467         assert(_stream !is null);
1468 
1469         if ( !_stream.isConnected ) {
1470             debug(requests) trace("disconnected stream on enter");
1471             if ( !restartedRequest ) {
1472                 debug(requests) trace("disconnected stream on enter: retry");
1473                 assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
1474 
1475                 _cm.del(_uri.scheme, _uri.host, _uri.port);
1476                 _stream.close();
1477                 _stream = null;
1478 
1479                 restartedRequest = true;
1480                 goto connect;
1481             }
1482             debug(requests) trace("disconnected stream on enter: return response");
1483             //_stream = null;
1484             return _response;
1485         }
1486         _response._connectedAt = Clock.currTime;
1487 
1488         auto h = requestHeaders();
1489 
1490         Appender!string req;
1491 
1492         string encoded;
1493 
1494         switch (_method) {
1495             case "POST","PUT","PATCH":
1496                 encoded = params2query(params);
1497                 safeSetHeader(h, _userHeaders.ContentType, "Content-Type", "application/x-www-form-urlencoded");
1498                 if ( encoded.length > 0) {
1499                     safeSetHeader(h, _userHeaders.ContentLength, "Content-Length", to!string(encoded.length));
1500                 }
1501                 req.put(requestString());
1502                 break;
1503             default:
1504                 req.put(requestString(params));
1505         }
1506 
1507         h.byKeyValue.
1508             map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n").
1509             each!(h => req.put(h));
1510         req.put("\r\n");
1511         if ( encoded ) {
1512             req.put(encoded);
1513         }
1514 
1515         debug(requests) trace(req.data);
1516         if ( _verbosity >= 1 ) {
1517             req.data.splitLines.each!(a => writeln("> " ~ a));
1518         }
1519         //
1520         // Now send request and receive response
1521         //
1522         try {
1523             _stream.send(req.data());
1524             _response._requestSentAt = Clock.currTime;
1525             debug(requests) trace("starting receive response");
1526             receiveResponse(_stream);
1527             debug(requests) trace("done receive response");
1528             _response._finishedAt = Clock.currTime;
1529         }
1530         catch (NetworkException e) {
1531             // On SEND this can means:
1532             // we started to send request to the server, but it closed connection because of keepalive timeout.
1533             // We have to restart request if possible.
1534 
1535             // On RECEIVE - if we received something - then this exception is real and unexpected error.
1536             // If we didn't receive anything - we can restart request again as it can be
1537             debug(requests) tracef("Exception on receive response: %s", e.msg);
1538             if ( _response._responseHeaders.length != 0 )
1539             {
1540                 _stream.close();
1541                 throw new RequestException("Unexpected network error");
1542             }
1543         }
1544 
1545         if ( serverPrematurelyClosedConnection()
1546             && !restartedRequest
1547             && isIdempotent(_method)
1548             ) {
1549             ///
1550             /// We didn't receive any data (keepalive connectioin closed?)
1551             /// and we can restart this request.
1552             /// Go ahead.
1553             ///
1554             debug(requests) tracef("Server closed keepalive connection");
1555 
1556             assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
1557 
1558             _cm.del(_uri.scheme, _uri.host, _uri.port);
1559             _stream.close();
1560             _stream = null;
1561 
1562             restartedRequest = true;
1563             goto connect;
1564         }
1565 
1566         if ( _useStreaming ) {
1567             if ( _response._receiveAsRange.activated ) {
1568                 debug(requests) trace("streaming_in activated");
1569                 return _response;
1570             } else {
1571                 // this can happen if whole response body received together with headers
1572                 _response._receiveAsRange.data = _response.responseBody.data;
1573             }
1574         }
1575 
1576         close_connection_if_not_keepalive(_stream);
1577 
1578         if ( _verbosity >= 1 ) {
1579             writeln(">> Connect time: ", _response._connectedAt - _response._startedAt);
1580             writeln(">> Request send time: ", _response._requestSentAt - _response._connectedAt);
1581             writeln(">> Response recv time: ", _response._finishedAt - _response._requestSentAt);
1582         }
1583 
1584         if ( willFollowRedirect ) {
1585             debug(requests) trace("going to follow redirect");
1586             if ( _history.length >= _maxRedirects ) {
1587                 _stream = null;
1588                 throw new MaxRedirectsException("%d redirects reached maxRedirects %d.".format(_history.length, _maxRedirects));
1589             }
1590             // "location" in response already checked in canFollowRedirect
1591             immutable new_location = *("location" in _response.responseHeaders);
1592             immutable current_uri = _uri, next_uri = uriFromLocation(_uri, new_location);
1593 
1594             if ( _method == "GET" && _response.code == 301 ) {
1595                 _permanent_redirects[_uri] = new_location;
1596             }
1597 
1598             // save current response for history
1599             _history ~= _response;
1600 
1601             // prepare new response (for redirected request)
1602             _response = new HTTPResponse;
1603             _response.uri = current_uri;
1604             _response.finalURI = next_uri;
1605             _stream = null;
1606 
1607             // set new uri
1608             _uri = next_uri;
1609             debug(requests) tracef("Redirected to %s", next_uri);
1610             if ( _method != "GET" && _response.code != 307 && _response.code != 308 ) {
1611                 // 307 and 308 do not change method
1612                 return this.get();
1613             }
1614             if ( restartedRequest ) {
1615                 debug(requests) trace("Rare event: clearing 'restartedRequest' on redirect");
1616                 restartedRequest = false;
1617             }
1618             goto connect;
1619         }
1620 
1621         _response._history = _history;
1622         return _response;
1623     }
1624 
1625     /// WRAPPERS
1626     ///
1627     /// send file(s) using POST and multipart form.
1628     /// This wrapper will be deprecated, use post with MultipartForm - it is more general and clear.
1629     /// Parameters:
1630     ///     url = url
1631     ///     files = array of PostFile structures
1632     /// Returns:
1633     ///     Response
1634     /// Each PostFile structure contain path to file, and optional field name and content type.
1635     /// If no field name provided, then basename of the file will be used.
1636     /// application/octet-stream is default when no content type provided.
1637     /// Example:
1638     /// ---------------------------------------------------------------
1639     ///    PostFile[] files = [
1640     ///                   {fileName:"tests/abc.txt", fieldName:"abc", contentType:"application/octet-stream"},
1641     ///                   {fileName:"tests/test.txt"}
1642     ///               ];
1643     ///    rs = rq.exec!"POST"("http://httpbin.org/post", files);
1644     /// ---------------------------------------------------------------
1645     ///
1646     deprecated("Use Request() instead of HTTPRequest(); will be removed 2019-07")
1647     HTTPResponse exec(string method="POST")(string url, PostFile[] files) if (method=="POST") {
1648         MultipartForm multipart;
1649         File[]        toClose;
1650         foreach(ref f; files) {
1651             File file = File(f.fileName, "rb");
1652             toClose ~= file;
1653             string fileName = f.fileName ? f.fileName : f.fieldName;
1654             string contentType = f.contentType ? f.contentType : "application/octetstream";
1655             multipart.add(f.fieldName, new FormDataFile(file), ["filename":fileName, "Content-Type": contentType]);
1656         }
1657         auto res = exec!"POST"(url, multipart);
1658         toClose.each!"a.close";
1659         return res;
1660     }
1661     ///
1662     /// exec request with parameters when you can use dictionary (when you have no duplicates in parameter names)
1663     /// Consider switch to exec(url, QueryParams) as it more generic and clear.
1664     /// Parameters:
1665     ///     url = url
1666     ///     params = dictionary with field names as keys and field values as values.
1667     /// Returns:
1668     ///     Response
1669     deprecated("Use Request() instead of HTTPRequest(); will be removed 2019-07")
1670     HTTPResponse exec(string method="GET")(string url, string[string] params) {
1671         return exec!method(url, params.byKeyValue.map!(p => QueryParam(p.key, p.value)).array);
1672     }
1673     ///
1674     /// GET request. Simple wrapper over exec!"GET"
1675     /// Params:
1676     /// args = request parameters. see exec docs.
1677     ///
1678     deprecated("Use Request() instead of HTTPRequest; will be removed 2019-07")
1679     HTTPResponse get(A...)(A args) {
1680         return exec!"GET"(args);
1681     }
1682     ///
1683     /// POST request. Simple wrapper over exec!"POST"
1684     /// Params:
1685     /// uri = endpoint uri
1686     /// args = request parameters. see exec docs.
1687     ///
1688     deprecated("Use Request() instead of HTTPRequest; will be removed 2019-07")
1689     HTTPResponse post(A...)(string uri, A args) {
1690         return exec!"POST"(uri, args);
1691     }
1692 
1693     import requests.request;
1694 
1695     // we use this if we send from ubyte[][] and user provided Content-Length
1696     private void sendFlattenContent(NetworkStream _stream) {
1697         while ( !_postData.empty ) {
1698             auto chunk = _postData.front;
1699             _stream.send(chunk);
1700             _postData.popFront;
1701         }
1702         debug(requests) tracef("sent");
1703     }
1704     // we use this if we send from ubyte[][] as chunked content
1705     private void sendChunkedContent(NetworkStream _stream) {
1706         while ( !_postData.empty ) {
1707             auto chunk = _postData.front;
1708             auto chunkHeader = "%x\r\n".format(chunk.length);
1709             debug(requests) tracef("sending %s%s", chunkHeader, cast(string)chunk);
1710             _stream.send(chunkHeader);
1711             _stream.send(chunk);
1712             _stream.send("\r\n");
1713             debug(requests) tracef("chunk sent");
1714             _postData.popFront;
1715         }
1716         debug(requests) tracef("sent");
1717         _stream.send("0\r\n\r\n");
1718     }
1719 
1720     HTTPResponse exec_from_range(InputRangeAdapter postData)
1721     do {
1722 
1723         _postData = postData;
1724 
1725         debug(requests) tracef("exec from range");
1726 
1727         NetworkStream _stream;
1728         _response = new HTTPResponse;
1729         _history.length = 0;
1730         _response.uri = _uri;
1731         _response.finalURI = _uri;
1732         bool restartedRequest = false;
1733         bool send_flat;
1734 
1735     connect:
1736         _contentReceived = 0;
1737         _response._startedAt = Clock.currTime;
1738 
1739         assert(_stream is null);
1740 
1741         _stream = _cm.get(_uri.scheme, _uri.host, _uri.port);
1742 
1743         if ( _stream is null ) {
1744             debug(requests) trace("create new connection");
1745             _stream = setupConnection();
1746         } else {
1747             debug(requests) trace("reuse old connection");
1748         }
1749 
1750         assert(_stream !is null);
1751 
1752         if ( !_stream.isConnected ) {
1753             debug(requests) trace("disconnected stream on enter");
1754             if ( !restartedRequest ) {
1755                 debug(requests) trace("disconnected stream on enter: retry");
1756                 assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
1757 
1758                 _cm.del(_uri.scheme, _uri.host, _uri.port);
1759                 _stream.close();
1760                 _stream = null;
1761 
1762                 restartedRequest = true;
1763                 goto connect;
1764             }
1765             debug(requests) trace("disconnected stream on enter: return response");
1766             //_stream = null;
1767             return _response;
1768         }
1769         _response._connectedAt = Clock.currTime;
1770 
1771         Appender!string req;
1772         req.put(requestString());
1773 
1774         auto h = requestHeaders;
1775         if ( _contentType ) {
1776             safeSetHeader(h, _userHeaders.ContentType, "Content-Type", _contentType);
1777         }
1778 
1779         if ( _postData.length >= 0 )
1780         {
1781             // we know t
1782             safeSetHeader(h, _userHeaders.ContentLength, "Content-Length", to!string(_postData.length));
1783         }
1784 
1785         if ( _userHeaders.ContentLength || "Content-Length" in h )
1786         {
1787             debug(requests) tracef("User provided content-length for chunked content");
1788             send_flat = true;
1789         }
1790         else
1791         {
1792             h["Transfer-Encoding"] = "chunked";
1793             send_flat = false;
1794         }
1795         h.byKeyValue.
1796             map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n").
1797             each!(h => req.put(h));
1798         req.put("\r\n");
1799 
1800         debug(requests) tracef("send <%s>", req.data);
1801         if ( _verbosity >= 1 ) {
1802             req.data.splitLines.each!(a => writeln("> " ~ a));
1803         }
1804 
1805         try {
1806             // send headers
1807             _stream.send(req.data());
1808             // send body
1809             if ( send_flat ) {
1810                 sendFlattenContent(_stream);
1811             } else {
1812                 sendChunkedContent(_stream);
1813             }
1814             _response._requestSentAt = Clock.currTime;
1815             debug(requests) trace("starting receive response");
1816             receiveResponse(_stream);
1817             debug(requests) trace("finished receive response");
1818             _response._finishedAt = Clock.currTime;
1819         }
1820         catch (NetworkException e)
1821         {
1822             _stream.close();
1823             throw new RequestException("Network error during data exchange");
1824         }
1825         if ( serverPrematurelyClosedConnection()
1826         && !restartedRequest
1827         && isIdempotent(_method)
1828         ) {
1829             ///
1830             /// We didn't receive any data (keepalive connectioin closed?)
1831             /// and we can restart this request.
1832             /// Go ahead.
1833             ///
1834             debug(requests) tracef("Server closed keepalive connection");
1835 
1836             assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
1837 
1838             _cm.del(_uri.scheme, _uri.host, _uri.port);
1839             _stream.close();
1840             _stream = null;
1841 
1842             restartedRequest = true;
1843             goto connect;
1844         }
1845 
1846         if ( _useStreaming ) {
1847             if ( _response._receiveAsRange.activated ) {
1848                 debug(requests) trace("streaming_in activated");
1849                 return _response;
1850             } else {
1851                 // this can happen if whole response body received together with headers
1852                 _response._receiveAsRange.data = _response.responseBody.data;
1853             }
1854         }
1855 
1856         close_connection_if_not_keepalive(_stream);
1857 
1858         if ( _verbosity >= 1 ) {
1859             writeln(">> Connect time: ", _response._connectedAt - _response._startedAt);
1860             writeln(">> Request send time: ", _response._requestSentAt - _response._connectedAt);
1861             writeln(">> Response recv time: ", _response._finishedAt - _response._requestSentAt);
1862         }
1863 
1864 
1865         if ( willFollowRedirect ) {
1866             if ( _history.length >= _maxRedirects ) {
1867                 _stream = null;
1868                 throw new MaxRedirectsException("%d redirects reached maxRedirects %d.".format(_history.length, _maxRedirects));
1869             }
1870             // "location" in response already checked in canFollowRedirect
1871             immutable new_location = *("location" in _response.responseHeaders);
1872             immutable current_uri = _uri, next_uri = uriFromLocation(_uri, new_location);
1873 
1874             immutable get_or_head = _method == "GET" || _method == "HEAD";
1875             immutable code = _response.code;
1876 
1877             // save current response for history
1878             _history ~= _response;
1879 
1880             if ( code == 301 )
1881             {
1882                 // permanent redirect and change method
1883                 _permanent_redirects[_uri] = new_location;
1884                 if ( !get_or_head )
1885                 {
1886                     _method = "GET";
1887                 }
1888             }
1889             if ( (code == 302 || code == 303) && !get_or_head)
1890             {
1891                 // only change method
1892                 _method = "GET";
1893             }
1894             if ( code == 307 )
1895             {
1896                 // no change method, no permanent
1897             }
1898             if ( code == 308 )
1899             {
1900                 // permanent redirection and do not change method
1901                 _permanent_redirects[_uri] = new_location;
1902             }
1903 
1904             // prepare new response (for redirected request)
1905             _response = new HTTPResponse;
1906             _response.uri = current_uri;
1907             _response.finalURI = next_uri;
1908 
1909             _stream = null;
1910 
1911             // set new uri
1912             this._uri = next_uri;
1913             debug(requests) tracef("Redirected to %s", next_uri);
1914             if ( restartedRequest ) {
1915                 debug(requests) trace("Rare event: clearing 'restartedRequest' on redirect");
1916                 restartedRequest = false;
1917             }
1918             if ( _method == "GET")
1919             {
1920                 return exec_from_parameters();
1921             }
1922             goto connect;
1923         }
1924 
1925         _response._history = _history;
1926         return _response;
1927     }
1928 
1929     HTTPResponse exec_from_multipart_form(MultipartForm form) {
1930         import std.uuid;
1931         import std.file;
1932 
1933         _multipartForm = form;
1934 
1935         debug(requests) tracef("exec from multipart form");
1936 
1937         NetworkStream _stream;
1938         _response = new HTTPResponse;
1939         _response.uri = _uri;
1940         _response.finalURI = _uri;
1941         bool restartedRequest = false;
1942 
1943     connect:
1944         _contentReceived = 0;
1945         _response._startedAt = Clock.currTime;
1946 
1947         assert(_stream is null);
1948 
1949         _stream = _cm.get(_uri.scheme, _uri.host, _uri.port);
1950 
1951         if ( _stream is null ) {
1952             debug(requests) trace("create new connection");
1953             _stream = setupConnection();
1954         } else {
1955             debug(requests) trace("reuse old connection");
1956         }
1957 
1958         assert(_stream !is null);
1959 
1960         if ( !_stream.isConnected ) {
1961             debug(requests) trace("disconnected stream on enter");
1962             if ( !restartedRequest ) {
1963                 debug(requests) trace("disconnected stream on enter: retry");
1964                 assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
1965 
1966                 _cm.del(_uri.scheme, _uri.host, _uri.port);
1967                 _stream.close();
1968                 _stream = null;
1969 
1970                 restartedRequest = true;
1971                 goto connect;
1972             }
1973             debug(requests) trace("disconnected stream on enter: return response");
1974             //_stream = null;
1975             return _response;
1976         }
1977         _response._connectedAt = Clock.currTime;
1978 
1979         Appender!string req;
1980         req.put(requestString());
1981 
1982         string   boundary = randomUUID().toString;
1983         string[] partHeaders;
1984         size_t   contentLength;
1985 
1986         foreach(ref part; _multipartForm._sources) {
1987             string h = "--" ~ boundary ~ "\r\n";
1988             string disposition = `form-data; name="%s"`.format(part.name);
1989             string optionals = part.
1990             parameters.byKeyValue().
1991             filter!(p => p.key!="Content-Type").
1992             map!   (p => "%s=%s".format(p.key, p.value)).
1993             join("; ");
1994 
1995             h ~= `Content-Disposition: ` ~ [disposition, optionals].join("; ") ~ "\r\n";
1996 
1997             auto contentType = "Content-Type" in part.parameters;
1998             if ( contentType ) {
1999                 h ~= "Content-Type: " ~ *contentType ~ "\r\n";
2000             }
2001 
2002             h ~= "\r\n";
2003             partHeaders ~= h;
2004             contentLength += h.length + part.input.getSize() + "\r\n".length;
2005         }
2006         contentLength += "--".length + boundary.length + "--\r\n".length;
2007 
2008         auto h = requestHeaders();
2009         safeSetHeader(h, _userHeaders.ContentType, "Content-Type", "multipart/form-data; boundary=" ~ boundary);
2010         safeSetHeader(h, _userHeaders.ContentLength, "Content-Length", to!string(contentLength));
2011 
2012         h.byKeyValue.
2013         map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n").
2014         each!(h => req.put(h));
2015         req.put("\r\n");
2016 
2017         debug(requests) trace(req.data);
2018         if ( _verbosity >= 1 ) req.data.splitLines.each!(a => writeln("> " ~ a));
2019 
2020         try {
2021             _stream.send(req.data());
2022             foreach(ref source; _multipartForm._sources) {
2023                 debug(requests) tracef("sending part headers <%s>", partHeaders.front);
2024                 _stream.send(partHeaders.front);
2025                 partHeaders.popFront;
2026                 while (true) {
2027                     auto chunk = source.input.read();
2028                     if ( chunk.length <= 0 ) {
2029                         break;
2030                     }
2031                     _stream.send(chunk);
2032                 }
2033                 _stream.send("\r\n");
2034             }
2035             _stream.send("--" ~ boundary ~ "--\r\n");
2036             _response._requestSentAt = Clock.currTime;
2037             receiveResponse(_stream);
2038             _response._finishedAt = Clock.currTime;
2039         }
2040         catch (NetworkException e) {
2041             errorf("Error sending request: ", e.msg);
2042             _stream.close();
2043             return _response;
2044         }
2045 
2046         if ( serverPrematurelyClosedConnection()
2047         && !restartedRequest
2048         && isIdempotent(_method)
2049         ) {
2050             ///
2051             /// We didn't receive any data (keepalive connectioin closed?)
2052             /// and we can restart this request.
2053             /// Go ahead.
2054             ///
2055             debug(requests) tracef("Server closed keepalive connection");
2056 
2057             assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
2058 
2059             _cm.del(_uri.scheme, _uri.host, _uri.port);
2060             _stream.close();
2061             _stream = null;
2062 
2063             restartedRequest = true;
2064             goto connect;
2065         }
2066 
2067         if ( _useStreaming ) {
2068             if ( _response._receiveAsRange.activated ) {
2069                 debug(requests) trace("streaming_in activated");
2070                 return _response;
2071             } else {
2072                 // this can happen if whole response body received together with headers
2073                 _response._receiveAsRange.data = _response.responseBody.data;
2074             }
2075         }
2076 
2077         close_connection_if_not_keepalive(_stream);
2078 
2079         if ( _verbosity >= 1 ) {
2080             writeln(">> Connect time: ", _response._connectedAt - _response._startedAt);
2081             writeln(">> Request send time: ", _response._requestSentAt - _response._connectedAt);
2082             writeln(">> Response recv time: ", _response._finishedAt - _response._requestSentAt);
2083         }
2084 
2085         if ( willFollowRedirect ) {
2086             if ( _history.length >= _maxRedirects ) {
2087                 _stream = null;
2088                 throw new MaxRedirectsException("%d redirects reached maxRedirects %d.".format(_history.length, _maxRedirects));
2089             }
2090             // "location" in response already checked in canFollowRedirect
2091             immutable new_location = *("location" in _response.responseHeaders);
2092             immutable current_uri = _uri;
2093             immutable next_uri = uriFromLocation(_uri, new_location);
2094 
2095             immutable get_or_head = _method == "GET" || _method == "HEAD";
2096             immutable code = _response.code;
2097 
2098             // save current response for history
2099             _history ~= _response;
2100 
2101             if ( code == 301 )
2102             {
2103                 // permanent redirect and change method
2104                 _permanent_redirects[_uri] = new_location;
2105                 if ( !get_or_head )
2106                 {
2107                     _method = "GET";
2108                 }
2109             }
2110             if ( (code == 302 || code == 303) && !get_or_head)
2111             {
2112                 // only change method
2113                 _method = "GET";
2114             }
2115             if ( code == 307 )
2116             {
2117                 // no change method, no permanent
2118             }
2119             if ( code == 308 )
2120             {
2121                 // permanent redirection and do not change method
2122                 _permanent_redirects[_uri] = new_location;
2123             }
2124 
2125             // prepare new response (for redirected request)
2126             _response = new HTTPResponse;
2127             _response.uri = current_uri;
2128             _response.finalURI = next_uri;
2129             _stream = null;
2130 
2131             // set new uri
2132             this._uri = next_uri;
2133             debug(requests) tracef("Redirected to %s", next_uri);
2134             if ( restartedRequest ) {
2135                 debug(requests) trace("Rare event: clearing 'restartedRequest' on redirect");
2136                 restartedRequest = false;
2137             }
2138             if ( _method == "GET")
2139             {
2140                 return exec_from_parameters();
2141             }
2142             goto connect;
2143         }
2144 
2145         _response._history = _history;
2146         return _response;
2147     }
2148 
2149     HTTPResponse exec_from_parameters() {
2150 
2151         debug(requests) tracef("exec from parameters request");
2152 
2153         assert(_uri != URI.init);
2154         NetworkStream _stream;
2155         _response = new HTTPResponse;
2156         _history.length = 0;
2157         _response.uri = _uri;
2158         _response.finalURI = _uri;
2159         bool restartedRequest = false; // True if this is restarted keepAlive request
2160 
2161     connect:
2162         if ( _method == "GET" && _uri in _permanent_redirects ) {
2163             debug(requests) trace("use parmanent redirects cache");
2164             _uri = uriFromLocation(_uri, _permanent_redirects[_uri]);
2165             _response._finalURI = _uri;
2166         }
2167         _contentReceived = 0;
2168         _response._startedAt = Clock.currTime;
2169 
2170         assert(_stream is null);
2171 
2172         _stream = _cm.get(_uri.scheme, _uri.host, _uri.port);
2173 
2174         if ( _stream is null ) {
2175             debug(requests) trace("create new connection");
2176             _stream = setupConnection();
2177         } else {
2178             debug(requests) trace("reuse old connection");
2179         }
2180 
2181         assert(_stream !is null);
2182 
2183         if ( !_stream.isConnected ) {
2184             debug(requests) trace("disconnected stream on enter");
2185             if ( !restartedRequest ) {
2186                 debug(requests) trace("disconnected stream on enter: retry");
2187                 assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
2188 
2189                 _cm.del(_uri.scheme, _uri.host, _uri.port);
2190                 _stream.close();
2191                 _stream = null;
2192 
2193                 restartedRequest = true;
2194                 goto connect;
2195             }
2196             debug(requests) trace("disconnected stream on enter: return response");
2197             //_stream = null;
2198             return _response;
2199         }
2200         _response._connectedAt = Clock.currTime;
2201 
2202         auto h = requestHeaders();
2203 
2204         Appender!string req;
2205 
2206         string encoded;
2207 
2208         switch (_method) {
2209             case "POST","PUT","PATCH":
2210                 encoded = params2query(_params);
2211                 safeSetHeader(h, _userHeaders.ContentType, "Content-Type", "application/x-www-form-urlencoded");
2212                 safeSetHeader(h, _userHeaders.ContentLength, "Content-Length", to!string(encoded.length));
2213                 req.put(requestString());
2214                 break;
2215             default:
2216                 req.put(requestString(_params));
2217         }
2218 
2219         h.byKeyValue.
2220         map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n").
2221         each!(h => req.put(h));
2222         req.put("\r\n");
2223         if ( encoded ) {
2224             req.put(encoded);
2225         }
2226 
2227         debug(requests) trace(req.data);
2228         if ( _verbosity >= 1 ) {
2229             req.data.splitLines.each!(a => writeln("> " ~ a));
2230         }
2231         //
2232         // Now send request and receive response
2233         //
2234         try {
2235             _stream.send(req.data());
2236             _response._requestSentAt = Clock.currTime;
2237             debug(requests) trace("starting receive response");
2238             receiveResponse(_stream);
2239             debug(requests) tracef("done receive response");
2240             _response._finishedAt = Clock.currTime;
2241         }
2242         catch (NetworkException e) {
2243             // On SEND this can means:
2244             // we started to send request to the server, but it closed connection because of keepalive timeout.
2245             // We have to restart request if possible.
2246 
2247             // On RECEIVE - if we received something - then this exception is real and unexpected error.
2248             // If we didn't receive anything - we can restart request again as it can be
2249             debug(requests) tracef("Exception on receive response: %s", e.msg);
2250             if ( _response._responseHeaders.length != 0 )
2251             {
2252                 _stream.close();
2253                 throw new RequestException("Unexpected network error");
2254             }
2255         }
2256 
2257         if ( serverPrematurelyClosedConnection()
2258             && !restartedRequest
2259             && isIdempotent(_method)
2260             ) {
2261             ///
2262             /// We didn't receive any data (keepalive connectioin closed?)
2263             /// and we can restart this request.
2264             /// Go ahead.
2265             ///
2266             debug(requests) tracef("Server closed keepalive connection");
2267 
2268             assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
2269 
2270             _cm.del(_uri.scheme, _uri.host, _uri.port);
2271             _stream.close();
2272             _stream = null;
2273 
2274             restartedRequest = true;
2275             goto connect;
2276         }
2277 
2278         if ( _useStreaming ) {
2279             if ( _response._receiveAsRange.activated ) {
2280                 debug(requests) trace("streaming_in activated");
2281                 return _response;
2282             } else {
2283                 // this can happen if whole response body received together with headers
2284                 _response._receiveAsRange.data = _response.responseBody.data;
2285             }
2286         }
2287 
2288         close_connection_if_not_keepalive(_stream);
2289 
2290         if ( _verbosity >= 1 ) {
2291             writeln(">> Connect time: ", _response._connectedAt - _response._startedAt);
2292             writeln(">> Request send time: ", _response._requestSentAt - _response._connectedAt);
2293             writeln(">> Response recv time: ", _response._finishedAt - _response._requestSentAt);
2294         }
2295 
2296         if ( willFollowRedirect ) {
2297             debug(requests) trace("going to follow redirect");
2298             if ( _history.length >= _maxRedirects ) {
2299                 _stream = null;
2300                 throw new MaxRedirectsException("%d redirects reached maxRedirects %d.".format(_history.length, _maxRedirects));
2301             }
2302             // "location" in response already checked in canFollowRedirect
2303             immutable new_location = *("location" in _response.responseHeaders);
2304             immutable current_uri = _uri;
2305             immutable next_uri = uriFromLocation(_uri, new_location);
2306 
2307             immutable get_or_head = _method == "GET" || _method == "HEAD";
2308             immutable code = _response.code;
2309 
2310             // save current response for history
2311             _history ~= _response;
2312 
2313             if ( code == 301 )
2314             {
2315                 // permanent redirect and change method
2316                 _permanent_redirects[_uri] = new_location;
2317                 if ( !get_or_head )
2318                 {
2319                     _method = "GET";
2320                 }
2321             }
2322             if ( (code == 302 || code == 303) && !get_or_head)
2323             {
2324                 // only change method
2325                 _method = "GET";
2326             }
2327             if ( code == 307 )
2328             {
2329                 // no change method, no permanent
2330             }
2331             if ( code == 308 )
2332             {
2333                 // permanent redirection and do not change method
2334                 _permanent_redirects[_uri] = new_location;
2335             }
2336 
2337             // prepare new response (for redirected request)
2338             _response = new HTTPResponse;
2339             _response.uri = current_uri;
2340             _response.finalURI = next_uri;
2341             _stream = null;
2342 
2343             // set new uri
2344             _uri = next_uri;
2345             debug(requests) tracef("Redirected to %s", next_uri);
2346             //if ( _method != "GET" && _response.code != 307 && _response.code != 308 ) {
2347             //    // 307 and 308 do not change method
2348             //    return exec_from_parameters(r);
2349             //}
2350             if ( restartedRequest ) {
2351                 debug(requests) trace("Rare event: clearing 'restartedRequest' on redirect");
2352                 restartedRequest = false;
2353             }
2354             goto connect;
2355         }
2356 
2357         _response._history = _history;
2358         return _response;
2359     }
2360     HTTPResponse execute(Request r)
2361     {
2362         _method = r.method;
2363         _uri = r.uri;
2364         _useStreaming = r.useStreaming;
2365         _permanent_redirects = r.permanent_redirects;
2366         _maxRedirects = r.maxRedirects;
2367         _authenticator = r.authenticator;
2368         _maxHeadersLength = r.maxHeadersLength;
2369         _maxContentLength = r.maxContentLength;
2370         _verbosity = r.verbosity;
2371         _keepAlive = r.keepAlive;
2372         _bufferSize = r.bufferSize;
2373         _proxy = r.proxy;
2374         _timeout = r.timeout;
2375         _contentType = r.contentType;
2376         _socketFactory = r.socketFactory;
2377         _sslOptions = r.sslOptions;
2378         _bind = r.bind;
2379         _headers = r.headers;
2380         _userHeaders = r.userHeaders;
2381 
2382         _params = r.params;
2383 
2384         // this assignments increments refCounts, so we can't use const Request
2385         // but Request is anyway struct and called by-value
2386         _cm = r.cm;
2387         _cookie = r.cookie;
2388 
2389         debug(requests) trace("serving %s".format(r));
2390         if ( !r.postData.empty)
2391         {
2392             return exec_from_range(r.postData);
2393         }
2394         if ( r.hasMultipartForm )
2395         {
2396             return exec_from_multipart_form(r.multipartForm);
2397         }
2398         auto rs = exec_from_parameters();
2399         return rs;
2400     }
2401 }
2402 
2403 version(vibeD) {
2404     import std.json;
2405     package string httpTestServer() {
2406         return "http://httpbin.org/";
2407     }
2408     package string fromJsonArrayToStr(JSONValue v) {
2409         return v.str;
2410     }
2411 }
2412 else {
2413     import std.json;
2414     package string httpTestServer() {
2415         return "http://127.0.0.1:8081/";
2416     }
2417     package string fromJsonArrayToStr(JSONValue v) {
2418         return cast(string)(v.array.map!"cast(ubyte)a.integer".array);
2419     }
2420 }