1 module requests.http;
2 
3 private:
4 import std.algorithm;
5 import std.array;
6 import std.ascii;
7 import std.conv;
8 import std.datetime;
9 import std.exception;
10 import std.format;
11 import std.stdio;
12 import std.range;
13 import std.string;
14 import std.traits;
15 import std.typecons;
16 import std.experimental.logger;
17 import core.thread;
18 
19 import requests.streams;
20 import requests.uri;
21 import requests.utils;
22 import requests.base;
23 import requests.connmanager;
24 import requests.rangeadapter;
25 
26 static immutable ushort[] redirectCodes = [301, 302, 303, 307, 308];
27 
28 enum   HTTP11 = 101;
29 enum   HTTP10 = 100;
30 
31 static immutable string[string] proxies;
32 shared static this() {
33     import std.process;
34     import std.string;
35     foreach(p; ["http", "https", "all"])
36     {
37         auto v = environment.get(p ~ "_proxy", environment.get(p.toUpper() ~ "_PROXY"));
38         if ( v !is null && v.length > 0 )
39         {
40             debug(requests) tracef("will use %s for %s as proxy", v, p);
41             proxies[p] = v;
42         }
43     }
44 }
45 
46 public class MaxRedirectsException: Exception {
47     this(string message, string file = __FILE__, size_t line = __LINE__, Throwable next = null) @safe pure nothrow {
48         super(message, file, line, next);
49     }
50 }
51 
52 ///
53 ///
54 ///
55 //public auto queryParams(T...)(T params) pure nothrow @safe
56 //{
57 //    static assert (T.length % 2 == 0, "wrong args count");
58 //
59 //    QueryParam[] output;
60 //    output.reserve = T.length / 2;
61 //
62 //    void queryParamsHelper(T...)(T params, ref QueryParam[] output)
63 //    {
64 //        static if (T.length > 0)
65 //        {
66 //            output ~= QueryParam(params[0].to!string, params[1].to!string);
67 //            queryParamsHelper(params[2..$], output);
68 //        }
69 //    }
70 //
71 //    queryParamsHelper(params, output);
72 //    return output;
73 //}
74 
75 ///
76 /// Response - result of request execution.
77 ///
78 /// Response.code - response HTTP code.
79 /// Response.status_line - received HTTP status line.
80 /// Response.responseHeaders - received headers.
81 /// Response.responseBody - container for received body
82 /// Response.history - for redirected responses contain all history
83 ///
84 public class HTTPResponse : Response {
85     private {
86         string              _status_line;
87 
88         HTTPResponse[]      _history; // redirects history
89 
90         mixin(Setter!string("status_line"));
91 
92         int                 _version;
93     }
94 
95     ~this() {
96         _responseHeaders = null;
97         _history.length = 0;
98     }
99 
100     mixin(Getter("status_line"));
101 
102     @property final string[string] responseHeaders() @safe @nogc nothrow {
103         return _responseHeaders;
104     }
105     @property final HTTPResponse[] history() @safe @nogc nothrow {
106         return _history;
107     }
108 
109     private int parse_version(in string v) pure const nothrow @safe {
110         // try to parse HTTP/1.x to version
111         try if ( v.length > 5 ) {
112             return (v[5..$].split(".").map!"to!int(a)".array[0..2].reduce!((a,b) => a*100 + b));
113         } catch (Exception e) {
114         }
115         return 0;
116     }
117     unittest {
118         auto r = new HTTPResponse();
119         assert(r.parse_version("HTTP/1.1") == 101);
120         assert(r.parse_version("HTTP/1.0") == 100);
121         assert(r.parse_version("HTTP/0.9") == 9);
122         assert(r.parse_version("HTTP/xxx") == 0);
123     }
124 }
125 
126 ///
127 /// Request.
128 /// Configurable parameters:
129 /// $(B method) - string, method to use (GET, POST, ...)
130 /// $(B headers) - string[string], add any additional headers you'd like to send.
131 /// $(B authenticator) - class Auth, class to send auth headers.
132 /// $(B keepAlive) - bool, set true for keepAlive requests. default true.
133 /// $(B maxRedirects) - uint, maximum number of redirects. default 10.
134 /// $(B maxHeadersLength) - size_t, maximum length of server response headers. default = 32KB.
135 /// $(B maxContentLength) - size_t, maximun content length. delault - 0 = unlimited.
136 /// $(B bufferSize) - size_t, send and receive buffer size. default = 16KB.
137 /// $(B verbosity) - uint, level of verbosity(0 - nothing, 1 - headers, 2 - headers and body progress). default = 0.
138 /// $(B proxy) - string, set proxy url if needed. default - null.
139 /// $(B cookie) - Tuple Cookie, Read/Write cookie You can get cookie setted by server, or set cookies before doing request.
140 /// $(B timeout) - Duration, Set timeout value for connect/receive/send.
141 ///
142 public struct HTTPRequest {
143     private {
144         string         _method = "GET";
145         URI            _uri;
146         string[string] _headers;
147         string[]       _filteredHeaders;
148         Auth           _authenticator;
149         bool           _keepAlive = true;
150         uint           _maxRedirects = 10;
151         size_t         _maxHeadersLength = 32 * 1024;   // 32 KB
152         size_t         _maxContentLength;               // 0 - Unlimited
153         string         _proxy;
154         uint           _verbosity = 0;                  // 0 - no output, 1 - headers, 2 - headers+body info
155         Duration       _timeout = 30.seconds;
156         size_t         _bufferSize = 16*1024; // 16k
157         bool           _useStreaming;                   // return iterator instead of completed request
158 
159         HTTPResponse[] _history;                        // redirects history
160         DataPipe!ubyte _bodyDecoder;
161         DecodeChunked  _unChunker;
162         long           _contentLength;
163         long           _contentReceived;
164         SSLOptions     _sslOptions;
165         string         _bind;
166         _UH            _userHeaders;
167 
168         RefCounted!ConnManager      _cm;
169         RefCounted!Cookies          _cookie;
170         string[URI]                 _permanent_redirects;            // cache 301 redirects for GET requests
171         MultipartForm               _multipartForm;
172 
173         NetStreamFactory  _socketFactory;
174 
175         QueryParam[]        _params;
176         string              _contentType;
177         InputRangeAdapter   _postData;
178     }
179     package HTTPResponse   _response;
180 
181     mixin(Getter_Setter!string     ("method"));
182     mixin(Getter_Setter!bool       ("keepAlive"));
183     mixin(Getter_Setter!size_t     ("maxContentLength"));
184     mixin(Getter_Setter!size_t     ("maxHeadersLength"));
185     mixin(Getter_Setter!size_t     ("bufferSize"));
186     mixin(Getter_Setter!uint       ("maxRedirects"));
187     mixin(Getter_Setter!uint       ("verbosity"));
188     mixin(Getter                   ("proxy"));
189     mixin(Getter_Setter!Duration   ("timeout"));
190     mixin(Setter!Auth              ("authenticator"));
191     mixin(Getter_Setter!bool       ("useStreaming"));
192     mixin(Getter                   ("contentLength"));
193     mixin(Getter                   ("contentReceived"));
194     mixin(Getter_Setter!SSLOptions ("sslOptions"));
195     mixin(Getter_Setter!string     ("bind"));
196     mixin(Setter!NetStreamFactory  ("socketFactory"));
197 
198     @property void sslSetVerifyPeer(bool v) pure @safe nothrow @nogc {
199         _sslOptions.setVerifyPeer(v);
200     }
201     @property void sslSetKeyFile(string p, SSLOptions.filetype t = SSLOptions.filetype.pem) pure @safe nothrow @nogc {
202         _sslOptions.setKeyFile(p, t);
203     }
204     @property void sslSetCertFile(string p, SSLOptions.filetype t = SSLOptions.filetype.pem) pure @safe nothrow @nogc {
205         _sslOptions.setCertFile(p, t);
206     }
207     @property void sslSetCaCert(string path) pure @safe nothrow @nogc {
208         _sslOptions.setCaCert(path);
209     }
210     //@property final void cookie(Cookie[] s) pure @safe @nogc nothrow {
211     //    _cookie = s;
212     //}
213     @property final void proxy(string v) {
214         if ( v != _proxy ) {
215             _cm.clear();
216         }
217         _proxy = v;
218     }
219     //@property final Cookie[] cookie() pure @safe @nogc nothrow {
220     //    return _cookie;
221     //}
222 
223     this(string uri) {
224         _uri = URI(uri);
225         _cm = ConnManager(10);
226     }
227     ~this() {
228         _headers = null;
229         _authenticator = null;
230         _history = null;
231         _bodyDecoder = null;
232         _unChunker = null;
233         //if ( _cm ) {
234         //    _cm.clear();
235         //}
236     }
237     string toString() const {
238         return "HTTPRequest(%s, %s)".format(_method, _uri.uri());
239     }
240     string format(string fmt) const {
241         import std.array;
242         import std.stdio;
243         auto a = appender!string();
244         auto f = FormatSpec!char(fmt);
245         while (f.writeUpToNextSpec(a)) {
246             switch(f.spec) {
247                 case 'h':
248                     // Remote hostname.
249                     a.put(_uri.host);
250                     break;
251                 case 'm':
252                     // method.
253                     a.put(_method);
254                     break;
255                 case 'p':
256                     // Remote port.
257                     a.put("%d".format(_uri.port));
258                     break;
259                 case 'P':
260                     // Path
261                     a.put(_uri.path);
262                     break;
263                 case 'q':
264                     // query parameters supplied with url.
265                     a.put(_uri.query);
266                     break;
267                 case 'U':
268                     a.put(_uri.uri());
269                     break;
270                 default:
271                     throw new FormatException("Unknown Request format spec " ~ f.spec);
272             }
273         }
274         return a.data();
275     }
276     string select_proxy(string scheme) {
277         if ( _proxy is null && proxies.length == 0 ) {
278             debug(requests) tracef("proxy=null");
279             return null;
280         }
281         if ( _proxy ) {
282             debug(requests) tracef("proxy=%s", _proxy);
283             return _proxy;
284         }
285         auto p = scheme in proxies;
286         if ( p !is null && *p != "") {
287             debug(requests) tracef("proxy=%s", *p);
288             return *p;
289         }
290         p = "all" in proxies;
291         if ( p !is null && *p != "") {
292             debug(requests) tracef("proxy=%s", *p);
293             return *p;
294         }
295         debug(requests) tracef("proxy=null");
296         return null;
297     }
298     void clearHeaders() {
299         _headers = null;
300     }
301     @property void uri(in URI newURI) {
302         //handleURLChange(_uri, newURI);
303         _uri = newURI;
304     }
305     /// Add headers to request
306     /// Params:
307     /// headers = headers to send.
308     void addHeaders(in string[string] headers) {
309         foreach(pair; headers.byKeyValue) {
310             string _h = pair.key;
311             switch(toLower(_h)) {
312             case "host":
313                 _userHeaders.Host = true;
314                 break;
315             case "user-agent":
316                 _userHeaders.UserAgent = true;
317                 break;
318             case "content-length":
319                 _userHeaders.ContentLength = true;
320                 break;
321             case "content-type":
322                 _userHeaders.ContentType = true;
323                 break;
324             case "connection":
325                 _userHeaders.Connection = true;
326                 break;
327             case "cookie":
328                 _userHeaders.Cookie = true;
329                 break;
330             default:
331                 break;
332             }
333             _headers[pair.key] = pair.value;
334         }
335     }
336     private void safeSetHeader(ref string[string] headers, bool userAdded, string h, string v) pure @safe {
337         if ( !userAdded ) {
338             headers[h] = v;
339         }
340     }
341     /// Remove headers from request
342     /// Params:
343     /// headers = headers to remove.
344     void removeHeaders(in string[] headers) pure {
345         _filteredHeaders ~= headers;
346     }
347     ///
348     /// compose headers to send
349     ///
350     private string[string] requestHeaders() {
351 
352         string[string] generatedHeaders;
353 
354         if ( _authenticator ) {
355             _authenticator.
356                 authHeaders(_uri.host).
357                 byKeyValue.
358                 each!(pair => generatedHeaders[pair.key] = pair.value);
359         }
360 
361         _headers.byKey.each!(h => generatedHeaders[h] = _headers[h]);
362 
363         safeSetHeader(generatedHeaders, _userHeaders.AcceptEncoding, "Accept-Encoding", "gzip,deflate");
364         safeSetHeader(generatedHeaders, _userHeaders.UserAgent, "User-Agent", "dlang-requests");
365         safeSetHeader(generatedHeaders, _userHeaders.Connection, "Connection", _keepAlive?"Keep-Alive":"Close");
366 
367         if ( !_userHeaders.Host )
368         {
369             generatedHeaders["Host"] = _uri.host;
370             if ( _uri.scheme !in standard_ports || _uri.port != standard_ports[_uri.scheme] ) {
371                 generatedHeaders["Host"] ~= ":%d".format(_uri.port);
372             }
373         }
374 
375         if ( _cookie._array.length && !_userHeaders.Cookie ) {
376             auto cs = _cookie._array.
377                 filter!(c => _uri.path.pathMatches(c.path) && _uri.host.domainMatches(c.domain)).
378                 map!(c => "%s=%s".format(c.attr, c.value)).
379                 joiner(";");
380             if ( ! cs.empty )
381             {
382                 generatedHeaders["Cookie"] = to!string(cs);
383             }
384         }
385 
386         _filteredHeaders.each!(h => generatedHeaders.remove(h));
387 
388         return generatedHeaders;
389     }
390     ///
391     /// Build request string.
392     /// Handle proxy and query parameters.
393     ///
394     private @property string requestString(QueryParam[] params = null) {
395         auto query = _uri.query.dup;
396         if ( params ) {
397             query ~= params2query(params);
398             if ( query[0] != '?' ) {
399                 query = "?" ~ query;
400             }
401         }
402         string actual_proxy = select_proxy(_uri.scheme);
403         if ( actual_proxy && _uri.scheme != "https" ) {
404             return "%s %s%s HTTP/1.1\r\n".format(_method, _uri.uri(No.params), query);
405         }
406         return "%s %s%s HTTP/1.1\r\n".format(_method, _uri.path, query);
407     }
408 
409     unittest {
410         HTTPRequest request;
411         request._uri = URI("https://www.blub.de/");
412         import std.stdio;
413         auto expected = "GET /?a=b&c=d HTTP/1.1\r\n";
414         auto actual = request.requestString([QueryParam("a", "b"), QueryParam("c", "d")]);
415         assert(expected == actual, "Expected\n  '%s' but got\n  '%s'".format(expected, actual));
416     }
417 
418     ///
419     /// encode parameters and build query part of the url
420     ///
421     private static string params2query(in QueryParam[] params) pure @safe {
422         return params.
423                 map!(a => "%s=%s".format(a.key.urlEncoded, a.value.urlEncoded)).
424                 join("&");
425     }
426     //
427     package unittest {
428         assert(params2query(queryParams("a","b", "c", " d "))=="a=b&c=%20d%20");
429     }
430     ///
431     /// Analyze received headers, take appropriate actions:
432     /// check content length, attach unchunk and uncompress
433     ///
434     private void analyzeHeaders(in string[string] headers) {
435 
436         _contentLength = -1;
437         _unChunker = null;
438         auto contentLength = "content-length" in headers;
439         if ( contentLength ) {
440             try {
441                 string l = *contentLength;
442                 _contentLength = parse!long(l);
443                 // TODO: maybe add a strict mode that checks if l was parsed completely
444                 if ( _maxContentLength && _contentLength > _maxContentLength) {
445                     throw new RequestException("ContentLength > maxContentLength (%d>%d)".
446                                 format(_contentLength, _maxContentLength));
447                 }
448             } catch (ConvException e) {
449                 throw new RequestException("Can't convert Content-Length from %s".format(*contentLength));
450             }
451         }
452         auto transferEncoding = "transfer-encoding" in headers;
453         if ( transferEncoding ) {
454             debug(requests) tracef("transferEncoding: %s", *transferEncoding);
455             if ( (*transferEncoding).toLower == "chunked") {
456                 _unChunker = new DecodeChunked();
457                 _bodyDecoder.insert(_unChunker);
458             }
459         }
460         auto contentEncoding = "content-encoding" in headers;
461         if ( contentEncoding ) switch (*contentEncoding) {
462             default:
463                 throw new RequestException("Unknown content-encoding " ~ *contentEncoding);
464             case "gzip":
465             case "deflate":
466                 _bodyDecoder.insert(new Decompressor!ubyte);
467         }
468 
469     }
470     ///
471     /// Called when we know that all headers already received in buffer.
472     /// This routine does not interpret headers content (see analyzeHeaders).
473     /// 1. Split headers on lines
474     /// 2. store status line, store response code
475     /// 3. unfold headers if needed
476     /// 4. store headers
477     ///
478     private void parseResponseHeaders(in ubyte[] input, string lineSep) {
479         string lastHeader;
480         auto buffer = cast(string)input;
481 
482         foreach(line; buffer.split(lineSep).map!(l => l.stripRight)) {
483             if ( ! _response.status_line.length ) {
484                 debug (requests) tracef("statusLine: %s", line);
485                 _response.status_line = line;
486                 if ( _verbosity >= 1 ) {
487                     writefln("< %s", line);
488                 }
489                 auto parsed = line.split(" ");
490                 if ( parsed.length >= 2 ) {
491                     _response.code = parsed[1].to!ushort;
492                     _response._version = _response.parse_version(parsed[0]);
493                 }
494                 continue;
495             }
496             if ( line[0] == ' ' || line[0] == '\t' ) {
497                 // unfolding https://tools.ietf.org/html/rfc822#section-3.1
498                 if ( auto stored = lastHeader in _response._responseHeaders) {
499                     *stored ~= line;
500                 }
501                 continue;
502             }
503             auto parsed = line.findSplit(":");
504             auto header = parsed[0].toLower;
505             auto value = parsed[2].strip;
506 
507             if ( _verbosity >= 1 ) {
508                 writefln("< %s: %s", header, value);
509             }
510 
511             lastHeader = header;
512             debug (requests) tracef("Header %s = %s", header, value);
513 
514             if ( header != "set-cookie" ) {
515                 auto stored = _response.responseHeaders.get(header, null);
516                 if ( stored ) {
517                     value = stored ~ "," ~ value;
518                 }
519                 _response._responseHeaders[header] = value;
520                 continue;
521             }
522             _cookie._array ~= processCookie(value);
523         }
524     }
525 
526     ///
527     /// Process Set-Cookie header from server response
528     ///
529     private Cookie[] processCookie(string value ) pure {
530         // cookie processing
531         //
532         // as we can't join several set-cookie lines in single line
533         // < Set-Cookie: idx=7f2800f63c112a65ef5082957bcca24b; expires=Mon, 29-May-2017 00:31:25 GMT; path=/; domain=example.com
534         // < Set-Cookie: idx=7f2800f63c112a65ef5082957bcca24b; expires=Mon, 29-May-2017 00:31:25 GMT; path=/; domain=example.com, cs=ip764-RgKqc-HvSkxRxdQQAKW8LA; path=/; domain=.example.com; HttpOnly
535         //
536         Cookie[] res;
537         string[string] kv;
538         auto fields = value.split(";").map!strip;
539         while(!fields.empty) {
540             auto s = fields.front.findSplit("=");
541             fields.popFront;
542             if ( s[1] != "=" ) {
543                 continue;
544             }
545             auto k = s[0];
546             auto v = s[2];
547             switch(k.toLower()) {
548                 case "domain":
549                     k = "domain";
550                     break;
551                 case "path":
552                     k = "path";
553                     break;
554                 case "expires":
555                     continue;
556                 case "max-age":
557                     continue;
558                 default:
559                     break;
560             }
561             kv[k] = v;
562         }
563         if ( "domain" !in kv ) {
564             kv["domain"] = _uri.host;
565         }
566         if ( "path" !in kv ) {
567             kv["path"] = _uri.path;
568         }
569         auto domain = kv["domain"]; kv.remove("domain");
570         auto path   = kv["path"];   kv.remove("path");
571         foreach(pair; kv.byKeyValue) {
572             auto _attr = pair.key;
573             auto _value = pair.value;
574             auto cookie = Cookie(path, domain, _attr, _value);
575             res ~= cookie;
576         }
577         return res;
578     }
579 
580     private bool willFollowRedirect() {
581         if ( !canFind(redirectCodes, _response.code) ) {
582             return false;
583         }
584         if ( !_maxRedirects ) {
585             return false;
586         }
587         if ( "location" !in _response.responseHeaders ) {
588             return false;
589         }
590         return true;
591     }
592     private URI uriFromLocation(const ref URI uri, in string location) {
593         URI newURI = uri;
594         try {
595             newURI = URI(location);
596         } catch (UriException e) {
597             debug(requests) trace("Can't parse Location:, try relative uri");
598             newURI.path = location;
599             newURI.uri = newURI.recalc_uri;
600         }
601         return newURI;
602     }
603     ///
604     /// if we have new uri, then we need to check if we have to reopen existent connection
605     ///
606     private void checkURL(string url, string file=__FILE__, size_t line=__LINE__) {
607         if (url is null && _uri.uri == "" ) {
608             throw new RequestException("No url configured", file, line);
609         }
610 
611         if ( url !is null ) {
612             URI newURI = URI(url);
613             //handleURLChange(_uri, newURI);
614             _uri = newURI;
615         }
616     }
617     ///
618     /// Setup connection. Handle proxy and https case
619     ///
620     /// Place new connection in ConnManager cache
621     ///
622     private NetworkStream setupConnection()
623     do {
624 
625         debug(requests) tracef("Set up new connection");
626         NetworkStream stream;
627 
628         // on exit
629         // place created connection to conn. manager
630         // close connection purged from manager (if any)
631         //
632         scope(exit) {
633             if ( stream )
634             {
635                 if ( auto purged_connection = _cm.put(_uri.scheme, _uri.host, _uri.port, stream) )
636                 {
637                     debug(requests) tracef("closing purged connection %s", purged_connection);
638                     purged_connection.close();
639                 }
640             }
641         }
642 
643         if ( _socketFactory )
644         {
645             debug(requests) tracef("use socketFactory");
646             stream = _socketFactory(_uri.scheme, _uri.host, _uri.port);
647         }
648 
649         if ( stream ) // socket factory created connection
650         {
651             return stream;
652         }
653 
654         URI   uri; // this URI will be used temporarry if we need proxy
655         string actual_proxy = select_proxy(_uri.scheme);
656         final switch (_uri.scheme) {
657             case"http":
658                 if ( actual_proxy ) {
659                     uri.uri_parse(actual_proxy);
660                     uri.idn_encode();
661                 } else {
662                     // use original uri
663                     uri = _uri;
664                 }
665                 stream = new TCPStream();
666                 stream.bind(_bind);
667                 stream.connect(uri.host, uri.port, _timeout);
668                 break ;
669             case"https":
670                 if ( actual_proxy ) {
671                     uri.uri_parse(actual_proxy);
672                     uri.idn_encode();
673                     stream = new TCPStream();
674                     stream.bind(_bind);
675                     stream.connect(uri.host, uri.port, _timeout);
676                     if ( verbosity>=1 ) {
677                         writeln("> CONNECT %s:%d HTTP/1.1".format(_uri.host, _uri.port));
678                     }
679                     stream.send("CONNECT %s:%d HTTP/1.1\r\n".format(_uri.host, _uri.port));
680                     if (uri.username)
681                     {
682                         debug(requests) tracef("Add Proxy-Authorization header");
683                         auto auth = new BasicAuthentication(uri.username, uri.password);
684                         auto header = auth.authHeaders("");
685                         stream.send("Proxy-Authorization: %s\r\n".format(header["Authorization"]));
686                     }
687                     stream.send("\r\n");
688                     while ( stream.isConnected ) {
689                         ubyte[1024] b;
690                         auto read = stream.receive(b);
691                         if ( verbosity>=1) {
692                             writefln("< %s", cast(string)b[0..read]);
693                         }
694                         debug(requests) tracef("read: %d", read);
695                         if ( b[0..read].canFind("\r\n\r\n") || b[0..read].canFind("\n\n") ) {
696                             debug(requests) tracef("proxy connection ready");
697                             // convert connection to ssl
698                             stream = new SSLStream(stream, _sslOptions, _uri.host);
699                             break ;
700                         } else {
701                             debug(requests) tracef("still wait for proxy connection");
702                         }
703                     }
704                 } else {
705                     uri = _uri;
706                     stream = new SSLStream(_sslOptions);
707                     stream.bind(_bind);
708                     stream.connect(uri.host, uri.port, _timeout);
709                     debug(requests) tracef("ssl connection to origin server ready");
710                 }
711                 break ;
712         }
713 
714         return stream;
715     }
716     ///
717     /// Request sent, now receive response.
718     /// Find headers, split on headers and body, continue to receive body
719     ///
720     private void receiveResponse(NetworkStream _stream) {
721 
722         try {
723             _stream.readTimeout = timeout;
724         } catch (Exception e) {
725             debug(requests) tracef("Failed to set read timeout for stream: %s", e.msg);
726             return;
727         }
728         // Commented this out as at exit we can have alreade closed socket
729         // scope(exit) {
730         //     if ( _stream && _stream.isOpen ) {
731         //         _stream.readTimeout = 0.seconds;
732         //     }
733         // }
734 
735         _bodyDecoder = new DataPipe!ubyte();
736         scope(exit) {
737             if ( !_useStreaming ) {
738                 _bodyDecoder = null;
739                 _unChunker = null;
740             }
741         }
742 
743         auto buffer = Buffer!ubyte();
744         Buffer!ubyte partialBody;
745         ptrdiff_t read;
746         string lineSep = null, headersEnd = null;
747         bool headersHaveBeenReceived;
748 
749         while( !headersHaveBeenReceived ) {
750 
751             auto b = new ubyte[_bufferSize];
752             read = _stream.receive(b);
753 
754             debug(requests) tracef("read: %d", read);
755             if ( read == 0 ) {
756                 break;
757             }
758             auto data = b[0..read];
759             buffer.putNoCopy(data);
760             if ( verbosity>=3 ) {
761                 writeln(data.dump.join("\n"));
762             }
763 
764             if ( buffer.length > maxHeadersLength ) {
765                 throw new RequestException("Headers length > maxHeadersLength (%d > %d)".format(buffer.length, maxHeadersLength));
766             }
767 
768             // Proper HTTP uses "\r\n" as a line separator, but broken servers sometimes use "\n".
769             // Servers that use "\r\n" might have "\n" inside a header.
770             // For any half-sane server, the first '\n' should be at the end of the status line, so this can be used to detect the line separator.
771             // In any case, all the interesting points in the header for now are at '\n' characters, so scan the newly read data for them.
772             foreach (idx; buffer.length-read..buffer.length)
773             {
774                 if ( buffer[idx] == '\n' )
775                 {
776                     if ( lineSep is null )
777                     {
778                         // First '\n'. Detect line/header endings.
779                         // HTTP header sections end with a double line separator
780                         lineSep = "\n";
781                         headersEnd = "\n\n";
782                         if ( idx > 0 && buffer[idx-1] == '\r' )
783                         {
784                             lineSep = "\r\n";
785                             headersEnd = "\r\n\r\n";
786                         }
787                     }
788                     else
789                     {
790                         // Potential header ending.
791                         if ( buffer.data[0..idx+1].endsWith(headersEnd) )
792                         {
793                             auto ResponseHeaders = buffer.data[0..idx+1-headersEnd.length];
794                             partialBody = buffer[idx+1..$];
795                             _contentReceived += partialBody.length;
796                             parseResponseHeaders(ResponseHeaders, lineSep);
797                             headersHaveBeenReceived = true;
798                             break;
799                         }
800                     }
801                 }
802             }
803         }
804 
805         analyzeHeaders(_response._responseHeaders);
806 
807         _bodyDecoder.putNoCopy(partialBody.data);
808 
809         auto v = _bodyDecoder.get();
810         _response._responseBody.putNoCopy(v);
811 
812         // https://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.4
813         if ( (_method == "HEAD") || responseMustNotIncludeBody(_response.code) || (_contentLength < 0 && _unChunker is null) )
814         {
815             debug(requests) tracef("response without body");
816             return;
817         }
818 
819         _response._contentLength = _contentLength;
820         _response._contentReceived = _contentReceived;
821 
822         if ( _verbosity >= 2 ) writefln("< %d bytes of body received", partialBody.length);
823 
824         while( true ) {
825             if ( _contentLength >= 0 && _contentReceived >= _contentLength ) {
826                 debug(requests) trace("Body received.");
827                 break;
828             }
829             if ( _unChunker && _unChunker.done ) {
830                 break;
831             }
832 
833             if ( _useStreaming && _response._responseBody.length && !redirectCodes.canFind(_response.code) ) {
834                 debug(requests) trace("streaming requested");
835                 // save _stream in closure
836                 auto __stream = _stream;
837                 auto __bodyDecoder = _bodyDecoder;
838                 auto __unChunker = _unChunker;
839                 auto __contentReceived = _contentReceived;
840                 auto __contentLength = _contentLength;
841                 auto __bufferSize = _bufferSize;
842                 auto __response = _response;
843                 auto __verbosity = _verbosity;
844 
845                 // set up response
846                 _response._contentLength = _contentLength;
847                 _response.receiveAsRange.activated = true;
848                 _response.receiveAsRange.data = _response._responseBody.data;
849                 _response.receiveAsRange.cm = _cm;
850                 _response.receiveAsRange.read = delegate ubyte[] () {
851 
852                     while(true) {
853                         // check if we received everything we need
854                         if ( ( __unChunker && __unChunker.done )
855                             || !__stream.isConnected()
856                             || (__contentLength > 0 && __contentReceived >= __contentLength) )
857                         {
858                             debug(requests) trace("streaming_in receive completed");
859                             __bodyDecoder.flush();
860                             return __bodyDecoder.get();
861                         }
862                         // have to continue
863                         auto b = new ubyte[__bufferSize];
864                         try {
865                             read = __stream.receive(b);
866                         }
867                         catch (Exception e) {
868                             throw new RequestException("streaming_in error reading from socket", __FILE__, __LINE__, e);
869                         }
870                         debug(requests) tracef("streaming_in received %d bytes", read);
871 
872                         if ( read == 0 ) {
873                             debug(requests) tracef("streaming_in: server closed connection");
874                             __bodyDecoder.flush();
875                             return __bodyDecoder.get();
876                         }
877 
878                         if ( __verbosity>=3 ) {
879                             writeln(b[0..read].dump.join("\n"));
880                         }
881                         __response._contentReceived += read;
882                         __contentReceived += read;
883                         __bodyDecoder.putNoCopy(b[0..read]);
884                         auto res = __bodyDecoder.getNoCopy();
885                         if ( res.length == 0 ) {
886                             // there were nothing to produce (beginning of the chunk or no decompressed data)
887                             continue;
888                         }
889                         if (res.length == 1) {
890                             return res[0];
891                         }
892                         //
893                         // I'd like to "return _bodyDecoder.getNoCopy().join;" but it is slower
894                         //
895                         auto total = res.map!(b=>b.length).sum;
896                         // create buffer for joined bytes
897                         ubyte[] joined = new ubyte[total];
898                         size_t p;
899                         // memcopy
900                         foreach(ref _; res) {
901                             joined[p .. p + _.length] = _;
902                             p += _.length;
903                         }
904                         return joined;
905                     }
906                     assert(0);
907                 };
908                 // we prepared for streaming
909                 return;
910             }
911 
912             auto b = new ubyte[_bufferSize];
913             read = _stream.receive(b);
914 
915             if ( read == 0 ) {
916                 debug(requests) trace("read done");
917                 break;
918             }
919             if ( _verbosity >= 2 ) {
920                 writefln("< %d bytes of body received", read);
921             }
922 
923             if ( verbosity>=3 ) {
924                 writeln(b[0..read].dump.join("\n"));
925             }
926 
927             debug(requests) tracef("read: %d", read);
928             _contentReceived += read;
929             if ( _maxContentLength && _contentReceived > _maxContentLength ) {
930                 throw new RequestException("ContentLength > maxContentLength (%d>%d)".
931                     format(_contentLength, _maxContentLength));
932             }
933 
934             _bodyDecoder.putNoCopy(b[0..read]); // send buffer to all decoders
935 
936             _bodyDecoder.getNoCopy.             // fetch result and place to body
937                 each!(b => _response._responseBody.putNoCopy(b));
938 
939             debug(requests) tracef("receivedTotal: %d, contentLength: %d, bodyLength: %d", _contentReceived, _contentLength, _response._responseBody.length);
940 
941         }
942         _bodyDecoder.flush();
943         _response._responseBody.putNoCopy(_bodyDecoder.get());
944         _response._contentReceived = _contentReceived;
945     }
946     ///
947     /// Check that we received anything.
948     /// Server can close previous connection (keepalive or not)
949     ///
950     private bool serverPrematurelyClosedConnection() pure @safe {
951         immutable server_closed_connection = _response._responseHeaders.length == 0 && _response._status_line.length == 0;
952         // debug(requests) tracef("server closed connection = %s (headers.length=%s, status_line.length=%s)",
953         //     server_closed_connection, _response._responseHeaders.length,  _response._status_line.length);
954         return server_closed_connection;
955     }
956     private bool isIdempotent(in string method) pure @safe nothrow {
957         return ["GET", "HEAD"].canFind(method);
958     }
959     ///
960     /// If we do not want keepalive request,
961     /// or server signalled to close connection,
962     /// then close it
963     ///
964     void close_connection_if_not_keepalive(NetworkStream _stream) {
965         auto connection = "connection" in _response._responseHeaders;
966         if ( !_keepAlive ) {
967             _stream.close();
968         } else switch(_response._version) {
969             case HTTP11:
970                 // HTTP/1.1 defines the "close" connection option for the sender to signal that the connection
971                 // will be closed after completion of the response. For example,
972                 //        Connection: close
973                 // in either the request or the response header fields indicates that the connection
974                 // SHOULD NOT be considered `persistent' (section 8.1) after the current request/response is complete.
975                 // HTTP/1.1 applications that do not support persistent connections MUST include the "close" connection
976                 // option in every message.
977                 if ( connection && (*connection).toLower.split(",").canFind("close") ) {
978                     _stream.close();
979                 }
980                 break;
981             default:
982                 // for anything else close connection if there is no keep-alive in Connection
983                 if ( connection && !(*connection).toLower.split(",").canFind("keep-alive") ) {
984                     _stream.close();
985                 }
986                 break;
987         }
988     }
989     ///
990     /// Send multipart for request.
991     /// You would like to use this method for sending large portions of mixed data or uploading files to forms.
992     /// Content of the posted form consist of sources. Each source have at least name and value (can be string-like object or opened file, see more docs for MultipartForm struct)
993     /// Params:
994     ///     url = url
995     ///     sources = array of sources.
996     deprecated("Use Request() instead of HTTPRequest(); will be removed 2019-07")
997     HTTPResponse exec(string method="POST")(string url, MultipartForm sources) {
998         import std.uuid;
999         import std.file;
1000 
1001         checkURL(url);
1002         //if ( _cm is null ) {
1003         //    _cm = new ConnManager();
1004         //}
1005 
1006         NetworkStream _stream;
1007         _method = method;
1008         _response = new HTTPResponse;
1009         _response.uri = _uri;
1010         _response.finalURI = _uri;
1011         bool restartedRequest = false;
1012 
1013     connect:
1014         _contentReceived = 0;
1015         _response._startedAt = Clock.currTime;
1016 
1017         assert(_stream is null);
1018 
1019         _stream = _cm.get(_uri.scheme, _uri.host, _uri.port);
1020 
1021         if ( _stream is null ) {
1022             debug(requests) trace("create new connection");
1023             _stream = setupConnection();
1024         } else {
1025             debug(requests) trace("reuse old connection");
1026         }
1027 
1028         assert(_stream !is null);
1029 
1030         if ( !_stream.isConnected ) {
1031             debug(requests) trace("disconnected stream on enter");
1032             if ( !restartedRequest ) {
1033                 debug(requests) trace("disconnected stream on enter: retry");
1034                 assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
1035 
1036                 _cm.del(_uri.scheme, _uri.host, _uri.port);
1037                 _stream.close();
1038                 _stream = null;
1039 
1040                 restartedRequest = true;
1041                 goto connect;
1042             }
1043             debug(requests) trace("disconnected stream on enter: return response");
1044             //_stream = null;
1045             return _response;
1046         }
1047         _response._connectedAt = Clock.currTime;
1048 
1049         Appender!string req;
1050         req.put(requestString());
1051 
1052         string   boundary = randomUUID().toString;
1053         string[] partHeaders;
1054         size_t   contentLength;
1055 
1056         foreach(ref part; sources._sources) {
1057             string h = "--" ~ boundary ~ "\r\n";
1058             string disposition = `form-data; name="%s"`.format(part.name);
1059             string optionals = part.
1060                 parameters.byKeyValue().
1061                 filter!(p => p.key!="Content-Type").
1062                 map!   (p => "%s=%s".format(p.key, p.value)).
1063                 join("; ");
1064 
1065             h ~= `Content-Disposition: ` ~ [disposition, optionals].join("; ") ~ "\r\n";
1066 
1067             auto contentType = "Content-Type" in part.parameters;
1068             if ( contentType ) {
1069                 h ~= "Content-Type: " ~ *contentType ~ "\r\n";
1070             }
1071 
1072             h ~= "\r\n";
1073             partHeaders ~= h;
1074             contentLength += h.length + part.input.getSize() + "\r\n".length;
1075         }
1076         contentLength += "--".length + boundary.length + "--\r\n".length;
1077 
1078         auto h = requestHeaders();
1079         safeSetHeader(h, _userHeaders.ContentType, "Content-Type", "multipart/form-data; boundary=" ~ boundary);
1080         safeSetHeader(h, _userHeaders.ContentLength, "Content-Length", to!string(contentLength));
1081 
1082         h.byKeyValue.
1083             map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n").
1084                 each!(h => req.put(h));
1085         req.put("\r\n");
1086 
1087         debug(requests) trace(req.data);
1088         if ( _verbosity >= 1 ) req.data.splitLines.each!(a => writeln("> " ~ a));
1089 
1090         try {
1091             _stream.send(req.data());
1092             foreach(ref source; sources._sources) {
1093                 debug(requests) tracef("sending part headers <%s>", partHeaders.front);
1094                 _stream.send(partHeaders.front);
1095                 partHeaders.popFront;
1096                 while (true) {
1097                     auto chunk = source.input.read();
1098                     if ( chunk.length <= 0 ) {
1099                         break;
1100                     }
1101                     _stream.send(chunk);
1102                 }
1103                 _stream.send("\r\n");
1104             }
1105             _stream.send("--" ~ boundary ~ "--\r\n");
1106             _response._requestSentAt = Clock.currTime;
1107             receiveResponse(_stream);
1108             _response._finishedAt = Clock.currTime;
1109         }
1110         catch (NetworkException e) {
1111             errorf("Error sending request: ", e.msg);
1112             _stream.close();
1113             return _response;
1114         }
1115 
1116         if ( serverPrematurelyClosedConnection()
1117         && !restartedRequest
1118         && isIdempotent(_method)
1119         ) {
1120             ///
1121             /// We didn't receive any data (keepalive connectioin closed?)
1122             /// and we can restart this request.
1123             /// Go ahead.
1124             ///
1125             debug(requests) tracef("Server closed keepalive connection");
1126 
1127             assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
1128 
1129             _cm.del(_uri.scheme, _uri.host, _uri.port);
1130             _stream.close();
1131             _stream = null;
1132 
1133             restartedRequest = true;
1134             goto connect;
1135         }
1136 
1137         if ( _useStreaming ) {
1138             if ( _response._receiveAsRange.activated ) {
1139                 debug(requests) trace("streaming_in activated");
1140                 return _response;
1141             } else {
1142                 // this can happen if whole response body received together with headers
1143                 _response._receiveAsRange.data = _response.responseBody.data;
1144             }
1145         }
1146 
1147         close_connection_if_not_keepalive(_stream);
1148 
1149         if ( _verbosity >= 1 ) {
1150             writeln(">> Connect time: ", _response._connectedAt - _response._startedAt);
1151             writeln(">> Request send time: ", _response._requestSentAt - _response._connectedAt);
1152             writeln(">> Response recv time: ", _response._finishedAt - _response._requestSentAt);
1153         }
1154 
1155         if ( willFollowRedirect ) {
1156             if ( _history.length >= _maxRedirects ) {
1157                 _stream = null;
1158                 throw new MaxRedirectsException("%d redirects reached maxRedirects %d.".format(_history.length, _maxRedirects));
1159             }
1160             // "location" in response already checked in canFollowRedirect
1161             immutable new_location = *("location" in _response.responseHeaders);
1162             immutable current_uri = _uri, next_uri = uriFromLocation(_uri, new_location);
1163 
1164             // save current response for history
1165             _history ~= _response;
1166 
1167             // prepare new response (for redirected request)
1168             _response = new HTTPResponse;
1169             _response.uri = current_uri;
1170             _response.finalURI = next_uri;
1171             _stream = null;
1172 
1173             // set new uri
1174             this._uri = next_uri;
1175             debug(requests) tracef("Redirected to %s", next_uri);
1176             if ( _method != "GET" && _response.code != 307 && _response.code != 308 ) {
1177                 // 307 and 308 do not change method
1178                 return this.get();
1179             }
1180             if ( restartedRequest ) {
1181                 debug(requests) trace("Rare event: clearing 'restartedRequest' on redirect");
1182                 restartedRequest = false;
1183             }
1184             goto connect;
1185         }
1186 
1187         _response._history = _history;
1188         return _response;
1189     }
1190 
1191     // we use this if we send from ubyte[][] and user provided Content-Length
1192     private void sendFlattenContent(T)(NetworkStream _stream, T content) {
1193         while ( !content.empty ) {
1194             auto chunk = content.front;
1195             _stream.send(chunk);
1196             content.popFront;
1197         }
1198         debug(requests) tracef("sent");
1199     }
1200     // we use this if we send from ubyte[][] as chunked content
1201     private void sendChunkedContent(T)(NetworkStream _stream, T content) {
1202         while ( !content.empty ) {
1203             auto chunk = content.front;
1204             auto chunkHeader = "%x\r\n".format(chunk.length);
1205             debug(requests) tracef("sending %s%s", chunkHeader, chunk);
1206             _stream.send(chunkHeader);
1207             _stream.send(chunk);
1208             _stream.send("\r\n");
1209             content.popFront;
1210         }
1211         debug(requests) tracef("sent");
1212         _stream.send("0\r\n\r\n");
1213     }
1214     ///
1215     /// POST/PUT/... data from some string(with Content-Length), or from range of strings/bytes (use Transfer-Encoding: chunked).
1216     /// When rank 1 (flat array) used as content it must have length. In that case "content" will be sent directly to network, and Content-Length headers will be added.
1217     /// If you are goung to send some range and do not know length at the moment when you start to send request, then you can send chunks of chars or ubyte.
1218     /// Try not to send too short chunks as this will put additional load on client and server. Chunks of length 2048 or 4096 are ok.
1219     ///
1220     /// Parameters:
1221     ///    url = url
1222     ///    content = string or input range
1223     ///    contentType = content type
1224     ///  Returns:
1225     ///     Response
1226     ///  Examples:
1227     ///  ---------------------------------------------------------------------------------------------------------
1228     ///      rs = rq.exec!"POST"("http://httpbin.org/post", "привiт, свiт!", "application/octet-stream");
1229     ///
1230     ///      auto s = lineSplitter("one,\ntwo,\nthree.");
1231     ///      rs = rq.exec!"POST"("http://httpbin.org/post", s, "application/octet-stream");
1232     ///
1233     ///      auto s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
1234     ///      rs = rq.exec!"POST"("http://httpbin.org/post", s.representation.chunks(10), "application/octet-stream");
1235     ///
1236     ///      auto f = File("tests/test.txt", "rb");
1237     ///      rs = rq.exec!"POST"("http://httpbin.org/post", f.byChunk(3), "application/octet-stream");
1238     ///  --------------------------------------------------------------------------------------------------------
1239     deprecated("Use Request() instead of HTTPRequest(); will be removed 2019-07")
1240     HTTPResponse exec(string method="POST", R)(string url, R content, string contentType="application/octet-stream")
1241         if ( (rank!R == 1)
1242             || (rank!R == 2 && isSomeChar!(Unqual!(typeof(content.front.front))))
1243             || (rank!R == 2 && (is(Unqual!(typeof(content.front.front)) == ubyte)))
1244         )
1245     do {
1246         debug(requests) tracef("started url=%s, this._uri=%s", url, _uri);
1247 
1248         checkURL(url);
1249         //if ( _cm is null ) {
1250         //    _cm = new ConnManager();
1251         //}
1252 
1253         NetworkStream _stream;
1254         _method = method;
1255         _response = new HTTPResponse;
1256         _history.length = 0;
1257         _response.uri = _uri;
1258         _response.finalURI = _uri;
1259         bool restartedRequest = false;
1260         bool send_flat;
1261 
1262     connect:
1263         _contentReceived = 0;
1264         _response._startedAt = Clock.currTime;
1265 
1266         assert(_stream is null);
1267 
1268         _stream = _cm.get(_uri.scheme, _uri.host, _uri.port);
1269 
1270         if ( _stream is null ) {
1271             debug(requests) trace("create new connection");
1272             _stream = setupConnection();
1273         } else {
1274             debug(requests) trace("reuse old connection");
1275         }
1276 
1277         assert(_stream !is null);
1278 
1279         if ( !_stream.isConnected ) {
1280             debug(requests) trace("disconnected stream on enter");
1281             if ( !restartedRequest ) {
1282                 debug(requests) trace("disconnected stream on enter: retry");
1283                 assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
1284 
1285                 _cm.del(_uri.scheme, _uri.host, _uri.port);
1286                 _stream.close();
1287                 _stream = null;
1288 
1289                 restartedRequest = true;
1290                 goto connect;
1291             }
1292             debug(requests) trace("disconnected stream on enter: return response");
1293             //_stream = null;
1294             return _response;
1295         }
1296         _response._connectedAt = Clock.currTime;
1297 
1298         Appender!string req;
1299         req.put(requestString());
1300 
1301         auto h = requestHeaders;
1302         if ( contentType ) {
1303             safeSetHeader(h, _userHeaders.ContentType, "Content-Type", contentType);
1304         }
1305         static if ( rank!R == 1 ) {
1306             safeSetHeader(h, _userHeaders.ContentLength, "Content-Length", to!string(content.length));
1307         } else {
1308             if ( _userHeaders.ContentLength ) {
1309                 debug(requests) tracef("User provided content-length for chunked content");
1310                 send_flat = true;
1311             } else {
1312                 h["Transfer-Encoding"] = "chunked";
1313                 send_flat = false;
1314             }
1315         }
1316         h.byKeyValue.
1317             map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n").
1318             each!(h => req.put(h));
1319         req.put("\r\n");
1320 
1321         debug(requests) trace(req.data);
1322         if ( _verbosity >= 1 ) {
1323             req.data.splitLines.each!(a => writeln("> " ~ a));
1324         }
1325 
1326         try {
1327             // send headers
1328             _stream.send(req.data());
1329             // send body
1330             static if ( rank!R == 1) {
1331                 _stream.send(content);
1332             } else {
1333                 if ( send_flat ) {
1334                     sendFlattenContent(_stream, content);
1335                 } else {
1336                     sendChunkedContent(_stream, content);
1337                 }
1338             }
1339             _response._requestSentAt = Clock.currTime;
1340             debug(requests) trace("starting receive response");
1341             receiveResponse(_stream);
1342             debug(requests) trace("finished receive response");
1343             _response._finishedAt = Clock.currTime;
1344         } catch (NetworkException e) {
1345             _stream.close();
1346             throw new RequestException("Network error during data exchange");
1347         }
1348 
1349         if ( serverPrematurelyClosedConnection()
1350             && !restartedRequest
1351             && isIdempotent(_method))
1352         {
1353             ///
1354             /// We didn't receive any data (keepalive connectioin closed?)
1355             /// and we can restart this request.
1356             /// Go ahead.
1357             ///
1358             debug(requests) tracef("Server closed keepalive connection");
1359 
1360             assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
1361 
1362             _cm.del(_uri.scheme, _uri.host, _uri.port);
1363             _stream.close();
1364             _stream = null;
1365 
1366             restartedRequest = true;
1367             goto connect;
1368         }
1369 
1370         if ( _useStreaming ) {
1371             if ( _response._receiveAsRange.activated ) {
1372                 debug(requests) trace("streaming_in activated");
1373                 return _response;
1374             } else {
1375                 // this can happen if whole response body received together with headers
1376                 _response._receiveAsRange.data = _response.responseBody.data;
1377             }
1378         }
1379 
1380         close_connection_if_not_keepalive(_stream);
1381 
1382         if ( _verbosity >= 1 ) {
1383             writeln(">> Connect time: ", _response._connectedAt - _response._startedAt);
1384             writeln(">> Request send time: ", _response._requestSentAt - _response._connectedAt);
1385             writeln(">> Response recv time: ", _response._finishedAt - _response._requestSentAt);
1386         }
1387 
1388 
1389         if ( willFollowRedirect ) {
1390             if ( _history.length >= _maxRedirects ) {
1391                 _stream = null;
1392                 throw new MaxRedirectsException("%d redirects reached maxRedirects %d.".format(_history.length, _maxRedirects));
1393             }
1394             // "location" in response already checked in canFollowRedirect
1395             immutable new_location = *("location" in _response.responseHeaders);
1396             immutable current_uri = _uri, next_uri = uriFromLocation(_uri, new_location);
1397 
1398             // save current response for history
1399             _history ~= _response;
1400 
1401             // prepare new response (for redirected request)
1402             _response = new HTTPResponse;
1403             _response.uri = current_uri;
1404             _response.finalURI = next_uri;
1405 
1406             _stream = null;
1407 
1408             // set new uri
1409             this._uri = next_uri;
1410             debug(requests) tracef("Redirected to %s", next_uri);
1411             if ( _method != "GET" && _response.code != 307 && _response.code != 308 ) {
1412                 // 307 and 308 do not change method
1413                 return this.get();
1414             }
1415             if ( restartedRequest ) {
1416                 debug(requests) trace("Rare event: clearing 'restartedRequest' on redirect");
1417                 restartedRequest = false;
1418             }
1419             goto connect;
1420         }
1421 
1422         _response._history = _history;
1423         return _response;
1424     }
1425     ///
1426     /// Send request with parameters.
1427     /// If used for POST or PUT requests then application/x-www-form-urlencoded used.
1428     /// Request parameters will be encoded into request string or placed in request body for POST/PUT
1429     /// requests.
1430     /// Parameters:
1431     ///     url = url
1432     ///     params = request parameters
1433     ///  Returns:
1434     ///     Response
1435     ///  Examples:
1436     ///  ---------------------------------------------------------------------------------
1437     ///     rs = Request().exec!"GET"("http://httpbin.org/get", ["c":"d", "a":"b"]);
1438     ///  ---------------------------------------------------------------------------------
1439     ///
1440     deprecated("Use Request() instead of HTTPRequest; will be removed 2019-07")
1441     HTTPResponse exec(string method="GET")(string url = null, QueryParam[] params = null)
1442     do {
1443         debug(requests) tracef("started url=%s, this._uri=%s", url, _uri);
1444 
1445         checkURL(url);
1446         //if ( _cm is null ) {
1447         //    _cm = new ConnManager();
1448         //}
1449 
1450         NetworkStream _stream;
1451         _method = method;
1452         _response = new HTTPResponse;
1453         _history.length = 0;
1454         _response.uri = _uri;
1455         _response.finalURI = _uri;
1456         bool restartedRequest = false; // True if this is restarted keepAlive request
1457 
1458     connect:
1459         if ( _method == "GET" && _uri in _permanent_redirects ) {
1460             debug(requests) trace("use parmanent redirects cache");
1461             _uri = uriFromLocation(_uri, _permanent_redirects[_uri]);
1462             _response._finalURI = _uri;
1463         }
1464         _contentReceived = 0;
1465         _response._startedAt = Clock.currTime;
1466 
1467         assert(_stream is null);
1468 
1469         _stream = _cm.get(_uri.scheme, _uri.host, _uri.port);
1470 
1471         if ( _stream is null ) {
1472             debug(requests) trace("create new connection");
1473             _stream = setupConnection();
1474         } else {
1475             debug(requests) trace("reuse old connection");
1476         }
1477 
1478         assert(_stream !is null);
1479 
1480         if ( !_stream.isConnected ) {
1481             debug(requests) trace("disconnected stream on enter");
1482             if ( !restartedRequest ) {
1483                 debug(requests) trace("disconnected stream on enter: retry");
1484                 assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
1485 
1486                 _cm.del(_uri.scheme, _uri.host, _uri.port);
1487                 _stream.close();
1488                 _stream = null;
1489 
1490                 restartedRequest = true;
1491                 goto connect;
1492             }
1493             debug(requests) trace("disconnected stream on enter: return response");
1494             //_stream = null;
1495             return _response;
1496         }
1497         _response._connectedAt = Clock.currTime;
1498 
1499         auto h = requestHeaders();
1500 
1501         Appender!string req;
1502 
1503         string encoded;
1504 
1505         switch (_method) {
1506             case "POST","PUT","PATCH":
1507                 encoded = params2query(params);
1508                 safeSetHeader(h, _userHeaders.ContentType, "Content-Type", "application/x-www-form-urlencoded");
1509                 if ( encoded.length > 0) {
1510                     safeSetHeader(h, _userHeaders.ContentLength, "Content-Length", to!string(encoded.length));
1511                 }
1512                 req.put(requestString());
1513                 break;
1514             default:
1515                 req.put(requestString(params));
1516         }
1517 
1518         h.byKeyValue.
1519             map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n").
1520             each!(h => req.put(h));
1521         req.put("\r\n");
1522         if ( encoded ) {
1523             req.put(encoded);
1524         }
1525 
1526         debug(requests) trace(req.data);
1527         if ( _verbosity >= 1 ) {
1528             req.data.splitLines.each!(a => writeln("> " ~ a));
1529         }
1530         //
1531         // Now send request and receive response
1532         //
1533         try {
1534             _stream.send(req.data());
1535             _response._requestSentAt = Clock.currTime;
1536             debug(requests) trace("starting receive response");
1537             receiveResponse(_stream);
1538             debug(requests) trace("done receive response");
1539             _response._finishedAt = Clock.currTime;
1540         }
1541         catch (NetworkException e) {
1542             // On SEND this can means:
1543             // we started to send request to the server, but it closed connection because of keepalive timeout.
1544             // We have to restart request if possible.
1545 
1546             // On RECEIVE - if we received something - then this exception is real and unexpected error.
1547             // If we didn't receive anything - we can restart request again as it can be
1548             debug(requests) tracef("Exception on receive response: %s", e.msg);
1549             if ( _response._responseHeaders.length != 0 )
1550             {
1551                 _stream.close();
1552                 throw new RequestException("Unexpected network error");
1553             }
1554         }
1555 
1556         if ( serverPrematurelyClosedConnection()
1557             && !restartedRequest
1558             && isIdempotent(_method)
1559             ) {
1560             ///
1561             /// We didn't receive any data (keepalive connectioin closed?)
1562             /// and we can restart this request.
1563             /// Go ahead.
1564             ///
1565             debug(requests) tracef("Server closed keepalive connection");
1566 
1567             assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
1568 
1569             _cm.del(_uri.scheme, _uri.host, _uri.port);
1570             _stream.close();
1571             _stream = null;
1572 
1573             restartedRequest = true;
1574             goto connect;
1575         }
1576 
1577         if ( _useStreaming ) {
1578             if ( _response._receiveAsRange.activated ) {
1579                 debug(requests) trace("streaming_in activated");
1580                 return _response;
1581             } else {
1582                 // this can happen if whole response body received together with headers
1583                 _response._receiveAsRange.data = _response.responseBody.data;
1584             }
1585         }
1586 
1587         close_connection_if_not_keepalive(_stream);
1588 
1589         if ( _verbosity >= 1 ) {
1590             writeln(">> Connect time: ", _response._connectedAt - _response._startedAt);
1591             writeln(">> Request send time: ", _response._requestSentAt - _response._connectedAt);
1592             writeln(">> Response recv time: ", _response._finishedAt - _response._requestSentAt);
1593         }
1594 
1595         if ( willFollowRedirect ) {
1596             debug(requests) trace("going to follow redirect");
1597             if ( _history.length >= _maxRedirects ) {
1598                 _stream = null;
1599                 throw new MaxRedirectsException("%d redirects reached maxRedirects %d.".format(_history.length, _maxRedirects));
1600             }
1601             // "location" in response already checked in canFollowRedirect
1602             immutable new_location = *("location" in _response.responseHeaders);
1603             immutable current_uri = _uri, next_uri = uriFromLocation(_uri, new_location);
1604 
1605             if ( _method == "GET" && _response.code == 301 ) {
1606                 _permanent_redirects[_uri] = new_location;
1607             }
1608 
1609             // save current response for history
1610             _history ~= _response;
1611 
1612             // prepare new response (for redirected request)
1613             _response = new HTTPResponse;
1614             _response.uri = current_uri;
1615             _response.finalURI = next_uri;
1616             _stream = null;
1617 
1618             // set new uri
1619             _uri = next_uri;
1620             debug(requests) tracef("Redirected to %s", next_uri);
1621             if ( _method != "GET" && _response.code != 307 && _response.code != 308 ) {
1622                 // 307 and 308 do not change method
1623                 return this.get();
1624             }
1625             if ( restartedRequest ) {
1626                 debug(requests) trace("Rare event: clearing 'restartedRequest' on redirect");
1627                 restartedRequest = false;
1628             }
1629             goto connect;
1630         }
1631 
1632         _response._history = _history;
1633         return _response;
1634     }
1635 
1636     /// WRAPPERS
1637     ///
1638     /// send file(s) using POST and multipart form.
1639     /// This wrapper will be deprecated, use post with MultipartForm - it is more general and clear.
1640     /// Parameters:
1641     ///     url = url
1642     ///     files = array of PostFile structures
1643     /// Returns:
1644     ///     Response
1645     /// Each PostFile structure contain path to file, and optional field name and content type.
1646     /// If no field name provided, then basename of the file will be used.
1647     /// application/octet-stream is default when no content type provided.
1648     /// Example:
1649     /// ---------------------------------------------------------------
1650     ///    PostFile[] files = [
1651     ///                   {fileName:"tests/abc.txt", fieldName:"abc", contentType:"application/octet-stream"},
1652     ///                   {fileName:"tests/test.txt"}
1653     ///               ];
1654     ///    rs = rq.exec!"POST"("http://httpbin.org/post", files);
1655     /// ---------------------------------------------------------------
1656     ///
1657     deprecated("Use Request() instead of HTTPRequest(); will be removed 2019-07")
1658     HTTPResponse exec(string method="POST")(string url, PostFile[] files) if (method=="POST") {
1659         MultipartForm multipart;
1660         File[]        toClose;
1661         foreach(ref f; files) {
1662             File file = File(f.fileName, "rb");
1663             toClose ~= file;
1664             string fileName = f.fileName ? f.fileName : f.fieldName;
1665             string contentType = f.contentType ? f.contentType : "application/octetstream";
1666             multipart.add(f.fieldName, new FormDataFile(file), ["filename":fileName, "Content-Type": contentType]);
1667         }
1668         auto res = exec!"POST"(url, multipart);
1669         toClose.each!"a.close";
1670         return res;
1671     }
1672     ///
1673     /// exec request with parameters when you can use dictionary (when you have no duplicates in parameter names)
1674     /// Consider switch to exec(url, QueryParams) as it more generic and clear.
1675     /// Parameters:
1676     ///     url = url
1677     ///     params = dictionary with field names as keys and field values as values.
1678     /// Returns:
1679     ///     Response
1680     deprecated("Use Request() instead of HTTPRequest(); will be removed 2019-07")
1681     HTTPResponse exec(string method="GET")(string url, string[string] params) {
1682         return exec!method(url, params.byKeyValue.map!(p => QueryParam(p.key, p.value)).array);
1683     }
1684     ///
1685     /// GET request. Simple wrapper over exec!"GET"
1686     /// Params:
1687     /// args = request parameters. see exec docs.
1688     ///
1689     deprecated("Use Request() instead of HTTPRequest; will be removed 2019-07")
1690     HTTPResponse get(A...)(A args) {
1691         return exec!"GET"(args);
1692     }
1693     ///
1694     /// POST request. Simple wrapper over exec!"POST"
1695     /// Params:
1696     /// uri = endpoint uri
1697     /// args = request parameters. see exec docs.
1698     ///
1699     deprecated("Use Request() instead of HTTPRequest; will be removed 2019-07")
1700     HTTPResponse post(A...)(string uri, A args) {
1701         return exec!"POST"(uri, args);
1702     }
1703 
1704     import requests.request;
1705 
1706     // we use this if we send from ubyte[][] and user provided Content-Length
1707     private void sendFlattenContent(NetworkStream _stream) {
1708         while ( !_postData.empty ) {
1709             auto chunk = _postData.front;
1710             _stream.send(chunk);
1711             _postData.popFront;
1712         }
1713         debug(requests) tracef("sent");
1714     }
1715     // we use this if we send from ubyte[][] as chunked content
1716     private void sendChunkedContent(NetworkStream _stream) {
1717         while ( !_postData.empty ) {
1718             auto chunk = _postData.front;
1719             auto chunkHeader = "%x\r\n".format(chunk.length);
1720             debug(requests) tracef("sending %s%s", chunkHeader, cast(string)chunk);
1721             _stream.send(chunkHeader);
1722             _stream.send(chunk);
1723             _stream.send("\r\n");
1724             debug(requests) tracef("chunk sent");
1725             _postData.popFront;
1726         }
1727         debug(requests) tracef("sent");
1728         _stream.send("0\r\n\r\n");
1729     }
1730 
1731     HTTPResponse exec_from_range(InputRangeAdapter postData)
1732     do {
1733 
1734         _postData = postData;
1735 
1736         debug(requests) tracef("exec from range");
1737 
1738         NetworkStream _stream;
1739         _response = new HTTPResponse;
1740         _history.length = 0;
1741         _response.uri = _uri;
1742         _response.finalURI = _uri;
1743         bool restartedRequest = false;
1744         bool send_flat;
1745 
1746     connect:
1747         _contentReceived = 0;
1748         _response._startedAt = Clock.currTime;
1749 
1750         assert(_stream is null);
1751 
1752         _stream = _cm.get(_uri.scheme, _uri.host, _uri.port);
1753 
1754         if ( _stream is null ) {
1755             debug(requests) trace("create new connection");
1756             _stream = setupConnection();
1757         } else {
1758             debug(requests) trace("reuse old connection");
1759         }
1760 
1761         assert(_stream !is null);
1762 
1763         if ( !_stream.isConnected ) {
1764             debug(requests) trace("disconnected stream on enter");
1765             if ( !restartedRequest ) {
1766                 debug(requests) trace("disconnected stream on enter: retry");
1767                 assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
1768 
1769                 _cm.del(_uri.scheme, _uri.host, _uri.port);
1770                 _stream.close();
1771                 _stream = null;
1772 
1773                 restartedRequest = true;
1774                 goto connect;
1775             }
1776             debug(requests) trace("disconnected stream on enter: return response");
1777             //_stream = null;
1778             return _response;
1779         }
1780         _response._connectedAt = Clock.currTime;
1781 
1782         Appender!string req;
1783         req.put(requestString());
1784 
1785         auto h = requestHeaders;
1786         if ( _contentType ) {
1787             safeSetHeader(h, _userHeaders.ContentType, "Content-Type", _contentType);
1788         }
1789 
1790         if ( _postData.length >= 0 )
1791         {
1792             // we know t
1793             safeSetHeader(h, _userHeaders.ContentLength, "Content-Length", to!string(_postData.length));
1794         }
1795 
1796         if ( _userHeaders.ContentLength || "Content-Length" in h )
1797         {
1798             debug(requests) tracef("User provided content-length for chunked content");
1799             send_flat = true;
1800         }
1801         else
1802         {
1803             h["Transfer-Encoding"] = "chunked";
1804             send_flat = false;
1805         }
1806         h.byKeyValue.
1807             map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n").
1808             each!(h => req.put(h));
1809         req.put("\r\n");
1810 
1811         debug(requests) tracef("send <%s>", req.data);
1812         if ( _verbosity >= 1 ) {
1813             req.data.splitLines.each!(a => writeln("> " ~ a));
1814         }
1815 
1816         try {
1817             // send headers
1818             _stream.send(req.data());
1819             // send body
1820             if ( send_flat ) {
1821                 sendFlattenContent(_stream);
1822             } else {
1823                 sendChunkedContent(_stream);
1824             }
1825             _response._requestSentAt = Clock.currTime;
1826             debug(requests) trace("starting receive response");
1827             receiveResponse(_stream);
1828             debug(requests) trace("finished receive response");
1829             _response._finishedAt = Clock.currTime;
1830         }
1831         catch (NetworkException e)
1832         {
1833             _stream.close();
1834             throw new RequestException("Network error during data exchange");
1835         }
1836         if ( serverPrematurelyClosedConnection()
1837         && !restartedRequest
1838         && isIdempotent(_method)
1839         ) {
1840             ///
1841             /// We didn't receive any data (keepalive connectioin closed?)
1842             /// and we can restart this request.
1843             /// Go ahead.
1844             ///
1845             debug(requests) tracef("Server closed keepalive connection");
1846 
1847             assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
1848 
1849             _cm.del(_uri.scheme, _uri.host, _uri.port);
1850             _stream.close();
1851             _stream = null;
1852 
1853             restartedRequest = true;
1854             goto connect;
1855         }
1856 
1857         if ( _useStreaming ) {
1858             if ( _response._receiveAsRange.activated ) {
1859                 debug(requests) trace("streaming_in activated");
1860                 return _response;
1861             } else {
1862                 // this can happen if whole response body received together with headers
1863                 _response._receiveAsRange.data = _response.responseBody.data;
1864             }
1865         }
1866 
1867         close_connection_if_not_keepalive(_stream);
1868 
1869         if ( _verbosity >= 1 ) {
1870             writeln(">> Connect time: ", _response._connectedAt - _response._startedAt);
1871             writeln(">> Request send time: ", _response._requestSentAt - _response._connectedAt);
1872             writeln(">> Response recv time: ", _response._finishedAt - _response._requestSentAt);
1873         }
1874 
1875 
1876         if ( willFollowRedirect ) {
1877             if ( _history.length >= _maxRedirects ) {
1878                 _stream = null;
1879                 throw new MaxRedirectsException("%d redirects reached maxRedirects %d.".format(_history.length, _maxRedirects));
1880             }
1881             // "location" in response already checked in canFollowRedirect
1882             immutable new_location = *("location" in _response.responseHeaders);
1883             immutable current_uri = _uri, next_uri = uriFromLocation(_uri, new_location);
1884 
1885             immutable get_or_head = _method == "GET" || _method == "HEAD";
1886             immutable code = _response.code;
1887 
1888             // save current response for history
1889             _history ~= _response;
1890 
1891             if ( code == 301 )
1892             {
1893                 // permanent redirect and change method
1894                 _permanent_redirects[_uri] = new_location;
1895                 if ( !get_or_head )
1896                 {
1897                     _method = "GET";
1898                 }
1899             }
1900             if ( (code == 302 || code == 303) && !get_or_head)
1901             {
1902                 // only change method
1903                 _method = "GET";
1904             }
1905             if ( code == 307 )
1906             {
1907                 // no change method, no permanent
1908             }
1909             if ( code == 308 )
1910             {
1911                 // permanent redirection and do not change method
1912                 _permanent_redirects[_uri] = new_location;
1913             }
1914 
1915             // prepare new response (for redirected request)
1916             _response = new HTTPResponse;
1917             _response.uri = current_uri;
1918             _response.finalURI = next_uri;
1919 
1920             _stream = null;
1921 
1922             // set new uri
1923             this._uri = next_uri;
1924             debug(requests) tracef("Redirected to %s", next_uri);
1925             if ( restartedRequest ) {
1926                 debug(requests) trace("Rare event: clearing 'restartedRequest' on redirect");
1927                 restartedRequest = false;
1928             }
1929             if ( _method == "GET")
1930             {
1931                 return exec_from_parameters();
1932             }
1933             goto connect;
1934         }
1935 
1936         _response._history = _history;
1937         return _response;
1938     }
1939 
1940     HTTPResponse exec_from_multipart_form(MultipartForm form) {
1941         import std.uuid;
1942         import std.file;
1943 
1944         _multipartForm = form;
1945 
1946         debug(requests) tracef("exec from multipart form");
1947 
1948         NetworkStream _stream;
1949         _response = new HTTPResponse;
1950         _response.uri = _uri;
1951         _response.finalURI = _uri;
1952         bool restartedRequest = false;
1953 
1954     connect:
1955         _contentReceived = 0;
1956         _response._startedAt = Clock.currTime;
1957 
1958         assert(_stream is null);
1959 
1960         _stream = _cm.get(_uri.scheme, _uri.host, _uri.port);
1961 
1962         if ( _stream is null ) {
1963             debug(requests) trace("create new connection");
1964             _stream = setupConnection();
1965         } else {
1966             debug(requests) trace("reuse old connection");
1967         }
1968 
1969         assert(_stream !is null);
1970 
1971         if ( !_stream.isConnected ) {
1972             debug(requests) trace("disconnected stream on enter");
1973             if ( !restartedRequest ) {
1974                 debug(requests) trace("disconnected stream on enter: retry");
1975                 assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
1976 
1977                 _cm.del(_uri.scheme, _uri.host, _uri.port);
1978                 _stream.close();
1979                 _stream = null;
1980 
1981                 restartedRequest = true;
1982                 goto connect;
1983             }
1984             debug(requests) trace("disconnected stream on enter: return response");
1985             //_stream = null;
1986             return _response;
1987         }
1988         _response._connectedAt = Clock.currTime;
1989 
1990         Appender!string req;
1991         req.put(requestString());
1992 
1993         string   boundary = randomUUID().toString;
1994         string[] partHeaders;
1995         size_t   contentLength;
1996 
1997         foreach(ref part; _multipartForm._sources) {
1998             string h = "--" ~ boundary ~ "\r\n";
1999             string disposition = `form-data; name="%s"`.format(part.name);
2000             string optionals = part.
2001             parameters.byKeyValue().
2002             filter!(p => p.key!="Content-Type").
2003             map!   (p => "%s=%s".format(p.key, p.value)).
2004             join("; ");
2005 
2006             h ~= `Content-Disposition: ` ~ [disposition, optionals].join("; ") ~ "\r\n";
2007 
2008             auto contentType = "Content-Type" in part.parameters;
2009             if ( contentType ) {
2010                 h ~= "Content-Type: " ~ *contentType ~ "\r\n";
2011             }
2012 
2013             h ~= "\r\n";
2014             partHeaders ~= h;
2015             contentLength += h.length + part.input.getSize() + "\r\n".length;
2016         }
2017         contentLength += "--".length + boundary.length + "--\r\n".length;
2018 
2019         auto h = requestHeaders();
2020         safeSetHeader(h, _userHeaders.ContentType, "Content-Type", "multipart/form-data; boundary=" ~ boundary);
2021         safeSetHeader(h, _userHeaders.ContentLength, "Content-Length", to!string(contentLength));
2022 
2023         h.byKeyValue.
2024         map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n").
2025         each!(h => req.put(h));
2026         req.put("\r\n");
2027 
2028         debug(requests) trace(req.data);
2029         if ( _verbosity >= 1 ) req.data.splitLines.each!(a => writeln("> " ~ a));
2030 
2031         try {
2032             _stream.send(req.data());
2033             foreach(ref source; _multipartForm._sources) {
2034                 debug(requests) tracef("sending part headers <%s>", partHeaders.front);
2035                 _stream.send(partHeaders.front);
2036                 partHeaders.popFront;
2037                 while (true) {
2038                     auto chunk = source.input.read();
2039                     if ( chunk.length <= 0 ) {
2040                         break;
2041                     }
2042                     _stream.send(chunk);
2043                 }
2044                 _stream.send("\r\n");
2045             }
2046             _stream.send("--" ~ boundary ~ "--\r\n");
2047             _response._requestSentAt = Clock.currTime;
2048             receiveResponse(_stream);
2049             _response._finishedAt = Clock.currTime;
2050         }
2051         catch (NetworkException e) {
2052             errorf("Error sending request: ", e.msg);
2053             _stream.close();
2054             return _response;
2055         }
2056 
2057         if ( serverPrematurelyClosedConnection()
2058         && !restartedRequest
2059         && isIdempotent(_method)
2060         ) {
2061             ///
2062             /// We didn't receive any data (keepalive connectioin closed?)
2063             /// and we can restart this request.
2064             /// Go ahead.
2065             ///
2066             debug(requests) tracef("Server closed keepalive connection");
2067 
2068             assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
2069 
2070             _cm.del(_uri.scheme, _uri.host, _uri.port);
2071             _stream.close();
2072             _stream = null;
2073 
2074             restartedRequest = true;
2075             goto connect;
2076         }
2077 
2078         if ( _useStreaming ) {
2079             if ( _response._receiveAsRange.activated ) {
2080                 debug(requests) trace("streaming_in activated");
2081                 return _response;
2082             } else {
2083                 // this can happen if whole response body received together with headers
2084                 _response._receiveAsRange.data = _response.responseBody.data;
2085             }
2086         }
2087 
2088         close_connection_if_not_keepalive(_stream);
2089 
2090         if ( _verbosity >= 1 ) {
2091             writeln(">> Connect time: ", _response._connectedAt - _response._startedAt);
2092             writeln(">> Request send time: ", _response._requestSentAt - _response._connectedAt);
2093             writeln(">> Response recv time: ", _response._finishedAt - _response._requestSentAt);
2094         }
2095 
2096         if ( willFollowRedirect ) {
2097             if ( _history.length >= _maxRedirects ) {
2098                 _stream = null;
2099                 throw new MaxRedirectsException("%d redirects reached maxRedirects %d.".format(_history.length, _maxRedirects));
2100             }
2101             // "location" in response already checked in canFollowRedirect
2102             immutable new_location = *("location" in _response.responseHeaders);
2103             immutable current_uri = _uri;
2104             immutable next_uri = uriFromLocation(_uri, new_location);
2105 
2106             immutable get_or_head = _method == "GET" || _method == "HEAD";
2107             immutable code = _response.code;
2108 
2109             // save current response for history
2110             _history ~= _response;
2111 
2112             if ( code == 301 )
2113             {
2114                 // permanent redirect and change method
2115                 _permanent_redirects[_uri] = new_location;
2116                 if ( !get_or_head )
2117                 {
2118                     _method = "GET";
2119                 }
2120             }
2121             if ( (code == 302 || code == 303) && !get_or_head)
2122             {
2123                 // only change method
2124                 _method = "GET";
2125             }
2126             if ( code == 307 )
2127             {
2128                 // no change method, no permanent
2129             }
2130             if ( code == 308 )
2131             {
2132                 // permanent redirection and do not change method
2133                 _permanent_redirects[_uri] = new_location;
2134             }
2135 
2136             // prepare new response (for redirected request)
2137             _response = new HTTPResponse;
2138             _response.uri = current_uri;
2139             _response.finalURI = next_uri;
2140             _stream = null;
2141 
2142             // set new uri
2143             this._uri = next_uri;
2144             debug(requests) tracef("Redirected to %s", next_uri);
2145             if ( restartedRequest ) {
2146                 debug(requests) trace("Rare event: clearing 'restartedRequest' on redirect");
2147                 restartedRequest = false;
2148             }
2149             if ( _method == "GET")
2150             {
2151                 return exec_from_parameters();
2152             }
2153             goto connect;
2154         }
2155 
2156         _response._history = _history;
2157         return _response;
2158     }
2159 
2160     HTTPResponse exec_from_parameters() {
2161 
2162         debug(requests) tracef("exec from parameters request");
2163 
2164         assert(_uri != URI.init);
2165         NetworkStream _stream;
2166         _response = new HTTPResponse;
2167         _history.length = 0;
2168         _response.uri = _uri;
2169         _response.finalURI = _uri;
2170         bool restartedRequest = false; // True if this is restarted keepAlive request
2171 
2172     connect:
2173         if ( _method == "GET" && _uri in _permanent_redirects ) {
2174             debug(requests) trace("use parmanent redirects cache");
2175             _uri = uriFromLocation(_uri, _permanent_redirects[_uri]);
2176             _response._finalURI = _uri;
2177         }
2178         _contentReceived = 0;
2179         _response._startedAt = Clock.currTime;
2180 
2181         assert(_stream is null);
2182 
2183         _stream = _cm.get(_uri.scheme, _uri.host, _uri.port);
2184 
2185         if ( _stream is null ) {
2186             debug(requests) trace("create new connection");
2187             _stream = setupConnection();
2188         } else {
2189             debug(requests) trace("reuse old connection");
2190         }
2191 
2192         assert(_stream !is null);
2193 
2194         if ( !_stream.isConnected ) {
2195             debug(requests) trace("disconnected stream on enter");
2196             if ( !restartedRequest ) {
2197                 debug(requests) trace("disconnected stream on enter: retry");
2198                 assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
2199 
2200                 _cm.del(_uri.scheme, _uri.host, _uri.port);
2201                 _stream.close();
2202                 _stream = null;
2203 
2204                 restartedRequest = true;
2205                 goto connect;
2206             }
2207             debug(requests) trace("disconnected stream on enter: return response");
2208             //_stream = null;
2209             return _response;
2210         }
2211         _response._connectedAt = Clock.currTime;
2212 
2213         auto h = requestHeaders();
2214 
2215         Appender!string req;
2216 
2217         string encoded;
2218 
2219         switch (_method) {
2220             case "POST","PUT","PATCH":
2221                 encoded = params2query(_params);
2222                 safeSetHeader(h, _userHeaders.ContentType, "Content-Type", "application/x-www-form-urlencoded");
2223                 safeSetHeader(h, _userHeaders.ContentLength, "Content-Length", to!string(encoded.length));
2224                 req.put(requestString());
2225                 break;
2226             default:
2227                 req.put(requestString(_params));
2228         }
2229 
2230         h.byKeyValue.
2231         map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n").
2232         each!(h => req.put(h));
2233         req.put("\r\n");
2234         if ( encoded ) {
2235             req.put(encoded);
2236         }
2237 
2238         debug(requests) trace(req.data);
2239         if ( _verbosity >= 1 ) {
2240             req.data.splitLines.each!(a => writeln("> " ~ a));
2241         }
2242         //
2243         // Now send request and receive response
2244         //
2245         try {
2246             _stream.send(req.data());
2247             _response._requestSentAt = Clock.currTime;
2248             debug(requests) trace("starting receive response");
2249             receiveResponse(_stream);
2250             debug(requests) tracef("done receive response");
2251             _response._finishedAt = Clock.currTime;
2252         }
2253         catch (NetworkException e) {
2254             // On SEND this can means:
2255             // we started to send request to the server, but it closed connection because of keepalive timeout.
2256             // We have to restart request if possible.
2257 
2258             // On RECEIVE - if we received something - then this exception is real and unexpected error.
2259             // If we didn't receive anything - we can restart request again as it can be
2260             debug(requests) tracef("Exception on receive response: %s", e.msg);
2261             if ( _response._responseHeaders.length != 0 )
2262             {
2263                 _stream.close();
2264                 throw new RequestException("Unexpected network error");
2265             }
2266         }
2267 
2268         if ( serverPrematurelyClosedConnection()
2269             && !restartedRequest
2270             && isIdempotent(_method)
2271             ) {
2272             ///
2273             /// We didn't receive any data (keepalive connectioin closed?)
2274             /// and we can restart this request.
2275             /// Go ahead.
2276             ///
2277             debug(requests) tracef("Server closed keepalive connection");
2278 
2279             assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
2280 
2281             _cm.del(_uri.scheme, _uri.host, _uri.port);
2282             _stream.close();
2283             _stream = null;
2284 
2285             restartedRequest = true;
2286             goto connect;
2287         }
2288 
2289         if ( _useStreaming ) {
2290             if ( _response._receiveAsRange.activated ) {
2291                 debug(requests) trace("streaming_in activated");
2292                 return _response;
2293             } else {
2294                 // this can happen if whole response body received together with headers
2295                 _response._receiveAsRange.data = _response.responseBody.data;
2296             }
2297         }
2298 
2299         close_connection_if_not_keepalive(_stream);
2300 
2301         if ( _verbosity >= 1 ) {
2302             writeln(">> Connect time: ", _response._connectedAt - _response._startedAt);
2303             writeln(">> Request send time: ", _response._requestSentAt - _response._connectedAt);
2304             writeln(">> Response recv time: ", _response._finishedAt - _response._requestSentAt);
2305         }
2306 
2307         if ( willFollowRedirect ) {
2308             debug(requests) trace("going to follow redirect");
2309             if ( _history.length >= _maxRedirects ) {
2310                 _stream = null;
2311                 throw new MaxRedirectsException("%d redirects reached maxRedirects %d.".format(_history.length, _maxRedirects));
2312             }
2313             // "location" in response already checked in canFollowRedirect
2314             immutable new_location = *("location" in _response.responseHeaders);
2315             immutable current_uri = _uri;
2316             immutable next_uri = uriFromLocation(_uri, new_location);
2317 
2318             immutable get_or_head = _method == "GET" || _method == "HEAD";
2319             immutable code = _response.code;
2320 
2321             // save current response for history
2322             _history ~= _response;
2323 
2324             if ( code == 301 )
2325             {
2326                 // permanent redirect and change method
2327                 _permanent_redirects[_uri] = new_location;
2328                 if ( !get_or_head )
2329                 {
2330                     _method = "GET";
2331                 }
2332             }
2333             if ( (code == 302 || code == 303) && !get_or_head)
2334             {
2335                 // only change method
2336                 _method = "GET";
2337             }
2338             if ( code == 307 )
2339             {
2340                 // no change method, no permanent
2341             }
2342             if ( code == 308 )
2343             {
2344                 // permanent redirection and do not change method
2345                 _permanent_redirects[_uri] = new_location;
2346             }
2347 
2348             // prepare new response (for redirected request)
2349             _response = new HTTPResponse;
2350             _response.uri = current_uri;
2351             _response.finalURI = next_uri;
2352             _stream = null;
2353 
2354             // set new uri
2355             _uri = next_uri;
2356             debug(requests) tracef("Redirected to %s", next_uri);
2357             //if ( _method != "GET" && _response.code != 307 && _response.code != 308 ) {
2358             //    // 307 and 308 do not change method
2359             //    return exec_from_parameters(r);
2360             //}
2361             if ( restartedRequest ) {
2362                 debug(requests) trace("Rare event: clearing 'restartedRequest' on redirect");
2363                 restartedRequest = false;
2364             }
2365             goto connect;
2366         }
2367 
2368         _response._history = _history;
2369         return _response;
2370     }
2371     HTTPResponse execute(Request r)
2372     {
2373         _method = r.method;
2374         _uri = r.uri;
2375         _useStreaming = r.useStreaming;
2376         _permanent_redirects = r.permanent_redirects;
2377         _maxRedirects = r.maxRedirects;
2378         _authenticator = r.authenticator;
2379         _maxHeadersLength = r.maxHeadersLength;
2380         _maxContentLength = r.maxContentLength;
2381         _verbosity = r.verbosity;
2382         _keepAlive = r.keepAlive;
2383         _bufferSize = r.bufferSize;
2384         _proxy = r.proxy;
2385         _timeout = r.timeout;
2386         _contentType = r.contentType;
2387         _socketFactory = r.socketFactory;
2388         _sslOptions = r.sslOptions;
2389         _bind = r.bind;
2390         _headers = r.headers;
2391         _userHeaders = r.userHeaders;
2392 
2393         _params = r.params;
2394 
2395         // this assignments increments refCounts, so we can't use const Request
2396         // but Request is anyway struct and called by-value
2397         _cm = r.cm;
2398         _cookie = r.cookie;
2399 
2400         debug(requests) trace("serving %s".format(r));
2401         if ( !r.postData.empty)
2402         {
2403             return exec_from_range(r.postData);
2404         }
2405         if ( r.hasMultipartForm )
2406         {
2407             return exec_from_multipart_form(r.multipartForm);
2408         }
2409         auto rs = exec_from_parameters();
2410         return rs;
2411     }
2412 }
2413 
2414 version(vibeD) {
2415     import std.json;
2416     package string httpTestServer() {
2417         return "http://httpbin.org/";
2418     }
2419     package string fromJsonArrayToStr(JSONValue v) {
2420         return v.str;
2421     }
2422 }
2423 else {
2424     import std.json;
2425     package string httpTestServer() {
2426         return "http://127.0.0.1:8081/";
2427     }
2428     package string fromJsonArrayToStr(JSONValue v) {
2429         return cast(string)(v.array.map!"cast(ubyte)a.integer".array);
2430     }
2431 }