requests.http source code

1 module requests.http;
2 
3 private:
4 import std.algorithm;
5 import std.array;
6 import std.ascii;
7 import std.conv;
8 import std.datetime;
9 import std.exception;
10 import std.format;
11 import std.stdio;
12 import std.range;
13 import std.string;
14 import std.traits;
15 import std.typecons;
16 import std.bitmanip;
17 import std.experimental.logger;
18 import core.thread;
19 
20 import requests.streams;
21 import requests.uri;
22 import requests.utils;
23 import requests.base;
24 import requests.connmanager;
25 
26 static immutable ushort[] redirectCodes = [301, 302, 303, 307, 308];
27 static immutable uint     defaultBufferSize = 12*1024;
28 enum   HTTP11 = 101;
29 enum   HTTP10 = 100;
30 
31 static immutable string[string] proxies;
32 static this() {
33     import std.process;
34     proxies["http"] = environment.get("http_proxy", environment.get("HTTP_PROXY"));
35     proxies["https"] = environment.get("https_proxy", environment.get("HTTPS_PROXY"));
36     proxies["all"] = environment.get("all_proxy", environment.get("ALL_PROXY"));
37     foreach(p; proxies.byKey()) {
38         if (proxies[p] is null) {
39             continue;
40         }
41         URI u = URI(proxies[p]);
42     }
43 }
44 
45 public class MaxRedirectsException: Exception {
46     this(string message, string file = __FILE__, size_t line = __LINE__, Throwable next = null) @safe pure nothrow {
47         super(message, file, line, next);
48     }
49 }
50 
51 /**
52  * Basic authentication.
53  * Adds $(B Authorization: Basic) header to request.
54  */
55 public class BasicAuthentication: Auth {
56     private {
57         string   _username, _password;
58         string[] _domains;
59     }
60     /// Constructor.
61     /// Params:
62     /// username = username
63     /// password = password
64     /// domains = not used now
65     ///
66     this(string username, string password, string[] domains = []) {
67         _username = username;
68         _password = password;
69         _domains = domains;
70     }
71     override string[string] authHeaders(string domain) {
72         import std.base64;
73         string[string] auth;
74         auth["Authorization"] = "Basic " ~ to!string(Base64.encode(cast(ubyte[])"%s:%s".format(_username, _password)));
75         return auth;
76     }
77     override string userName() {
78         return _username;
79     }
80     override string password() {
81         return _password;
82     }
83 }
84 ///
85 ///
86 ///
87 public auto queryParams(T...)(T params) pure nothrow @safe
88 {
89     static assert (T.length % 2 == 0, "wrong args count");
90 
91     QueryParam[] output;
92     output.reserve = T.length / 2;
93 
94     void queryParamsHelper(T...)(T params, ref QueryParam[] output)
95     {
96         static if (T.length > 0)
97         {
98             output ~= QueryParam(params[0].to!string, params[1].to!string);
99             queryParamsHelper(params[2..$], output);
100         }
101     }
102 
103     queryParamsHelper(params, output);
104     return output;
105 }
106 
107 ///
108 /// Response - result of request execution.
109 ///
110 /// Response.code - response HTTP code.
111 /// Response.status_line - received HTTP status line.
112 /// Response.responseHeaders - received headers.
113 /// Response.responseBody - container for received body
114 /// Response.history - for redirected responses contain all history
115 ///
116 public class HTTPResponse : Response {
117     private {
118         string              _status_line;
119 
120         HTTPResponse[]      _history; // redirects history
121 
122         mixin(Setter!string("status_line"));
123 
124         int                 _version;
125     }
126 
127     ~this() {
128         _responseHeaders = null;
129         _history.length = 0;
130     }
131 
132     mixin(Getter!string("status_line"));
133     @property final string[string] responseHeaders() @safe @nogc nothrow {
134         return _responseHeaders;
135     }
136     @property final HTTPResponse[] history() @safe @nogc nothrow {
137         return _history;
138     }
139 
140     @property auto getStats() const pure @safe {
141         alias statTuple = Tuple!(Duration, "connectTime",
142                                  Duration, "sendTime",
143                                  Duration, "recvTime");
144         statTuple stat;
145         stat.connectTime = _connectedAt - _startedAt;
146         stat.sendTime = _requestSentAt - _connectedAt;
147         stat.recvTime = _finishedAt - _requestSentAt;
148         return stat;
149     }
150     private int parse_version(in string v) pure const nothrow @safe {
151         // try to parse HTTP/1.x to version
152         try if ( v.length > 5 ) {
153             return (v[5..$].split(".").map!"to!int(a)".array[0..2].reduce!((a,b) => a*100 + b));
154         } catch (Exception e) {
155         }
156         return 0;
157     }
158     unittest {
159         auto r = new HTTPResponse();
160         assert(r.parse_version("HTTP/1.1") == 101);
161         assert(r.parse_version("HTTP/1.0") == 100);
162         assert(r.parse_version("HTTP/0.9") == 9);
163         assert(r.parse_version("HTTP/xxx") == 0);
164     }
165 }
166 /**
167  * Struct to send multiple files in POST request.
168  */
169 public struct PostFile {
170     /// Path to the file to send.
171     string fileName;
172     /// Name of the field (if empty - send file base name)
173     string fieldName;
174     /// contentType of the file if not empty
175     string contentType;
176 }
177 ///
178 /// This is File-like interface for sending data to multipart fotms
179 ///
180 public interface FiniteReadable {
181     /// size of the content
182     abstract ulong  getSize();
183     /// file-like read()
184     abstract ubyte[] read();
185 }
186 ///
187 /// Helper to create form elements from File.
188 /// Params:
189 /// name = name of the field in form
190 /// f = opened std.stio.File to send to server
191 /// parameters = optional parameters (most important are "filename" and "Content-Type")
192 ///
193 public auto formData(string name, File f, string[string] parameters = null) {
194     return MultipartForm.FormData(name, new FormDataFile(f), parameters);
195 }
196 ///
197 /// Helper to create form elements from ubyte[].
198 /// Params:
199 /// name = name of the field in form
200 /// b = data to send to server
201 /// parameters = optional parameters (can be "filename" and "Content-Type")
202 ///
203 public auto formData(string name, ubyte[] b, string[string] parameters = null) {
204     return MultipartForm.FormData(name, new FormDataBytes(b), parameters);
205 }
206 public auto formData(string name, string b, string[string] parameters = null) {
207     return MultipartForm.FormData(name, new FormDataBytes(b.dup.representation), parameters);
208 }
209 public class FormDataBytes : FiniteReadable {
210     private {
211         ulong   _size;
212         ubyte[] _data;
213         size_t  _offset;
214         bool    _exhausted;
215     }
216     this(ubyte[] data) {
217         _data = data;
218         _size = data.length;
219     }
220     final override ulong getSize() {
221         return _size;
222     }
223     final override ubyte[] read() {
224         enforce( !_exhausted, "You can't read froum exhausted source" );
225         size_t toRead = min(defaultBufferSize, _size - _offset);
226         auto result = _data[_offset.._offset+toRead];
227         _offset += toRead;
228         if ( toRead == 0 ) {
229             _exhausted = true;
230         }
231         return result;
232     }
233 }
234 public class FormDataFile : FiniteReadable {
235     import  std.file;
236     private {
237         File    _fileHandle;
238         ulong   _fileSize;
239         size_t  _processed;
240         bool    _exhausted;
241     }
242     this(File file) {
243         import std.file;
244         _fileHandle = file;
245         _fileSize = std.file.getSize(file.name);
246     }
247     final override ulong getSize() pure nothrow @safe {
248         return _fileSize;
249     }
250     final override ubyte[] read() {
251         enforce( !_exhausted, "You can't read froum exhausted source" );
252         auto b = new ubyte[defaultBufferSize];
253         auto r = _fileHandle.rawRead(b);
254         auto toRead = min(r.length, _fileSize - _processed);
255         if ( toRead == 0 ) {
256             _exhausted = true;
257         }
258         _processed += toRead;
259         return r[0..toRead];
260     }
261 }
262 ///
263 /// This struct used to bulld POST's to forms.
264 /// Each part have name and data. data is something that can be read-ed and have size.
265 /// For example this can be string-like object (wrapped for reading) or opened File.
266 ///
267 public struct MultipartForm {
268     package struct FormData {
269         FiniteReadable  input;
270         string          name;
271         string[string]  parameters;
272         this(string name, FiniteReadable i, string[string] parameters = null) {
273             this.input = i;
274             this.name = name;
275             this.parameters = parameters;
276         }
277     }
278 
279     private FormData[] _sources;
280     auto add(FormData d) {
281         _sources ~= d;
282         return this;
283     }
284     auto add(string name, FiniteReadable i, string[string]parameters = null) {
285         _sources ~= FormData(name, i, parameters);
286         return this;
287     }
288 }
289 ///
290 
291 ///
292 /// Request.
293 /// Configurable parameters:
294 /// $(B method) - string, method to use (GET, POST, ...)
295 /// $(B headers) - string[string], add any additional headers you'd like to send.
296 /// $(B authenticator) - class Auth, class to send auth headers.
297 /// $(B keepAlive) - bool, set true for keepAlive requests. default true.
298 /// $(B maxRedirects) - uint, maximum number of redirects. default 10.
299 /// $(B maxHeadersLength) - size_t, maximum length of server response headers. default = 32KB.
300 /// $(B maxContentLength) - size_t, maximun content length. delault - 0 = unlimited.
301 /// $(B bufferSize) - size_t, send and receive buffer size. default = 16KB.
302 /// $(B verbosity) - uint, level of verbosity(0 - nothing, 1 - headers, 2 - headers and body progress). default = 0.
303 /// $(B proxy) - string, set proxy url if needed. default - null.
304 /// $(B cookie) - Tuple Cookie, Read/Write cookie You can get cookie setted by server, or set cookies before doing request.
305 /// $(B timeout) - Duration, Set timeout value for connect/receive/send.
306 ///
307 public struct HTTPRequest {
308     alias NetStrFactory = NetworkStream delegate(string, string, ushort);
309     private {
310         struct _UH {
311             // flags for each important header, added by user using addHeaders
312             mixin(bitfields!(
313                 bool, "Host", 1,
314                 bool, "UserAgent", 1,
315                 bool, "ContentLength", 1,
316                 bool, "Connection", 1,
317                 bool, "AcceptEncoding", 1,
318                 bool, "ContentType", 1,
319                 bool, "Cookie", 1,
320                 uint, "", 1
321             ));
322         }
323         string         _method = "GET";
324         URI            _uri;
325         string[string] _headers;
326         string[]       _filteredHeaders;
327         Auth           _authenticator;
328         bool           _keepAlive = true;
329         uint           _maxRedirects = 10;
330         size_t         _maxHeadersLength = 32 * 1024;   // 32 KB
331         size_t         _maxContentLength;               // 0 - Unlimited
332         string         _proxy;
333         uint           _verbosity = 0;                  // 0 - no output, 1 - headers, 2 - headers+body info
334         Duration       _timeout = 30.seconds;
335         size_t         _bufferSize = defaultBufferSize; // 16k
336         bool           _useStreaming;                   // return iterator instead of completed request
337 
338         string[URI]     _permanent_redirects;            // cache 301 redirects for GET requests
339         HTTPResponse[] _history;                        // redirects history
340         DataPipe!ubyte _bodyDecoder;
341         DecodeChunked  _unChunker;
342         long           _contentLength;
343         long           _contentReceived;
344         Cookie[]       _cookie;
345         SSLOptions     _sslOptions;
346         string         _bind;
347         _UH            _userHeaders;
348         ConnManager    _cm;
349         NetStrFactory  _socketFactory;
350     }
351     package HTTPResponse   _response;
352 
353     mixin(Getter_Setter!string     ("method"));
354     mixin(Getter_Setter!bool       ("keepAlive"));
355     mixin(Getter_Setter!size_t     ("maxContentLength"));
356     mixin(Getter_Setter!size_t     ("maxHeadersLength"));
357     mixin(Getter_Setter!size_t     ("bufferSize"));
358     mixin(Getter_Setter!uint       ("maxRedirects"));
359     mixin(Getter_Setter!uint       ("verbosity"));
360     mixin(Getter!string            ("proxy"));
361     mixin(Getter_Setter!Duration   ("timeout"));
362     mixin(Setter!Auth              ("authenticator"));
363     mixin(Getter_Setter!bool       ("useStreaming"));
364     mixin(Getter!long              ("contentLength"));
365     mixin(Getter!long              ("contentReceived"));
366     mixin(Getter_Setter!SSLOptions ("sslOptions"));
367     mixin(Getter_Setter!string     ("bind"));
368     mixin(Setter!NetStrFactory     ("socketFactory"));
369 
370     @property void sslSetVerifyPeer(bool v) pure @safe nothrow @nogc {
371         _sslOptions.setVerifyPeer(v);
372     }
373     @property void sslSetKeyFile(string p, SSLOptions.filetype t = SSLOptions.filetype.pem) pure @safe nothrow @nogc {
374         _sslOptions.setKeyFile(p, t);
375     }
376     @property void sslSetCertFile(string p, SSLOptions.filetype t = SSLOptions.filetype.pem) pure @safe nothrow @nogc {
377         _sslOptions.setCertFile(p, t);
378     }
379     @property void sslSetCaCert(string path) pure @safe nothrow @nogc {
380         _sslOptions.setCaCert(path);
381     }
382     @property final void cookie(Cookie[] s) pure @safe @nogc nothrow {
383         _cookie = s;
384     }
385     @property final void proxy(string v) {
386         if ( v != _proxy ) {
387             _cm.clear();
388         }
389         _proxy = v;
390     }
391     @property final Cookie[] cookie() pure @safe @nogc nothrow {
392         return _cookie;
393     }
394 
395     this(string uri) {
396         _uri = URI(uri);
397         _cm = new ConnManager;
398     }
399     ~this() {
400         _headers = null;
401         _authenticator = null;
402         _history = null;
403         _bodyDecoder = null;
404         _unChunker = null;
405         if ( _cm ) {
406             _cm.clear();
407         }
408     }
409     string toString() const {
410         return "HTTPRequest(%s, %s)".format(_method, _uri.uri());
411     }
412     string format(string fmt) const {
413         import std.array;
414         import std.stdio;
415         auto a = appender!string();
416         auto f = FormatSpec!char(fmt);
417         while (f.writeUpToNextSpec(a)) {
418             switch(f.spec) {
419                 case 'h':
420                     // Remote hostname.
421                     a.put(_uri.host);
422                     break;
423                 case 'm':
424                     // method.
425                     a.put(_method);
426                     break;
427                 case 'p':
428                     // Remote port.
429                     a.put("%d".format(_uri.port));
430                     break;
431                 case 'P':
432                     // Path
433                     a.put(_uri.path);
434                     break;
435                 case 'q':
436                     // query parameters supplied with url.
437                     a.put(_uri.query);
438                     break;
439                 case 'U':
440                     a.put(_uri.uri());
441                     break;
442                 default:
443                     throw new FormatException("Unknown Request format spec " ~ f.spec);
444             }
445         }
446         return a.data();
447     }
448     string select_proxy(string scheme) {
449         if ( _proxy is null && proxies.length == 0 ) {
450             debug(requests) tracef("proxy=null");
451             return null;
452         }
453         if ( _proxy ) {
454             debug(requests) tracef("proxy=%s", _proxy);
455             return _proxy;
456         }
457         auto p = scheme in proxies;
458         if ( p !is null && *p != "") {
459             debug(requests) tracef("proxy=%s", *p);
460             return *p;
461         }
462         p = "all" in proxies;
463         if ( p !is null && *p != "") {
464             debug(requests) tracef("proxy=%s", *p);
465             return *p;
466         }
467         debug(requests) tracef("proxy=null");
468         return null;
469     }
470     void clearHeaders() {
471         _headers = null;
472     }
473     @property void uri(in URI newURI) {
474         //handleURLChange(_uri, newURI);
475         _uri = newURI;
476     }
477     /// Add headers to request
478     /// Params:
479     /// headers = headers to send.
480     void addHeaders(in string[string] headers) {
481         foreach(pair; headers.byKeyValue) {
482             string _h = pair.key;
483             switch(toLower(_h)) {
484             case "host":
485                 _userHeaders.Host = true;
486                 break;
487             case "user-agent":
488                 _userHeaders.UserAgent = true;
489                 break;
490             case "content-length":
491                 _userHeaders.ContentLength = true;
492                 break;
493             case "content-type":
494                 _userHeaders.ContentType = true;
495                 break;
496             case "connection":
497                 _userHeaders.Connection = true;
498                 break;
499             case "cookie":
500                 _userHeaders.Cookie = true;
501                 break;
502             default:
503                 break;
504             }
505             _headers[pair.key] = pair.value;
506         }
507     }
508     private void safeSetHeader(ref string[string] headers, bool userAdded, string h, string v) pure @safe {
509         if ( !userAdded ) {
510             headers[h] = v;
511         }
512     }
513     /// Remove headers from request
514     /// Params:
515     /// headers = headers to remove.
516     void removeHeaders(in string[] headers) pure {
517         _filteredHeaders ~= headers;
518     }
519     ///
520     /// compose headers to send
521     ///
522     private string[string] requestHeaders() {
523 
524         string[string] generatedHeaders;
525 
526         if ( _authenticator ) {
527             _authenticator.
528                 authHeaders(_uri.host).
529                 byKeyValue.
530                 each!(pair => generatedHeaders[pair.key] = pair.value);
531         }
532 
533         _headers.byKey.each!(h => generatedHeaders[h] = _headers[h]);
534 
535         safeSetHeader(generatedHeaders, _userHeaders.AcceptEncoding, "Accept-Encoding", "gzip,deflate");
536         safeSetHeader(generatedHeaders, _userHeaders.UserAgent, "User-Agent", "dlang-requests");
537         safeSetHeader(generatedHeaders, _userHeaders.Connection, "Connection", _keepAlive?"Keep-Alive":"Close");
538 
539         if ( !_userHeaders.Host )
540         {
541             generatedHeaders["Host"] = _uri.host;
542             if ( _uri.scheme !in standard_ports || _uri.port != standard_ports[_uri.scheme] ) {
543                 generatedHeaders["Host"] ~= ":%d".format(_uri.port);
544             }
545         }
546 
547         if ( _cookie.length && !_userHeaders.Cookie ) {
548             auto cs = _cookie.
549                 filter!(c => _uri.path.pathMatches(c.path) && _uri.host.domainMatches(c.domain)).
550                 map!(c => "%s=%s".format(c.attr, c.value)).
551                 joiner(";");
552             if ( ! cs.empty )
553             {
554                 generatedHeaders["Cookie"] = to!string(cs);
555             }
556         }
557 
558         _filteredHeaders.each!(h => generatedHeaders.remove(h));
559 
560         return generatedHeaders;
561     }
562     ///
563     /// Build request string.
564     /// Handle proxy and query parameters.
565     ///
566     private @property string requestString(QueryParam[] params = null) {
567         auto query = _uri.query.dup;
568         if ( params ) {
569             query ~= "&" ~ params2query(params);
570             if ( query[0] != '?' ) {
571                 query = "?" ~ query;
572             }
573         }
574         string actual_proxy = select_proxy(_uri.scheme);
575         if ( actual_proxy && _uri.scheme != "https" ) {
576             return "%s %s%s HTTP/1.1\r\n".format(_method, _uri.uri(No.params), query);
577         }
578         return "%s %s%s HTTP/1.1\r\n".format(_method, _uri.path, query);
579     }
580     ///
581     /// encode parameters and build query part of the url
582     ///
583     private static string params2query(in QueryParam[] params) pure @safe {
584         return params.
585                 map!(a => "%s=%s".format(a.key.urlEncoded, a.value.urlEncoded)).
586                 join("&");
587     }
588     //
589     package unittest {
590         assert(params2query(queryParams("a","b", "c", " d "))=="a=b&c=%20d%20");
591     }
592     ///
593     /// Analyze received headers, take appropriate actions:
594     /// check content length, attach unchunk and uncompress
595     ///
596     private void analyzeHeaders(in string[string] headers) {
597 
598         _contentLength = -1;
599         _unChunker = null;
600         auto contentLength = "content-length" in headers;
601         if ( contentLength ) {
602             try {
603                 _contentLength = to!long(*contentLength);
604                 if ( _maxContentLength && _contentLength > _maxContentLength) {
605                     throw new RequestException("ContentLength > maxContentLength (%d>%d)".
606                                 format(_contentLength, _maxContentLength));
607                 }
608             } catch (ConvException e) {
609                 throw new RequestException("Can't convert Content-Length from %s".format(*contentLength));
610             }
611         }
612         auto transferEncoding = "transfer-encoding" in headers;
613         if ( transferEncoding ) {
614             debug(requests) tracef("transferEncoding: %s", *transferEncoding);
615             if ( (*transferEncoding).toLower == "chunked") {
616                 _unChunker = new DecodeChunked();
617                 _bodyDecoder.insert(_unChunker);
618             }
619         }
620         auto contentEncoding = "content-encoding" in headers;
621         if ( contentEncoding ) switch (*contentEncoding) {
622             default:
623                 throw new RequestException("Unknown content-encoding " ~ *contentEncoding);
624             case "gzip":
625             case "deflate":
626                 _bodyDecoder.insert(new Decompressor!ubyte);
627         }
628 
629     }
630     ///
631     /// Called when we know that all headers already received in buffer.
632     /// This routine does not interpret headers content (see analyzeHeaders).
633     /// 1. Split headers on lines
634     /// 2. store status line, store response code
635     /// 3. unfold headers if needed
636     /// 4. store headers
637     ///
638     private void parseResponseHeaders(in ubyte[] input) {
639         string lastHeader;
640         auto buffer = cast(string)input;
641 
642         foreach(line; buffer.split("\n").map!(l => l.stripRight)) {
643             if ( ! _response.status_line.length ) {
644                 debug (requests) tracef("statusLine: %s", line);
645                 _response.status_line = line;
646                 if ( _verbosity >= 1 ) {
647                     writefln("< %s", line);
648                 }
649                 auto parsed = line.split(" ");
650                 if ( parsed.length >= 2 ) {
651                     _response.code = parsed[1].to!ushort;
652                     _response._version = _response.parse_version(parsed[0]);
653                 }
654                 continue;
655             }
656             if ( line[0] == ' ' || line[0] == '\t' ) {
657                 // unfolding https://tools.ietf.org/html/rfc822#section-3.1
658                 if ( auto stored = lastHeader in _response._responseHeaders) {
659                     *stored ~= line;
660                 }
661                 continue;
662             }
663             auto parsed = line.findSplit(":");
664             auto header = parsed[0].toLower;
665             auto value = parsed[2].strip;
666 
667             if ( _verbosity >= 1 ) {
668                 writefln("< %s: %s", header, value);
669             }
670 
671             lastHeader = header;
672             debug (requests) tracef("Header %s = %s", header, value);
673 
674             if ( header != "set-cookie" ) {
675                 auto stored = _response.responseHeaders.get(header, null);
676                 if ( stored ) {
677                     value = stored ~ "," ~ value;
678                 }
679                 _response._responseHeaders[header] = value;
680                 continue;
681             }
682             _cookie ~= processCookie(value);
683         }
684     }
685 
686     ///
687     /// Process Set-Cookie header from server response
688     ///
689     private Cookie[] processCookie(string value ) pure {
690         // cookie processing
691         //
692         // as we can't join several set-cookie lines in single line
693         // < Set-Cookie: idx=7f2800f63c112a65ef5082957bcca24b; expires=Mon, 29-May-2017 00:31:25 GMT; path=/; domain=example.com
694         // < Set-Cookie: idx=7f2800f63c112a65ef5082957bcca24b; expires=Mon, 29-May-2017 00:31:25 GMT; path=/; domain=example.com, cs=ip764-RgKqc-HvSkxRxdQQAKW8LA; path=/; domain=.example.com; HttpOnly
695         //
696         Cookie[] res;
697         string[string] kv;
698         auto fields = value.split(";").map!strip;
699         while(!fields.empty) {
700             auto s = fields.front.findSplit("=");
701             fields.popFront;
702             if ( s[1] != "=" ) {
703                 continue;
704             }
705             auto k = s[0];
706             auto v = s[2];
707             switch(k.toLower()) {
708                 case "domain":
709                     k = "domain";
710                     break;
711                 case "path":
712                     k = "path";
713                     break;
714                 case "expires":
715                     continue;
716                 default:
717                     break;
718             }
719             kv[k] = v;
720         }
721         if ( "domain" !in kv ) {
722             kv["domain"] = _uri.host;
723         }
724         if ( "path" !in kv ) {
725             kv["path"] = _uri.path;
726         }
727         auto domain = kv["domain"]; kv.remove("domain");
728         auto path   = kv["path"];   kv.remove("path");
729         foreach(pair; kv.byKeyValue) {
730             auto _attr = pair.key;
731             auto _value = pair.value;
732             auto cookie = Cookie(path, domain, _attr, _value);
733             res ~= cookie;
734         }
735         return res;
736     }
737     ///
738     /// Do we received \r\n\r\n?
739     ///
740     private bool headersHaveBeenReceived(in ubyte[] data, ref Buffer!ubyte buffer, out string separator) const @safe {
741         foreach(s; ["\r\n\r\n", "\n\n"]) {
742             if ( data.canFind(s) || buffer.canFind(s) ) {
743                 separator = s;
744                 return true;
745             }
746         }
747         return false;
748     }
749 
750     private bool willFollowRedirect() {
751         if ( !canFind(redirectCodes, _response.code) ) {
752             return false;
753         }
754         if ( !_maxRedirects ) {
755             return false;
756         }
757         if ( "location" !in _response.responseHeaders ) {
758             return false;
759         }
760         return true;
761     }
762     private URI uriFromLocation(const ref URI uri, in string location) {
763         URI newURI = uri;
764         try {
765             newURI = URI(location);
766         } catch (UriException e) {
767             debug(requests) trace("Can't parse Location:, try relative uri");
768             newURI.path = location;
769             newURI.uri = newURI.recalc_uri;
770         }
771         return newURI;
772     }
773     ///
774     /// if we have new uri, then we need to check if we have to reopen existent connection
775     ///
776     private void checkURL(string url, string file=__FILE__, size_t line=__LINE__) {
777         if (url is null && _uri.uri == "" ) {
778             throw new RequestException("No url configured", file, line);
779         }
780 
781         if ( url !is null ) {
782             URI newURI = URI(url);
783             //handleURLChange(_uri, newURI);
784             _uri = newURI;
785         }
786     }
787     ///
788     /// Setup connection. Handle proxy and https case
789     ///
790     /// Place new connection in ConnManager cache
791     ///
792     private NetworkStream setupConnection()
793     do {
794 
795         debug(requests) tracef("Set up new connection");
796         NetworkStream stream;
797 
798         // on exit
799         // place created connection to conn. manager
800         // close connection purged from manager (if any)
801         //
802         scope(exit) {
803             if ( stream )
804             {
805                 if ( auto purged_connection = _cm.put(_uri.scheme, _uri.host, _uri.port, stream) )
806                 {
807                     debug(requests) tracef("closing purged connection %s", purged_connection);
808                     purged_connection.close();
809                 }
810             }
811         }
812 
813         if ( _socketFactory )
814         {
815             debug(requests) tracef("use socketFactory");
816             stream = _socketFactory(_uri.scheme, _uri.host, _uri.port);
817         }
818 
819         if ( stream ) // socket factory created connection
820         {
821             return stream;
822         }
823 
824         URI   uri; // this URI will be used temporarry if we need proxy
825         string actual_proxy = select_proxy(_uri.scheme);
826         final switch (_uri.scheme) {
827             case"http":
828                 if ( actual_proxy ) {
829                     uri.uri_parse(actual_proxy);
830                     uri.idn_encode();
831                 } else {
832                     // use original uri
833                     uri = _uri;
834                 }
835                 stream = new TCPStream();
836                 stream.bind(_bind);
837                 stream.connect(uri.host, uri.port, _timeout);
838                 break ;
839             case"https":
840                 if ( actual_proxy ) {
841                     uri.uri_parse(actual_proxy);
842                     uri.idn_encode();
843                     stream = new TCPStream();
844                     stream.bind(_bind);
845                     stream.connect(uri.host, uri.port, _timeout);
846                     if ( verbosity>=1 ) {
847                         writeln("> CONNECT %s:%d HTTP/1.1".format(_uri.host, _uri.port));
848                     }
849                     stream.send("CONNECT %s:%d HTTP/1.1\r\n\r\n".format(_uri.host, _uri.port));
850                     while ( stream.isConnected ) {
851                         ubyte[1024] b;
852                         auto read = stream.receive(b);
853                         if ( verbosity>=1) {
854                             writefln("< %s", cast(string)b[0..read]);
855                         }
856                         debug(requests) tracef("read: %d", read);
857                         if ( b[0..read].canFind("\r\n\r\n") || b[0..read].canFind("\n\n") ) {
858                             debug(requests) tracef("proxy connection ready");
859                             // convert connection to ssl
860                             stream = new SSLStream(stream, _sslOptions, _uri.host);
861                             break ;
862                         } else {
863                             debug(requests) tracef("still wait for proxy connection");
864                         }
865                     }
866                 } else {
867                     uri = _uri;
868                     stream = new SSLStream(_sslOptions);
869                     stream.bind(_bind);
870                     stream.connect(uri.host, uri.port, _timeout);
871                     debug(requests) tracef("ssl connection to origin server ready");
872                 }
873                 break ;
874         }
875 
876         return stream;
877     }
878     ///
879     /// Request sent, now receive response.
880     /// Find headers, split on headers and body, continue to receive body
881     ///
882     private void receiveResponse(NetworkStream _stream) {
883 
884         try {
885             _stream.readTimeout = timeout;
886         } catch (Exception e) {
887             debug(requests) tracef("Failed to set read timeout for stream: %s", e.msg);
888             return;
889         }
890         // Commented this out as at exit we can have alreade closed socket
891         // scope(exit) {
892         //     if ( _stream && _stream.isOpen ) {
893         //         _stream.readTimeout = 0.seconds;
894         //     }
895         // }
896 
897         _bodyDecoder = new DataPipe!ubyte();
898         scope(exit) {
899             if ( !_useStreaming ) {
900                 _bodyDecoder = null;
901                 _unChunker = null;
902             }
903         }
904 
905         auto buffer = Buffer!ubyte();
906         Buffer!ubyte partialBody;
907         ptrdiff_t read;
908         string separator;
909 
910         while(true) {
911 
912             auto b = new ubyte[_bufferSize];
913             read = _stream.receive(b);
914 
915             debug(requests) tracef("read: %d", read);
916             if ( read == 0 ) {
917                 break;
918             }
919             auto data = b[0..read];
920             buffer.putNoCopy(data);
921             if ( verbosity>=3 ) {
922                 writeln(data.dump.join("\n"));
923             }
924 
925             if ( buffer.length > maxHeadersLength ) {
926                 throw new RequestException("Headers length > maxHeadersLength (%d > %d)".format(buffer.length, maxHeadersLength));
927             }
928             if ( headersHaveBeenReceived(data, buffer, separator) ) {
929                 auto s = buffer.data.findSplit(separator);
930                 auto ResponseHeaders = s[0];
931                 partialBody = Buffer!ubyte(s[2]);
932                 _contentReceived += partialBody.length;
933                 parseResponseHeaders(ResponseHeaders);
934                 break;
935             }
936         }
937 
938         analyzeHeaders(_response._responseHeaders);
939 
940         _bodyDecoder.putNoCopy(partialBody.data);
941 
942         auto v = _bodyDecoder.get();
943         _response._responseBody.putNoCopy(v);
944 
945         if ( _verbosity >= 2 ) writefln("< %d bytes of body received", partialBody.length);
946 
947         if ( _method == "HEAD" ) {
948             // HEAD response have ContentLength, but have no body
949             return;
950         }
951 
952         while( true ) {
953             if ( _contentLength >= 0 && _contentReceived >= _contentLength ) {
954                 debug(requests) trace("Body received.");
955                 break;
956             }
957             if ( _unChunker && _unChunker.done ) {
958                 break;
959             }
960 
961             if ( _useStreaming && _response._responseBody.length && !redirectCodes.canFind(_response.code) ) {
962                 debug(requests) trace("streaming requested");
963                 // save _stream in closure
964                 auto stream = _stream;
965                 // set up response
966                 _response.receiveAsRange.activated = true;
967                 _response.receiveAsRange.data = _response._responseBody.data;
968                 _response.receiveAsRange.read = delegate ubyte[] () {
969 
970                     while(true) {
971                         // check if we received everything we need
972                         if ( ( _unChunker && _unChunker.done )
973                             || !stream.isConnected()
974                             || (_contentLength > 0 && _contentReceived >= _contentLength) )
975                         {
976                             debug(requests) trace("streaming_in receive completed");
977                             _bodyDecoder.flush();
978                             return _bodyDecoder.get();
979                         }
980                         // have to continue
981                         auto b = new ubyte[_bufferSize];
982                         try {
983                             read = stream.receive(b);
984                         }
985                         catch (Exception e) {
986                             throw new RequestException("streaming_in error reading from socket", __FILE__, __LINE__, e);
987                         }
988                         debug(requests) tracef("streaming_in received %d bytes", read);
989 
990                         if ( read == 0 ) {
991                             debug(requests) tracef("streaming_in: server closed connection");
992                             _bodyDecoder.flush();
993                             return _bodyDecoder.get();
994                         }
995 
996                         if ( verbosity>=3 ) {
997                             writeln(b[0..read].dump.join("\n"));
998                         }
999 
1000                         _contentReceived += read;
1001                         _bodyDecoder.putNoCopy(b[0..read]);
1002                         auto res = _bodyDecoder.getNoCopy();
1003                         if ( res.length == 0 ) {
1004                             // there were nothing to produce (beginning of the chunk or no decompressed data)
1005                             continue;
1006                         }
1007                         if (res.length == 1) {
1008                             return res[0];
1009                         }
1010                         //
1011                         // I'd like to "return _bodyDecoder.getNoCopy().join;" but it is slower
1012                         //
1013                         auto total = res.map!(b=>b.length).sum;
1014                         // create buffer for joined bytes
1015                         ubyte[] joined = new ubyte[total];
1016                         size_t p;
1017                         // memcopy
1018                         foreach(ref _; res) {
1019                             joined[p .. p + _.length] = _;
1020                             p += _.length;
1021                         }
1022                         return joined;
1023                     }
1024                     assert(0);
1025                 };
1026                 // we prepared for streaming
1027                 return;
1028             }
1029 
1030             auto b = new ubyte[_bufferSize];
1031             read = _stream.receive(b);
1032 
1033             if ( read == 0 ) {
1034                 debug(requests) trace("read done");
1035                 break;
1036             }
1037             if ( _verbosity >= 2 ) {
1038                 writefln("< %d bytes of body received", read);
1039             }
1040 
1041             if ( verbosity>=3 ) {
1042                 writeln(b[0..read].dump.join("\n"));
1043             }
1044 
1045             debug(requests) tracef("read: %d", read);
1046             _contentReceived += read;
1047             if ( _maxContentLength && _contentReceived > _maxContentLength ) {
1048                 throw new RequestException("ContentLength > maxContentLength (%d>%d)".
1049                     format(_contentLength, _maxContentLength));
1050             }
1051 
1052             _bodyDecoder.putNoCopy(b[0..read]); // send buffer to all decoders
1053 
1054             _bodyDecoder.getNoCopy.             // fetch result and place to body
1055                 each!(b => _response._responseBody.putNoCopy(b));
1056 
1057             debug(requests) tracef("receivedTotal: %d, contentLength: %d, bodyLength: %d", _contentReceived, _contentLength, _response._responseBody.length);
1058 
1059         }
1060         _bodyDecoder.flush();
1061         _response._responseBody.putNoCopy(_bodyDecoder.get());
1062     }
1063     ///
1064     /// Check that we received anything.
1065     /// Server can close previous connection (keepalive or not)
1066     ///
1067     private bool serverPrematurelyClosedConnection() pure @safe {
1068         immutable server_closed_connection = _response._responseHeaders.length == 0 && _response._status_line.length == 0;
1069         // debug(requests) tracef("server closed connection = %s (headers.length=%s, status_line.length=%s)",
1070         //     server_closed_connection, _response._responseHeaders.length,  _response._status_line.length);
1071         return server_closed_connection;
1072     }
1073     private bool isIdempotent(in string method) pure @safe nothrow {
1074         return ["GET", "HEAD"].canFind(method);
1075     }
1076     ///
1077     /// If we do not want keepalive request,
1078     /// or server signalled to close connection,
1079     /// then close it
1080     ///
1081     void close_connection_if_not_keepalive(NetworkStream _stream) {
1082         auto connection = "connection" in _response._responseHeaders;
1083         if ( !_keepAlive ) {
1084             _stream.close();
1085         } else switch(_response._version) {
1086             case HTTP11:
1087                 // HTTP/1.1 defines the "close" connection option for the sender to signal that the connection
1088                 // will be closed after completion of the response. For example,
1089                 //        Connection: close
1090                 // in either the request or the response header fields indicates that the connection
1091                 // SHOULD NOT be considered `persistent' (section 8.1) after the current request/response is complete.
1092                 // HTTP/1.1 applications that do not support persistent connections MUST include the "close" connection
1093                 // option in every message.
1094                 if ( connection && (*connection).toLower.split(",").canFind("close") ) {
1095                     _stream.close();
1096                 }
1097                 break;
1098             default:
1099                 // for anything else close connection if there is no keep-alive in Connection
1100                 if ( connection && !(*connection).toLower.split(",").canFind("keep-alive") ) {
1101                     _stream.close();
1102                 }
1103                 break;
1104         }
1105     }
1106     ///
1107     /// Send multipart for request.
1108     /// You would like to use this method for sending large portions of mixed data or uploading files to forms.
1109     /// Content of the posted form consist of sources. Each source have at least name and value (can be string-like object or opened file, see more docs for MultipartForm struct)
1110     /// Params:
1111     ///     url = url
1112     ///     sources = array of sources.
1113     HTTPResponse exec(string method="POST")(string url, MultipartForm sources) {
1114         import std.uuid;
1115         import std.file;
1116 
1117         checkURL(url);
1118         if ( _cm is null ) {
1119             _cm = new ConnManager();
1120         }
1121 
1122         NetworkStream _stream;
1123         _method = method;
1124         _response = new HTTPResponse;
1125         _response.uri = _uri;
1126         _response.finalURI = _uri;
1127         bool restartedRequest = false;
1128 
1129     connect:
1130         _contentReceived = 0;
1131         _response._startedAt = Clock.currTime;
1132 
1133         assert(_stream is null);
1134 
1135         _stream = _cm.get(_uri.scheme, _uri.host, _uri.port);
1136 
1137         if ( _stream is null ) {
1138             debug(requests) trace("create new connection");
1139             _stream = setupConnection();
1140         } else {
1141             debug(requests) trace("reuse old connection");
1142         }
1143 
1144         assert(_stream !is null);
1145 
1146         if ( !_stream.isConnected ) {
1147             debug(requests) trace("disconnected stream on enter");
1148             if ( !restartedRequest ) {
1149                 debug(requests) trace("disconnected stream on enter: retry");
1150                 assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
1151 
1152                 _cm.del(_uri.scheme, _uri.host, _uri.port);
1153                 _stream.close();
1154                 _stream = null;
1155 
1156                 restartedRequest = true;
1157                 goto connect;
1158             }
1159             debug(requests) trace("disconnected stream on enter: return response");
1160             //_stream = null;
1161             return _response;
1162         }
1163         _response._connectedAt = Clock.currTime;
1164 
1165         Appender!string req;
1166         req.put(requestString());
1167 
1168         string   boundary = randomUUID().toString;
1169         string[] partHeaders;
1170         size_t   contentLength;
1171 
1172         foreach(ref part; sources._sources) {
1173             string h = "--" ~ boundary ~ "\r\n";
1174             string disposition = `form-data; name="%s"`.format(part.name);
1175             string optionals = part.
1176                 parameters.byKeyValue().
1177                 filter!(p => p.key!="Content-Type").
1178                 map!   (p => "%s=%s".format(p.key, p.value)).
1179                 join("; ");
1180 
1181             h ~= `Content-Disposition: ` ~ [disposition, optionals].join("; ") ~ "\r\n";
1182 
1183             auto contentType = "Content-Type" in part.parameters;
1184             if ( contentType ) {
1185                 h ~= "Content-Type: " ~ *contentType ~ "\r\n";
1186             }
1187 
1188             h ~= "\r\n";
1189             partHeaders ~= h;
1190             contentLength += h.length + part.input.getSize() + "\r\n".length;
1191         }
1192         contentLength += "--".length + boundary.length + "--\r\n".length;
1193 
1194         auto h = requestHeaders();
1195         safeSetHeader(h, _userHeaders.ContentType, "Content-Type", "multipart/form-data; boundary=" ~ boundary);
1196         safeSetHeader(h, _userHeaders.ContentLength, "Content-Length", to!string(contentLength));
1197 
1198         h.byKeyValue.
1199             map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n").
1200                 each!(h => req.put(h));
1201         req.put("\r\n");
1202 
1203         debug(requests) trace(req.data);
1204         if ( _verbosity >= 1 ) req.data.splitLines.each!(a => writeln("> " ~ a));
1205 
1206         try {
1207             _stream.send(req.data());
1208             foreach(ref source; sources._sources) {
1209                 debug(requests) tracef("sending part headers <%s>", partHeaders.front);
1210                 _stream.send(partHeaders.front);
1211                 partHeaders.popFront;
1212                 while (true) {
1213                     auto chunk = source.input.read();
1214                     if ( chunk.length <= 0 ) {
1215                         break;
1216                     }
1217                     _stream.send(chunk);
1218                 }
1219                 _stream.send("\r\n");
1220             }
1221             _stream.send("--" ~ boundary ~ "--\r\n");
1222             _response._requestSentAt = Clock.currTime;
1223             receiveResponse(_stream);
1224             _response._finishedAt = Clock.currTime;
1225         }
1226         catch (NetworkException e) {
1227             errorf("Error sending request: ", e.msg);
1228             _stream.close();
1229             return _response;
1230         }
1231 
1232         if ( serverPrematurelyClosedConnection()
1233         && !restartedRequest
1234         && isIdempotent(_method)
1235         ) {
1236             ///
1237             /// We didn't receive any data (keepalive connectioin closed?)
1238             /// and we can restart this request.
1239             /// Go ahead.
1240             ///
1241             debug(requests) tracef("Server closed keepalive connection");
1242 
1243             assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
1244 
1245             _cm.del(_uri.scheme, _uri.host, _uri.port);
1246             _stream.close();
1247             _stream = null;
1248 
1249             restartedRequest = true;
1250             goto connect;
1251         }
1252 
1253         if ( _useStreaming ) {
1254             if ( _response._receiveAsRange.activated ) {
1255                 debug(requests) trace("streaming_in activated");
1256                 return _response;
1257             } else {
1258                 // this can happen if whole response body received together with headers
1259                 _response._receiveAsRange.data = _response.responseBody.data;
1260             }
1261         }
1262 
1263         close_connection_if_not_keepalive(_stream);
1264 
1265         if ( _verbosity >= 1 ) {
1266             writeln(">> Connect time: ", _response._connectedAt - _response._startedAt);
1267             writeln(">> Request send time: ", _response._requestSentAt - _response._connectedAt);
1268             writeln(">> Response recv time: ", _response._finishedAt - _response._requestSentAt);
1269         }
1270 
1271         if ( willFollowRedirect ) {
1272             if ( _history.length >= _maxRedirects ) {
1273                 _stream = null;
1274                 throw new MaxRedirectsException("%d redirects reached maxRedirects %d.".format(_history.length, _maxRedirects));
1275             }
1276             // "location" in response already checked in canFollowRedirect
1277             immutable new_location = *("location" in _response.responseHeaders);
1278             immutable current_uri = _uri, next_uri = uriFromLocation(_uri, new_location);
1279 
1280             // save current response for history
1281             _history ~= _response;
1282 
1283             // prepare new response (for redirected request)
1284             _response = new HTTPResponse;
1285             _response.uri = current_uri;
1286             _response.finalURI = next_uri;
1287             _stream = null;
1288 
1289             // set new uri
1290             this._uri = next_uri;
1291             debug(requests) tracef("Redirected to %s", next_uri);
1292             if ( _method != "GET" && _response.code != 307 && _response.code != 308 ) {
1293                 // 307 and 308 do not change method
1294                 return this.get();
1295             }
1296             if ( restartedRequest ) {
1297                 debug(requests) trace("Rare event: clearing 'restartedRequest' on redirect");
1298                 restartedRequest = false;
1299             }
1300             goto connect;
1301         }
1302 
1303         _response._history = _history;
1304         return _response;
1305     }
1306 
1307     // we use this if we send from ubyte[][] and user provided Content-Length
1308     private void sendFlattenContent(T)(NetworkStream _stream, T content) {
1309         while ( !content.empty ) {
1310             auto chunk = content.front;
1311             _stream.send(chunk);
1312             content.popFront;
1313         }
1314         debug(requests) tracef("sent");
1315     }
1316     // we use this if we send from ubyte[][] as chunked content
1317     private void sendChunkedContent(T)(NetworkStream _stream, T content) {
1318         while ( !content.empty ) {
1319             auto chunk = content.front;
1320             auto chunkHeader = "%x\r\n".format(chunk.length);
1321             debug(requests) tracef("sending %s%s", chunkHeader, chunk);
1322             _stream.send(chunkHeader);
1323             _stream.send(chunk);
1324             _stream.send("\r\n");
1325             content.popFront;
1326         }
1327         debug(requests) tracef("sent");
1328         _stream.send("0\r\n\r\n");
1329     }
1330     ///
1331     /// POST/PUT/... data from some string(with Content-Length), or from range of strings/bytes (use Transfer-Encoding: chunked).
1332     /// When rank 1 (flat array) used as content it must have length. In that case "content" will be sent directly to network, and Content-Length headers will be added.
1333     /// If you are goung to send some range and do not know length at the moment when you start to send request, then you can send chunks of chars or ubyte.
1334     /// Try not to send too short chunks as this will put additional load on client and server. Chunks of length 2048 or 4096 are ok.
1335     ///
1336     /// Parameters:
1337     ///    url = url
1338     ///    content = string or input range
1339     ///    contentType = content type
1340     ///  Returns:
1341     ///     Response
1342     ///  Examples:
1343     ///  ---------------------------------------------------------------------------------------------------------
1344     ///      rs = rq.exec!"POST"("http://httpbin.org/post", "привiт, свiт!", "application/octet-stream");
1345     ///
1346     ///      auto s = lineSplitter("one,\ntwo,\nthree.");
1347     ///      rs = rq.exec!"POST"("http://httpbin.org/post", s, "application/octet-stream");
1348     ///
1349     ///      auto s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
1350     ///      rs = rq.exec!"POST"("http://httpbin.org/post", s.representation.chunks(10), "application/octet-stream");
1351     ///
1352     ///      auto f = File("tests/test.txt", "rb");
1353     ///      rs = rq.exec!"POST"("http://httpbin.org/post", f.byChunk(3), "application/octet-stream");
1354     ///  --------------------------------------------------------------------------------------------------------
1355     HTTPResponse exec(string method="POST", R)(string url, R content, string contentType="application/octet-stream")
1356         if ( (rank!R == 1)
1357             || (rank!R == 2 && isSomeChar!(Unqual!(typeof(content.front.front))))
1358             || (rank!R == 2 && (is(Unqual!(typeof(content.front.front)) == ubyte)))
1359         )
1360     do {
1361         debug(requests) tracef("started url=%s, this._uri=%s", url, _uri);
1362 
1363         checkURL(url);
1364         if ( _cm is null ) {
1365             _cm = new ConnManager();
1366         }
1367 
1368         NetworkStream _stream;
1369         _method = method;
1370         _response = new HTTPResponse;
1371         _history.length = 0;
1372         _response.uri = _uri;
1373         _response.finalURI = _uri;
1374         bool restartedRequest = false;
1375         bool send_flat;
1376 
1377     connect:
1378         _contentReceived = 0;
1379         _response._startedAt = Clock.currTime;
1380 
1381         assert(_stream is null);
1382 
1383         _stream = _cm.get(_uri.scheme, _uri.host, _uri.port);
1384 
1385         if ( _stream is null ) {
1386             debug(requests) trace("create new connection");
1387             _stream = setupConnection();
1388         } else {
1389             debug(requests) trace("reuse old connection");
1390         }
1391 
1392         assert(_stream !is null);
1393 
1394         if ( !_stream.isConnected ) {
1395             debug(requests) trace("disconnected stream on enter");
1396             if ( !restartedRequest ) {
1397                 debug(requests) trace("disconnected stream on enter: retry");
1398                 assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
1399 
1400                 _cm.del(_uri.scheme, _uri.host, _uri.port);
1401                 _stream.close();
1402                 _stream = null;
1403 
1404                 restartedRequest = true;
1405                 goto connect;
1406             }
1407             debug(requests) trace("disconnected stream on enter: return response");
1408             //_stream = null;
1409             return _response;
1410         }
1411         _response._connectedAt = Clock.currTime;
1412 
1413         Appender!string req;
1414         req.put(requestString());
1415 
1416         auto h = requestHeaders;
1417         if ( contentType ) {
1418             safeSetHeader(h, _userHeaders.ContentType, "Content-Type", contentType);
1419         }
1420         static if ( rank!R == 1 ) {
1421             safeSetHeader(h, _userHeaders.ContentLength, "Content-Length", to!string(content.length));
1422         } else {
1423             if ( _userHeaders.ContentLength ) {
1424                 debug(requests) tracef("User provided content-length for chunked content");
1425                 send_flat = true;
1426             } else {
1427                 h["Transfer-Encoding"] = "chunked";
1428                 send_flat = false;
1429             }
1430         }
1431         h.byKeyValue.
1432             map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n").
1433             each!(h => req.put(h));
1434         req.put("\r\n");
1435 
1436         debug(requests) trace(req.data);
1437         if ( _verbosity >= 1 ) {
1438             req.data.splitLines.each!(a => writeln("> " ~ a));
1439         }
1440 
1441         try {
1442             // send headers
1443             _stream.send(req.data());
1444             // send body
1445             static if ( rank!R == 1) {
1446                 _stream.send(content);
1447             } else {
1448                 if ( send_flat ) {
1449                     sendFlattenContent(_stream, content);
1450                 } else {
1451                     sendChunkedContent(_stream, content);
1452                 }
1453             }
1454             _response._requestSentAt = Clock.currTime;
1455             debug(requests) trace("starting receive response");
1456             receiveResponse(_stream);
1457             debug(requests) trace("finished receive response");
1458             _response._finishedAt = Clock.currTime;
1459         } catch (NetworkException e) {
1460             _stream.close();
1461             throw new RequestException("Network error during data exchange");
1462         }
1463 
1464         if ( serverPrematurelyClosedConnection()
1465         && !restartedRequest
1466         && isIdempotent(_method)
1467         ) {
1468             ///
1469             /// We didn't receive any data (keepalive connectioin closed?)
1470             /// and we can restart this request.
1471             /// Go ahead.
1472             ///
1473             debug(requests) tracef("Server closed keepalive connection");
1474 
1475             assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
1476 
1477             _cm.del(_uri.scheme, _uri.host, _uri.port);
1478             _stream.close();
1479             _stream = null;
1480 
1481             restartedRequest = true;
1482             goto connect;
1483         }
1484 
1485         if ( _useStreaming ) {
1486             if ( _response._receiveAsRange.activated ) {
1487                 debug(requests) trace("streaming_in activated");
1488                 return _response;
1489             } else {
1490                 // this can happen if whole response body received together with headers
1491                 _response._receiveAsRange.data = _response.responseBody.data;
1492             }
1493         }
1494 
1495         close_connection_if_not_keepalive(_stream);
1496 
1497         if ( _verbosity >= 1 ) {
1498             writeln(">> Connect time: ", _response._connectedAt - _response._startedAt);
1499             writeln(">> Request send time: ", _response._requestSentAt - _response._connectedAt);
1500             writeln(">> Response recv time: ", _response._finishedAt - _response._requestSentAt);
1501         }
1502 
1503 
1504         if ( willFollowRedirect ) {
1505             if ( _history.length >= _maxRedirects ) {
1506                 _stream = null;
1507                 throw new MaxRedirectsException("%d redirects reached maxRedirects %d.".format(_history.length, _maxRedirects));
1508             }
1509             // "location" in response already checked in canFollowRedirect
1510             immutable new_location = *("location" in _response.responseHeaders);
1511             immutable current_uri = _uri, next_uri = uriFromLocation(_uri, new_location);
1512 
1513             // save current response for history
1514             _history ~= _response;
1515 
1516             // prepare new response (for redirected request)
1517             _response = new HTTPResponse;
1518             _response.uri = current_uri;
1519             _response.finalURI = next_uri;
1520 
1521             _stream = null;
1522 
1523             // set new uri
1524             this._uri = next_uri;
1525             debug(requests) tracef("Redirected to %s", next_uri);
1526             if ( _method != "GET" && _response.code != 307 && _response.code != 308 ) {
1527                 // 307 and 308 do not change method
1528                 return this.get();
1529             }
1530             if ( restartedRequest ) {
1531                 debug(requests) trace("Rare event: clearing 'restartedRequest' on redirect");
1532                 restartedRequest = false;
1533             }
1534             goto connect;
1535         }
1536 
1537         _response._history = _history;
1538         return _response;
1539     }
1540     ///
1541     /// Send request with parameters.
1542     /// If used for POST or PUT requests then application/x-www-form-urlencoded used.
1543     /// Request parameters will be encoded into request string or placed in request body for POST/PUT
1544     /// requests.
1545     /// Parameters:
1546     ///     url = url
1547     ///     params = request parameters
1548     ///  Returns:
1549     ///     Response
1550     ///  Examples:
1551     ///  ---------------------------------------------------------------------------------
1552     ///     rs = Request().exec!"GET"("http://httpbin.org/get", ["c":"d", "a":"b"]);
1553     ///  ---------------------------------------------------------------------------------
1554     ///
1555     HTTPResponse exec(string method="GET")(string url = null, QueryParam[] params = null)
1556     do {
1557         debug(requests) tracef("started url=%s, this._uri=%s", url, _uri);
1558 
1559         checkURL(url);
1560         if ( _cm is null ) {
1561             _cm = new ConnManager();
1562         }
1563 
1564         NetworkStream _stream;
1565         _method = method;
1566         _response = new HTTPResponse;
1567         _history.length = 0;
1568         _response.uri = _uri;
1569         _response.finalURI = _uri;
1570         bool restartedRequest = false; // True if this is restarted keepAlive request
1571 
1572     connect:
1573         if ( _method == "GET" && _uri in _permanent_redirects ) {
1574             debug(requests) trace("use parmanent redirects cache");
1575             _uri = uriFromLocation(_uri, _permanent_redirects[_uri]);
1576             _response._finalURI = _uri;
1577         }
1578         _contentReceived = 0;
1579         _response._startedAt = Clock.currTime;
1580 
1581         assert(_stream is null);
1582 
1583         _stream = _cm.get(_uri.scheme, _uri.host, _uri.port);
1584 
1585         if ( _stream is null ) {
1586             debug(requests) trace("create new connection");
1587             _stream = setupConnection();
1588         } else {
1589             debug(requests) trace("reuse old connection");
1590         }
1591 
1592         assert(_stream !is null);
1593 
1594         if ( !_stream.isConnected ) {
1595             debug(requests) trace("disconnected stream on enter");
1596             if ( !restartedRequest ) {
1597                 debug(requests) trace("disconnected stream on enter: retry");
1598                 assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
1599 
1600                 _cm.del(_uri.scheme, _uri.host, _uri.port);
1601                 _stream.close();
1602                 _stream = null;
1603 
1604                 restartedRequest = true;
1605                 goto connect;
1606             }
1607             debug(requests) trace("disconnected stream on enter: return response");
1608             //_stream = null;
1609             return _response;
1610         }
1611         _response._connectedAt = Clock.currTime;
1612 
1613         auto h = requestHeaders();
1614 
1615         Appender!string req;
1616 
1617         string encoded;
1618 
1619         switch (_method) {
1620             case "POST","PUT":
1621                 encoded = params2query(params);
1622                 safeSetHeader(h, _userHeaders.ContentType, "Content-Type", "application/x-www-form-urlencoded");
1623                 if ( encoded.length > 0) {
1624                     safeSetHeader(h, _userHeaders.ContentLength, "Content-Length", to!string(encoded.length));
1625                 }
1626                 req.put(requestString());
1627                 break;
1628             default:
1629                 req.put(requestString(params));
1630         }
1631 
1632         h.byKeyValue.
1633             map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n").
1634             each!(h => req.put(h));
1635         req.put("\r\n");
1636         if ( encoded ) {
1637             req.put(encoded);
1638         }
1639 
1640         debug(requests) trace(req.data);
1641         if ( _verbosity >= 1 ) {
1642             req.data.splitLines.each!(a => writeln("> " ~ a));
1643         }
1644         //
1645         // Now send request and receive response
1646         //
1647         try {
1648             _stream.send(req.data());
1649             _response._requestSentAt = Clock.currTime;
1650             debug(requests) trace("starting receive response");
1651             receiveResponse(_stream);
1652             debug(requests) trace("done receive response");
1653             _response._finishedAt = Clock.currTime;
1654         }
1655         catch (NetworkException e) {
1656             // On SEND this can means:
1657             // we started to send request to the server, but it closed connection because of keepalive timeout.
1658             // We have to restart request if possible.
1659 
1660             // On RECEIVE - if we received something - then this exception is real and unexpected error.
1661             // If we didn't receive anything - we can restart request again as it can be
1662             debug(requests) tracef("Exception on receive response: %s", e.msg);
1663             if ( _response._responseHeaders.length != 0 )
1664             {
1665                 _stream.close();
1666                 throw new RequestException("Unexpected network error");
1667             }
1668         }
1669 
1670         if ( serverPrematurelyClosedConnection()
1671             && !restartedRequest
1672             && isIdempotent(_method)
1673             ) {
1674             ///
1675             /// We didn't receive any data (keepalive connectioin closed?)
1676             /// and we can restart this request.
1677             /// Go ahead.
1678             ///
1679             debug(requests) tracef("Server closed keepalive connection");
1680 
1681             assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
1682 
1683             _cm.del(_uri.scheme, _uri.host, _uri.port);
1684             _stream.close();
1685             _stream = null;
1686 
1687             restartedRequest = true;
1688             goto connect;
1689         }
1690 
1691         if ( _useStreaming ) {
1692             if ( _response._receiveAsRange.activated ) {
1693                 debug(requests) trace("streaming_in activated");
1694                 return _response;
1695             } else {
1696                 // this can happen if whole response body received together with headers
1697                 _response._receiveAsRange.data = _response.responseBody.data;
1698             }
1699         }
1700 
1701         close_connection_if_not_keepalive(_stream);
1702 
1703         if ( _verbosity >= 1 ) {
1704             writeln(">> Connect time: ", _response._connectedAt - _response._startedAt);
1705             writeln(">> Request send time: ", _response._requestSentAt - _response._connectedAt);
1706             writeln(">> Response recv time: ", _response._finishedAt - _response._requestSentAt);
1707         }
1708 
1709         if ( willFollowRedirect ) {
1710             debug(requests) trace("going to follow redirect");
1711             if ( _history.length >= _maxRedirects ) {
1712                 _stream = null;
1713                 throw new MaxRedirectsException("%d redirects reached maxRedirects %d.".format(_history.length, _maxRedirects));
1714             }
1715             // "location" in response already checked in canFollowRedirect
1716             immutable new_location = *("location" in _response.responseHeaders);
1717             immutable current_uri = _uri, next_uri = uriFromLocation(_uri, new_location);
1718 
1719             if ( _method == "GET" && _response.code == 301 ) {
1720                 _permanent_redirects[_uri] = new_location;
1721             }
1722 
1723             // save current response for history
1724             _history ~= _response;
1725 
1726             // prepare new response (for redirected request)
1727             _response = new HTTPResponse;
1728             _response.uri = current_uri;
1729             _response.finalURI = next_uri;
1730             _stream = null;
1731 
1732             // set new uri
1733             _uri = next_uri;
1734             debug(requests) tracef("Redirected to %s", next_uri);
1735             if ( _method != "GET" && _response.code != 307 && _response.code != 308 ) {
1736                 // 307 and 308 do not change method
1737                 return this.get();
1738             }
1739             if ( restartedRequest ) {
1740                 debug(requests) trace("Rare event: clearing 'restartedRequest' on redirect");
1741                 restartedRequest = false;
1742             }
1743             goto connect;
1744         }
1745 
1746         _response._history = _history;
1747         return _response;
1748     }
1749 
1750     /// WRAPPERS
1751     ///
1752     /// send file(s) using POST and multipart form.
1753     /// This wrapper will be deprecated, use post with MultipartForm - it is more general and clear.
1754     /// Parameters:
1755     ///     url = url
1756     ///     files = array of PostFile structures
1757     /// Returns:
1758     ///     Response
1759     /// Each PostFile structure contain path to file, and optional field name and content type.
1760     /// If no field name provided, then basename of the file will be used.
1761     /// application/octet-stream is default when no content type provided.
1762     /// Example:
1763     /// ---------------------------------------------------------------
1764     ///    PostFile[] files = [
1765     ///                   {fileName:"tests/abc.txt", fieldName:"abc", contentType:"application/octet-stream"},
1766     ///                   {fileName:"tests/test.txt"}
1767     ///               ];
1768     ///    rs = rq.exec!"POST"("http://httpbin.org/post", files);
1769     /// ---------------------------------------------------------------
1770     ///
1771     HTTPResponse exec(string method="POST")(string url, PostFile[] files) if (method=="POST") {
1772         MultipartForm multipart;
1773         File[]        toClose;
1774         foreach(ref f; files) {
1775             File file = File(f.fileName, "rb");
1776             toClose ~= file;
1777             string fileName = f.fileName ? f.fileName : f.fieldName;
1778             string contentType = f.contentType ? f.contentType : "application/octetstream";
1779             multipart.add(f.fieldName, new FormDataFile(file), ["filename":fileName, "Content-Type": contentType]);
1780         }
1781         auto res = exec!"POST"(url, multipart);
1782         toClose.each!"a.close";
1783         return res;
1784     }
1785     ///
1786     /// exec request with parameters when you can use dictionary (when you have no duplicates in parameter names)
1787     /// Consider switch to exec(url, QueryParams) as it more generic and clear.
1788     /// Parameters:
1789     ///     url = url
1790     ///     params = dictionary with field names as keys and field values as values.
1791     /// Returns:
1792     ///     Response
1793     HTTPResponse exec(string method="GET")(string url, string[string] params) {
1794         return exec!method(url, params.byKeyValue.map!(p => QueryParam(p.key, p.value)).array);
1795     }
1796     ///
1797     /// GET request. Simple wrapper over exec!"GET"
1798     /// Params:
1799     /// args = request parameters. see exec docs.
1800     ///
1801     HTTPResponse get(A...)(A args) {
1802         return exec!"GET"(args);
1803     }
1804     ///
1805     /// POST request. Simple wrapper over exec!"POST"
1806     /// Params:
1807     /// uri = endpoint uri
1808     /// args = request parameters. see exec docs.
1809     ///
1810     HTTPResponse post(A...)(string uri, A args) {
1811         return exec!"POST"(uri, args);
1812     }
1813 }
1814 
1815 version(vibeD) {
1816     import std.json;
1817     package string httpTestServer() {
1818         return "http://httpbin.org/";
1819     }
1820     package string fromJsonArrayToStr(JSONValue v) {
1821         return v.str;
1822     }
1823 }
1824 else {
1825     import std.json;
1826     package string httpTestServer() {
1827         return "http://127.0.0.1:8081/";
1828     }
1829     package string fromJsonArrayToStr(JSONValue v) {
1830         return cast(string)(v.array.map!"cast(ubyte)a.integer".array);
1831     }
1832 }
1833 
1834 
1835 package unittest {
1836     import std.json;
1837     import std.array;
1838 
1839     globalLogLevel(LogLevel.info);
1840 
1841     string httpbinUrl = httpTestServer();
1842     version(vibeD) {
1843     }
1844     else {
1845         import httpbin;
1846         auto server = httpbinApp();
1847         server.start();
1848         scope(exit) {
1849             server.stop();
1850         }
1851     }
1852     HTTPRequest  rq;
1853     HTTPResponse rs;
1854     info("Check GET");
1855     URI uri = URI(httpbinUrl);
1856     rs = rq.get(httpbinUrl);
1857     assert(rs.code==200);
1858     assert(rs.responseBody.length > 0);
1859     assert(rq.format("%m|%h|%p|%P|%q|%U") ==
1860             "GET|%s|%d|%s||%s"
1861             .format(uri.host, uri.port, uri.path, httpbinUrl));
1862     info("Check GET with AA params");
1863     {
1864         rs = rq.get(httpbinUrl ~ "get", ["c":" d", "a":"b"]);
1865         assert(rs.code == 200);
1866         auto json = parseJSON(cast(string)rs.responseBody.data).object["args"].object;
1867         assert(json["c"].str == " d");
1868         assert(json["a"].str == "b");
1869     }
1870     rq.keepAlive = false; // disable keepalive on non-idempotent requests
1871     info("Check POST files");
1872     {
1873         import std.file;
1874         import std.path;
1875         auto tmpd = tempDir();
1876         auto tmpfname = tmpd ~ dirSeparator ~ "request_test.txt";
1877         auto f = File(tmpfname, "wb");
1878         f.rawWrite("abcdefgh\n12345678\n");
1879         f.close();
1880         // files
1881         PostFile[] files = [
1882             {fileName: tmpfname, fieldName:"abc", contentType:"application/octet-stream"},
1883             {fileName: tmpfname}
1884         ];
1885         rs = rq.post(httpbinUrl ~ "post", files);
1886         assert(rs.code==200);
1887     }
1888     info("Check POST chunked from file.byChunk");
1889     {
1890         import std.file;
1891         import std.path;
1892         auto tmpd = tempDir();
1893         auto tmpfname = tmpd ~ dirSeparator ~ "request_test.txt";
1894         auto f = File(tmpfname, "wb");
1895         f.rawWrite("abcdefgh\n12345678\n");
1896         f.close();
1897         f = File(tmpfname, "rb");
1898         rs = rq.post(httpbinUrl ~ "post", f.byChunk(3), "application/octet-stream");
1899         if (httpbinUrl != "http://httpbin.org/") {
1900             assert(rs.code==200);
1901             auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]);
1902             assert(data=="abcdefgh\n12345678\n");
1903         }
1904         f.close();
1905     }
1906     info("Check POST chunked from lineSplitter");
1907     {
1908         auto s = lineSplitter("one,\ntwo,\nthree.");
1909         rs = rq.exec!"POST"(httpbinUrl ~ "post", s, "application/octet-stream");
1910         if (httpbinUrl != "http://httpbin.org/") {
1911             assert(rs.code==200);
1912             auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]);
1913             assert(data=="one,two,three.");
1914         }
1915     }
1916     info("Check POST chunked from array");
1917     {
1918         auto s = ["one,", "two,", "three."];
1919         rs = rq.post(httpbinUrl ~ "post", s, "application/octet-stream");
1920         if (httpbinUrl != "http://httpbin.org/") {
1921             assert(rs.code==200);
1922             auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]);
1923             assert(data=="one,two,three.");
1924         }
1925     }
1926     info("Check POST chunked using std.range.chunks()");
1927     {
1928         auto s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
1929         rs = rq.post(httpbinUrl ~ "post", s.representation.chunks(10), "application/octet-stream");
1930         if (httpbinUrl != "http://httpbin.org/") {
1931             assert(rs.code==200);
1932             auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody.data).object["data"]);
1933             assert(data==s);
1934         }
1935     }
1936     info("Check POST from QueryParams");
1937     {
1938         rs = rq.post(httpbinUrl ~ "post", queryParams("name[]", "first", "name[]", 2));
1939         assert(rs.code==200);
1940         auto data = parseJSON(cast(string)rs.responseBody).object["form"].object;
1941         string[] a;
1942         try {
1943             a = to!(string[])(data["name[]"].str);
1944         }
1945         catch (JSONException e) {
1946             a = data["name[]"].array.map!"a.str".array;
1947         }
1948         assert(equal(["first", "2"], a));
1949     }
1950     info("Check POST from AA");
1951     {
1952         rs = rq.post(httpbinUrl ~ "post", ["a":"b ", "c":"d"]);
1953         assert(rs.code==200);
1954         auto form = parseJSON(cast(string)rs.responseBody.data).object["form"].object;
1955         assert(form["a"].str == "b ");
1956         assert(form["c"].str == "d");
1957     }
1958     info("Check POST json");
1959     {
1960         rs = rq.post(httpbinUrl ~ "post?b=x", `{"a":"a b", "c":[1,2,3]}`, "application/json");
1961         assert(rs.code==200);
1962         auto json = parseJSON(cast(string)rs.responseBody).object["args"].object;
1963         assert(json["b"].str == "x");
1964         json = parseJSON(cast(string)rs.responseBody).object["json"].object;
1965         assert(json["a"].str == "a b");
1966         assert(json["c"].array.map!(a=>a.integer).array == [1,2,3]);
1967     }
1968     info("Check DELETE");
1969     rs = rq.exec!"DELETE"(httpbinUrl ~ "delete");
1970     assert(rs.code==200);
1971     info("Check PUT");
1972     rs = rq.exec!"PUT"(httpbinUrl ~ "put",  `{"a":"b", "c":[1,2,3]}`, "application/json");
1973     assert(rs.code==200);
1974     assert(parseJSON(cast(string)rs.responseBody).object["json"].object["a"].str=="b");
1975     info("Check PATCH");
1976     rs = rq.exec!"PATCH"(httpbinUrl ~ "patch", "привiт, свiт!", "application/octet-stream");
1977     assert(rs.code==200);
1978     info("Check HEAD");
1979     rs = rq.exec!"HEAD"(httpbinUrl);
1980     assert(rs.code==200);
1981 
1982     rq._keepAlive = true;
1983     info("Check compressed content");
1984     rs = rq.get(httpbinUrl ~ "gzip");
1985     assert(rs.code==200);
1986     bool gzipped = parseJSON(cast(string)rs.responseBody).object["gzipped"].type == JSON_TYPE.TRUE;
1987     assert(gzipped);
1988     info("gzip - ok");
1989     rs = rq.get(httpbinUrl ~ "deflate");
1990     assert(rs.code==200);
1991     bool deflated = parseJSON(cast(string)rs.responseBody).object["deflated"].type == JSON_TYPE.TRUE;
1992     assert(deflated);
1993     info("deflate - ok");
1994 
1995     info("Check redirects");
1996     rs = rq.get(httpbinUrl ~ "relative-redirect/2");
1997     assert(rs.history.length == 2);
1998     assert(rs.code==200);
1999     rs = rq.get(httpbinUrl ~ "absolute-redirect/2");
2000     assert(rs.history.length == 2);
2001     assert(rs.code==200);
2002 
2003     rq.maxRedirects = 2;
2004     assertThrown!MaxRedirectsException(rq.get(httpbinUrl ~ "absolute-redirect/3"));
2005 
2006     rq.maxRedirects = 0;
2007     rs = rq.get(httpbinUrl ~ "absolute-redirect/1");
2008     assert(rs.code==302);
2009 
2010     info("Check cookie");
2011     {
2012         rq.maxRedirects = 10;
2013         rs = rq.get(httpbinUrl ~ "cookies/set?A=abcd&b=cdef");
2014         assert(rs.code == 200);
2015         auto json = parseJSON(cast(string)rs.responseBody.data).object["cookies"].object;
2016         assert(json["A"].str == "abcd");
2017         assert(json["b"].str == "cdef");
2018         foreach(c; rq.cookie) {
2019             final switch(c.attr) {
2020                 case "A":
2021                     assert(c.value == "abcd");
2022                     break;
2023                 case "b":
2024                     assert(c.value == "cdef");
2025                     break;
2026             }
2027         }
2028     }
2029     info("Check chunked content");
2030     rs = rq.get(httpbinUrl ~ "range/1024");
2031     assert(rs.code==200);
2032     assert(rs.responseBody.length==1024);
2033 
2034     info("Check basic auth");
2035     rq.authenticator = new BasicAuthentication("user", "passwd");
2036     rs = rq.get(httpbinUrl ~ "basic-auth/user/passwd");
2037     assert(rs.code==200);
2038 
2039     info("Check limits");
2040     rq = HTTPRequest();
2041     rq.maxContentLength = 1;
2042     assertThrown!RequestException(rq.get(httpbinUrl));
2043 
2044     rq = HTTPRequest();
2045     rq.maxHeadersLength = 1;
2046     assertThrown!RequestException(rq.get(httpbinUrl));
2047 
2048     rq = HTTPRequest();
2049     info("Check POST multiPartForm");
2050     {
2051         /// This is example on usage files with MultipartForm data.
2052         /// For this example we have to create files which will be sent.
2053         import std.file;
2054         import std.path;
2055         /// preapare files
2056         auto tmpd = tempDir();
2057         auto tmpfname1 = tmpd ~ dirSeparator ~ "request_test1.txt";
2058         auto f = File(tmpfname1, "wb");
2059         f.rawWrite("file1 content\n");
2060         f.close();
2061         auto tmpfname2 = tmpd ~ dirSeparator ~ "request_test2.txt";
2062         f = File(tmpfname2, "wb");
2063         f.rawWrite("file2 content\n");
2064         f.close();
2065         ///
2066         /// Ok, files ready.
2067         /// Now we will prepare Form data
2068         ///
2069         File f1 = File(tmpfname1, "rb");
2070         File f2 = File(tmpfname2, "rb");
2071         scope(exit) {
2072             f1.close();
2073             f2.close();
2074         }
2075         ///
2076         /// for each part we have to set field name, source (ubyte array or opened file) and optional filename and content-type
2077         ///
2078         MultipartForm mForm = MultipartForm().
2079             add(formData("Field1", cast(ubyte[])"form field from memory")).
2080                 add(formData("Field2", cast(ubyte[])"file field from memory", ["filename":"data2"])).
2081                 add(formData("File1", f1, ["filename":"file1", "Content-Type": "application/octet-stream"])).
2082                 add(formData("File2", f2, ["filename":"file2", "Content-Type": "application/octet-stream"]));
2083         /// everything ready, send request
2084         rs = rq.post(httpbinUrl ~ "post", mForm);
2085     }
2086     info("Check exception handling, error messages and timeous are OK");
2087     rq.timeout = 1.seconds;
2088     assertThrown!TimeoutException(rq.get(httpbinUrl ~ "delay/3"));
2089 //    assertThrown!ConnectError(rq.get("http://0.0.0.0:65000/"));
2090 //    assertThrown!ConnectError(rq.get("http://1.1.1.1/"));
2091 //    assertThrown!ConnectError(rq.get("http://gkhgkhgkjhgjhgfjhgfjhgf/"));
2092 }
2093