requests.http source code

1 module requests.http;
2 
3 private:
4 import std.algorithm;
5 import std.array;
6 import std.ascii;
7 import std.conv;
8 import std.datetime;
9 import std.exception;
10 import std.format;
11 import std.stdio;
12 import std.range;
13 import std.string;
14 import std.traits;
15 import std.typecons;
16 import std.bitmanip;
17 import std.experimental.logger;
18 import core.thread;
19 
20 import requests.streams;
21 import requests.uri;
22 import requests.utils;
23 import requests.base;
24 import requests.connmanager;
25 
26 static immutable ushort[] redirectCodes = [301, 302, 303, 307, 308];
27 static immutable uint     defaultBufferSize = 12*1024;
28 enum   HTTP11 = 101;
29 enum   HTTP10 = 100;
30 
31 static immutable string[string] proxies;
32 static this() {
33     import std.process;
34     proxies["http"] = environment.get("http_proxy", environment.get("HTTP_PROXY"));
35     proxies["https"] = environment.get("https_proxy", environment.get("HTTPS_PROXY"));
36     proxies["all"] = environment.get("all_proxy", environment.get("ALL_PROXY"));
37     foreach(p; proxies.byKey()) {
38         if (proxies[p] is null) {
39             continue;
40         }
41         URI u = URI(proxies[p]);
42     }
43 }
44 
45 public class MaxRedirectsException: Exception {
46     this(string message, string file = __FILE__, size_t line = __LINE__, Throwable next = null) @safe pure nothrow {
47         super(message, file, line, next);
48     }
49 }
50 
51 /**
52  * Basic authentication.
53  * Adds $(B Authorization: Basic) header to request.
54  */
55 public class BasicAuthentication: Auth {
56     private {
57         string   _username, _password;
58         string[] _domains;
59     }
60     /// Constructor.
61     /// Params:
62     /// username = username
63     /// password = password
64     /// domains = not used now
65     ///
66     this(string username, string password, string[] domains = []) {
67         _username = username;
68         _password = password;
69         _domains = domains;
70     }
71     override string[string] authHeaders(string domain) {
72         import std.base64;
73         string[string] auth;
74         auth["Authorization"] = "Basic " ~ to!string(Base64.encode(cast(ubyte[])"%s:%s".format(_username, _password)));
75         return auth;
76     }
77     override string userName() {
78         return _username;
79     }
80     override string password() {
81         return _password;
82     }
83 }
84 ///
85 ///
86 ///
87 public auto queryParams(T...)(T params) pure nothrow @safe
88 {
89     static assert (T.length % 2 == 0, "wrong args count");
90 
91     QueryParam[] output;
92     output.reserve = T.length / 2;
93 
94     void queryParamsHelper(T...)(T params, ref QueryParam[] output)
95     {
96         static if (T.length > 0)
97         {
98             output ~= QueryParam(params[0].to!string, params[1].to!string);
99             queryParamsHelper(params[2..$], output);
100         }
101     }
102 
103     queryParamsHelper(params, output);
104     return output;
105 }
106 
107 ///
108 /// Response - result of request execution.
109 ///
110 /// Response.code - response HTTP code.
111 /// Response.status_line - received HTTP status line.
112 /// Response.responseHeaders - received headers.
113 /// Response.responseBody - container for received body
114 /// Response.history - for redirected responses contain all history
115 ///
116 public class HTTPResponse : Response {
117     private {
118         string              _status_line;
119 
120         HTTPResponse[]      _history; // redirects history
121 
122         mixin(Setter!string("status_line"));
123 
124         int                 _version;
125     }
126 
127     ~this() {
128         _responseHeaders = null;
129         _history.length = 0;
130     }
131 
132     mixin(Getter!string("status_line"));
133     @property final string[string] responseHeaders() @safe @nogc nothrow {
134         return _responseHeaders;
135     }
136     @property final HTTPResponse[] history() @safe @nogc nothrow {
137         return _history;
138     }
139 
140     @property auto getStats() const pure @safe {
141         alias statTuple = Tuple!(Duration, "connectTime",
142                                  Duration, "sendTime",
143                                  Duration, "recvTime");
144         statTuple stat;
145         stat.connectTime = _connectedAt - _startedAt;
146         stat.sendTime = _requestSentAt - _connectedAt;
147         stat.recvTime = _finishedAt - _requestSentAt;
148         return stat;
149     }
150     private int parse_version(in string v) pure const nothrow @safe {
151         // try to parse HTTP/1.x to version
152         try if ( v.length > 5 ) {
153             return (v[5..$].split(".").map!"to!int(a)".array[0..2].reduce!((a,b) => a*100 + b));
154         } catch (Exception e) {
155         }
156         return 0;
157     }
158     unittest {
159         auto r = new HTTPResponse();
160         assert(r.parse_version("HTTP/1.1") == 101);
161         assert(r.parse_version("HTTP/1.0") == 100);
162         assert(r.parse_version("HTTP/0.9") == 9);
163         assert(r.parse_version("HTTP/xxx") == 0);
164     }
165 }
166 /**
167  * Struct to send multiple files in POST request.
168  */
169 public struct PostFile {
170     /// Path to the file to send.
171     string fileName;
172     /// Name of the field (if empty - send file base name)
173     string fieldName;
174     /// contentType of the file if not empty
175     string contentType;
176 }
177 ///
178 /// This is File-like interface for sending data to multipart fotms
179 ///
180 public interface FiniteReadable {
181     /// size of the content
182     abstract ulong  getSize();
183     /// file-like read()
184     abstract ubyte[] read();
185 }
186 ///
187 /// Helper to create form elements from File.
188 /// Params:
189 /// name = name of the field in form
190 /// f = opened std.stio.File to send to server
191 /// parameters = optional parameters (most important are "filename" and "Content-Type")
192 ///
193 public auto formData(string name, File f, string[string] parameters = null) {
194     return MultipartForm.FormData(name, new FormDataFile(f), parameters);
195 }
196 ///
197 /// Helper to create form elements from ubyte[].
198 /// Params:
199 /// name = name of the field in form
200 /// b = data to send to server
201 /// parameters = optional parameters (can be "filename" and "Content-Type")
202 ///
203 public auto formData(string name, ubyte[] b, string[string] parameters = null) {
204     return MultipartForm.FormData(name, new FormDataBytes(b), parameters);
205 }
206 public auto formData(string name, string b, string[string] parameters = null) {
207     return MultipartForm.FormData(name, new FormDataBytes(b.dup.representation), parameters);
208 }
209 public class FormDataBytes : FiniteReadable {
210     private {
211         ulong   _size;
212         ubyte[] _data;
213         size_t  _offset;
214         bool    _exhausted;
215     }
216     this(ubyte[] data) {
217         _data = data;
218         _size = data.length;
219     }
220     final override ulong getSize() {
221         return _size;
222     }
223     final override ubyte[] read() {
224         enforce( !_exhausted, "You can't read froum exhausted source" );
225         size_t toRead = min(defaultBufferSize, _size - _offset);
226         auto result = _data[_offset.._offset+toRead];
227         _offset += toRead;
228         if ( toRead == 0 ) {
229             _exhausted = true;
230         }
231         return result;
232     }
233 }
234 public class FormDataFile : FiniteReadable {
235     import  std.file;
236     private {
237         File    _fileHandle;
238         ulong   _fileSize;
239         size_t  _processed;
240         bool    _exhausted;
241     }
242     this(File file) {
243         import std.file;
244         _fileHandle = file;
245         _fileSize = std.file.getSize(file.name);
246     }
247     final override ulong getSize() pure nothrow @safe {
248         return _fileSize;
249     }
250     final override ubyte[] read() {
251         enforce( !_exhausted, "You can't read froum exhausted source" );
252         auto b = new ubyte[defaultBufferSize];
253         auto r = _fileHandle.rawRead(b);
254         auto toRead = min(r.length, _fileSize - _processed);
255         if ( toRead == 0 ) {
256             _exhausted = true;
257         }
258         _processed += toRead;
259         return r[0..toRead];
260     }
261 }
262 ///
263 /// This struct used to bulld POST's to forms.
264 /// Each part have name and data. data is something that can be read-ed and have size.
265 /// For example this can be string-like object (wrapped for reading) or opened File.
266 ///
267 public struct MultipartForm {
268     package struct FormData {
269         FiniteReadable  input;
270         string          name;
271         string[string]  parameters;
272         this(string name, FiniteReadable i, string[string] parameters = null) {
273             this.input = i;
274             this.name = name;
275             this.parameters = parameters;
276         }
277     }
278 
279     private FormData[] _sources;
280     auto add(FormData d) {
281         _sources ~= d;
282         return this;
283     }
284     auto add(string name, FiniteReadable i, string[string]parameters = null) {
285         _sources ~= FormData(name, i, parameters);
286         return this;
287     }
288 }
289 ///
290 
291 ///
292 /// Request.
293 /// Configurable parameters:
294 /// $(B method) - string, method to use (GET, POST, ...)
295 /// $(B headers) - string[string], add any additional headers you'd like to send.
296 /// $(B authenticator) - class Auth, class to send auth headers.
297 /// $(B keepAlive) - bool, set true for keepAlive requests. default true.
298 /// $(B maxRedirects) - uint, maximum number of redirects. default 10.
299 /// $(B maxHeadersLength) - size_t, maximum length of server response headers. default = 32KB.
300 /// $(B maxContentLength) - size_t, maximun content length. delault - 0 = unlimited.
301 /// $(B bufferSize) - size_t, send and receive buffer size. default = 16KB.
302 /// $(B verbosity) - uint, level of verbosity(0 - nothing, 1 - headers, 2 - headers and body progress). default = 0.
303 /// $(B proxy) - string, set proxy url if needed. default - null.
304 /// $(B cookie) - Tuple Cookie, Read/Write cookie You can get cookie setted by server, or set cookies before doing request.
305 /// $(B timeout) - Duration, Set timeout value for connect/receive/send.
306 ///
307 public struct HTTPRequest {
308     alias NetStrFactory = NetworkStream delegate(string, string, ushort);
309     private {
310         struct _UH {
311             // flags for each important header, added by user using addHeaders
312             mixin(bitfields!(
313                 bool, "Host", 1,
314                 bool, "UserAgent", 1,
315                 bool, "ContentLength", 1,
316                 bool, "Connection", 1,
317                 bool, "AcceptEncoding", 1,
318                 bool, "ContentType", 1,
319                 bool, "Cookie", 1,
320                 uint, "", 1
321             ));
322         }
323         string         _method = "GET";
324         URI            _uri;
325         string[string] _headers;
326         string[]       _filteredHeaders;
327         Auth           _authenticator;
328         bool           _keepAlive = true;
329         uint           _maxRedirects = 10;
330         size_t         _maxHeadersLength = 32 * 1024;   // 32 KB
331         size_t         _maxContentLength;               // 0 - Unlimited
332         string         _proxy;
333         uint           _verbosity = 0;                  // 0 - no output, 1 - headers, 2 - headers+body info
334         Duration       _timeout = 30.seconds;
335         size_t         _bufferSize = defaultBufferSize; // 16k
336         bool           _useStreaming;                   // return iterator instead of completed request
337 
338         string[URI]     _permanent_redirects;            // cache 301 redirects for GET requests
339         HTTPResponse[] _history;                        // redirects history
340         DataPipe!ubyte _bodyDecoder;
341         DecodeChunked  _unChunker;
342         long           _contentLength;
343         long           _contentReceived;
344         Cookie[]       _cookie;
345         SSLOptions     _sslOptions;
346         string         _bind;
347         _UH            _userHeaders;
348         ConnManager    _cm;
349         NetStrFactory  _socketFactory;
350     }
351     package HTTPResponse   _response;
352 
353     mixin(Getter_Setter!string     ("method"));
354     mixin(Getter_Setter!bool       ("keepAlive"));
355     mixin(Getter_Setter!size_t     ("maxContentLength"));
356     mixin(Getter_Setter!size_t     ("maxHeadersLength"));
357     mixin(Getter_Setter!size_t     ("bufferSize"));
358     mixin(Getter_Setter!uint       ("maxRedirects"));
359     mixin(Getter_Setter!uint       ("verbosity"));
360     mixin(Getter!string            ("proxy"));
361     mixin(Getter_Setter!Duration   ("timeout"));
362     mixin(Setter!Auth              ("authenticator"));
363     mixin(Getter_Setter!bool       ("useStreaming"));
364     mixin(Getter!long              ("contentLength"));
365     mixin(Getter!long              ("contentReceived"));
366     mixin(Getter_Setter!SSLOptions ("sslOptions"));
367     mixin(Getter_Setter!string     ("bind"));
368     mixin(Setter!NetStrFactory     ("socketFactory"));
369 
370     @property void sslSetVerifyPeer(bool v) pure @safe nothrow @nogc {
371         _sslOptions.setVerifyPeer(v);
372     }
373     @property void sslSetKeyFile(string p, SSLOptions.filetype t = SSLOptions.filetype.pem) pure @safe nothrow @nogc {
374         _sslOptions.setKeyFile(p, t);
375     }
376     @property void sslSetCertFile(string p, SSLOptions.filetype t = SSLOptions.filetype.pem) pure @safe nothrow @nogc {
377         _sslOptions.setCertFile(p, t);
378     }
379     @property void sslSetCaCert(string path) pure @safe nothrow @nogc {
380         _sslOptions.setCaCert(path);
381     }
382     @property final void cookie(Cookie[] s) pure @safe @nogc nothrow {
383         _cookie = s;
384     }
385     @property final void proxy(string v) {
386         if ( v != _proxy ) {
387             _cm.clear();
388         }
389         _proxy = v;
390     }
391     @property final Cookie[] cookie() pure @safe @nogc nothrow {
392         return _cookie;
393     }
394 
395     this(string uri) {
396         _uri = URI(uri);
397         _cm = new ConnManager;
398     }
399     ~this() {
400         _headers = null;
401         _authenticator = null;
402         _history = null;
403         _bodyDecoder = null;
404         _unChunker = null;
405         if ( _cm ) {
406             _cm.clear();
407         }
408     }
409     string toString() const {
410         return "HTTPRequest(%s, %s)".format(_method, _uri.uri());
411     }
412     string format(string fmt) const {
413         import std.array;
414         import std.stdio;
415         auto a = appender!string();
416         auto f = FormatSpec!char(fmt);
417         while (f.writeUpToNextSpec(a)) {
418             switch(f.spec) {
419                 case 'h':
420                     // Remote hostname.
421                     a.put(_uri.host);
422                     break;
423                 case 'm':
424                     // method.
425                     a.put(_method);
426                     break;
427                 case 'p':
428                     // Remote port.
429                     a.put("%d".format(_uri.port));
430                     break;
431                 case 'P':
432                     // Path
433                     a.put(_uri.path);
434                     break;
435                 case 'q':
436                     // query parameters supplied with url.
437                     a.put(_uri.query);
438                     break;
439                 case 'U':
440                     a.put(_uri.uri());
441                     break;
442                 default:
443                     throw new FormatException("Unknown Request format spec " ~ f.spec);
444             }
445         }
446         return a.data();
447     }
448     string select_proxy(string scheme) {
449         if ( _proxy is null && proxies.length == 0 ) {
450             debug(requests) tracef("proxy=null");
451             return null;
452         }
453         if ( _proxy ) {
454             debug(requests) tracef("proxy=%s", _proxy);
455             return _proxy;
456         }
457         auto p = scheme in proxies;
458         if ( p !is null && *p != "") {
459             debug(requests) tracef("proxy=%s", *p);
460             return *p;
461         }
462         p = "all" in proxies;
463         if ( p !is null && *p != "") {
464             debug(requests) tracef("proxy=%s", *p);
465             return *p;
466         }
467         debug(requests) tracef("proxy=null");
468         return null;
469     }
470     void clearHeaders() {
471         _headers = null;
472     }
473     @property void uri(in URI newURI) {
474         //handleURLChange(_uri, newURI);
475         _uri = newURI;
476     }
477     /// Add headers to request
478     /// Params:
479     /// headers = headers to send.
480     void addHeaders(in string[string] headers) {
481         foreach(pair; headers.byKeyValue) {
482             string _h = pair.key;
483             switch(toLower(_h)) {
484             case "host":
485                 _userHeaders.Host = true;
486                 break;
487             case "user-agent":
488                 _userHeaders.UserAgent = true;
489                 break;
490             case "content-length":
491                 _userHeaders.ContentLength = true;
492                 break;
493             case "content-type":
494                 _userHeaders.ContentType = true;
495                 break;
496             case "connection":
497                 _userHeaders.Connection = true;
498                 break;
499             case "cookie":
500                 _userHeaders.Cookie = true;
501                 break;
502             default:
503                 break;
504             }
505             _headers[pair.key] = pair.value;
506         }
507     }
508     private void safeSetHeader(ref string[string] headers, bool userAdded, string h, string v) pure @safe {
509         if ( !userAdded ) {
510             headers[h] = v;
511         }
512     }
513     /// Remove headers from request
514     /// Params:
515     /// headers = headers to remove.
516     void removeHeaders(in string[] headers) pure {
517         _filteredHeaders ~= headers;
518     }
519     ///
520     /// compose headers to send
521     ///
522     private string[string] requestHeaders() {
523 
524         string[string] generatedHeaders;
525 
526         if ( _authenticator ) {
527             _authenticator.
528                 authHeaders(_uri.host).
529                 byKeyValue.
530                 each!(pair => generatedHeaders[pair.key] = pair.value);
531         }
532 
533         _headers.byKey.each!(h => generatedHeaders[h] = _headers[h]);
534 
535         safeSetHeader(generatedHeaders, _userHeaders.AcceptEncoding, "Accept-Encoding", "gzip,deflate");
536         safeSetHeader(generatedHeaders, _userHeaders.UserAgent, "User-Agent", "dlang-requests");
537         safeSetHeader(generatedHeaders, _userHeaders.Connection, "Connection", _keepAlive?"Keep-Alive":"Close");
538 
539         if ( !_userHeaders.Host )
540         {
541             generatedHeaders["Host"] = _uri.host;
542             if ( _uri.scheme !in standard_ports || _uri.port != standard_ports[_uri.scheme] ) {
543                 generatedHeaders["Host"] ~= ":%d".format(_uri.port);
544             }
545         }
546 
547         if ( _cookie.length && !_userHeaders.Cookie ) {
548             auto cs = _cookie.
549                 filter!(c => _uri.path.pathMatches(c.path) && _uri.host.domainMatches(c.domain)).
550                 map!(c => "%s=%s".format(c.attr, c.value)).
551                 joiner(";");
552             if ( ! cs.empty )
553             {
554                 generatedHeaders["Cookie"] = to!string(cs);
555             }
556         }
557 
558         _filteredHeaders.each!(h => generatedHeaders.remove(h));
559 
560         return generatedHeaders;
561     }
562     ///
563     /// Build request string.
564     /// Handle proxy and query parameters.
565     ///
566     private @property string requestString(QueryParam[] params = null) {
567         auto query = _uri.query.dup;
568         if ( params ) {
569             query ~= "&" ~ params2query(params);
570             if ( query[0] != '?' ) {
571                 query = "?" ~ query;
572             }
573         }
574         string actual_proxy = select_proxy(_uri.scheme);
575         if ( actual_proxy && _uri.scheme != "https" ) {
576             return "%s %s%s HTTP/1.1\r\n".format(_method, _uri.uri(No.params), query);
577         }
578         return "%s %s%s HTTP/1.1\r\n".format(_method, _uri.path, query);
579     }
580     ///
581     /// encode parameters and build query part of the url
582     ///
583     private static string params2query(in QueryParam[] params) pure @safe {
584         return params.
585                 map!(a => "%s=%s".format(a.key.urlEncoded, a.value.urlEncoded)).
586                 join("&");
587     }
588     //
589     package unittest {
590         assert(params2query(queryParams("a","b", "c", " d "))=="a=b&c=%20d%20");
591     }
592     ///
593     /// Analyze received headers, take appropriate actions:
594     /// check content length, attach unchunk and uncompress
595     ///
596     private void analyzeHeaders(in string[string] headers) {
597 
598         _contentLength = -1;
599         _unChunker = null;
600         auto contentLength = "content-length" in headers;
601         if ( contentLength ) {
602             try {
603                 _contentLength = to!long(*contentLength);
604                 if ( _maxContentLength && _contentLength > _maxContentLength) {
605                     throw new RequestException("ContentLength > maxContentLength (%d>%d)".
606                                 format(_contentLength, _maxContentLength));
607                 }
608             } catch (ConvException e) {
609                 throw new RequestException("Can't convert Content-Length from %s".format(*contentLength));
610             }
611         }
612         auto transferEncoding = "transfer-encoding" in headers;
613         if ( transferEncoding ) {
614             debug(requests) tracef("transferEncoding: %s", *transferEncoding);
615             if ( (*transferEncoding).toLower == "chunked") {
616                 _unChunker = new DecodeChunked();
617                 _bodyDecoder.insert(_unChunker);
618             }
619         }
620         auto contentEncoding = "content-encoding" in headers;
621         if ( contentEncoding ) switch (*contentEncoding) {
622             default:
623                 throw new RequestException("Unknown content-encoding " ~ *contentEncoding);
624             case "gzip":
625             case "deflate":
626                 _bodyDecoder.insert(new Decompressor!ubyte);
627         }
628 
629     }
630     ///
631     /// Called when we know that all headers already received in buffer.
632     /// This routine does not interpret headers content (see analyzeHeaders).
633     /// 1. Split headers on lines
634     /// 2. store status line, store response code
635     /// 3. unfold headers if needed
636     /// 4. store headers
637     ///
638     private void parseResponseHeaders(in ubyte[] input) {
639         string lastHeader;
640         auto buffer = cast(string)input;
641 
642         foreach(line; buffer.split("\n").map!(l => l.stripRight)) {
643             if ( ! _response.status_line.length ) {
644                 debug (requests) tracef("statusLine: %s", line);
645                 _response.status_line = line;
646                 if ( _verbosity >= 1 ) {
647                     writefln("< %s", line);
648                 }
649                 auto parsed = line.split(" ");
650                 if ( parsed.length >= 2 ) {
651                     _response.code = parsed[1].to!ushort;
652                     _response._version = _response.parse_version(parsed[0]);
653                 }
654                 continue;
655             }
656             if ( line[0] == ' ' || line[0] == '\t' ) {
657                 // unfolding https://tools.ietf.org/html/rfc822#section-3.1
658                 if ( auto stored = lastHeader in _response._responseHeaders) {
659                     *stored ~= line;
660                 }
661                 continue;
662             }
663             auto parsed = line.findSplit(":");
664             auto header = parsed[0].toLower;
665             auto value = parsed[2].strip;
666 
667             if ( _verbosity >= 1 ) {
668                 writefln("< %s: %s", header, value);
669             }
670 
671             lastHeader = header;
672             debug (requests) tracef("Header %s = %s", header, value);
673 
674             if ( header != "set-cookie" ) {
675                 auto stored = _response.responseHeaders.get(header, null);
676                 if ( stored ) {
677                     value = stored ~ "," ~ value;
678                 }
679                 _response._responseHeaders[header] = value;
680                 continue;
681             }
682             _cookie ~= processCookie(value);
683         }
684     }
685 
686     ///
687     /// Process Set-Cookie header from server response
688     ///
689     private Cookie[] processCookie(string value ) pure {
690         // cookie processing
691         //
692         // as we can't join several set-cookie lines in single line
693         // < Set-Cookie: idx=7f2800f63c112a65ef5082957bcca24b; expires=Mon, 29-May-2017 00:31:25 GMT; path=/; domain=example.com
694         // < Set-Cookie: idx=7f2800f63c112a65ef5082957bcca24b; expires=Mon, 29-May-2017 00:31:25 GMT; path=/; domain=example.com, cs=ip764-RgKqc-HvSkxRxdQQAKW8LA; path=/; domain=.example.com; HttpOnly
695         //
696         Cookie[] res;
697         string[string] kv;
698         auto fields = value.split(";").map!strip;
699         while(!fields.empty) {
700             auto s = fields.front.findSplit("=");
701             fields.popFront;
702             if ( s[1] != "=" ) {
703                 continue;
704             }
705             auto k = s[0];
706             auto v = s[2];
707             switch(k.toLower()) {
708                 case "domain":
709                     k = "domain";
710                     break;
711                 case "path":
712                     k = "path";
713                     break;
714                 case "expires":
715                     continue;
716                 default:
717                     break;
718             }
719             kv[k] = v;
720         }
721         if ( "domain" !in kv ) {
722             kv["domain"] = _uri.host;
723         }
724         if ( "path" !in kv ) {
725             kv["path"] = _uri.path;
726         }
727         auto domain = kv["domain"]; kv.remove("domain");
728         auto path   = kv["path"];   kv.remove("path");
729         foreach(pair; kv.byKeyValue) {
730             auto _attr = pair.key;
731             auto _value = pair.value;
732             auto cookie = Cookie(path, domain, _attr, _value);
733             res ~= cookie;
734         }
735         return res;
736     }
737     ///
738     /// Do we received \r\n\r\n?
739     ///
740     private bool headersHaveBeenReceived(in ubyte[] data, ref Buffer!ubyte buffer, out string separator) const @safe {
741         foreach(s; ["\r\n\r\n", "\n\n"]) {
742             if ( data.canFind(s) || buffer.canFind(s) ) {
743                 separator = s;
744                 return true;
745             }
746         }
747         return false;
748     }
749 
750     private bool willFollowRedirect() {
751         if ( !canFind(redirectCodes, _response.code) ) {
752             return false;
753         }
754         if ( !_maxRedirects ) {
755             return false;
756         }
757         if ( "location" !in _response.responseHeaders ) {
758             return false;
759         }
760         return true;
761     }
762     private URI uriFromLocation(const ref URI uri, in string location) {
763         URI newURI = uri;
764         try {
765             newURI = URI(location);
766         } catch (UriException e) {
767             debug(requests) trace("Can't parse Location:, try relative uri");
768             newURI.path = location;
769             newURI.uri = newURI.recalc_uri;
770         }
771         return newURI;
772     }
773     ///
774     /// if we have new uri, then we need to check if we have to reopen existent connection
775     ///
776     private void checkURL(string url, string file=__FILE__, size_t line=__LINE__) {
777         if (url is null && _uri.uri == "" ) {
778             throw new RequestException("No url configured", file, line);
779         }
780 
781         if ( url !is null ) {
782             URI newURI = URI(url);
783             //handleURLChange(_uri, newURI);
784             _uri = newURI;
785         }
786     }
787     ///
788     /// Setup connection. Handle proxy and https case
789     ///
790     /// Place new connection in ConnManager cache
791     ///
792     private NetworkStream setupConnection()
793     do {
794 
795         debug(requests) tracef("Set up new connection");
796         NetworkStream stream;
797 
798         if ( _socketFactory )
799         {
800             debug(requests) tracef("use socketFactory");
801             stream = _socketFactory(_uri.scheme, _uri.host, _uri.port);
802         }
803         else
804         {
805             URI   uri; // this URI will be used temporarry if we need proxy
806             string actual_proxy = select_proxy(_uri.scheme);
807             final switch (_uri.scheme) {
808                 case"http":
809                 if ( actual_proxy ) {
810                     uri.uri_parse(actual_proxy);
811                     uri.idn_encode();
812                 } else {
813                     // use original uri
814                     uri = _uri;
815                 }
816                 stream = new TCPStream();
817                 stream.bind(_bind);
818                 stream.connect(uri.host, uri.port, _timeout);
819                 break ;
820                 case"https":
821                 if ( actual_proxy ) {
822                     uri.uri_parse(actual_proxy);
823                     uri.idn_encode();
824                     stream = new TCPStream();
825                     stream.bind(_bind);
826                     stream.connect(uri.host, uri.port, _timeout);
827                     if ( verbosity>=1 ) {
828                         writeln("> CONNECT %s:%d HTTP/1.1".format(_uri.host, _uri.port));
829                     }
830                     stream.send("CONNECT %s:%d HTTP/1.1\r\n\r\n".format(_uri.host, _uri.port));
831                     while ( stream.isConnected ) {
832                         ubyte[1024] b;
833                         auto read = stream.receive(b);
834                         if ( verbosity>=1) {
835                             writefln("< %s", cast(string)b[0..read]);
836                         }
837                         debug(requests) tracef("read: %d", read);
838                         if ( b[0..read].canFind("\r\n\r\n") || b[0..read].canFind("\n\n") ) {
839                             debug(requests) tracef("proxy connection ready");
840                             // convert connection to ssl
841                             stream = new SSLStream(stream, _sslOptions, _uri.host);
842                             break ;
843                         } else {
844                             debug(requests) tracef("still wait for proxy connection");
845                         }
846                     }
847                 } else {
848                     uri = _uri;
849                     stream = new SSLStream(_sslOptions);
850                     stream.bind(_bind);
851                     stream.connect(uri.host, uri.port, _timeout);
852                     debug(requests) tracef("ssl connection to origin server ready");
853                 }
854                 break ;
855             }
856         }
857 
858         if ( stream ) {
859             auto purged_connection = _cm.put(_uri.scheme, _uri.host, _uri.port, stream);
860             if ( purged_connection ) {
861                 debug(requests) tracef("closing purged connection %s", purged_connection);
862                 purged_connection.close();
863             }
864         }
865 
866         return stream;
867     }
868     ///
869     /// Request sent, now receive response.
870     /// Find headers, split on headers and body, continue to receive body
871     ///
872     private void receiveResponse(NetworkStream _stream) {
873 
874         try {
875             _stream.readTimeout = timeout;
876         } catch (Exception e) {
877             debug(requests) tracef("Failed to set read timeout for stream: %s", e.msg);
878             return;
879         }
880         // Commented this out as at exit we can have alreade closed socket
881         // scope(exit) {
882         //     if ( _stream && _stream.isOpen ) {
883         //         _stream.readTimeout = 0.seconds;
884         //     }
885         // }
886 
887         _bodyDecoder = new DataPipe!ubyte();
888         scope(exit) {
889             if ( !_useStreaming ) {
890                 _bodyDecoder = null;
891                 _unChunker = null;
892             }
893         }
894 
895         auto buffer = Buffer!ubyte();
896         Buffer!ubyte partialBody;
897         ptrdiff_t read;
898         string separator;
899 
900         while(true) {
901 
902             auto b = new ubyte[_bufferSize];
903             read = _stream.receive(b);
904 
905             debug(requests) tracef("read: %d", read);
906             if ( read == 0 ) {
907                 break;
908             }
909             auto data = b[0..read];
910             buffer.putNoCopy(data);
911             if ( verbosity>=3 ) {
912                 writeln(data.dump.join("\n"));
913             }
914 
915             if ( buffer.length > maxHeadersLength ) {
916                 throw new RequestException("Headers length > maxHeadersLength (%d > %d)".format(buffer.length, maxHeadersLength));
917             }
918             if ( headersHaveBeenReceived(data, buffer, separator) ) {
919                 auto s = buffer.data.findSplit(separator);
920                 auto ResponseHeaders = s[0];
921                 partialBody = Buffer!ubyte(s[2]);
922                 _contentReceived += partialBody.length;
923                 parseResponseHeaders(ResponseHeaders);
924                 break;
925             }
926         }
927 
928         analyzeHeaders(_response._responseHeaders);
929 
930         _bodyDecoder.putNoCopy(partialBody.data);
931 
932         auto v = _bodyDecoder.get();
933         _response._responseBody.putNoCopy(v);
934 
935         if ( _verbosity >= 2 ) writefln("< %d bytes of body received", partialBody.length);
936 
937         if ( _method == "HEAD" ) {
938             // HEAD response have ContentLength, but have no body
939             return;
940         }
941 
942         while( true ) {
943             if ( _contentLength >= 0 && _contentReceived >= _contentLength ) {
944                 debug(requests) trace("Body received.");
945                 break;
946             }
947             if ( _unChunker && _unChunker.done ) {
948                 break;
949             }
950 
951             if ( _useStreaming && _response._responseBody.length && !redirectCodes.canFind(_response.code) ) {
952                 debug(requests) trace("streaming requested");
953                 // save _stream in closure
954                 auto stream = _stream;
955                 // set up response
956                 _response.receiveAsRange.activated = true;
957                 _response.receiveAsRange.data = _response._responseBody.data;
958                 _response.receiveAsRange.read = delegate ubyte[] () {
959 
960                     while(true) {
961                         // check if we received everything we need
962                         if ( ( _unChunker && _unChunker.done )
963                             || !stream.isConnected()
964                             || (_contentLength > 0 && _contentReceived >= _contentLength) )
965                         {
966                             debug(requests) trace("streaming_in receive completed");
967                             _bodyDecoder.flush();
968                             return _bodyDecoder.get();
969                         }
970                         // have to continue
971                         auto b = new ubyte[_bufferSize];
972                         try {
973                             read = stream.receive(b);
974                         }
975                         catch (Exception e) {
976                             throw new RequestException("streaming_in error reading from socket", __FILE__, __LINE__, e);
977                         }
978                         debug(requests) tracef("streaming_in received %d bytes", read);
979 
980                         if ( read == 0 ) {
981                             debug(requests) tracef("streaming_in: server closed connection");
982                             _bodyDecoder.flush();
983                             return _bodyDecoder.get();
984                         }
985 
986                         if ( verbosity>=3 ) {
987                             writeln(b[0..read].dump.join("\n"));
988                         }
989 
990                         _contentReceived += read;
991                         _bodyDecoder.putNoCopy(b[0..read]);
992                         auto res = _bodyDecoder.getNoCopy();
993                         if ( res.length == 0 ) {
994                             // there were nothing to produce (beginning of the chunk or no decompressed data)
995                             continue;
996                         }
997                         if (res.length == 1) {
998                             return res[0];
999                         }
1000                         //
1001                         // I'd like to "return _bodyDecoder.getNoCopy().join;" but it is slower
1002                         //
1003                         auto total = res.map!(b=>b.length).sum;
1004                         // create buffer for joined bytes
1005                         ubyte[] joined = new ubyte[total];
1006                         size_t p;
1007                         // memcopy
1008                         foreach(ref _; res) {
1009                             joined[p .. p + _.length] = _;
1010                             p += _.length;
1011                         }
1012                         return joined;
1013                     }
1014                     assert(0);
1015                 };
1016                 // we prepared for streaming
1017                 return;
1018             }
1019 
1020             auto b = new ubyte[_bufferSize];
1021             read = _stream.receive(b);
1022 
1023             if ( read == 0 ) {
1024                 debug(requests) trace("read done");
1025                 break;
1026             }
1027             if ( _verbosity >= 2 ) {
1028                 writefln("< %d bytes of body received", read);
1029             }
1030 
1031             if ( verbosity>=3 ) {
1032                 writeln(b[0..read].dump.join("\n"));
1033             }
1034 
1035             debug(requests) tracef("read: %d", read);
1036             _contentReceived += read;
1037             if ( _maxContentLength && _contentReceived > _maxContentLength ) {
1038                 throw new RequestException("ContentLength > maxContentLength (%d>%d)".
1039                     format(_contentLength, _maxContentLength));
1040             }
1041 
1042             _bodyDecoder.putNoCopy(b[0..read]); // send buffer to all decoders
1043 
1044             _bodyDecoder.getNoCopy.             // fetch result and place to body
1045                 each!(b => _response._responseBody.putNoCopy(b));
1046 
1047             debug(requests) tracef("receivedTotal: %d, contentLength: %d, bodyLength: %d", _contentReceived, _contentLength, _response._responseBody.length);
1048 
1049         }
1050         _bodyDecoder.flush();
1051         _response._responseBody.putNoCopy(_bodyDecoder.get());
1052     }
1053     ///
1054     /// Check that we received anything.
1055     /// Server can close previous connection (keepalive or not)
1056     ///
1057     private bool serverPrematurelyClosedConnection() pure @safe {
1058         immutable server_closed_connection = _response._responseHeaders.length == 0 && _response._status_line.length == 0;
1059         // debug(requests) tracef("server closed connection = %s (headers.length=%s, status_line.length=%s)",
1060         //     server_closed_connection, _response._responseHeaders.length,  _response._status_line.length);
1061         return server_closed_connection;
1062     }
1063     private bool isIdempotent(in string method) pure @safe nothrow {
1064         return ["GET", "HEAD"].canFind(method);
1065     }
1066     ///
1067     /// If we do not want keepalive request,
1068     /// or server signalled to close connection,
1069     /// then close it
1070     ///
1071     void close_connection_if_not_keepalive(NetworkStream _stream) {
1072         auto connection = "connection" in _response._responseHeaders;
1073         if ( !_keepAlive ) {
1074             _stream.close();
1075         } else switch(_response._version) {
1076             case HTTP11:
1077                 // HTTP/1.1 defines the "close" connection option for the sender to signal that the connection
1078                 // will be closed after completion of the response. For example,
1079                 //        Connection: close
1080                 // in either the request or the response header fields indicates that the connection
1081                 // SHOULD NOT be considered `persistent' (section 8.1) after the current request/response is complete.
1082                 // HTTP/1.1 applications that do not support persistent connections MUST include the "close" connection
1083                 // option in every message.
1084                 if ( connection && (*connection).toLower.split(",").canFind("close") ) {
1085                     _stream.close();
1086                 }
1087                 break;
1088             default:
1089                 // for anything else close connection if there is no keep-alive in Connection
1090                 if ( connection && !(*connection).toLower.split(",").canFind("keep-alive") ) {
1091                     _stream.close();
1092                 }
1093                 break;
1094         }
1095     }
1096     ///
1097     /// Send multipart for request.
1098     /// You would like to use this method for sending large portions of mixed data or uploading files to forms.
1099     /// Content of the posted form consist of sources. Each source have at least name and value (can be string-like object or opened file, see more docs for MultipartForm struct)
1100     /// Params:
1101     ///     url = url
1102     ///     sources = array of sources.
1103     HTTPResponse exec(string method="POST")(string url, MultipartForm sources) {
1104         import std.uuid;
1105         import std.file;
1106 
1107         checkURL(url);
1108         if ( _cm is null ) {
1109             _cm = new ConnManager();
1110         }
1111 
1112         NetworkStream _stream;
1113         _method = method;
1114         _response = new HTTPResponse;
1115         _response.uri = _uri;
1116         _response.finalURI = _uri;
1117         bool restartedRequest = false;
1118 
1119     connect:
1120         _contentReceived = 0;
1121         _response._startedAt = Clock.currTime;
1122 
1123         assert(_stream is null);
1124 
1125         _stream = _cm.get(_uri.scheme, _uri.host, _uri.port);
1126 
1127         if ( _stream is null ) {
1128             debug(requests) trace("create new connection");
1129             _stream = setupConnection();
1130         } else {
1131             debug(requests) trace("reuse old connection");
1132         }
1133 
1134         assert(_stream !is null);
1135 
1136         if ( !_stream.isConnected ) {
1137             debug(requests) trace("disconnected stream on enter");
1138             if ( !restartedRequest ) {
1139                 debug(requests) trace("disconnected stream on enter: retry");
1140                 assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
1141 
1142                 _cm.del(_uri.scheme, _uri.host, _uri.port);
1143                 _stream.close();
1144                 _stream = null;
1145 
1146                 restartedRequest = true;
1147                 goto connect;
1148             }
1149             debug(requests) trace("disconnected stream on enter: return response");
1150             //_stream = null;
1151             return _response;
1152         }
1153         _response._connectedAt = Clock.currTime;
1154 
1155         Appender!string req;
1156         req.put(requestString());
1157 
1158         string   boundary = randomUUID().toString;
1159         string[] partHeaders;
1160         size_t   contentLength;
1161 
1162         foreach(ref part; sources._sources) {
1163             string h = "--" ~ boundary ~ "\r\n";
1164             string disposition = `form-data; name="%s"`.format(part.name);
1165             string optionals = part.
1166                 parameters.byKeyValue().
1167                 filter!(p => p.key!="Content-Type").
1168                 map!   (p => "%s=%s".format(p.key, p.value)).
1169                 join("; ");
1170 
1171             h ~= `Content-Disposition: ` ~ [disposition, optionals].join("; ") ~ "\r\n";
1172 
1173             auto contentType = "Content-Type" in part.parameters;
1174             if ( contentType ) {
1175                 h ~= "Content-Type: " ~ *contentType ~ "\r\n";
1176             }
1177 
1178             h ~= "\r\n";
1179             partHeaders ~= h;
1180             contentLength += h.length + part.input.getSize() + "\r\n".length;
1181         }
1182         contentLength += "--".length + boundary.length + "--\r\n".length;
1183 
1184         auto h = requestHeaders();
1185         safeSetHeader(h, _userHeaders.ContentType, "Content-Type", "multipart/form-data; boundary=" ~ boundary);
1186         safeSetHeader(h, _userHeaders.ContentLength, "Content-Length", to!string(contentLength));
1187 
1188         h.byKeyValue.
1189             map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n").
1190                 each!(h => req.put(h));
1191         req.put("\r\n");
1192 
1193         debug(requests) trace(req.data);
1194         if ( _verbosity >= 1 ) req.data.splitLines.each!(a => writeln("> " ~ a));
1195 
1196         try {
1197             _stream.send(req.data());
1198             foreach(ref source; sources._sources) {
1199                 debug(requests) tracef("sending part headers <%s>", partHeaders.front);
1200                 _stream.send(partHeaders.front);
1201                 partHeaders.popFront;
1202                 while (true) {
1203                     auto chunk = source.input.read();
1204                     if ( chunk.length <= 0 ) {
1205                         break;
1206                     }
1207                     _stream.send(chunk);
1208                 }
1209                 _stream.send("\r\n");
1210             }
1211             _stream.send("--" ~ boundary ~ "--\r\n");
1212             _response._requestSentAt = Clock.currTime;
1213             receiveResponse(_stream);
1214             _response._finishedAt = Clock.currTime;
1215         }
1216         catch (NetworkException e) {
1217             errorf("Error sending request: ", e.msg);
1218             _stream.close();
1219             return _response;
1220         }
1221 
1222         if ( serverPrematurelyClosedConnection()
1223         && !restartedRequest
1224         && isIdempotent(_method)
1225         ) {
1226             ///
1227             /// We didn't receive any data (keepalive connectioin closed?)
1228             /// and we can restart this request.
1229             /// Go ahead.
1230             ///
1231             debug(requests) tracef("Server closed keepalive connection");
1232 
1233             assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
1234 
1235             _cm.del(_uri.scheme, _uri.host, _uri.port);
1236             _stream.close();
1237             _stream = null;
1238 
1239             restartedRequest = true;
1240             goto connect;
1241         }
1242 
1243         if ( _useStreaming ) {
1244             if ( _response._receiveAsRange.activated ) {
1245                 debug(requests) trace("streaming_in activated");
1246                 return _response;
1247             } else {
1248                 // this can happen if whole response body received together with headers
1249                 _response._receiveAsRange.data = _response.responseBody.data;
1250             }
1251         }
1252 
1253         close_connection_if_not_keepalive(_stream);
1254 
1255         if ( _verbosity >= 1 ) {
1256             writeln(">> Connect time: ", _response._connectedAt - _response._startedAt);
1257             writeln(">> Request send time: ", _response._requestSentAt - _response._connectedAt);
1258             writeln(">> Response recv time: ", _response._finishedAt - _response._requestSentAt);
1259         }
1260 
1261         if ( willFollowRedirect ) {
1262             if ( _history.length >= _maxRedirects ) {
1263                 _stream = null;
1264                 throw new MaxRedirectsException("%d redirects reached maxRedirects %d.".format(_history.length, _maxRedirects));
1265             }
1266             // "location" in response already checked in canFollowRedirect
1267             immutable new_location = *("location" in _response.responseHeaders);
1268             immutable current_uri = _uri, next_uri = uriFromLocation(_uri, new_location);
1269 
1270             // save current response for history
1271             _history ~= _response;
1272 
1273             // prepare new response (for redirected request)
1274             _response = new HTTPResponse;
1275             _response.uri = current_uri;
1276             _response.finalURI = next_uri;
1277             _stream = null;
1278 
1279             // set new uri
1280             this._uri = next_uri;
1281             debug(requests) tracef("Redirected to %s", next_uri);
1282             if ( _method != "GET" && _response.code != 307 && _response.code != 308 ) {
1283                 // 307 and 308 do not change method
1284                 return this.get();
1285             }
1286             if ( restartedRequest ) {
1287                 debug(requests) trace("Rare event: clearing 'restartedRequest' on redirect");
1288                 restartedRequest = false;
1289             }
1290             goto connect;
1291         }
1292 
1293         _response._history = _history;
1294         return _response;
1295     }
1296 
1297     // we use this if we send from ubyte[][] and user provided Content-Length
1298     private void sendFlattenContent(T)(NetworkStream _stream, T content) {
1299         while ( !content.empty ) {
1300             auto chunk = content.front;
1301             _stream.send(chunk);
1302             content.popFront;
1303         }
1304         debug(requests) tracef("sent");
1305     }
1306     // we use this if we send from ubyte[][] as chunked content
1307     private void sendChunkedContent(T)(NetworkStream _stream, T content) {
1308         while ( !content.empty ) {
1309             auto chunk = content.front;
1310             auto chunkHeader = "%x\r\n".format(chunk.length);
1311             debug(requests) tracef("sending %s%s", chunkHeader, chunk);
1312             _stream.send(chunkHeader);
1313             _stream.send(chunk);
1314             _stream.send("\r\n");
1315             content.popFront;
1316         }
1317         debug(requests) tracef("sent");
1318         _stream.send("0\r\n\r\n");
1319     }
1320     ///
1321     /// POST/PUT/... data from some string(with Content-Length), or from range of strings/bytes (use Transfer-Encoding: chunked).
1322     /// When rank 1 (flat array) used as content it must have length. In that case "content" will be sent directly to network, and Content-Length headers will be added.
1323     /// If you are goung to send some range and do not know length at the moment when you start to send request, then you can send chunks of chars or ubyte.
1324     /// Try not to send too short chunks as this will put additional load on client and server. Chunks of length 2048 or 4096 are ok.
1325     ///
1326     /// Parameters:
1327     ///    url = url
1328     ///    content = string or input range
1329     ///    contentType = content type
1330     ///  Returns:
1331     ///     Response
1332     ///  Examples:
1333     ///  ---------------------------------------------------------------------------------------------------------
1334     ///      rs = rq.exec!"POST"("http://httpbin.org/post", "привiт, свiт!", "application/octet-stream");
1335     ///
1336     ///      auto s = lineSplitter("one,\ntwo,\nthree.");
1337     ///      rs = rq.exec!"POST"("http://httpbin.org/post", s, "application/octet-stream");
1338     ///
1339     ///      auto s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
1340     ///      rs = rq.exec!"POST"("http://httpbin.org/post", s.representation.chunks(10), "application/octet-stream");
1341     ///
1342     ///      auto f = File("tests/test.txt", "rb");
1343     ///      rs = rq.exec!"POST"("http://httpbin.org/post", f.byChunk(3), "application/octet-stream");
1344     ///  --------------------------------------------------------------------------------------------------------
1345     HTTPResponse exec(string method="POST", R)(string url, R content, string contentType="application/octet-stream")
1346         if ( (rank!R == 1)
1347             || (rank!R == 2 && isSomeChar!(Unqual!(typeof(content.front.front))))
1348             || (rank!R == 2 && (is(Unqual!(typeof(content.front.front)) == ubyte)))
1349         )
1350     do {
1351         debug(requests) tracef("started url=%s, this._uri=%s", url, _uri);
1352 
1353         checkURL(url);
1354         if ( _cm is null ) {
1355             _cm = new ConnManager();
1356         }
1357 
1358         NetworkStream _stream;
1359         _method = method;
1360         _response = new HTTPResponse;
1361         _history.length = 0;
1362         _response.uri = _uri;
1363         _response.finalURI = _uri;
1364         bool restartedRequest = false;
1365         bool send_flat;
1366 
1367     connect:
1368         _contentReceived = 0;
1369         _response._startedAt = Clock.currTime;
1370 
1371         assert(_stream is null);
1372 
1373         _stream = _cm.get(_uri.scheme, _uri.host, _uri.port);
1374 
1375         if ( _stream is null ) {
1376             debug(requests) trace("create new connection");
1377             _stream = setupConnection();
1378         } else {
1379             debug(requests) trace("reuse old connection");
1380         }
1381 
1382         assert(_stream !is null);
1383 
1384         if ( !_stream.isConnected ) {
1385             debug(requests) trace("disconnected stream on enter");
1386             if ( !restartedRequest ) {
1387                 debug(requests) trace("disconnected stream on enter: retry");
1388                 assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
1389 
1390                 _cm.del(_uri.scheme, _uri.host, _uri.port);
1391                 _stream.close();
1392                 _stream = null;
1393 
1394                 restartedRequest = true;
1395                 goto connect;
1396             }
1397             debug(requests) trace("disconnected stream on enter: return response");
1398             //_stream = null;
1399             return _response;
1400         }
1401         _response._connectedAt = Clock.currTime;
1402 
1403         Appender!string req;
1404         req.put(requestString());
1405 
1406         auto h = requestHeaders;
1407         if ( contentType ) {
1408             safeSetHeader(h, _userHeaders.ContentType, "Content-Type", contentType);
1409         }
1410         static if ( rank!R == 1 ) {
1411             safeSetHeader(h, _userHeaders.ContentLength, "Content-Length", to!string(content.length));
1412         } else {
1413             if ( _userHeaders.ContentLength ) {
1414                 debug(requests) tracef("User provided content-length for chunked content");
1415                 send_flat = true;
1416             } else {
1417                 h["Transfer-Encoding"] = "chunked";
1418                 send_flat = false;
1419             }
1420         }
1421         h.byKeyValue.
1422             map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n").
1423             each!(h => req.put(h));
1424         req.put("\r\n");
1425 
1426         debug(requests) trace(req.data);
1427         if ( _verbosity >= 1 ) {
1428             req.data.splitLines.each!(a => writeln("> " ~ a));
1429         }
1430 
1431         try {
1432             // send headers
1433             _stream.send(req.data());
1434             // send body
1435             static if ( rank!R == 1) {
1436                 _stream.send(content);
1437             } else {
1438                 if ( send_flat ) {
1439                     sendFlattenContent(_stream, content);
1440                 } else {
1441                     sendChunkedContent(_stream, content);
1442                 }
1443             }
1444             _response._requestSentAt = Clock.currTime;
1445             receiveResponse(_stream);
1446             _response._finishedAt = Clock.currTime;
1447         } catch (NetworkException e) {
1448             _stream.close();
1449             throw new RequestException("Network error during data exchange");
1450         }
1451 
1452         if ( serverPrematurelyClosedConnection()
1453         && !restartedRequest
1454         && isIdempotent(_method)
1455         ) {
1456             ///
1457             /// We didn't receive any data (keepalive connectioin closed?)
1458             /// and we can restart this request.
1459             /// Go ahead.
1460             ///
1461             debug(requests) tracef("Server closed keepalive connection");
1462 
1463             assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
1464 
1465             _cm.del(_uri.scheme, _uri.host, _uri.port);
1466             _stream.close();
1467             _stream = null;
1468 
1469             restartedRequest = true;
1470             goto connect;
1471         }
1472 
1473         if ( _useStreaming ) {
1474             if ( _response._receiveAsRange.activated ) {
1475                 debug(requests) trace("streaming_in activated");
1476                 return _response;
1477             } else {
1478                 // this can happen if whole response body received together with headers
1479                 _response._receiveAsRange.data = _response.responseBody.data;
1480             }
1481         }
1482 
1483         close_connection_if_not_keepalive(_stream);
1484 
1485         if ( _verbosity >= 1 ) {
1486             writeln(">> Connect time: ", _response._connectedAt - _response._startedAt);
1487             writeln(">> Request send time: ", _response._requestSentAt - _response._connectedAt);
1488             writeln(">> Response recv time: ", _response._finishedAt - _response._requestSentAt);
1489         }
1490 
1491 
1492         if ( willFollowRedirect ) {
1493             if ( _history.length >= _maxRedirects ) {
1494                 _stream = null;
1495                 throw new MaxRedirectsException("%d redirects reached maxRedirects %d.".format(_history.length, _maxRedirects));
1496             }
1497             // "location" in response already checked in canFollowRedirect
1498             immutable new_location = *("location" in _response.responseHeaders);
1499             immutable current_uri = _uri, next_uri = uriFromLocation(_uri, new_location);
1500 
1501             // save current response for history
1502             _history ~= _response;
1503 
1504             // prepare new response (for redirected request)
1505             _response = new HTTPResponse;
1506             _response.uri = current_uri;
1507             _response.finalURI = next_uri;
1508 
1509             _stream = null;
1510 
1511             // set new uri
1512             this._uri = next_uri;
1513             debug(requests) tracef("Redirected to %s", next_uri);
1514             if ( _method != "GET" && _response.code != 307 && _response.code != 308 ) {
1515                 // 307 and 308 do not change method
1516                 return this.get();
1517             }
1518             if ( restartedRequest ) {
1519                 debug(requests) trace("Rare event: clearing 'restartedRequest' on redirect");
1520                 restartedRequest = false;
1521             }
1522             goto connect;
1523         }
1524 
1525         _response._history = _history;
1526         return _response;
1527     }
1528     ///
1529     /// Send request with parameters.
1530     /// If used for POST or PUT requests then application/x-www-form-urlencoded used.
1531     /// Request parameters will be encoded into request string or placed in request body for POST/PUT
1532     /// requests.
1533     /// Parameters:
1534     ///     url = url
1535     ///     params = request parameters
1536     ///  Returns:
1537     ///     Response
1538     ///  Examples:
1539     ///  ---------------------------------------------------------------------------------
1540     ///     rs = Request().exec!"GET"("http://httpbin.org/get", ["c":"d", "a":"b"]);
1541     ///  ---------------------------------------------------------------------------------
1542     ///
1543     HTTPResponse exec(string method="GET")(string url = null, QueryParam[] params = null)
1544     do {
1545         debug(requests) tracef("started url=%s, this._uri=%s", url, _uri);
1546 
1547         checkURL(url);
1548         if ( _cm is null ) {
1549             _cm = new ConnManager();
1550         }
1551 
1552         NetworkStream _stream;
1553         _method = method;
1554         _response = new HTTPResponse;
1555         _history.length = 0;
1556         _response.uri = _uri;
1557         _response.finalURI = _uri;
1558         bool restartedRequest = false; // True if this is restarted keepAlive request
1559 
1560     connect:
1561         if ( _method == "GET" && _uri in _permanent_redirects ) {
1562             debug(requests) trace("use parmanent redirects cache");
1563             _uri = uriFromLocation(_uri, _permanent_redirects[_uri]);
1564             _response._finalURI = _uri;
1565         }
1566         _contentReceived = 0;
1567         _response._startedAt = Clock.currTime;
1568 
1569         assert(_stream is null);
1570 
1571         _stream = _cm.get(_uri.scheme, _uri.host, _uri.port);
1572 
1573         if ( _stream is null ) {
1574             debug(requests) trace("create new connection");
1575             _stream = setupConnection();
1576         } else {
1577             debug(requests) trace("reuse old connection");
1578         }
1579 
1580         assert(_stream !is null);
1581 
1582         if ( !_stream.isConnected ) {
1583             debug(requests) trace("disconnected stream on enter");
1584             if ( !restartedRequest ) {
1585                 debug(requests) trace("disconnected stream on enter: retry");
1586                 assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
1587 
1588                 _cm.del(_uri.scheme, _uri.host, _uri.port);
1589                 _stream.close();
1590                 _stream = null;
1591 
1592                 restartedRequest = true;
1593                 goto connect;
1594             }
1595             debug(requests) trace("disconnected stream on enter: return response");
1596             //_stream = null;
1597             return _response;
1598         }
1599         _response._connectedAt = Clock.currTime;
1600 
1601         auto h = requestHeaders();
1602 
1603         Appender!string req;
1604 
1605         string encoded;
1606 
1607         switch (_method) {
1608             case "POST","PUT":
1609                 encoded = params2query(params);
1610                 safeSetHeader(h, _userHeaders.ContentType, "Content-Type", "application/x-www-form-urlencoded");
1611                 if ( encoded.length > 0) {
1612                     safeSetHeader(h, _userHeaders.ContentLength, "Content-Length", to!string(encoded.length));
1613                 }
1614                 req.put(requestString());
1615                 break;
1616             default:
1617                 req.put(requestString(params));
1618         }
1619 
1620         h.byKeyValue.
1621             map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n").
1622             each!(h => req.put(h));
1623         req.put("\r\n");
1624         if ( encoded ) {
1625             req.put(encoded);
1626         }
1627 
1628         debug(requests) trace(req.data);
1629         if ( _verbosity >= 1 ) {
1630             req.data.splitLines.each!(a => writeln("> " ~ a));
1631         }
1632         //
1633         // Now send request and receive response
1634         //
1635         try {
1636             _stream.send(req.data());
1637             _response._requestSentAt = Clock.currTime;
1638             debug(requests) trace("starting receive response");
1639             receiveResponse(_stream);
1640             debug(requests) trace("done receive response");
1641             _response._finishedAt = Clock.currTime;
1642         }
1643         catch (NetworkException e) {
1644             // On SEND this can means:
1645             // we started to send request to the server, but it closed connection because of keepalive timeout.
1646             // We have to restart request if possible.
1647 
1648             // On RECEIVE - if we received something - then this exception is real and unexpected error.
1649             // If we didn't receive anything - we can restart request again as it can be
1650             debug(requests) tracef("Exception on receive response: %s", e.msg);
1651             if ( _response._responseHeaders.length != 0 )
1652             {
1653                 _stream.close();
1654                 throw new RequestException("Unexpected network error");
1655             }
1656         }
1657 
1658         if ( serverPrematurelyClosedConnection()
1659             && !restartedRequest
1660             && isIdempotent(_method)
1661             ) {
1662             ///
1663             /// We didn't receive any data (keepalive connectioin closed?)
1664             /// and we can restart this request.
1665             /// Go ahead.
1666             ///
1667             debug(requests) tracef("Server closed keepalive connection");
1668 
1669             assert(_cm.get(_uri.scheme, _uri.host, _uri.port) == _stream);
1670 
1671             _cm.del(_uri.scheme, _uri.host, _uri.port);
1672             _stream.close();
1673             _stream = null;
1674 
1675             restartedRequest = true;
1676             goto connect;
1677         }
1678 
1679         if ( _useStreaming ) {
1680             if ( _response._receiveAsRange.activated ) {
1681                 debug(requests) trace("streaming_in activated");
1682                 return _response;
1683             } else {
1684                 // this can happen if whole response body received together with headers
1685                 _response._receiveAsRange.data = _response.responseBody.data;
1686             }
1687         }
1688 
1689         close_connection_if_not_keepalive(_stream);
1690 
1691         if ( _verbosity >= 1 ) {
1692             writeln(">> Connect time: ", _response._connectedAt - _response._startedAt);
1693             writeln(">> Request send time: ", _response._requestSentAt - _response._connectedAt);
1694             writeln(">> Response recv time: ", _response._finishedAt - _response._requestSentAt);
1695         }
1696 
1697         if ( willFollowRedirect ) {
1698             debug(requests) trace("going to follow redirect");
1699             if ( _history.length >= _maxRedirects ) {
1700                 _stream = null;
1701                 throw new MaxRedirectsException("%d redirects reached maxRedirects %d.".format(_history.length, _maxRedirects));
1702             }
1703             // "location" in response already checked in canFollowRedirect
1704             immutable new_location = *("location" in _response.responseHeaders);
1705             immutable current_uri = _uri, next_uri = uriFromLocation(_uri, new_location);
1706 
1707             if ( _method == "GET" && _response.code == 301 ) {
1708                 _permanent_redirects[_uri] = new_location;
1709             }
1710 
1711             // save current response for history
1712             _history ~= _response;
1713 
1714             // prepare new response (for redirected request)
1715             _response = new HTTPResponse;
1716             _response.uri = current_uri;
1717             _response.finalURI = next_uri;
1718             _stream = null;
1719 
1720             // set new uri
1721             _uri = next_uri;
1722             debug(requests) tracef("Redirected to %s", next_uri);
1723             if ( _method != "GET" && _response.code != 307 && _response.code != 308 ) {
1724                 // 307 and 308 do not change method
1725                 return this.get();
1726             }
1727             if ( restartedRequest ) {
1728                 debug(requests) trace("Rare event: clearing 'restartedRequest' on redirect");
1729                 restartedRequest = false;
1730             }
1731             goto connect;
1732         }
1733 
1734         _response._history = _history;
1735         return _response;
1736     }
1737 
1738     /// WRAPPERS
1739     ///
1740     /// send file(s) using POST and multipart form.
1741     /// This wrapper will be deprecated, use post with MultipartForm - it is more general and clear.
1742     /// Parameters:
1743     ///     url = url
1744     ///     files = array of PostFile structures
1745     /// Returns:
1746     ///     Response
1747     /// Each PostFile structure contain path to file, and optional field name and content type.
1748     /// If no field name provided, then basename of the file will be used.
1749     /// application/octet-stream is default when no content type provided.
1750     /// Example:
1751     /// ---------------------------------------------------------------
1752     ///    PostFile[] files = [
1753     ///                   {fileName:"tests/abc.txt", fieldName:"abc", contentType:"application/octet-stream"},
1754     ///                   {fileName:"tests/test.txt"}
1755     ///               ];
1756     ///    rs = rq.exec!"POST"("http://httpbin.org/post", files);
1757     /// ---------------------------------------------------------------
1758     ///
1759     HTTPResponse exec(string method="POST")(string url, PostFile[] files) if (method=="POST") {
1760         MultipartForm multipart;
1761         File[]        toClose;
1762         foreach(ref f; files) {
1763             File file = File(f.fileName, "rb");
1764             toClose ~= file;
1765             string fileName = f.fileName ? f.fileName : f.fieldName;
1766             string contentType = f.contentType ? f.contentType : "application/octetstream";
1767             multipart.add(f.fieldName, new FormDataFile(file), ["filename":fileName, "Content-Type": contentType]);
1768         }
1769         auto res = exec!"POST"(url, multipart);
1770         toClose.each!"a.close";
1771         return res;
1772     }
1773     ///
1774     /// exec request with parameters when you can use dictionary (when you have no duplicates in parameter names)
1775     /// Consider switch to exec(url, QueryParams) as it more generic and clear.
1776     /// Parameters:
1777     ///     url = url
1778     ///     params = dictionary with field names as keys and field values as values.
1779     /// Returns:
1780     ///     Response
1781     HTTPResponse exec(string method="GET")(string url, string[string] params) {
1782         return exec!method(url, params.byKeyValue.map!(p => QueryParam(p.key, p.value)).array);
1783     }
1784     ///
1785     /// GET request. Simple wrapper over exec!"GET"
1786     /// Params:
1787     /// args = request parameters. see exec docs.
1788     ///
1789     HTTPResponse get(A...)(A args) {
1790         return exec!"GET"(args);
1791     }
1792     ///
1793     /// POST request. Simple wrapper over exec!"POST"
1794     /// Params:
1795     /// uri = endpoint uri
1796     /// args = request parameters. see exec docs.
1797     ///
1798     HTTPResponse post(A...)(string uri, A args) {
1799         return exec!"POST"(uri, args);
1800     }
1801 }
1802 
1803 version(vibeD) {
1804     import std.json;
1805     package string httpTestServer() {
1806         return "http://httpbin.org/";
1807     }
1808     package string fromJsonArrayToStr(JSONValue v) {
1809         return v.str;
1810     }
1811 }
1812 else {
1813     import std.json;
1814     package string httpTestServer() {
1815         return "http://127.0.0.1:8081/";
1816     }
1817     package string fromJsonArrayToStr(JSONValue v) {
1818         return cast(string)(v.array.map!"cast(ubyte)a.integer".array);
1819     }
1820 }
1821 
1822 
1823 package unittest {
1824     import std.json;
1825     import std.array;
1826 
1827     globalLogLevel(LogLevel.info);
1828 
1829     string httpbinUrl = httpTestServer();
1830     version(vibeD) {
1831     }
1832     else {
1833         import httpbin;
1834         auto server = httpbinApp();
1835         server.start();
1836         scope(exit) {
1837             server.stop();
1838         }
1839     }
1840     HTTPRequest  rq;
1841     HTTPResponse rs;
1842     info("Check GET");
1843     URI uri = URI(httpbinUrl);
1844     rs = rq.get(httpbinUrl);
1845     assert(rs.code==200);
1846     assert(rs.responseBody.length > 0);
1847     assert(rq.format("%m|%h|%p|%P|%q|%U") ==
1848             "GET|%s|%d|%s||%s"
1849             .format(uri.host, uri.port, uri.path, httpbinUrl));
1850     info("Check GET with AA params");
1851     {
1852         rs = rq.get(httpbinUrl ~ "get", ["c":" d", "a":"b"]);
1853         assert(rs.code == 200);
1854         auto json = parseJSON(cast(string)rs.responseBody.data).object["args"].object;
1855         assert(json["c"].str == " d");
1856         assert(json["a"].str == "b");
1857     }
1858     rq.keepAlive = false; // disable keepalive on non-idempotent requests
1859     info("Check POST files");
1860     {
1861         import std.file;
1862         import std.path;
1863         auto tmpd = tempDir();
1864         auto tmpfname = tmpd ~ dirSeparator ~ "request_test.txt";
1865         auto f = File(tmpfname, "wb");
1866         f.rawWrite("abcdefgh\n12345678\n");
1867         f.close();
1868         // files
1869         PostFile[] files = [
1870             {fileName: tmpfname, fieldName:"abc", contentType:"application/octet-stream"},
1871             {fileName: tmpfname}
1872         ];
1873         rs = rq.post(httpbinUrl ~ "post", files);
1874         assert(rs.code==200);
1875     }
1876     info("Check POST chunked from file.byChunk");
1877     {
1878         import std.file;
1879         import std.path;
1880         auto tmpd = tempDir();
1881         auto tmpfname = tmpd ~ dirSeparator ~ "request_test.txt";
1882         auto f = File(tmpfname, "wb");
1883         f.rawWrite("abcdefgh\n12345678\n");
1884         f.close();
1885         f = File(tmpfname, "rb");
1886         rs = rq.post(httpbinUrl ~ "post", f.byChunk(3), "application/octet-stream");
1887         if (httpbinUrl != "http://httpbin.org/") {
1888             assert(rs.code==200);
1889             auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]);
1890             assert(data=="abcdefgh\n12345678\n");
1891         }
1892         f.close();
1893     }
1894     info("Check POST chunked from lineSplitter");
1895     {
1896         auto s = lineSplitter("one,\ntwo,\nthree.");
1897         rs = rq.exec!"POST"(httpbinUrl ~ "post", s, "application/octet-stream");
1898         if (httpbinUrl != "http://httpbin.org/") {
1899             assert(rs.code==200);
1900             auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]);
1901             assert(data=="one,two,three.");
1902         }
1903     }
1904     info("Check POST chunked from array");
1905     {
1906         auto s = ["one,", "two,", "three."];
1907         rs = rq.post(httpbinUrl ~ "post", s, "application/octet-stream");
1908         if (httpbinUrl != "http://httpbin.org/") {
1909             assert(rs.code==200);
1910             auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]);
1911             assert(data=="one,two,three.");
1912         }
1913     }
1914     info("Check POST chunked using std.range.chunks()");
1915     {
1916         auto s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
1917         rs = rq.post(httpbinUrl ~ "post", s.representation.chunks(10), "application/octet-stream");
1918         if (httpbinUrl != "http://httpbin.org/") {
1919             assert(rs.code==200);
1920             auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody.data).object["data"]);
1921             assert(data==s);
1922         }
1923     }
1924     info("Check POST from QueryParams");
1925     {
1926         rs = rq.post(httpbinUrl ~ "post", queryParams("name[]", "first", "name[]", 2));
1927         assert(rs.code==200);
1928         auto data = parseJSON(cast(string)rs.responseBody).object["form"].object;
1929         string[] a;
1930         try {
1931             a = to!(string[])(data["name[]"].str);
1932         }
1933         catch (JSONException e) {
1934             a = data["name[]"].array.map!"a.str".array;
1935         }
1936         assert(equal(["first", "2"], a));
1937     }
1938     info("Check POST from AA");
1939     {
1940         rs = rq.post(httpbinUrl ~ "post", ["a":"b ", "c":"d"]);
1941         assert(rs.code==200);
1942         auto form = parseJSON(cast(string)rs.responseBody.data).object["form"].object;
1943         assert(form["a"].str == "b ");
1944         assert(form["c"].str == "d");
1945     }
1946     info("Check POST json");
1947     {
1948         rs = rq.post(httpbinUrl ~ "post?b=x", `{"a":"a b", "c":[1,2,3]}`, "application/json");
1949         assert(rs.code==200);
1950         auto json = parseJSON(cast(string)rs.responseBody).object["args"].object;
1951         assert(json["b"].str == "x");
1952         json = parseJSON(cast(string)rs.responseBody).object["json"].object;
1953         assert(json["a"].str == "a b");
1954         assert(json["c"].array.map!(a=>a.integer).array == [1,2,3]);
1955     }
1956     info("Check DELETE");
1957     rs = rq.exec!"DELETE"(httpbinUrl ~ "delete");
1958     assert(rs.code==200);
1959     info("Check PUT");
1960     rs = rq.exec!"PUT"(httpbinUrl ~ "put",  `{"a":"b", "c":[1,2,3]}`, "application/json");
1961     assert(rs.code==200);
1962     assert(parseJSON(cast(string)rs.responseBody).object["json"].object["a"].str=="b");
1963     info("Check PATCH");
1964     rs = rq.exec!"PATCH"(httpbinUrl ~ "patch", "привiт, свiт!", "application/octet-stream");
1965     assert(rs.code==200);
1966     info("Check HEAD");
1967     rs = rq.exec!"HEAD"(httpbinUrl);
1968     assert(rs.code==200);
1969 
1970     rq._keepAlive = true;
1971     info("Check compressed content");
1972     rs = rq.get(httpbinUrl ~ "gzip");
1973     assert(rs.code==200);
1974     bool gzipped = parseJSON(cast(string)rs.responseBody).object["gzipped"].type == JSON_TYPE.TRUE;
1975     assert(gzipped);
1976     info("gzip - ok");
1977     rs = rq.get(httpbinUrl ~ "deflate");
1978     assert(rs.code==200);
1979     bool deflated = parseJSON(cast(string)rs.responseBody).object["deflated"].type == JSON_TYPE.TRUE;
1980     assert(deflated);
1981     info("deflate - ok");
1982 
1983     info("Check redirects");
1984     rs = rq.get(httpbinUrl ~ "relative-redirect/2");
1985     assert(rs.history.length == 2);
1986     assert(rs.code==200);
1987     rs = rq.get(httpbinUrl ~ "absolute-redirect/2");
1988     assert(rs.history.length == 2);
1989     assert(rs.code==200);
1990 
1991     rq.maxRedirects = 2;
1992     assertThrown!MaxRedirectsException(rq.get(httpbinUrl ~ "absolute-redirect/3"));
1993 
1994     rq.maxRedirects = 0;
1995     rs = rq.get(httpbinUrl ~ "absolute-redirect/1");
1996     assert(rs.code==302);
1997 
1998     info("Check cookie");
1999     {
2000         rq.maxRedirects = 10;
2001         rs = rq.get(httpbinUrl ~ "cookies/set?A=abcd&b=cdef");
2002         assert(rs.code == 200);
2003         auto json = parseJSON(cast(string)rs.responseBody.data).object["cookies"].object;
2004         assert(json["A"].str == "abcd");
2005         assert(json["b"].str == "cdef");
2006         foreach(c; rq.cookie) {
2007             final switch(c.attr) {
2008                 case "A":
2009                     assert(c.value == "abcd");
2010                     break;
2011                 case "b":
2012                     assert(c.value == "cdef");
2013                     break;
2014             }
2015         }
2016     }
2017     info("Check chunked content");
2018     rs = rq.get(httpbinUrl ~ "range/1024");
2019     assert(rs.code==200);
2020     assert(rs.responseBody.length==1024);
2021 
2022     info("Check basic auth");
2023     rq.authenticator = new BasicAuthentication("user", "passwd");
2024     rs = rq.get(httpbinUrl ~ "basic-auth/user/passwd");
2025     assert(rs.code==200);
2026 
2027     info("Check limits");
2028     rq = HTTPRequest();
2029     rq.maxContentLength = 1;
2030     assertThrown!RequestException(rq.get(httpbinUrl));
2031 
2032     rq = HTTPRequest();
2033     rq.maxHeadersLength = 1;
2034     assertThrown!RequestException(rq.get(httpbinUrl));
2035 
2036     rq = HTTPRequest();
2037     info("Check POST multiPartForm");
2038     {
2039         /// This is example on usage files with MultipartForm data.
2040         /// For this example we have to create files which will be sent.
2041         import std.file;
2042         import std.path;
2043         /// preapare files
2044         auto tmpd = tempDir();
2045         auto tmpfname1 = tmpd ~ dirSeparator ~ "request_test1.txt";
2046         auto f = File(tmpfname1, "wb");
2047         f.rawWrite("file1 content\n");
2048         f.close();
2049         auto tmpfname2 = tmpd ~ dirSeparator ~ "request_test2.txt";
2050         f = File(tmpfname2, "wb");
2051         f.rawWrite("file2 content\n");
2052         f.close();
2053         ///
2054         /// Ok, files ready.
2055         /// Now we will prepare Form data
2056         ///
2057         File f1 = File(tmpfname1, "rb");
2058         File f2 = File(tmpfname2, "rb");
2059         scope(exit) {
2060             f1.close();
2061             f2.close();
2062         }
2063         ///
2064         /// for each part we have to set field name, source (ubyte array or opened file) and optional filename and content-type
2065         ///
2066         MultipartForm mForm = MultipartForm().
2067             add(formData("Field1", cast(ubyte[])"form field from memory")).
2068                 add(formData("Field2", cast(ubyte[])"file field from memory", ["filename":"data2"])).
2069                 add(formData("File1", f1, ["filename":"file1", "Content-Type": "application/octet-stream"])).
2070                 add(formData("File2", f2, ["filename":"file2", "Content-Type": "application/octet-stream"]));
2071         /// everything ready, send request
2072         rs = rq.post(httpbinUrl ~ "post", mForm);
2073     }
2074     info("Check exception handling, error messages and timeous are OK");
2075     rq.timeout = 1.seconds;
2076     assertThrown!TimeoutException(rq.get(httpbinUrl ~ "delay/3"));
2077 //    assertThrown!ConnectError(rq.get("http://0.0.0.0:65000/"));
2078 //    assertThrown!ConnectError(rq.get("http://1.1.1.1/"));
2079 //    assertThrown!ConnectError(rq.get("http://gkhgkhgkjhgjhgfjhgfjhgf/"));
2080 }
2081