1 module requests.http;
2 
3 private:
4 import std.algorithm;
5 import std.array;
6 import std.conv;
7 import std.datetime;
8 import std.exception;
9 import std.format;
10 import std.stdio;
11 import std.range;
12 import std.string;
13 import std.traits;
14 import std.typecons;
15 import std.experimental.logger;
16 import core.thread;
17 
18 import requests.streams;
19 import requests.uri;
20 import requests.utils;
21 import requests.base;
22 
23 static immutable ushort[] redirectCodes = [301, 302, 303];
24 static immutable uint     defaultBufferSize = 12*1024;
25 
26 public class MaxRedirectsException: Exception {
27     this(string message, string file = __FILE__, size_t line = __LINE__, Throwable next = null) @safe pure nothrow {
28         super(message, file, line, next);
29     }
30 }
31 
32 /**
33  * Basic authentication.
34  * Adds $(B Authorization: Basic) header to request.
35  */
36 public class BasicAuthentication: Auth {
37     private {
38         string   _username, _password;
39         string[] _domains;
40     }
41     /// Constructor.
42     /// Params:
43     /// username = username
44     /// password = password
45     /// domains = not used now
46     /// 
47     this(string username, string password, string[] domains = []) {
48         _username = username;
49         _password = password;
50         _domains = domains;
51     }
52     override string[string] authHeaders(string domain) {
53         import std.base64;
54         string[string] auth;
55         auth["Authorization"] = "Basic " ~ to!string(Base64.encode(cast(ubyte[])"%s:%s".format(_username, _password)));
56         return auth;
57     }
58     override string userName() {
59         return _username;
60     }
61     override string password() {
62         return _password;
63     }
64 }
65 ///
66 ///
67 ///
68 public auto queryParams(T...)(T params) pure nothrow @safe
69 {
70     static assert (T.length % 2 == 0, "wrong args count");
71     
72     QueryParam[] output;
73     output.reserve = T.length / 2;
74     
75     void queryParamsHelper(T...)(T params, ref QueryParam[] output)
76     {
77         static if (T.length > 0)
78         {
79             output ~= QueryParam(params[0].to!string, params[1].to!string);
80             queryParamsHelper(params[2..$], output);
81         }
82     }
83 
84     queryParamsHelper(params, output);
85     return output;
86 }
87 
88 ///
89 /// Response - result of request execution.
90 ///
91 /// Response.code - response HTTP code.
92 /// Response.status_line - received HTTP status line.
93 /// Response.responseHeaders - received headers.
94 /// Response.responseBody - container for received body
95 /// Response.history - for redirected responses contain all history
96 /// 
97 public class HTTPResponse : Response {
98     private {
99         string         _status_line;
100 
101         HTTPResponse[] _history; // redirects history
102         SysTime        _startedAt, _connectedAt, _requestSentAt, _finishedAt;
103 
104         mixin(Setter!string("status_line"));
105     }
106 
107     ~this() {
108         _responseHeaders = null;
109         _history.length = 0;
110     }
111 
112     mixin(Getter!string("status_line"));
113     @property final string[string] responseHeaders() @safe @nogc nothrow {
114         return _responseHeaders;
115     }
116     @property final HTTPResponse[] history() @safe @nogc nothrow {
117         return _history;
118     }
119 
120     @property auto getStats() const pure @safe {
121         alias statTuple = Tuple!(Duration, "connectTime",
122                                  Duration, "sendTime",
123                                  Duration, "recvTime");
124         statTuple stat;
125         stat.connectTime = _connectedAt - _startedAt;
126         stat.sendTime = _requestSentAt - _connectedAt;
127         stat.recvTime = _finishedAt - _requestSentAt;
128         return stat;
129     }
130 }
131 /**
132  * Struct to send multiple files in POST request.
133  */
134 public struct PostFile {
135     /// Path to the file to send.
136     string fileName;
137     /// Name of the field (if empty - send file base name)
138     string fieldName;
139     /// contentType of the file if not empty
140     string contentType;
141 }
142 ///
143 /// This is File-like interface for sending data to multipart fotms
144 /// 
145 public interface FiniteReadable {
146     /// size of the content
147     abstract ulong  getSize();
148     /// file-like read()
149     abstract ubyte[] read();
150 }
151 ///
152 /// Helper to create form elements from File.
153 /// Params:
154 /// name = name of the field in form
155 /// f = opened std.stio.File to send to server
156 /// parameters = optional parameters (most important are "filename" and "Content-Type")
157 /// 
158 public auto formData(string name, File f, string[string] parameters = null) {
159     return MultipartForm.FormData(name, new FormDataFile(f), parameters);
160 }
161 ///
162 /// Helper to create form elements from ubyte[].
163 /// Params:
164 /// name = name of the field in form
165 /// b = data to send to server
166 /// parameters = optional parameters (can be "filename" and "Content-Type")
167 /// 
168 public auto formData(string name, ubyte[] b, string[string] parameters = null) {
169     return MultipartForm.FormData(name, new FormDataBytes(b), parameters);
170 }
171 public auto formData(string name, string b, string[string] parameters = null) {
172     return MultipartForm.FormData(name, new FormDataBytes(b.dup.representation), parameters);
173 }
174 public class FormDataBytes : FiniteReadable {
175     private {
176         ulong   _size;
177         ubyte[] _data;
178         size_t  _offset;
179         bool    _exhausted;
180     }
181     this(ubyte[] data) {
182         _data = data;
183         _size = data.length;
184     }
185     final override ulong getSize() {
186         return _size;
187     }
188     final override ubyte[] read() {
189         enforce( !_exhausted, "You can't read froum exhausted source" );
190         size_t toRead = min(defaultBufferSize, _size - _offset);
191         auto result = _data[_offset.._offset+toRead];
192         _offset += toRead;
193         if ( toRead == 0 ) {
194             _exhausted = true;
195         }
196         return result;
197     }
198 }
199 public class FormDataFile : FiniteReadable {
200     import  std.file;
201     private {
202         File    _fileHandle;
203         ulong   _fileSize;
204         size_t  _processed;
205         bool    _exhausted;
206     }
207     this(File file) {
208         import std.file;
209         _fileHandle = file;
210         _fileSize = std.file.getSize(file.name);
211     }
212     final override ulong getSize() pure nothrow @safe {
213         return _fileSize;
214     }
215     final override ubyte[] read() {
216         enforce( !_exhausted, "You can't read froum exhausted source" );
217         auto b = new ubyte[defaultBufferSize];
218         auto r = _fileHandle.rawRead(b);
219         auto toRead = min(r.length, _fileSize - _processed);
220         if ( toRead == 0 ) {
221             _exhausted = true;
222         }
223         _processed += toRead;
224         return r[0..toRead];
225     }
226 }
227 ///
228 /// This struct used to bulld POST's to forms.
229 /// Each part have name and data. data is something that can be read-ed and have size.
230 /// For example this can be string-like object (wrapped for reading) or opened File.
231 /// 
232 public struct MultipartForm {
233     package struct FormData {
234         FiniteReadable  input;
235         string          name;
236         string[string]  parameters;
237         this(string name, FiniteReadable i, string[string] parameters = null) {
238             this.input = i;
239             this.name = name;
240             this.parameters = parameters;
241         }
242     }
243 
244     private FormData[] _sources;
245     auto add(FormData d) {
246         _sources ~= d;
247         return this;
248     }
249     auto add(string name, FiniteReadable i, string[string]parameters = null) {
250         _sources ~= FormData(name, i, parameters);
251         return this;
252     }
253 }
254 ///
255 
256 ///
257 /// Request.
258 /// Configurable parameters:
259 /// $(B method) - string, method to use (GET, POST, ...)
260 /// $(B headers) - string[string], add any additional headers you'd like to send.
261 /// $(B authenticator) - class Auth, class to send auth headers.
262 /// $(B keepAlive) - bool, set true for keepAlive requests. default true.
263 /// $(B maxRedirects) - uint, maximum number of redirects. default 10.
264 /// $(B maxHeadersLength) - size_t, maximum length of server response headers. default = 32KB.
265 /// $(B maxContentLength) - size_t, maximun content length. delault - 0 = unlimited.
266 /// $(B bufferSize) - size_t, send and receive buffer size. default = 16KB.
267 /// $(B verbosity) - uint, level of verbosity(0 - nothing, 1 - headers, 2 - headers and body progress). default = 0.
268 /// $(B proxy) - string, set proxy url if needed. default - null.
269 /// $(B cookie) - Tuple Cookie, Read/Write cookie You can get cookie setted by server, or set cookies before doing request.
270 /// $(B timeout) - Duration, Set timeout value for connect/receive/send.
271 /// 
272 public struct HTTPRequest {
273     private {
274         enum           _preHeaders = [
275                             "Accept-Encoding": "gzip, deflate",
276                             "User-Agent":      "dlang-requests"
277                         ];
278         string         _method = "GET";
279         URI            _uri;
280         string[string] _headers;
281         string[]       _filteredHeaders;
282         Auth           _authenticator;
283         bool           _keepAlive = true;
284         uint           _maxRedirects = 10;
285         size_t         _maxHeadersLength = 32 * 1024; // 32 KB
286         size_t         _maxContentLength; // 0 - Unlimited
287         string         _proxy;
288         uint           _verbosity = 0;  // 0 - no output, 1 - headers, 2 - headers+body info
289         Duration       _timeout = 30.seconds;
290         size_t         _bufferSize = defaultBufferSize; // 16k
291         bool           _useStreaming; // return iterator instead of completed request
292 
293         NetworkStream   _stream;
294         HTTPResponse[] _history; // redirects history
295         DataPipe!ubyte _bodyDecoder;
296         DecodeChunked  _unChunker;
297         long           _contentLength;
298         long           _contentReceived;
299         Cookie[]       _cookie;
300         SSLOptions     _sslOptions;
301     }
302     package HTTPResponse   _response;
303 
304     mixin(Getter_Setter!string     ("method"));
305     mixin(Getter_Setter!bool       ("keepAlive"));
306     mixin(Getter_Setter!size_t     ("maxContentLength"));
307     mixin(Getter_Setter!size_t     ("maxHeadersLength"));
308     mixin(Getter_Setter!size_t     ("bufferSize"));
309     mixin(Getter_Setter!uint       ("maxRedirects"));
310     mixin(Getter_Setter!uint       ("verbosity"));
311     mixin(Getter_Setter!string     ("proxy"));
312     mixin(Getter_Setter!Duration   ("timeout"));
313     mixin(Setter!Auth              ("authenticator"));
314     mixin(Getter_Setter!bool       ("useStreaming"));
315     mixin(Getter!long              ("contentLength"));
316     mixin(Getter!long              ("contentReceived"));
317     mixin(Getter_Setter!SSLOptions ("sslOptions"));
318 
319     @property void sslSetVerifyPeer(bool v) pure @safe nothrow @nogc {
320         _sslOptions.setVerifyPeer(v);
321     }
322     @property void sslSetKeyFile(string p, SSLOptions.filetype t = SSLOptions.filetype.pem) pure @safe nothrow @nogc {
323         _sslOptions.setKeyFile(p, t);
324     }
325     @property void sslSetCertFile(string p, SSLOptions.filetype t = SSLOptions.filetype.pem) pure @safe nothrow @nogc {
326         _sslOptions.setCertFile(p, t);
327     }
328     @property void sslSetCaCert(string path) pure @safe nothrow @nogc {
329         _sslOptions.setCaCert(path);
330     }
331     @property final void cookie(Cookie[] s) pure @safe @nogc nothrow {
332         _cookie = s;
333     }
334     
335     @property final Cookie[] cookie() pure @safe @nogc nothrow {
336         return _cookie;
337     }    
338 
339     this(string uri) {
340         _uri = URI(uri);
341     }
342    ~this() {
343         if ( _stream && _stream.isConnected) {
344             _stream.close();
345         }
346         _stream = null;
347         _headers = null;
348         _authenticator = null;
349         _history = null;
350         _bodyDecoder = null;
351         _unChunker = null;
352     }
353     void clearHeaders() {
354         _headers = null;
355     }
356     @property void uri(in URI newURI) {
357         handleURLChange(_uri, newURI);
358         _uri = newURI;
359     }
360     /// Add headers to request
361     /// Params:
362     /// headers = headers to send.
363     void addHeaders(in string[string] headers) {
364         foreach(pair; headers.byKeyValue) {
365             _headers[pair.key] = pair.value;
366         }
367     }
368     /// Remove headers from request
369     /// Params:
370     /// headers = headers to remove.
371     void removeHeaders(in string[] headers) pure {
372         _filteredHeaders ~= headers;
373     }
374     ///
375     /// compose headers to send
376     /// 
377     private string[string] requestHeaders() {
378         string[string] generatedHeaders = _preHeaders;
379 
380         if ( _authenticator ) {
381             _authenticator.
382                 authHeaders(_uri.host).
383                 byKeyValue.
384                 each!(pair => generatedHeaders[pair.key] = pair.value);
385         }
386 
387         generatedHeaders["Connection"] = _keepAlive?"Keep-Alive":"Close";
388         generatedHeaders["Host"] = _uri.host;
389 
390         if ( _uri.scheme !in standard_ports || _uri.port != standard_ports[_uri.scheme] ) {
391             generatedHeaders["Host"] ~= ":%d".format(_uri.port);
392         }
393 
394         _headers.byKey.each!(h => generatedHeaders[h] = _headers[h]);
395 
396         if ( _cookie.length ) {
397             auto cs = _cookie.
398                 filter!(c => _uri.path.pathMatches(c.path) && _uri.host.domainMatches(c.domain)).
399                 map!(c => "%s=%s".format(c.attr, c.value)).
400                 joiner(";");
401             generatedHeaders["Cookie"] = "%s".format(cs);
402         }
403 
404         _filteredHeaders.each!(h => generatedHeaders.remove(h));
405 
406         return generatedHeaders;
407     }
408     ///
409     /// Build request string.
410     /// Handle proxy and query parameters.
411     /// 
412     private @property string requestString(QueryParam[] params = null) {
413         if ( _proxy ) {
414             return "%s %s HTTP/1.1\r\n".format(_method, _uri.uri);
415         }
416         auto query = _uri.query.dup;
417         if ( params ) {
418             query ~= params2query(params);
419             if ( query[0] != '?' ) {
420                 query = "?" ~ query;
421             }
422         }
423         return "%s %s%s HTTP/1.1\r\n".format(_method, _uri.path, query);
424     }
425     ///
426     /// encode parameters and build query part of the url
427     /// 
428     private static string params2query(in QueryParam[] params) pure @safe {
429         return params.
430                 map!(a => "%s=%s".format(a.key.urlEncoded, a.value.urlEncoded)).
431                 join("&");
432     }
433     //
434     package unittest {
435         assert(params2query(queryParams("a","b", "c", " d "))=="a=b&c=%20d%20");
436     }
437     ///
438     /// Analyze received headers, take appropriate actions:
439     /// check content length, attach unchunk and uncompress
440     /// 
441     private void analyzeHeaders(in string[string] headers) {
442 
443         _contentLength = -1;
444         _unChunker = null;
445         auto contentLength = "content-length" in headers;
446         if ( contentLength ) {
447             try {
448                 _contentLength = to!long(*contentLength);
449                 if ( _maxContentLength && _contentLength > _maxContentLength) {
450                     throw new RequestException("ContentLength > maxContentLength (%d>%d)".
451                                 format(_contentLength, _maxContentLength));
452                 }
453             } catch (ConvException e) {
454                 throw new RequestException("Can't convert Content-Length from %s".format(*contentLength));
455             }
456         }
457         auto transferEncoding = "transfer-encoding" in headers;
458         if ( transferEncoding ) {
459             debug(requests) tracef("transferEncoding: %s", *transferEncoding);
460             if ( *transferEncoding == "chunked") {
461                 _unChunker = new DecodeChunked();
462                 _bodyDecoder.insert(_unChunker);
463             }
464         }
465         auto contentEncoding = "content-encoding" in headers;
466         if ( contentEncoding ) switch (*contentEncoding) {
467             default:
468                 throw new RequestException("Unknown content-encoding " ~ *contentEncoding);
469             case "gzip":
470             case "deflate":
471                 _bodyDecoder.insert(new Decompressor!ubyte);
472         }
473 
474     }
475     ///
476     /// Called when we know that all headers already received in buffer.
477     /// This routine does not interpret headers content (see analyzeHeaders).
478     /// 1. Split headers on lines
479     /// 2. store status line, store response code
480     /// 3. unfold headers if needed
481     /// 4. store headers
482     /// 
483     private void parseResponseHeaders(in ubyte[] input) {
484         string lastHeader;
485         auto buffer = cast(string)input;
486  
487         foreach(line; buffer.split("\n").map!(l => l.stripRight)) {
488             if ( ! _response.status_line.length ) {
489                 debug (requests) tracef("statusLine: %s", line);
490                 _response.status_line = line;
491                 if ( _verbosity >= 1 ) {
492                     writefln("< %s", line);
493                 }
494                 auto parsed = line.split(" ");
495                 if ( parsed.length >= 3 ) {
496                     _response.code = parsed[1].to!ushort;
497                 }
498                 continue;
499             }
500             if ( line[0] == ' ' || line[0] == '\t' ) {
501                 // unfolding https://tools.ietf.org/html/rfc822#section-3.1
502                 if ( auto stored = lastHeader in _response._responseHeaders) {
503                     *stored ~= line;
504                 }
505                 continue;
506             }
507             auto parsed = line.findSplit(":");
508             auto header = parsed[0].toLower;
509             auto value = parsed[2].strip;
510 
511             if ( _verbosity >= 1 ) {
512                 writefln("< %s: %s", header, value);
513             }
514 
515             lastHeader = header;
516             debug (requests) tracef("Header %s = %s", header, value);
517 
518             if ( header != "set-cookie" ) {
519                 auto stored = _response.responseHeaders.get(header, null);
520                 if ( stored ) {
521                     value = stored ~ ", " ~ value;
522                 }
523                 _response._responseHeaders[header] = value;
524                 continue;
525             }
526             _cookie ~= processCookie(value);
527         }
528     }
529 
530     ///
531     /// Process Set-Cookie header from server response
532     /// 
533     private Cookie[] processCookie(string value ) pure {
534         // cookie processing
535         //
536         // as we can't join several set-cookie lines in single line
537         // < Set-Cookie: idx=7f2800f63c112a65ef5082957bcca24b; expires=Mon, 29-May-2017 00:31:25 GMT; path=/; domain=example.com
538         // < Set-Cookie: idx=7f2800f63c112a65ef5082957bcca24b; expires=Mon, 29-May-2017 00:31:25 GMT; path=/; domain=example.com, cs=ip764-RgKqc-HvSkxRxdQQAKW8LA; path=/; domain=.example.com; HttpOnly
539         //
540         Cookie[] res;
541         string[string] kv;
542         auto fields = value.split(";").map!strip;
543         while(!fields.empty) {
544             auto s = fields.front.findSplit("=");
545             fields.popFront;
546             if ( s[1] != "=" ) {
547                 continue;
548             }
549             auto k = s[0];
550             auto v = s[2];
551             switch(k.toLower()) {
552                 case "domain":
553                     k = "domain";
554                     break;
555                 case "path":
556                     k = "path";
557                     break;
558                 case "expires":
559                     continue;
560                 default:
561                     break;
562             }
563             kv[k] = v;
564         }
565         if ( "domain" !in kv ) {
566             kv["domain"] = _uri.host;
567         }
568         if ( "path" !in kv ) {
569             kv["path"] = _uri.path;
570         }
571         auto domain = kv["domain"]; kv.remove("domain");
572         auto path   = kv["path"];   kv.remove("path");
573         foreach(pair; kv.byKeyValue) {
574             auto _attr = pair.key;
575             auto _value = pair.value;
576             auto cookie = Cookie(path, domain, _attr, _value);
577             res ~= cookie;
578         }
579         return res;
580     }
581     ///
582     /// Do we received \r\n\r\n?
583     /// 
584     private bool headersHaveBeenReceived(in ubyte[] data, ref Buffer!ubyte buffer, out string separator) const @safe {
585         foreach(s; ["\r\n\r\n", "\n\n"]) {
586             if ( data.canFind(s) || buffer.canFind(s) ) {
587                 separator = s;
588                 return true;
589             }
590         }
591         return false;
592     }
593 
594     private bool followRedirectResponse() {
595         if ( _history.length >= _maxRedirects ) {
596             throw new MaxRedirectsException("%d redirects reached maxRedirects %d.".format(_history.length, _maxRedirects));
597         }
598         auto location = "location" in _response.responseHeaders;
599         if ( !location ) {
600             return false;
601         }
602         _history ~= _response;
603         auto connection = "connection" in _response._responseHeaders;
604         if ( !connection || *connection == "close" ) {
605             debug(requests) tracef("Closing connection because of 'Connection: close' or no 'Connection' header");
606             _stream.close();
607         }
608         URI oldURI = _uri;
609         URI newURI = oldURI;
610         try {
611             newURI = URI(*location);
612         } catch (UriException e) {
613             debug(requests) trace("Can't parse Location:, try relative uri");
614             newURI.path = *location;
615             newURI.uri = newURI.recalc_uri;
616         }
617         handleURLChange(oldURI, newURI);
618             oldURI = _response.uri;
619         _uri = newURI;
620         _response = new HTTPResponse;
621         _response.uri = oldURI;
622         _response.finalURI = newURI;
623         return true;
624     }
625     ///
626     /// If uri changed so that we have to change host or port, then we have to close socket stream
627     /// 
628     private void handleURLChange(in URI from, in URI to) {
629         if ( _stream !is null && _stream.isConnected && 
630             ( from.scheme != to.scheme || from.host != to.host || from.port != to.port) ) {
631             debug tracef("Have to reopen stream, because of URI change");
632             _stream.close();
633         }
634     }
635     ///
636     /// if we have new uri, then we need to check if we have to reopen existent connection
637     /// 
638     private void checkURL(string url, string file=__FILE__, size_t line=__LINE__) {
639         if (url is null && _uri.uri == "" ) {
640             throw new RequestException("No url configured", file, line);
641         }
642         
643         if ( url !is null ) {
644             URI newURI = URI(url);
645             handleURLChange(_uri, newURI);
646             _uri = newURI;
647         }
648     }
649     ///
650     /// Setup connection. Handle proxy and https case
651     /// 
652     private void setupConnection() {
653         if ( !_stream || !_stream.isConnected ) {
654             debug(requests) tracef("Set up new connection");
655             URI   uri;
656             if ( _proxy ) {
657                 // use proxy uri to connect
658                 uri.uri_parse(_proxy);
659             } else {
660                 // use original uri
661                 uri = _uri;
662             }
663             final switch (uri.scheme) {
664                 case "http":
665                     _stream = new TCPStream().connect(uri.host, uri.port, _timeout);
666                     break;
667                 case "https":
668                     _stream = new SSLStream(_sslOptions).connect(uri.host, uri.port, _timeout);
669                     break;
670             }
671         } else {
672             debug(requests) tracef("Use old connection");
673         }
674     }
675     ///
676     /// Request sent, now receive response.
677     /// Find headers, split on headers and body, continue to receive body
678     /// 
679     private void receiveResponse() {
680 
681         _stream.readTimeout = timeout;
682         scope(exit) {
683             if ( _stream && _stream.isOpen ) {
684                 _stream.readTimeout = 0.seconds;
685             }
686         }
687 
688         _bodyDecoder = new DataPipe!ubyte();
689         scope(exit) {
690             if ( !_useStreaming ) {
691                 _bodyDecoder = null;
692                 _unChunker = null;
693             }
694         }
695 
696         auto buffer = Buffer!ubyte();
697         Buffer!ubyte partialBody;
698         ptrdiff_t read;
699         string separator;
700         
701         while(true) {
702 
703             auto b = new ubyte[_bufferSize];
704             read = _stream.receive(b);
705 
706             debug(requests) tracef("read: %d", read);
707             if ( read == 0 ) {
708                 break;
709             }
710             auto data = b[0..read];
711             buffer.putNoCopy(data);
712             if ( verbosity>=3 ) {
713                 writeln(data.dump.join("\n"));
714             }
715 
716             if ( buffer.length > maxHeadersLength ) {
717                 throw new RequestException("Headers length > maxHeadersLength (%d > %d)".format(buffer.length, maxHeadersLength));
718             }
719             if ( headersHaveBeenReceived(data, buffer, separator) ) {
720                 auto s = buffer.data.findSplit(separator);
721                 auto ResponseHeaders = s[0];
722                 partialBody = Buffer!ubyte(s[2]);
723                 _contentReceived += partialBody.length;
724                 parseResponseHeaders(ResponseHeaders);
725                 break;
726             }
727         }
728         
729         analyzeHeaders(_response._responseHeaders);
730 
731         _bodyDecoder.putNoCopy(partialBody.data);
732 
733         if ( _verbosity >= 2 ) writefln("< %d bytes of body received", partialBody.length);
734 
735         if ( _method == "HEAD" ) {
736             // HEAD response have ContentLength, but have no body
737             return;
738         }
739 
740         while( true ) {
741             if ( _contentLength >= 0 && _contentReceived >= _contentLength ) {
742                 debug(requests) trace("Body received.");
743                 break;
744             }
745             if ( _unChunker && _unChunker.done ) {
746                 break;
747             }
748             if ( _useStreaming && _response._responseBody.length && !redirectCodes.canFind(_response.code) ) {
749                 debug(requests) trace("streaming requested");
750                 _response.receiveAsRange.activated = true;
751                 _response.receiveAsRange.data = _response._responseBody.data;
752                 _response.receiveAsRange.read = delegate ubyte[] () {
753                     while(true) {
754                         // check if we received everything we need
755                         if ( ( _unChunker && _unChunker.done )
756                             || !_stream.isConnected()
757                             || (_contentLength > 0 && _contentReceived >= _contentLength) ) 
758                         {
759                             debug(requests) trace("streaming_in receive completed");
760                             _bodyDecoder.flush();
761                             return _bodyDecoder.get();
762                         }
763                         // have to continue
764                         auto b = new ubyte[_bufferSize];
765                         try {
766                             read = _stream.receive(b);
767                         }
768                         catch (Exception e) {
769                             throw new RequestException("streaming_in error reading from socket", __FILE__, __LINE__, e);
770                         }
771                         debug(requests) tracef("streaming_in received %d bytes", read);
772 
773                         if ( read == 0 ) {
774                             debug(requests) tracef("streaming_in: server closed connection");
775                             _bodyDecoder.flush();
776                             return _bodyDecoder.get();
777                         }
778 
779                         if ( verbosity>=3 ) {
780                             writeln(b[0..read].dump.join("\n"));
781                         }
782 
783                         _contentReceived += read;
784                         _bodyDecoder.putNoCopy(b[0..read]);
785                         auto res = _bodyDecoder.getNoCopy();
786                         if ( res.length == 0 ) {
787                             // there were nothing to produce (beginning of the chunk or no decompressed data)
788                             continue;
789                         }
790                         if (res.length == 1) {
791                             return res[0];
792                         }
793                         //
794                         // I'd like to "return _bodyDecoder.getNoCopy().join;" but if is slower
795                         //
796                         auto total = res.map!(b=>b.length).sum;
797                         // create buffer for joined bytes
798                         ubyte[] joined = new ubyte[total];
799                         size_t p;
800                         // memcopy 
801                         foreach(ref _; res) {
802                             joined[p .. p + _.length] = _;
803                             p += _.length;
804                         }
805                         return joined;
806                     }
807                     assert(0);
808                 };
809                 // we prepared for streaming
810                 return;
811             }
812 
813             auto b = new ubyte[_bufferSize];
814             read = _stream.receive(b);
815 
816             if ( read == 0 ) {
817                 debug(requests) trace("read done");
818                 break;
819             }
820             if ( _verbosity >= 2 ) {
821                 writefln("< %d bytes of body received", read);
822             }
823 
824             if ( verbosity>=3 ) {
825                 writeln(b[0..read].dump.join("\n"));
826             }
827 
828             debug(requests) tracef("read: %d", read);
829             _contentReceived += read;
830             if ( _maxContentLength && _contentReceived > _maxContentLength ) {
831                 throw new RequestException("ContentLength > maxContentLength (%d>%d)".
832                     format(_contentLength, _maxContentLength));
833             }
834 
835             _bodyDecoder.putNoCopy(b[0..read]); // send buffer to all decoders
836 
837             _bodyDecoder.getNoCopy.             // fetch result and place to body
838                 each!(b => _response._responseBody.putNoCopy(b));
839 
840             debug(requests) tracef("receivedTotal: %d, contentLength: %d, bodyLength: %d", _contentReceived, _contentLength, _response._responseBody.length);
841 
842         }
843         _bodyDecoder.flush();
844         _response._responseBody.putNoCopy(_bodyDecoder.get());
845     }
846     private bool serverClosedKeepAliveConnection() pure @safe nothrow {
847         return _response._responseHeaders.length == 0 && _keepAlive;
848     }
849     private bool isIdempotent(in string method) pure @safe nothrow {
850         return ["GET", "HEAD"].canFind(method);
851     }
852     ///
853     /// Send multipart for request.
854     /// You would like to use this method for sending large portions of mixed data or uploading files to forms.
855     /// Content of the posted form consist of sources. Each source have at least name and value (can be string-like object or opened file, see more docs for MultipartForm struct)
856     /// Params:
857     ///     url = url
858     ///     sources = array of sources.
859     HTTPResponse exec(string method="POST")(string url, MultipartForm sources) {
860         import std.uuid;
861         import std.file;
862         //
863         // application/json
864         //
865         bool restartedRequest = false;
866         
867         _method = method;
868         
869         _response = new HTTPResponse;
870         checkURL(url);
871         _response.uri = _uri;
872         _response.finalURI = _uri;
873 
874     connect:
875         _contentReceived = 0;
876         _response._startedAt = Clock.currTime;
877         setupConnection();
878         
879         if ( !_stream.isConnected() ) {
880             return _response;
881         }
882         _response._connectedAt = Clock.currTime;
883         
884         Appender!string req;
885         req.put(requestString());
886         
887         string   boundary = randomUUID().toString;
888         string[] partHeaders;
889         size_t   contentLength;
890         
891         foreach(ref part; sources._sources) {
892             string h = "--" ~ boundary ~ "\r\n";
893             string disposition = `form-data; name="%s"`.format(part.name);
894             string optionals = part.
895                 parameters.byKeyValue().
896                 filter!(p => p.key!="Content-Type").
897                 map!   (p => "%s=%s".format(p.key, p.value)).
898                 join("; ");
899 
900             h ~= `Content-Disposition: ` ~ [disposition, optionals].join("; ") ~ "\r\n";
901 
902             auto contentType = "Content-Type" in part.parameters;
903             if ( contentType ) {
904                 h ~= "Content-Type: " ~ *contentType ~ "\r\n";
905             }
906 
907             h ~= "\r\n";
908             partHeaders ~= h;
909             contentLength += h.length + part.input.getSize() + "\r\n".length;
910         }
911         contentLength += "--".length + boundary.length + "--\r\n".length;
912         
913         auto h = requestHeaders();
914         h["Content-Type"] = "multipart/form-data; boundary=" ~ boundary;
915         h["Content-Length"] = to!string(contentLength);
916         h.byKeyValue.
917             map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n").
918                 each!(h => req.put(h));
919         req.put("\r\n");
920         
921         debug(requests) trace(req.data);
922         if ( _verbosity >= 1 ) req.data.splitLines.each!(a => writeln("> " ~ a));
923 
924         try {
925             _stream.send(req.data());
926             foreach(ref source; sources._sources) {
927                 debug(requests) tracef("sending part headers <%s>", partHeaders.front);
928                 _stream.send(partHeaders.front);
929                 partHeaders.popFront;
930                 while (true) {
931                     auto chunk = source.input.read();
932                     if ( chunk.length <= 0 ) {
933                         break;
934                     }
935                     _stream.send(chunk);
936                 }
937                 _stream.send("\r\n");
938             }
939             _stream.send("--" ~ boundary ~ "--\r\n");
940             _response._requestSentAt = Clock.currTime;
941             receiveResponse();
942             _response._finishedAt = Clock.currTime;
943         }
944         catch (NetworkException e) {
945             errorf("Error sending request: ", e.msg);
946             return _response;
947         }
948         if ( _useStreaming ) {
949             if ( _response._receiveAsRange.activated ) {
950                 debug(requests) trace("streaming_in activated");
951                 return _response;
952             } else {
953                 _response._receiveAsRange.data = _response.responseBody.data;
954             }
955         }
956         auto connection = "connection" in _response._responseHeaders;
957         if ( !connection || *connection == "close" ) {
958             debug(requests) tracef("Closing connection because of 'Connection: close' or no 'Connection' header");
959             _stream.close();
960         }
961         if ( canFind(redirectCodes, _response.code) && followRedirectResponse() ) {
962             if ( _method != "GET" ) {
963                 return this.get();
964             }
965             goto connect;
966         }
967         _response._history = _history;
968         ///
969         return _response;
970     }
971     ///
972     /// POST/PUT/... data from some string(with Content-Length), or from range of strings/bytes (use Transfer-Encoding: chunked).
973     /// When rank 1 (flat array) used as content it must have length. In that case "content" will be sent directly to network, and Content-Length headers will be added.
974     /// If you are goung to send some range and do not know length at the moment when you start to send request, then you can send chunks of chars or ubyte.
975     /// Try not to send too short chunks as this will put additional load on client and server. Chunks of length 2048 or 4096 are ok.
976     /// 
977     /// Parameters:
978     ///    url = url
979     ///    content = string or input range
980     ///    contentType = content type
981     ///  Returns:
982     ///     Response
983     ///  Examples:
984     ///  ---------------------------------------------------------------------------------------------------------
985     ///      rs = rq.exec!"POST"("http://httpbin.org/post", "привiт, свiт!", "application/octet-stream");
986     ///      
987     ///      auto s = lineSplitter("one,\ntwo,\nthree.");
988     ///      rs = rq.exec!"POST"("http://httpbin.org/post", s, "application/octet-stream");
989     ///      
990     ///      auto s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
991     ///      rs = rq.exec!"POST"("http://httpbin.org/post", s.representation.chunks(10), "application/octet-stream");
992     ///
993     ///      auto f = File("tests/test.txt", "rb");
994     ///      rs = rq.exec!"POST"("http://httpbin.org/post", f.byChunk(3), "application/octet-stream");
995     ///  --------------------------------------------------------------------------------------------------------
996     HTTPResponse exec(string method="POST", R)(string url, R content, string contentType="application/octet-stream")
997         if ( (rank!R == 1)
998             || (rank!R == 2 && isSomeChar!(Unqual!(typeof(content.front.front)))) 
999             || (rank!R == 2 && (is(Unqual!(typeof(content.front.front)) == ubyte)))
1000         ) {
1001         //
1002         // application/json
1003         //
1004         bool restartedRequest = false;
1005 
1006         _method = method;
1007         
1008         _response = new HTTPResponse;
1009         checkURL(url);
1010         _response.uri = _uri;
1011         _response.finalURI = _uri;
1012 
1013     connect:
1014         _contentReceived = 0;
1015         _response._startedAt = Clock.currTime;
1016         setupConnection();
1017         
1018         if ( !_stream.isConnected() ) {
1019             return _response;
1020         }
1021         _response._connectedAt = Clock.currTime;
1022 
1023         Appender!string req;
1024         req.put(requestString());
1025 
1026         auto h = requestHeaders;
1027         if ( contentType ) {
1028             h["Content-Type"] = contentType;
1029         }
1030         static if ( rank!R == 1 ) {
1031             h["Content-Length"] = to!string(content.length);
1032         } else {
1033             h["Transfer-Encoding"] = "chunked";
1034         }
1035         h.byKeyValue.
1036             map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n").
1037             each!(h => req.put(h));
1038         req.put("\r\n");
1039 
1040         debug(requests) trace(req.data);
1041         if ( _verbosity >= 1 ) {
1042             req.data.splitLines.each!(a => writeln("> " ~ a));
1043         }
1044 
1045         try {
1046             // send headers
1047             _stream.send(req.data());
1048             // send body
1049             static if ( rank!R == 1) {
1050                 _stream.send(content);
1051             } else {
1052                 while ( !content.empty ) {
1053                     auto chunk = content.front;
1054                     auto chunkHeader = "%x\r\n".format(chunk.length);
1055                     debug(requests) tracef("sending %s%s", chunkHeader, chunk);
1056                     _stream.send(chunkHeader);
1057                     _stream.send(chunk);
1058                     _stream.send("\r\n");
1059                     content.popFront;
1060                 }
1061                 debug(requests) tracef("sent");
1062                 _stream.send("0\r\n\r\n");
1063             }
1064             _response._requestSentAt = Clock.currTime;
1065             receiveResponse();
1066             _response._finishedAt = Clock.currTime;
1067         } catch (NetworkException e) {
1068             _stream.close();
1069             throw new RequestException("Network error during data exchange");
1070         }
1071 
1072         if ( _useStreaming ) {
1073             if ( _response._receiveAsRange.activated ) {
1074                 debug(requests) trace("streaming_in activated");
1075                 return _response;
1076             } else {
1077                 _response._receiveAsRange.data = _response.responseBody.data;
1078             }
1079         }
1080         auto connection = "connection" in _response._responseHeaders;
1081         if ( !connection || *connection == "close" ) {
1082             debug(requests) tracef("Closing connection because of 'Connection: close' or no 'Connection' header");
1083             _stream.close();
1084         }
1085         if ( canFind(redirectCodes, _response.code) && followRedirectResponse() ) {
1086             if ( _method != "GET" ) {
1087                 return this.get();
1088             }
1089             goto connect;
1090         }
1091         ///
1092         _response._history = _history;
1093         return _response;
1094     }
1095     ///
1096     /// Send request with pameters.
1097     /// If used for POST or PUT requests then application/x-www-form-urlencoded used.
1098     /// Request parameters will be encoded into request string or placed in request body for POST/PUT
1099     /// requests.
1100     /// Parameters:
1101     ///     url = url
1102     ///     params = request parameters
1103     ///  Returns:
1104     ///     Response
1105     ///  Examples:
1106     ///  ---------------------------------------------------------------------------------
1107     ///     rs = Request().exec!"GET"("http://httpbin.org/get", ["c":"d", "a":"b"]);
1108     ///  ---------------------------------------------------------------------------------
1109     ///     
1110     HTTPResponse exec(string method="GET")(string url = null, QueryParam[] params = null) {
1111 
1112         _method = method;
1113         _response = new HTTPResponse;
1114         _history.length = 0;
1115         bool restartedRequest = false; // True if this is restarted keepAlive request
1116         string encoded;
1117 
1118         checkURL(url);
1119         _response.uri = _uri;
1120         _response.finalURI = _uri;
1121 
1122     connect:
1123         _contentReceived = 0;
1124         _response._startedAt = Clock.currTime;
1125         setupConnection();
1126 
1127         if ( !_stream.isConnected() ) {
1128             return _response;
1129         }
1130         _response._connectedAt = Clock.currTime;
1131 
1132         auto h = requestHeaders();
1133 
1134         Appender!string req;
1135 
1136         switch (_method) {
1137             case "POST","PUT":
1138                 encoded = params2query(params);
1139                 h["Content-Type"] = "application/x-www-form-urlencoded";
1140                 h["Content-Length"] = to!string(encoded.length);
1141                 req.put(requestString());
1142                 break;
1143             default:
1144                 req.put(requestString(params));
1145         }
1146 
1147         h.byKeyValue.
1148             map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n").
1149             each!(h => req.put(h));
1150         req.put("\r\n");
1151         if ( encoded ) {
1152             req.put(encoded);
1153         }
1154 
1155         debug(requests) trace(req.data);
1156 
1157         if ( _verbosity >= 1 ) req.data.splitLines.each!(a => writeln("> " ~ a));
1158         //
1159         // Now send request and receive response
1160         //
1161         try {
1162             _stream.send(req.data());
1163             _response._requestSentAt = Clock.currTime;
1164             receiveResponse();
1165             _response._finishedAt = Clock.currTime;
1166         }
1167         catch (NetworkException e) {
1168             // On SEND this can means:
1169             // we started to send request to the server, but it closed connection because of keepalive timeout.
1170             // We have to restart request if possible.
1171 
1172             // On RECEIVE - if we received something - then this exception is real and unexpected error.
1173             // If we didn't receive anything - we can restart request again as it can be 
1174             if ( _response._responseHeaders.length != 0 ) {
1175                 _stream.close();
1176                 throw new RequestException("Unexpected network error");
1177             }
1178         }
1179 
1180         if ( serverClosedKeepAliveConnection()
1181             && !restartedRequest
1182             && isIdempotent(_method)
1183             ) {
1184             ///
1185             /// We didn't receive any data (keepalive connectioin closed?)
1186             /// and we can restart this request.
1187             /// Go ahead.
1188             ///
1189             debug(requests) tracef("Server closed keepalive connection");
1190             _stream.close();
1191             restartedRequest = true;
1192             goto connect;
1193         }
1194         
1195         if ( _useStreaming ) {
1196             if ( _response._receiveAsRange.activated ) {
1197                 debug(requests) trace("streaming_in activated");
1198                 return _response;
1199             } else {
1200                 // this can happen if whole response body received together with headers
1201                 _response._receiveAsRange.data = _response.responseBody.data;
1202             }
1203         }
1204 
1205         auto connection = "connection" in _response._responseHeaders;
1206         if ( !connection || *connection == "close" ) {
1207             debug(requests) tracef("Closing connection because of 'Connection: close' or no 'Connection' header");
1208             _stream.close();
1209         }
1210         if ( _verbosity >= 1 ) {
1211             writeln(">> Connect time: ", _response._connectedAt - _response._startedAt);
1212             writeln(">> Request send time: ", _response._requestSentAt - _response._connectedAt);
1213             writeln(">> Response recv time: ", _response._finishedAt - _response._requestSentAt);
1214         }
1215         if ( canFind(redirectCodes, _response.code) && followRedirectResponse() ) {
1216             if ( _method != "GET" ) {
1217                 return this.get();
1218             }
1219             goto connect;
1220         }
1221         ///
1222         _response._history = _history;
1223         return _response;
1224     }
1225 
1226     /// WRAPPERS
1227     ///
1228     /// send file(s) using POST and multipart form.
1229     /// This wrapper will be deprecated, use post with MultipartForm - it is more general and clear.
1230     /// Parameters:
1231     ///     url = url
1232     ///     files = array of PostFile structures
1233     /// Returns:
1234     ///     Response
1235     /// Each PostFile structure contain path to file, and optional field name and content type.
1236     /// If no field name provided, then basename of the file will be used.
1237     /// application/octet-stream is default when no content type provided.
1238     /// Example:
1239     /// ---------------------------------------------------------------
1240     ///    PostFile[] files = [
1241     ///                   {fileName:"tests/abc.txt", fieldName:"abc", contentType:"application/octet-stream"}, 
1242     ///                   {fileName:"tests/test.txt"}
1243     ///               ];
1244     ///    rs = rq.exec!"POST"("http://httpbin.org/post", files);
1245     /// ---------------------------------------------------------------
1246     /// 
1247     HTTPResponse exec(string method="POST")(string url, PostFile[] files) if (method=="POST") {
1248         MultipartForm multipart;
1249         File[]        toClose;
1250         foreach(ref f; files) {
1251             File file = File(f.fileName, "rb");
1252             toClose ~= file;
1253             string fileName = f.fileName ? f.fileName : f.fieldName;
1254             string contentType = f.contentType ? f.contentType : "application/octetstream";
1255             multipart.add(f.fieldName, new FormDataFile(file), ["filename":fileName, "Content-Type": contentType]);
1256         }
1257         auto res = exec!"POST"(url, multipart);
1258         toClose.each!"a.close";
1259         return res;
1260     }
1261     ///
1262     /// exec request with parameters when you can use dictionary (when you have no duplicates in parameter names)
1263     /// Consider switch to exec(url, QueryParams) as it more generic and clear.
1264     /// Parameters:
1265     ///     url = url
1266     ///     params = dictionary with field names as keys and field values as values.
1267     /// Returns:
1268     ///     Response
1269     HTTPResponse exec(string method="GET")(string url, string[string] params) {
1270         return exec!method(url, params.byKeyValue.map!(p => QueryParam(p.key, p.value)).array);
1271     }
1272     ///
1273     /// GET request. Simple wrapper over exec!"GET"
1274     /// Params:
1275     /// args = request parameters. see exec docs.
1276     ///
1277     HTTPResponse get(A...)(A args) {
1278         return exec!"GET"(args);
1279     }
1280     ///
1281     /// POST request. Simple wrapper over exec!"POST"
1282     /// Params:
1283     /// uri = endpoint uri
1284     /// args = request parameters. see exec docs.
1285     ///
1286     HTTPResponse post(A...)(string uri, A args) {
1287         return exec!"POST"(uri, args);
1288     }
1289 }
1290 
1291 version(vibeD) {
1292     import std.json;
1293     package string httpTestServer() {
1294         return "http://httpbin.org/";
1295     }
1296     package string fromJsonArrayToStr(JSONValue v) {
1297         return v.str;
1298     }
1299 }
1300 else {
1301     import std.json;
1302     package string httpTestServer() {
1303         return "http://127.0.0.1:8081/";
1304     }
1305     package string fromJsonArrayToStr(JSONValue v) {
1306         return cast(string)(v.array.map!"cast(ubyte)a.integer".array);
1307     }
1308 }
1309 
1310 
1311 package unittest {
1312     import std.json;
1313     import std.array;
1314 
1315     globalLogLevel(LogLevel.info);
1316 
1317     string httpbinUrl = httpTestServer();
1318     version(vibeD) {
1319     }
1320     else {
1321         import httpbin;
1322         auto server = httpbinApp();
1323         server.start();
1324         scope(exit) {
1325             server.stop();
1326         }
1327     }
1328     HTTPRequest  rq;
1329     HTTPResponse rs;
1330     info("Check GET");
1331     rs = rq.get(httpbinUrl);
1332     assert(rs.code==200);
1333     assert(rs.responseBody.length > 0);
1334 
1335     info("Check GET with AA params");
1336     {
1337         rs = HTTPRequest().get(httpbinUrl ~ "get", ["c":" d", "a":"b"]);
1338         assert(rs.code == 200);
1339         auto json = parseJSON(rs.responseBody.data).object["args"].object;
1340         assert(json["c"].str == " d");
1341         assert(json["a"].str == "b");
1342     }
1343     info("Check POST files");
1344     {
1345         import std.file;
1346         import std.path;
1347         auto tmpd = tempDir();
1348         auto tmpfname = tmpd ~ dirSeparator ~ "request_test.txt";
1349         auto f = File(tmpfname, "wb");
1350         f.rawWrite("abcdefgh\n12345678\n");
1351         f.close();
1352         // files
1353         PostFile[] files = [
1354             {fileName: tmpfname, fieldName:"abc", contentType:"application/octet-stream"}, 
1355             {fileName: tmpfname}
1356         ];
1357         rs = rq.post(httpbinUrl ~ "post", files);
1358         assert(rs.code==200);
1359     }
1360     info("Check POST chunked from file.byChunk");
1361     {
1362         import std.file;
1363         import std.path;
1364         auto tmpd = tempDir();
1365         auto tmpfname = tmpd ~ dirSeparator ~ "request_test.txt";
1366         auto f = File(tmpfname, "wb");
1367         f.rawWrite("abcdefgh\n12345678\n");
1368         f.close();
1369         f = File(tmpfname, "rb");
1370         rs = rq.post(httpbinUrl ~ "post", f.byChunk(3), "application/octet-stream");
1371         assert(rs.code==200);
1372         auto data = fromJsonArrayToStr(parseJSON(rs.responseBody).object["data"]);
1373         assert(data=="abcdefgh\n12345678\n");
1374         f.close();
1375     }
1376     info("Check POST chunked from lineSplitter");
1377     {
1378         auto s = lineSplitter("one,\ntwo,\nthree.");
1379         rs = rq.exec!"POST"(httpbinUrl ~ "post", s, "application/octet-stream");
1380         assert(rs.code==200);
1381         auto data = fromJsonArrayToStr(parseJSON(rs.responseBody).object["data"]);
1382         assert(data=="one,two,three.");
1383     }
1384     info("Check POST chunked from array");
1385     {
1386         auto s = ["one,", "two,", "three."];
1387         rs = rq.post(httpbinUrl ~ "post", s, "application/octet-stream");
1388         assert(rs.code==200);
1389         auto data = fromJsonArrayToStr(parseJSON(rs.responseBody).object["data"]);
1390         assert(data=="one,two,three.");
1391     }
1392     info("Check POST chunked using std.range.chunks()");
1393     {
1394         auto s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
1395         rs = rq.post(httpbinUrl ~ "post", s.representation.chunks(10), "application/octet-stream");
1396         assert(rs.code==200);
1397         auto data = fromJsonArrayToStr(parseJSON(rs.responseBody.data).object["data"]);
1398         assert(data==s);
1399     }
1400     info("Check POST from QueryParams");
1401     {
1402         rs = rq.post(httpbinUrl ~ "post", queryParams("name[]", "first", "name[]", 2));
1403         assert(rs.code==200);
1404         auto data = parseJSON(rs.responseBody).object["form"].object;
1405         string[] a;
1406         try {
1407             a = to!(string[])(data["name[]"].str);
1408         }
1409         catch (JSONException e) {
1410             a = data["name[]"].array.map!"a.str".array;
1411         }
1412         assert(equal(["first", "2"], a));
1413     }
1414     info("Check POST from AA");
1415     {
1416         rs = rq.post(httpbinUrl ~ "post", ["a":"b ", "c":"d"]);
1417         assert(rs.code==200);
1418         auto form = parseJSON(rs.responseBody.data).object["form"].object;
1419         assert(form["a"].str == "b ");
1420         assert(form["c"].str == "d");
1421     }
1422     info("Check POST json");
1423     {
1424         rs = rq.post(httpbinUrl ~ "post?b=x", `{"a":"a b", "c":[1,2,3]}`, "application/json");
1425         assert(rs.code==200);
1426         auto json = parseJSON(rs.responseBody).object["args"].object;
1427         assert(json["b"].str == "x");
1428         json = parseJSON(rs.responseBody).object["json"].object;
1429         assert(json["a"].str == "a b");
1430         assert(json["c"].array.map!(a=>a.integer).array == [1,2,3]);
1431     }
1432     info("Check HEAD");
1433     rs = rq.exec!"HEAD"(httpbinUrl);
1434     assert(rs.code==200);
1435     info("Check DELETE");
1436     rs = rq.exec!"DELETE"(httpbinUrl ~ "delete");
1437     assert(rs.code==200);
1438     info("Check PUT");
1439     rs = rq.exec!"PUT"(httpbinUrl ~ "put",  `{"a":"b", "c":[1,2,3]}`, "application/json");
1440     assert(rs.code==200);
1441     assert(parseJSON(rs.responseBody).object["json"].object["a"].str=="b");
1442     info("Check PATCH");
1443     rs = rq.exec!"PATCH"(httpbinUrl ~ "patch", "привiт, свiт!", "application/octet-stream");
1444     assert(rs.code==200);
1445     info("Check compressed content");
1446     rs = rq.get(httpbinUrl ~ "gzip");
1447     assert(rs.code==200);
1448     bool gzipped = parseJSON(rs.responseBody).object["gzipped"].type == JSON_TYPE.TRUE;
1449     assert(gzipped);
1450     info("gzip - ok");
1451     rs = rq.get(httpbinUrl ~ "deflate");
1452     assert(rs.code==200);
1453     bool deflated = parseJSON(rs.responseBody).object["deflated"].type == JSON_TYPE.TRUE;
1454     assert(deflated);
1455     info("deflate - ok");
1456 
1457     info("Check redirects");
1458     rs = rq.get(httpbinUrl ~ "relative-redirect/2");
1459     assert(rs.history.length == 2);
1460     assert(rs.code==200);
1461     rs = rq.get(httpbinUrl ~ "absolute-redirect/2");
1462     assert(rs.history.length == 2);
1463     assert(rs.code==200);
1464 
1465     rq.maxRedirects = 2;
1466     assertThrown!MaxRedirectsException(rq.get(httpbinUrl ~ "absolute-redirect/3"));
1467 
1468     info("Check cookie");
1469     {
1470         rs = rq.get(httpbinUrl ~ "cookies/set?A=abcd&b=cdef");
1471         assert(rs.code == 200);
1472         auto json = parseJSON(rs.responseBody.data).object["cookies"].object;
1473         assert(json["A"].str == "abcd");
1474         assert(json["b"].str == "cdef");
1475         foreach(c; rq.cookie) {
1476             final switch(c.attr) {
1477                 case "A":
1478                     assert(c.value == "abcd");
1479                     break;
1480                 case "b":
1481                     assert(c.value == "cdef");
1482                     break;
1483             }
1484         }
1485     }
1486     info("Check chunked content");
1487     rs = rq.get(httpbinUrl ~ "range/1024");
1488     assert(rs.code==200);
1489     assert(rs.responseBody.length==1024);
1490 
1491     info("Check basic auth");
1492     rq.authenticator = new BasicAuthentication("user", "passwd");
1493     rs = rq.get(httpbinUrl ~ "basic-auth/user/passwd");
1494     assert(rs.code==200);
1495 
1496     info("Check limits");
1497     rq = HTTPRequest();
1498     rq.maxContentLength = 1;
1499     assertThrown!RequestException(rq.get(httpbinUrl));
1500     rq = HTTPRequest();
1501     rq.maxHeadersLength = 1;
1502     assertThrown!RequestException(rq.get(httpbinUrl));
1503     rq = HTTPRequest();
1504     info("Check POST multiPartForm");
1505     {
1506         /// This is example on usage files with MultipartForm data.
1507         /// For this example we have to create files which will be sent.
1508         import std.file;
1509         import std.path;
1510         /// preapare files
1511         auto tmpd = tempDir();
1512         auto tmpfname1 = tmpd ~ dirSeparator ~ "request_test1.txt";
1513         auto f = File(tmpfname1, "wb");
1514         f.rawWrite("file1 content\n");
1515         f.close();
1516         auto tmpfname2 = tmpd ~ dirSeparator ~ "request_test2.txt";
1517         f = File(tmpfname2, "wb");
1518         f.rawWrite("file2 content\n");
1519         f.close();
1520         ///
1521         /// Ok, files ready.
1522         /// Now we will prepare Form data
1523         /// 
1524         File f1 = File(tmpfname1, "rb");
1525         File f2 = File(tmpfname2, "rb");
1526         scope(exit) {
1527             f1.close();
1528             f2.close();
1529         }
1530         ///
1531         /// for each part we have to set field name, source (ubyte array or opened file) and optional filename and content-type
1532         /// 
1533         MultipartForm mForm = MultipartForm().
1534             add(formData("Field1", cast(ubyte[])"form field from memory")).
1535                 add(formData("Field2", cast(ubyte[])"file field from memory", ["filename":"data2"])).
1536                 add(formData("File1", f1, ["filename":"file1", "Content-Type": "application/octet-stream"])).
1537                 add(formData("File2", f2, ["filename":"file2", "Content-Type": "application/octet-stream"]));
1538         /// everything ready, send request
1539         rs = rq.post(httpbinUrl ~ "post", mForm);
1540     }
1541     info("Check exception handling, error messages and timeous are OK");
1542     rq.timeout = 1.seconds;
1543     assertThrown!TimeoutException(rq.get(httpbinUrl ~ "delay/3"));
1544 //    assertThrown!ConnectError(rq.get("http://0.0.0.0:65000/"));
1545 //    assertThrown!ConnectError(rq.get("http://1.1.1.1/"));
1546 //    assertThrown!ConnectError(rq.get("http://gkhgkhgkjhgjhgfjhgfjhgf/"));
1547 }