1 module requests.server.httpd;
2 
3 import std.algorithm;
4 import std.array;
5 import std.conv;
6 import std.datetime;
7 import std.exception;
8 import std.experimental.logger;
9 import std.format;
10 import std.parallelism;
11 import std.range;
12 import std.regex;
13 import std.socket;
14 import std.stdio;
15 import std.string;
16 import std.traits;
17 import std.typecons;
18 import core.thread;
19 import requests.utils;
20 import requests.streams;
21 import requests.uri;
22 
23 version(vibeD){
24     pragma(msg, "httpd will not compile with vibeD");
25 }
26 else {
27     /*
28      ** This is small http server to run something like httpbin(http://httpbin.org) internally
29      ** for Requests unittest's.
30      */
31 
32     enum    DSBUFFSIZE = 16*1024;
33 
34     class HTTPD_RequestException: Exception {
35         this(string message, string file =__FILE__, size_t line = __LINE__, Throwable next = null) @safe pure nothrow {
36             super(message, file, line, next);
37         }
38     }
39 
40     struct HTTPD_Request {
41         private {
42             string          _requestLine;
43             string[string]  _requestHeaders;
44             Buffer!ubyte    _requestBody;
45             bool            _keepAlive;
46             URI             _uri;
47             string[string]  _query; // query in url
48             string          _method;
49             string          _path;
50             string          _json; // json for application/json
51             string[string]  _form; // form values for application/x-www-form-urlencoded
52             ubyte[][string] _files;
53             ubyte[]         _data; // raw data for unrecognized mime's
54             _DataSource     _dataSource;
55             string[string]  _cookies;
56         }
57         private mixin(Setter!(string[string])("requestHeaders"));
58         auto ref requestHeaders() inout @property @safe @nogc nothrow {
59             return _requestHeaders;
60         }
61         auto ref cookies() inout @property @safe @nogc nothrow {
62             return _cookies;
63         }
64         private mixin(Setter!(string[string])("query"));
65         auto ref query() inout @property @safe @nogc nothrow {
66             return _query;
67         }
68         auto ref requestBody() inout @property @safe @nogc nothrow {
69             return _requestBody;
70         }
71         private mixin(Setter!string("method"));
72         mixin(Getter("method"));
73         private mixin(Setter!string("requestLine"));
74         mixin(Getter("requestLine"));
75         private mixin(Setter!string("path"));
76         mixin(Getter("path"));
77         private mixin(Setter!bool("keepAlive"));
78         mixin(Getter("keepAlive"));
79         private mixin(Setter!URI("uri"));
80         mixin(Getter("uri"));
81 
82         @property string json() {
83             if ( _dataSource._readStarted ) {
84                 throw new HTTPD_RequestException("Request read() call already started.");
85             }
86             if ( _dataSource._requestHasBody && !_dataSource._requestBodyProcessed ) {
87                 debug(httpd) trace("receiving body on demand for json");
88                 loadBodyOnDemand(_dataSource);
89             }
90             return _json;
91         }
92         @property ubyte[] data() {
93             if ( _dataSource._readStarted ) {
94                 throw new HTTPD_RequestException("Request body read() already started.");
95             }
96             if ( _dataSource._requestHasBody && !_dataSource._requestBodyProcessed ) {
97                 debug(httpd) trace("receiving body on demand for data");
98                 loadBodyOnDemand(_dataSource);
99             }
100             return _data;
101         }
102         @property string[string] form() {
103             if ( _dataSource._readStarted ) {
104                 throw new HTTPD_RequestException("Request body read() already started.");
105             }
106             if ( _dataSource._requestHasBody && !_dataSource._requestBodyProcessed ) {
107                 debug(httpd) trace("receiving body on demand for form");
108                 loadBodyOnDemand(_dataSource);
109             }
110             return _form;
111         }
112         @property auto files() {
113             if ( _dataSource._readStarted ) {
114                 throw new HTTPD_RequestException("Request body read() already started.");
115             }
116             if ( _dataSource._requestHasBody && !_dataSource._requestBodyProcessed ) {
117                 debug(httpd) trace("receiving body on demand for form");
118                 loadBodyOnDemand(_dataSource);
119             }
120             return _files;
121         }
122 
123         @property bool requestHasBody() pure {
124             if ( "content-length" in _requestHeaders ) {
125                 return true;
126             }
127             if ( auto contentTransferEncoding = "transfer-encoding" in _requestHeaders ) {
128                 if ( *contentTransferEncoding=="chunked" ) {
129                     return true;
130                 }
131             }
132             return false;
133         }
134 
135         class _DataSource {
136             private {
137                 NetworkStream  _stream;
138                 DataPipe!ubyte _bodyDecoder;
139                 DecodeChunked  _unChunker;
140                 long           _contentLength =  0;
141                 long           _receivedLength = 0;
142                 ubyte[]        _content;
143                 bool           _requestHasBody;         // request has body
144                 bool           _requestBodyRecvInProgr; // loading body currently active
145                 bool           _requestBodyProcessed;   // we processed body - this can happens only once
146                 bool           _requestBodyReceived;    // all body data were received from network (we have to close socket if request terminated before all data received)
147                 bool           _readStarted;
148             }
149             bool empty() {
150                 debug(httpd) tracef("datasource empty: %s", _content.length==0);
151                 return _content.length==0;
152             }
153             ubyte[] front() {
154                 return _content;
155             }
156             void popFront() {
157                 debug(httpd) trace("datasource enters popFront");
158                 _content.length = 0;
159                 if ( !_requestBodyRecvInProgr ) {
160                     debug(httpd) trace("popFront called when dataSource is not active anymore");
161                     return;
162                 }
163                 while ( _bodyDecoder.empty && _stream && _stream.isOpen ) {
164                     auto b    = new ubyte[DSBUFFSIZE];
165                     auto read = _stream.receive(b);
166                     if ( read == 0 ) {
167                         debug(httpd) trace("stream closed when receiving in datasource");
168                         _bodyDecoder.flush();
169                         _requestBodyRecvInProgr = false;
170                         break;
171                     }
172                     debug(httpd) tracef("place %d bytes to datasource", read);
173                     _receivedLength += read;
174                     _bodyDecoder.putNoCopy(b[0..read]);
175                     if (   (_unChunker && _unChunker.done)
176                         || (_contentLength > 0 && _receivedLength >= _contentLength) )
177                     {
178                         debug(httpd) trace("request body reading complete (due contentLength or due last chunk consumed)");
179                         _bodyDecoder.flush();
180                         _requestBodyRecvInProgr = false;
181                         _requestBodyReceived = true;
182                         break;
183                     }
184                 }
185                 _content = _bodyDecoder.getNoCopy().join();
186                 debug(httpd) tracef("%d bytes in content after popFront", _content.length);
187             }
188             ///
189             /// raplace current front with another value
190             ///
191             void unPop(ubyte[] data) {
192                 assert(data.length > 0);
193                 _content = data;
194             }
195             ///
196             /// Scan over input stream,
197             /// can return data from stream
198             /// acc - accumulator for receiving needle
199             /// return empty data if we receiving needle
200             /// if needle found in stream, then acc == needle
201             /// if end of stream happened, then eos = true
202             ///
203             ubyte[] scanUntilR(string needle, ref ubyte[] acc, out bool eos) {
204                 auto d = needle.representation;
205                 ubyte[] l;
206 
207                 while (!this.empty) {
208                     auto c = this.front;
209                     debug(httpd) tracef("on scan: %s", cast(string)c);
210                     l = acc ~ c;
211                     auto s = l.findSplit(d);
212                     if ( s[1].length ) {
213                         if ( s[2].length ) {
214                             this.unPop(s[2]);
215                         } else {
216                             this.popFront;
217                         }
218                         acc = s[1];
219                         return s[0];
220                     }
221                     auto i = min(l.length, d.length);
222                     for(;i>0; i--) {
223                         if ( l.endsWith(d[0..i]) ) {
224                             acc = l[$-i..$];
225                             this.popFront;
226                             return l[0..$-i];
227                         }
228                     }
229                     if ( i == 0 ) {
230                         acc.length = 0;
231                         this.popFront;
232                         return l;
233                     }
234                 }
235                 eos = true; // end of stream
236                 acc.length = 0;
237                 return l;
238             }
239             void scanUntil(F)(string needle, F f) {
240                 auto d = needle.representation;
241                 ubyte[] acc;
242                 bool    eos; // end of stream
243 
244                 while( !eos ) {
245                     auto l = scanUntilR(needle, acc, eos);
246                     debug(httpd) tracef("scanr returned <%s> and <%s>", cast(string)l, cast(string)acc);
247                     f(l);
248                     if ( acc == needle) {
249                         return;
250                     }
251                 }
252             }
253             void skipUntil(string needle) {
254                 auto d = needle.representation;
255                 ubyte[] acc;
256                 bool    eos; // end of stream
257 
258                 while( !eos ) {
259                     auto l = scanUntilR(needle, acc, eos);
260                     debug(httpd) tracef("scanr returned <%s> and <%s>", cast(string)l, cast(string)acc);
261                     if ( acc == needle) {
262                         return;
263                     }
264                 }
265             }
266         }
267 
268         auto createDataSource(string partialBody, NetworkStream stream) {
269 
270             if ( !requestHasBody ) {
271                 return new _DataSource();
272             }
273 
274             auto ds = new _DataSource();
275 
276             ds._requestHasBody = true;
277             ds._requestBodyRecvInProgr = true;
278             ds._bodyDecoder = new DataPipe!ubyte;
279             ds._stream = stream;
280 
281             if ( auto contentLengthHeader = "content-length" in _requestHeaders ) {
282                 ds._contentLength = to!long(*contentLengthHeader);
283             }
284             else if ( auto contentTransferEncoding = "transfer-encoding" in _requestHeaders ) {
285                 if ( *contentTransferEncoding=="chunked" ) {
286                     ds._unChunker = new DecodeChunked();
287                     ds._bodyDecoder.insert(ds._unChunker);
288                 }
289             }
290             if ( partialBody.length ) {
291                 ds._bodyDecoder.putNoCopy(cast(ubyte[])partialBody);
292                 ds._receivedLength = (cast(ubyte[])partialBody).length;
293             }
294             while ( ds._bodyDecoder.empty ) {
295                 auto b    = new ubyte[DSBUFFSIZE];
296                 auto read = stream.receive(b);
297                 if ( read == 0 ) {
298                     debug(httpd) trace("stream closed when receiving in datasource");
299                     ds._requestBodyRecvInProgr = false;
300                     return ds;
301                 }
302                 debug(httpd) tracef("place %d bytes to datasource", read);
303                 ds._receivedLength += read;
304                 ds._bodyDecoder.putNoCopy(b[0..read]);
305             }
306             ds._content = ds._bodyDecoder.getNoCopy().join();
307             if (   ( ds._contentLength > 0 && ds._receivedLength >= ds._contentLength )
308                 || ( ds._unChunker && ds._unChunker.done) ) {
309                 // all data received we need not wait any data from network
310                 debug(httpd) trace("looks like we received complete request body together with request headers");
311                 ds._requestBodyRecvInProgr = false;
312                 ds._requestBodyReceived = true;
313             }
314             debug(httpd) tracef("initial content: %d bytes", ds._content.length);
315             return ds;
316         }
317         @property auto contentType() {
318             if ( auto ct = "content-type" in _requestHeaders ) {
319                 auto f = (*ct).split(";").map!strip;
320                 return f[0];
321             }
322             return null;
323         }
324 
325         struct PartData {
326             // handler for each part data stream
327             _DataSource    _ds;
328             string         _boundary;
329             ubyte[]        _content;
330             ubyte[]        _acc;
331             bool           _done;
332             bool           _eos;
333 
334             this(_DataSource ds, string boundary) {
335                 _ds = ds;
336                 _boundary = "\r\n" ~ boundary;
337                 _content = _ds.scanUntilR(_boundary, _acc, _eos);
338             }
339             bool empty() {
340                 return _content.length == 0;
341             }
342             auto front() {
343                 return _content;
344             }
345             void popFront() {
346                 _content.length = 0;
347                 if ( _done ) {
348                     return;
349                 }
350                 while( _content.length == 0 ) {
351                     _content = _ds.scanUntilR(_boundary, _acc, _eos);
352                     if ( _eos ) {
353                         return;
354                     }
355                     if (_acc == _boundary) {
356                         debug(httpd) tracef("part data done");
357                         _ds.skipUntil("\r\n");
358                         return;
359                     }
360                 }
361             }
362         }
363         struct Part {
364             _DataSource    _ds;
365             string[string] _headers;
366             string         _boundary;
367 
368             this(_DataSource ds, string[string] h, string boundary) {
369                 _ds = ds;
370                 _headers = h;
371                 _boundary = boundary;
372             }
373             @property string[string] headers() {
374                 return _headers;
375             }
376             @property disposition() {
377                 string[string] res;
378                 auto d = "content-disposition" in _headers;
379                 if ( !d ) {
380                     return res;
381                 }
382                 (*d).split(";").
383                     filter!"a.indexOf('=')>0".
384                         map!   "a.strip.split('=')".
385                         each!(p => res[p[0]] = urlDecode(p[1]).strip('"'));
386                 return res;
387             }
388             @property data() {
389                 return PartData(_ds, _boundary);
390             }
391         }
392         struct MultiPart {
393             string      _boundary;
394             _DataSource _ds;
395             Part        _part;
396             /*
397              --8a60ded0-ee76-4b6a-a1a0-dccaf93b92e7
398              Content-Disposition: form-data; name=Field1;
399 
400              form field from memory
401              --8a60ded0-ee76-4b6a-a1a0-dccaf93b92e7
402              Content-Disposition: form-data; name=Field2; filename=data2
403 
404              file field from memory
405              --8a60ded0-ee76-4b6a-a1a0-dccaf93b92e7
406              Content-Disposition: form-data; name=File1; filename=file1
407              Content-Type: application/octet-stream
408 
409              file1 content
410 
411              --8a60ded0-ee76-4b6a-a1a0-dccaf93b92e7
412              Content-Disposition: form-data; name=File2; filename=file2
413              Content-Type: application/octet-stream
414 
415              file2 content
416 
417              --8a60ded0-ee76-4b6a-a1a0-dccaf93b92e7--
418              */
419             int opApply(int delegate(Part p) dg) {
420                 int result = 0;
421                 while(!_ds.empty) {
422                     result = dg(_part);
423                     if ( result ) {
424                         break;
425                     }
426                     auto headers = skipHeaders();
427                     _part = Part(_ds, headers, _boundary);
428                 }
429                 return result;
430             }
431             auto skipHeaders() {
432                 ubyte[] buf;
433                 string[string] headers;
434 
435                 debug(httpd) tracef("Search for headers");
436                 _ds.scanUntil("\r\n\r\n", delegate void (ubyte[] data) {
437                         buf ~= data;
438                     });
439                 foreach(h; buf.split('\n').map!"cast(string)a".map!strip.filter!"a.length") {
440                     auto parsed = h.findSplit(":");
441                     headers[parsed[0].toLower] = parsed[2].strip;
442                 }
443                 debug(httpd) tracef("Headers: %s ", headers);
444                 return headers;
445             }
446             ///
447             /// Find boundary from request headers,
448             /// skip to begin of the first part,
449             /// create first part(read/parse headers, stop on the body begin)
450             ///
451             this(HTTPD_Request rq) {
452                 ubyte[] buf, rest;
453                 string separator;
454                 auto ct = "content-type" in rq._requestHeaders;
455                 auto b = (*ct).split(";").map!"a.strip.split(`=`)".filter!"a[0].toLower==`boundary`";
456                 if ( b.empty ) {
457                     throw new HTTPD_RequestException("Can't find 'boundary' in Content-Type %s".format(*ct));
458                 }
459                 _boundary = "--" ~ b.front[1];
460                 _ds = rq._dataSource;
461                 _ds.skipUntil(_boundary~"\r\n");
462                 auto headers = skipHeaders();
463                 _part = Part(_ds, headers, _boundary);
464             }
465         }
466 
467         auto multiPartRead() {
468             return MultiPart(this);
469         }
470 
471         auto read() {
472             if ( requestHasBody && _dataSource._requestBodyProcessed ) {
473                 throw new HTTPD_RequestException("Request body already consumed by call to data/form/json");
474             }
475             if ( _dataSource._readStarted ) {
476                 throw new HTTPD_RequestException("Request body read() already started.");
477             }
478             _dataSource._readStarted = true;
479             return _dataSource;
480         }
481 
482         void loadBodyOnDemand(ref _DataSource ds) {
483             ds._requestBodyProcessed = true;
484             debug(httpd) tracef("Process %s onDemand", contentType);
485             switch ( contentType ) {
486                 case "application/json":
487                     while(!ds.empty) {
488                         debug(httpd) tracef("add %d bytes to json from dataSource", ds.front.length);
489                         _json ~= cast(string)ds.front;
490                         ds.popFront;
491                     }
492                     break;
493                 case "application/x-www-form-urlencoded":
494                     string qBody;
495                     while(!ds.empty) {
496                         debug(httpd) tracef("add %d bytes to json from dataSource", ds.front.length);
497                         qBody ~= cast(string)ds.front;
498                         ds.popFront;
499                     }
500                     _form = parseQuery(qBody);
501                     break;
502                 case "multipart/form-data":
503                     debug(httpd) tracef("loading multiPart on demand");
504                     auto parts = multiPartRead();
505                     foreach(p; parts) {
506                         auto disposition = p.disposition;
507                         auto data = p.data.joiner.array;
508 
509                         if ( !("name" in disposition) ) {
510                             continue;
511                         }
512                         if ( auto fn = "filename" in disposition ) {
513                             _files[disposition["name"]] = data;
514                         } else {
515                             _form[disposition["name"]]  = cast(string)data;
516                         }
517                     }
518                     break;
519                 default:
520                     while(!ds.empty) {
521                         debug(httpd) tracef("add %d bytes to data from dataSource", ds.front.length);
522                         _data ~= ds.front;
523                         ds.popFront;
524                     }
525                     break;
526             }
527         }
528     }
529 
530     string[int] codes;
531     shared static this() {
532         codes = [
533             200: "OK",
534             302: "Found",
535             401: "Unauthorized",
536             404: "Not found",
537             405: "Method not allowed",
538             500: "Server error"
539         ];
540     }
541     enum Compression : int {
542         no    =   0,
543         gzip   =  1,
544         deflate = 2,
545         yes     = gzip|deflate,
546     };
547 
548     auto response(C)(HTTPD_Request rq, C content, ushort code = 200)
549         if ( isSomeString!C
550             || (__traits(compiles, cast(ubyte[])content))
551             || (__traits(compiles, cast(ubyte[])content.front))
552             )
553     {
554         return new HTTPD_Response!C(rq, content, code);
555     }
556 
557     class _Response {
558         abstract void send(NetworkStream);
559         abstract ref string[string] headers();
560     }
561 
562     class HTTPD_Response(C) : _Response {
563         ushort          _status = 200;
564         string          _status_reason = "Unspecified";
565         string[string]  _headers;
566         C               _content;
567         Compression     _compression = Compression.no;
568         HTTPD_Request   _request;
569         Cookie[]        _cookies;
570 
571         mixin(Getter_Setter!ushort("status"));
572         mixin(Getter("compression"));
573         @property void compress(Compression c = Compression.yes) {
574             _compression = c;
575         }
576         this(ref HTTPD_Request request, C content, ushort status = 200) {
577             _status  = status;
578             _request = request;
579             _content = content;
580         }
581         override ref string[string] headers() @property {
582             return _headers;
583         }
584         ref Cookie[] cookies() {
585             return _cookies;
586         }
587         void content(C)(C c) @property {
588             _content = makeContent(c);
589         }
590         auto selectCompression(in HTTPD_Request rq, in HTTPD_Response rs) {
591             if ( auto acceptEncodings = "accept-encoding" in rq.requestHeaders) {
592                 auto heAccept = (*acceptEncodings).split(",").map!strip;
593                 if ( (rs.compression & Compression.gzip) && heAccept.canFind("gzip")) {
594                     return "gzip";
595                 }
596                 if ( (compression & Compression.deflate) && heAccept.canFind("deflate")) {
597                     return "deflate";
598                 }
599             }
600             return null;
601         }
602         void sendCookies(NetworkStream stream) {
603             if ( _cookies.length ) {
604                 foreach(c; _cookies) {
605                     auto setCookie = "Set-Cookie: %s=%s; Path=%s\r\n".format(c.attr, c.value, c.path);
606                     stream.send(setCookie);
607                 }
608             }
609         }
610         final override void send(NetworkStream stream) {
611             import std.zlib;
612             auto    statusLine = "HTTP/1.1 " ~ to!string(_status) ~ " " ~ codes.get(_status, _status_reason) ~ " \r\n";
613 
614             if ( !stream.isOpen || !stream.isConnected ) {
615                 debug(httpd) tracef("Will not send to closed connection");
616                 return;
617             }
618             debug(httpd) tracef("sending statusLine: %s", statusLine.stripRight);
619             stream.send(statusLine);
620 
621             auto comp = selectCompression(_request, this);
622 
623             static if ( isSomeString!C || __traits(compiles, cast(ubyte[])_content) ) {
624                 ubyte[] data;
625                 if ( comp ) {
626                     _headers["content-encoding"] = comp;
627                     Compress compressor;
628                     final switch (comp) {
629                         case "gzip": // gzip
630                             compressor = new Compress(6, HeaderFormat.gzip);
631                             break;
632                         case "deflate": // deflate
633                             compressor = new Compress(6, HeaderFormat.deflate);
634                             break;
635                     }
636                     data = cast(ubyte[])compressor.compress(_content);
637                     data ~= cast(ubyte[])compressor.flush();
638                 }
639                 else {
640                     data = cast(ubyte[])_content;
641                 }
642                 _headers["content-length"] = to!string(data.length);
643                 foreach(p; _headers.byKeyValue) {
644                     stream.send(p.key ~ ": " ~ p.value ~ "\r\n");
645                 }
646                 if ( _cookies.length ) {
647                     sendCookies(stream);
648                 }
649                 stream.send("\r\n");
650                 if (_request.method == "HEAD") {
651                     return;
652                 }
653                 stream.send(data);
654             }
655             else {
656                 _headers["transfer-encoding"] = "chunked";
657                 Compress compressor;
658                 if ( comp !is null ) {
659                     _headers["content-encoding"] = comp;
660                     final switch (comp) {
661                         case "gzip": // gzip
662                             compressor = new Compress(6, HeaderFormat.gzip);
663                             break;
664                         case "deflate": // deflate
665                             compressor = new Compress(6, HeaderFormat.deflate);
666                             break;
667                     }
668                 }
669                 foreach(p; _headers.byKeyValue) {
670                     stream.send(p.key ~ ": " ~ p.value ~ "\r\n");
671                 }
672                 if ( _cookies.length ) {
673                     sendCookies(stream);
674                 }
675                 stream.send("\r\n");
676                 if (_request.method == "HEAD") {
677                     return;
678                 }
679                 ubyte[] data;
680                 while(!_content.empty) {
681                     auto chunk = cast(ubyte[])_content.front;
682                     _content.popFront;
683 
684                     if ( compressor ) {
685                         data ~= cast(ubyte[])compressor.compress(chunk);
686                         if ( data.length == 0 ) {
687                             continue;
688                         }
689                     } else {
690                         data = chunk;
691                     }
692                     stream.send("%x\r\n".format(data.length));
693                     stream.send(data);
694                     stream.send("\r\n");
695                     data.length = 0;
696                 }
697                 if ( compressor ) {
698                     data = cast(ubyte[])compressor.flush();
699                     stream.send("%x\r\n".format(data.length));
700                     stream.send(data);
701                     stream.send("\r\n");
702                 }
703                 stream.send("0\r\n\r\n");
704             }
705         }
706     }
707 
708     alias Handler = _Response delegate(in App app, ref HTTPD_Request, RequestArgs);
709 
710     struct RequestArgs {
711         private {
712             Captures!string _captures = void;
713             string          _string;
714         }
715         this(Captures!string c) @nogc @safe nothrow {
716             _captures = c;
717         }
718         this(string s) @nogc @safe pure nothrow {
719             _string = s;
720         }
721         bool empty() @nogc @safe pure nothrow {
722             return _captures.empty && _string is null;
723         }
724         string opIndex(string s) @safe pure {
725             return _captures[s];
726         }
727         string opIndex(size_t i) @safe pure {
728             if ( _string && i==0 ) {
729                 return _string;
730             }
731             return _captures[i];
732         }
733     }
734 
735     auto exactRoute(string s, Handler h) @safe pure nothrow {
736         return new ExactRoute(s, h);
737     }
738 
739     auto regexRoute(string s, Handler h) @safe {
740         return new RegexRoute(s, h);
741     }
742 
743     class Route {
744         Handler _handler;
745         string  _origin;
746 
747         abstract RequestArgs match(string) {
748             return RequestArgs();
749         };
750         final Handler handler() {
751             return _handler;
752         }
753         final string origin() {
754             return _origin;
755         }
756     }
757 
758     class ExactRoute: Route {
759 
760         this(string s, Handler h) @safe pure nothrow {
761             _origin = s;
762             _handler = h;
763         }
764         final override RequestArgs match(string input) {
765             if ( input == _origin ) {
766                 debug(httpd) tracef("%s matches %s", input, _origin);
767                 return RequestArgs(input);
768             }
769             return RequestArgs();
770         }
771     }
772     class RegexRoute: Route {
773         Regex!char        _re;
774 
775         this(string r, Handler h) @safe {
776             _origin = r;
777             _handler = h;
778             _re = regex(r);
779         }
780         final override RequestArgs match(string input) {
781             auto m = matchFirst(input, _re);
782             debug(httpd) if (!m.empty) {tracef("%s matches %s", input, _origin);}
783             return RequestArgs(m);
784         }
785     }
786 
787     struct Router {
788         alias RouteMatch = Tuple!(Handler, "handler", RequestArgs, "args");
789         private Route[] _routes;
790 
791         void addRoute(Route r) {
792             _routes ~= r;
793         }
794         auto getRoute(string path) {
795             RouteMatch match;
796             foreach(r; _routes) {
797                 auto args = r.match(path);
798                 if (!args.empty) {
799                     match.handler = r.handler;
800                     match.args = args;
801                     break;
802                 }
803             }
804             return match;
805         }
806     }
807 
808     private auto parseQuery(string query) {
809         /// TODO
810         /// switch to return dict of
811         /// struct QueryParam {
812         ///   private:
813         ///     string   name;
814         ///     string[] value;
815         ///   public:
816         ///     uint length() {return value.length;}
817         ///     string toString() {return value[0];}
818         ///     string[] toArray() {return value;}
819         /// }
820         debug (httpd) tracef("query: %s", query);
821         string[string] q;
822         if ( !query ) {
823             return q;
824         }
825         if ( query[0] == '?') {
826             query = query[1..$];
827         }
828         string[][] parsed = query.splitter("&").
829             map!(s => s.split("=")).
830                 filter!"a.length==2".
831                 map!(p => [urlDecode(p[0]), urlDecode(p[1])]).
832                 array;
833 
834         auto grouped = sort!"a[0]<b[0]"(parsed).assumeSorted!"a[0]<b[0]".groupBy();
835         foreach(g; grouped) {
836             string key = g.front[0];
837             string val;
838             auto vals = g.map!"a[1]".array;
839             if (vals.length == 1) {
840                 val = vals[0];
841             }
842             if (vals.length > 1) {
843                 val = to!string(vals);
844             }
845             q[key] = val;
846         }
847         return q;
848     }
849 
850     private bool headersReceived(in ubyte[] data, ref Buffer!ubyte buffer, out string separator) @safe {
851         foreach(s; ["\r\n\r\n", "\n\n"]) {
852             if ( data.canFind(s) || buffer.canFind(s) ) {
853                 separator = s;
854                 return true;
855             }
856         }
857         return false;
858     }
859 
860     private void parseRequestHeaders(in App app, ref HTTPD_Request rq, string buffer) {
861         string lastHeader;
862         auto   lines = buffer.splitLines.map!stripRight;
863         rq.requestLine = lines[0];
864         if ( lines.count == 1) {
865             return;
866         }
867         foreach(line; lines[1..$]) {
868             if ( !line.length ) {
869                 continue;
870             }
871             if ( line[0] == ' ' || line[0] == '\t' ) {
872                 // unfolding https://tools.ietf.org/html/rfc822#section-3.1
873                 if ( auto prevValue = lastHeader in rq.requestHeaders) {
874                     *prevValue ~= line;
875                 }
876                 continue;
877             }
878             auto parsed = line.findSplit(":");
879             auto header = parsed[0].toLower;
880             auto value =  parsed[2].strip;
881             lastHeader = header;
882             if ( auto h = header in rq.requestHeaders ) {
883                 *h ~= "; " ~ value;
884             } else {
885                 rq.requestHeaders[header] = value;
886             }
887             debug(httpd) tracef("%s: %s", header, value);
888         }
889         auto rqlFields = rq.requestLine.split(" ");
890         debug (httpd) tracef("rqLine %s", rq.requestLine);
891         rq.method = rqlFields[0];
892         auto scheme = app.useSSL?
893             "https://":
894                 "http://";
895         if ( "host" in rq.requestHeaders ) {
896             rq.uri = URI(scheme ~ rq.requestHeaders["host"] ~ rqlFields[1]);
897         } else {
898             rq.uri = URI(scheme ~ app.host ~ rqlFields[1]);
899         }
900         rq.path  = rq.uri.path;
901         rq.query = parseQuery(rq.uri.query);
902         debug (httpd) tracef("path: %s", rq.path);
903         debug (httpd) tracef("query: %s", rq.query);
904         //
905         // now analyze what we have
906         //
907         auto header = "connection" in rq.requestHeaders;
908         if ( header && toLower(*header) == "keep-alive") {
909             rq.keepAlive = true;
910         }
911         auto cookies = "cookie" in rq.requestHeaders;
912         if ( cookies ) {
913             (*cookies).split(';').
914                 map!"strip(a).split('=')".
915                     filter!(kv => kv.length==2).
916                     each!(kv => rq._cookies[kv[0]] = kv[1]);
917         }
918     }
919 
920     private auto read_request(in App app, NetworkStream stream) {
921         HTTPD_Request rq;
922         Buffer!ubyte  input;
923         string        separator;
924 
925         while( true ) {
926             ubyte[] b = new ubyte[app.bufferSize];
927             auto read = stream.receive(b);
928 
929             if ( read == 0 ) {
930                 return rq;
931             }
932             debug(httpd) tracef("received %d bytes", read);
933             input.putNoCopy(b[0..read]);
934 
935             if ( headersReceived(b, input, separator) ) {
936                 break;
937             }
938 
939             if ( input.length >= app.maxHeadersSize ) {
940                 throw new HTTPD_RequestException("Request headers length %d too large".format(input.length));
941             }
942         }
943         debug(httpd) trace("Headers received");
944         auto s = input.data!(string).findSplit(separator);
945         auto requestHeaders = s[0];
946         debug(httpd) tracef("Headers: %s", cast(string)requestHeaders);
947         parseRequestHeaders(app, rq, requestHeaders);
948         debug(httpd) trace("Headers parsed");
949 
950         rq._dataSource = rq.createDataSource(s[2], stream);
951 
952         return rq;
953     }
954 
955     void processor(in App app, HTTPD httpd, NetworkStream stream) {
956         stream.readTimeout = app.timeout;
957         HTTPD_Request  rq;
958         _Response      rs;
959         scope (exit) {
960             if ( stream.isOpen ) {
961                 stream.close();
962             }
963         }
964         uint rqLimit = max(app.rqLimit, 1);
965         try {
966             while ( rqLimit > 0 ) {
967                 rq = read_request(app, stream);
968                 if ( !httpd._running || !rq.requestLine.length ) {
969                     return;
970                 }
971                 auto match = httpd._router.getRoute(rq.path);
972                 if ( !match.handler ) {
973                     // return 404;
974                     debug (httpd) tracef("Route not found for %s", rq.path);
975                     rs = response(rq, "Requested path %s not found".format(rq.path), 404);
976                     break;
977                 }
978                 auto handler = match.handler;
979                 rs = handler(app, rq, match.args);
980                 if ( !stream.isOpen ) {
981                     debug(httpd) tracef("Request handler closed connection");
982                     return;
983                 }
984                 if ( rq.keepAlive && rqLimit > 1 ) {
985                     rs.headers["Connection"] = "Keep-Alive";
986                 }
987                 if ( rq._dataSource._requestHasBody && !rq._dataSource._requestBodyReceived ) {
988                     // for some reason some part of the request body still not received, and it will
989                     // stay on the way of next request if this is keep-Alive session,
990                     // so we must abort this connection anyway.
991                     debug(httpd) trace("Request handler did not consumed whole request body. We have to close connection after sending response.");
992                     rs.send(stream);
993                     return;
994                 }
995                 rs.send(stream);
996                 --rqLimit;
997                 if ( !rq.keepAlive || rqLimit==0 ) {
998                     debug(httpd) trace("Finished with that connection");
999                     return;
1000                 }
1001                 debug(httpd) trace("Continue with keepalive request");
1002                 rq = rq.init;
1003             }
1004         }
1005         catch (HTTPD_RequestException e) {
1006             debug(httpd)  error("Request exception: " ~ e.msg);
1007             rs = response(rq, "Request exception:\n" ~ e.msg, 500);
1008         }
1009         catch (TimeoutException e) {
1010             debug(httpd) {
1011                 if ( rq.requestLine ) {
1012                     error("Timeout reading/writing to client");
1013                 }
1014             }
1015         }
1016         catch (Exception e) {
1017             debug(httpd) error("Unexpected Exception " ~ e.msg);
1018             rs = response(rq, "Unexpected exception:\n" ~ e.msg, 500);
1019         }
1020         catch (Error e) {
1021             error(e.msg, e.info);
1022             rs = response(rq, "Unexpected error:\n" ~ e.msg, 500);
1023         }
1024         try {
1025             if ( stream.isOpen ) {
1026                 rs.send(stream);
1027             }
1028         }
1029         catch (Exception e) {
1030             infof("Exception when send %s", e.msg);
1031         }
1032         catch (Error e) {
1033             error("Error sending response: " ~ e.msg);
1034         }
1035     }
1036 
1037     class HTTPD
1038     {
1039         private {
1040             TaskPool              _server;
1041             __gshared bool        _running;
1042             Router                _router;
1043             App                   _app;
1044         }
1045         auto ref addRoute(Route r) {
1046             _router.addRoute(r);
1047             return this;
1048         }
1049         static NetworkStream openStream(in App app) {
1050             auto host = app.host;
1051             auto port = app.port;
1052             Address[] addresses;
1053             SSLOptions _sslOptions;
1054 
1055             try {
1056                 addresses = getAddress(host, port);
1057             } catch (Exception e) {
1058                 throw new ConnectError("Can't resolve name when connect to %s:%d: %s".format(host, port, e.msg));
1059             }
1060             auto tcpStream = app.useSSL?
1061                 new SSLStream(_sslOptions):
1062                 new TCPStream();
1063             tcpStream.open(addresses[0].addressFamily);
1064             return tcpStream;
1065         }
1066         static void run(in App app, HTTPD httpd) {
1067             Address[] addresses;
1068             try {
1069                 addresses = getAddress(app.host, app.port);
1070             } catch (Exception e) {
1071                 throw new ConnectError("Can't resolve name when connect to %s:%d: %s".format(app.host, app.port, e.msg));
1072             }
1073             auto tcpStream = openStream(app);
1074             tcpStream.reuseAddr(true);
1075             tcpStream.bind(addresses[0]);
1076             tcpStream.listen(128);
1077             defaultPoolThreads(64);
1078             auto pool = taskPool();
1079             _running = true;
1080             while ( _running ) {
1081                 auto stream = tcpStream.accept();
1082                 if ( _running ) {
1083                     auto connHandler = task!processor(app, httpd, stream);
1084                     pool.put(connHandler);
1085                 } else {
1086                     tcpStream.close();
1087                     break;
1088                 }
1089             }
1090         }
1091         void app(App a) {
1092             _app = a;
1093         }
1094         void start() {
1095             defaultPoolThreads(64);
1096             _server = taskPool();
1097             auto t = task!run(_app, this);
1098             _server.put(t);
1099             Thread.sleep(500.msecs);
1100         }
1101         void start(App app) {
1102             defaultPoolThreads(64);
1103             _app = app;
1104             _server = taskPool();
1105             auto t = task!run(_app, this);
1106             _server.put(t);
1107             Thread.sleep(500.msecs);
1108         }
1109         void stop() {
1110             if ( !_running ) {
1111                 return;
1112             }
1113             _running = false;
1114             try {
1115                 auto s = openStream(_app);
1116                 s.connect(_app.host, _app.port);
1117             } catch (Exception e) {
1118             }
1119             //        _server.stop();
1120         }
1121     }
1122 
1123     struct App {
1124         private {
1125             string   _name;
1126             string   _host;
1127             ushort   _port;
1128             Duration _timeout = 30.seconds;
1129             size_t   _bufferSize =     16*1024;
1130             size_t   _maxHeadersSize = 32*1024;
1131             bool     _useSSL = false;
1132             uint      _rqLimit = 10; // keepalive requestst per connection
1133             Router   _router;
1134         }
1135         mixin(Getter_Setter!string("name"));
1136         mixin(Getter_Setter!string("host"));
1137         mixin(Getter_Setter!ushort("port"));
1138         mixin(Getter_Setter!size_t("bufferSize"));
1139         mixin(Getter_Setter!size_t("maxHeadersSize"));
1140         mixin(Getter_Setter!Duration("timeout"));
1141         mixin(Getter_Setter!bool("useSSL"));
1142         mixin(Getter_Setter!uint("rqLimit"));
1143         this(string name) {
1144             _name = name;
1145         }
1146     }
1147 
1148 
1149     version(none) private unittest {
1150         import std.json;
1151         import std.conv;
1152         import requests.http: HTTPRequest, TimeoutException, BasicAuthentication, queryParams, MultipartForm, formData;
1153         globalLogLevel(LogLevel.info);
1154 
1155         static auto buildReply(ref HTTPD_Request rq) {
1156             auto args    = JSONValue(rq.query);
1157             auto headers = JSONValue(rq.requestHeaders);
1158             auto url     = JSONValue(rq.uri.uri);
1159             auto json    = JSONValue(rq.json);
1160             auto data    = JSONValue(rq.data);
1161             auto form    = JSONValue(rq.form);
1162             auto files   = JSONValue(rq.files);
1163             auto reply   = JSONValue(["args":args, "headers": headers, "json": json, "url": url, "data": data, "form": form, "files": files]);
1164             return reply.toString();
1165         }
1166 
1167         Router router;
1168         router.addRoute(exactRoute(r"/get", null));
1169         router.addRoute(regexRoute(r"/get/(?P<param>\d+)", null));
1170         auto r = router.getRoute(r"/get");
1171         assert(!r.args.empty);
1172         r = router.getRoute(r"/post");
1173         assert(r.args.empty);
1174 
1175         r = router.getRoute(r"/get/333");
1176         assert(!r.args.empty);
1177         assert(r.args["param"]=="333");
1178         r = router.getRoute(r"/get/aaa");
1179         assert(r.args.empty);
1180 
1181         HTTPD_Request rq;
1182         string headers = "GET /get?a=b&list[]=1&c=d&list[]=2 HTTP/1.1\n" ~
1183                          "Host: host\n" ~
1184                          "X-Test: test1\n" ~
1185                          " test2\n" ~
1186                          "Content-Length: 1\n";
1187         parseRequestHeaders(App(), rq, headers);
1188         assert(rq.requestHeaders["x-test"] == "test1 test2");
1189         assert(rq.requestHeaders["host"] == "host");
1190         assert(rq.path == "/get");
1191         assert(rq.query["a"] == "b");
1192         assert(rq.query["c"] == "d");
1193         assert(rq.query["list[]"] == `["1", "2"]`);
1194         auto root(in App app, ref HTTPD_Request rq,  RequestArgs args) {
1195             debug (httpd) trace("handler / called");
1196             auto rs = response(rq, buildReply(rq));
1197             rs.headers["Content-Type"] = "application/json";
1198             return rs;
1199         }
1200         auto get(in App app, ref HTTPD_Request rq,  RequestArgs args) {
1201             debug (httpd) trace("handler /get called");
1202             auto rs = response(rq, buildReply(rq));
1203             rs.headers["Content-Type"] = "application/json";
1204             return rs;
1205         }
1206         auto basicAuth(in App app, ref HTTPD_Request rq, RequestArgs args) {
1207             import std.base64;
1208             auto user    = args["user"];
1209             auto password= args["password"];
1210             auto auth    = cast(string)Base64.decode(rq.requestHeaders["authorization"].split()[1]);
1211             auto up      = auth.split(":");
1212             short status;
1213             if ( up[0]==user && up[1]==password) {
1214                 status = 200;
1215             } else {
1216                 status = 401;
1217             }
1218             auto rs = response(rq, buildReply(rq), status);
1219             rs.headers["Content-Type"] = "application/json";
1220             return rs;
1221         }
1222         auto rredir(in App app, ref HTTPD_Request rq,  RequestArgs args) {
1223             auto rs = response(rq, buildReply(rq));
1224             auto redirects = to!long(args["redirects"]);
1225             if ( redirects > 1 ) {
1226                 rs.headers["Location"] = "/relative-redirect/%d".format(redirects-1);
1227             } else {
1228                 rs.headers["Location"] = "/get";
1229             }
1230             rs.status    = 302;
1231             return rs;
1232         }
1233         auto aredir(in App app, ref HTTPD_Request rq,  RequestArgs args) {
1234             auto rs = response(rq, buildReply(rq));
1235             auto redirects = to!long(args["redirects"]);
1236             if ( redirects > 1 ) {
1237                 rs.headers["Location"] = "http://127.0.0.1:8081/absolute-redirect/%d".format(redirects-1);
1238             } else {
1239                 rs.headers["Location"] = "http://127.0.0.1:8081/get";
1240             }
1241             rs.status    = 302;
1242             return rs;
1243         }
1244         auto delay(in App app, ref HTTPD_Request rq, RequestArgs args) {
1245             auto delay = dur!"seconds"(to!long(args["delay"]));
1246             Thread.sleep(delay);
1247             auto rs = response(rq, buildReply(rq));
1248             rs.headers["Content-Type"] = "application/json";
1249             return rs;
1250         }
1251         auto gzip(in App app, ref HTTPD_Request rq, RequestArgs args) {
1252             auto rs = response(rq, buildReply(rq));
1253             rs.compress(Compression.gzip);
1254             rs.headers["Content-Type"] = "application/json";
1255             return rs;
1256         }
1257         auto deflate(in App app, ref HTTPD_Request rq, RequestArgs args) {
1258             auto rs = response(rq, buildReply(rq));
1259             rs.compress(Compression.deflate);
1260             return rs;
1261         }
1262         auto range(in App app, ref HTTPD_Request rq, RequestArgs args) {
1263             auto size = to!size_t(args["size"]);
1264             auto rs = response(rq, new ubyte[size].chunks(16));
1265             rs.compress(Compression.yes);
1266             return rs;
1267         }
1268         auto head(in App app, ref HTTPD_Request rq, RequestArgs args) {
1269             if ( rq.method != "HEAD") {
1270                 auto rs = response(rq, "Illegal method %s".format(rq.method), 405);
1271                 return rs;
1272             }
1273             else {
1274                 auto rs = response(rq, buildReply(rq));
1275                 rs.compress(Compression.yes);
1276                 return rs;
1277             }
1278         }
1279         auto del(in App app, ref HTTPD_Request rq, RequestArgs args) {
1280             if ( rq.method != "DELETE") {
1281                 auto rs = response(rq, "Illegal method %s".format(rq.method), 405);
1282                 return rs;
1283             }
1284             else {
1285                 auto rs = response(rq, buildReply(rq));
1286                 return rs;
1287             }
1288         }
1289         auto post(in App app, ref HTTPD_Request rq, RequestArgs args) {
1290             auto rs = response(rq, buildReply(rq));
1291             return rs;
1292         }
1293         auto postIter(in App app, ref HTTPD_Request rq, RequestArgs args) {
1294             int  c;
1295 
1296             if ( rq.contentType == "multipart/form-data" ) {
1297                 auto parts = rq.multiPartRead();
1298                 foreach(p; parts) {
1299                     auto disposition = p.disposition;
1300                     c += p.data.joiner.count;
1301                 }
1302                 auto rs = response(rq, "%d".format(c));
1303                 return rs;
1304             }
1305             else {
1306                 auto r = rq.read();
1307                 while ( !r.empty ) {
1308                     c += r.front.length;
1309                     r.popFront;
1310                 }
1311                 auto rs = response(rq, "%d".format(c));
1312                 return rs;
1313             }
1314         }
1315         auto read(in App app, ref HTTPD_Request rq, RequestArgs args) {
1316             auto r = rq.read();
1317             int  c;
1318             while ( !r.empty ) {
1319                 c += r.front.length;
1320                 r.popFront;
1321             }
1322             auto rs = response(rq, "%d".format(c));
1323             return rs;
1324         }
1325         auto readf1(in App app, ref HTTPD_Request rq, RequestArgs args) {
1326             // now call to read must throw exception
1327             auto r = rq.read();
1328             int  c;
1329             while ( !r.empty ) {
1330                 c += r.front.length;
1331                 r.popFront;
1332                 break;
1333             }
1334             auto rs = response(rq, "%d".format(c));
1335             return rs;
1336         }
1337         auto cookiesSet(in App app, ref HTTPD_Request rq, RequestArgs args) {
1338             Cookie[] cookies;
1339             foreach(p; rq.query.byKeyValue) {
1340                 cookies ~= Cookie("/cookies", rq.requestHeaders["host"], p.key, p.value);
1341             }
1342             auto rs = response(rq, buildReply(rq), 302);
1343             rs.headers["Location"] = "/cookies";
1344             rs.cookies = cookies;
1345             return rs;
1346         }
1347         auto cookies(in App app, ref HTTPD_Request rq, RequestArgs args) {
1348             auto cookies = ["cookies": JSONValue(rq.cookies)];
1349             auto rs = response(rq, JSONValue(cookies).toString);
1350             return rs;
1351         }
1352 
1353         auto httpbin = App("httpbin");
1354 
1355         httpbin.port = 8081;
1356         httpbin.host = "127.0.0.1";
1357 
1358         httpbin.timeout = 10.seconds;
1359         HTTPD server = new HTTPD();
1360 
1361         server.addRoute(exactRoute(r"/", &root)).
1362                 addRoute(exactRoute(r"/get", &get)).
1363                 addRoute(regexRoute(r"/delay/(?P<delay>\d+)", &delay)).
1364                 addRoute(regexRoute(r"/relative-redirect/(?P<redirects>\d+)", &rredir)).
1365                 addRoute(regexRoute(r"/absolute-redirect/(?P<redirects>\d+)", &aredir)).
1366                 addRoute(regexRoute(r"/basic-auth/(?P<user>[^/]+)/(?P<password>[^/]+)", &basicAuth)).
1367                 addRoute(exactRoute(r"/gzip", &gzip)).
1368                 addRoute(exactRoute(r"/deflate", &deflate)).
1369                 addRoute(regexRoute(r"/range/(?P<size>\d+)", &range)).
1370                 addRoute(exactRoute(r"/cookies/set", &cookiesSet)).
1371                 addRoute(exactRoute(r"/cookies", &cookies)).
1372                 addRoute(exactRoute(r"/head", &head)).
1373                 addRoute(exactRoute(r"/delete", &del)).
1374                 addRoute(exactRoute(r"/read", &read)).
1375                 addRoute(exactRoute(r"/readf1", &readf1)).
1376                 addRoute(exactRoute(r"/post", &post)).
1377                 addRoute(exactRoute(r"/postIter", &postIter));
1378 
1379         server.start(httpbin);
1380         scope(exit) {
1381             server.stop();
1382         }
1383         auto request = HTTPRequest();
1384 
1385         globalLogLevel(LogLevel.info);
1386         auto httpbin_url = "http://%s:%d/".format(httpbin.host, httpbin.port);
1387         request.timeout = 5.seconds;
1388         request.keepAlive = true;
1389         info("httpd Check GET");
1390         auto rs = request.get(httpbin_url);
1391         assert(rs.code == 200);
1392         assert(rs.responseBody.length > 0);
1393         auto content = rs.responseBody.data!string;
1394         auto json = parseJSON(cast(string)content);
1395         assert(json.object["url"].str == httpbin_url);
1396 
1397         info("httpd Check GET with parameters");
1398         rs = request.get(httpbin_url ~ "get", ["c":" d", "a":"b"]);
1399         assert(rs.code == 200);
1400         json = parseJSON(cast(string)rs.responseBody.data).object["args"].object;
1401         assert(json["a"].str == "b");
1402         assert(json["c"].str == " d");
1403 
1404         info("httpd Check relative redirect");
1405         rs = request.get(httpbin_url ~ "relative-redirect/2");
1406         assert(rs.history.length == 2);
1407         assert(rs.code==200);
1408 
1409         info("httpd Check absolute redirect");
1410         rs = request.get(httpbin_url ~ "absolute-redirect/2");
1411         assert(rs.history.length == 2);
1412         assert(rs.code==200);
1413 
1414         info("httpd Check basic auth");
1415         request.authenticator = new BasicAuthentication("user", "password");
1416         rs = request.get(httpbin_url ~ "basic-auth/user/password");
1417         assert(rs.code==200);
1418         request.authenticator = null;
1419 
1420         info("httpd Check timeout");
1421         request.timeout = 1.seconds;
1422         assertThrown!TimeoutException(request.get(httpbin_url ~ "delay/2"));
1423         Thread.sleep(1.seconds);
1424         request.timeout = 30.seconds;
1425 
1426         info("httpd Check gzip");
1427         rs = request.get(httpbin_url ~ "gzip");
1428         assert(rs.code==200);
1429         json = parseJSON(cast(string)rs.responseBody);
1430         assert(json.object["url"].str == httpbin_url ~ "gzip");
1431 
1432         info("httpd Check deflate");
1433         rs = request.get(httpbin_url ~ "deflate");
1434         assert(rs.code==200);
1435         json = parseJSON(cast(string)rs.responseBody);
1436         assert(json.object["url"].str == httpbin_url ~ "deflate");
1437 
1438         info("httpd Check range");
1439         rs = request.get(httpbin_url ~ "range/1023");
1440         assert(rs.code==200);
1441         assert(rs.responseBody.length == 1023);
1442 
1443         info("httpd Check HEAD");
1444         rs = request.exec!"HEAD"(httpbin_url ~ "head");
1445         assert(rs.code==200);
1446         assert(rs.responseBody.length == 0);
1447 
1448         info("httpd Check DELETE");
1449         rs = request.exec!"DELETE"(httpbin_url ~ "delete");
1450         assert(rs.code==200);
1451 
1452         info("httpd Check POST json");
1453         rs = request.post(httpbin_url ~ "post?b=x", `{"a":"b", "c":[1,2,3]}`, "application/json");
1454         json = parseJSON(cast(string)rs.responseBody);
1455         auto rqJson = parseJSON(json.object["json"].str);
1456         assert(rqJson.object["a"].str == "b");
1457         assert(equal([1,2,3], rqJson.object["c"].array.map!"a.integer"));
1458 
1459         info("httpd Check POST json/chunked body");
1460         rs = request.post(httpbin_url ~ "post?b=x", [`{"a":"b",`,` "c":[1,2,3]}`], "application/json");
1461         json = parseJSON(cast(string)rs.responseBody);
1462         assert(json.object["args"].object["b"].str == "x");
1463         rqJson = parseJSON(json.object["json"].str);
1464         assert(rqJson.object["a"].str == "b");
1465         assert(equal([1,2,3], rqJson.object["c"].array.map!"a.integer"));
1466 
1467         rs = request.post(httpbin_url ~ "post", "0123456789".repeat(32));
1468         json = parseJSON(cast(string)rs.responseBody);
1469         assert(equal(json.object["data"].array.map!"a.integer", "0123456789".repeat(32).join));
1470 
1471         info("httpd Check POST with params");
1472         rs = request.post(httpbin_url ~ "post", queryParams("b", 2, "a", "A"));
1473         assert(rs.code==200);
1474         auto data = parseJSON(cast(string)rs.responseBody).object["form"].object;
1475         assert((data["a"].str == "A"));
1476         assert((data["b"].str == "2"));
1477 
1478         // this is tests for httpd read() interface
1479         info("httpd Check POST/iterating over body");
1480         rs = request.post(httpbin_url ~ "read", "0123456789".repeat(1500));
1481         assert(equal(rs.responseBody, "15000"));
1482 
1483         {
1484             request.keepAlive = true;
1485             // this is test on how we can handle keepalive session when previous request leave unread data in socket
1486             try {
1487                 rs = request.post(httpbin_url ~ "readf1", "0123456789".repeat(1500));
1488             }
1489             catch (Exception e) {
1490                 // this can fail as httpd will close connection prematurely
1491             }
1492             // but next idempotent request must succeed
1493             rs = request.get(httpbin_url ~ "get");
1494             assert(rs.code == 200);
1495         }
1496         //
1497         {
1498             info("httpd Check POST/multipart form");
1499             import std.file;
1500             import std.path;
1501             auto tmpd = tempDir();
1502             auto tmpfname1 = tmpd ~ dirSeparator ~ "request_test1.txt";
1503             auto f = File(tmpfname1, "wb");
1504             f.rawWrite("file1 content\n");
1505             f.close();
1506             auto tmpfname2 = tmpd ~ dirSeparator ~ "request_test2.txt";
1507             f = File(tmpfname2, "wb");
1508             f.rawWrite("file2 content\n");
1509             f.close();
1510             ///
1511             /// Ok, files ready.
1512             /// Now we will prepare Form data
1513             ///
1514             File f1 = File(tmpfname1, "rb");
1515             File f2 = File(tmpfname2, "rb");
1516             scope(exit) {
1517                 f1.close();
1518                 f2.close();
1519             }
1520             ///
1521             /// for each part we have to set field name, source (ubyte array or opened file) and optional filename and content-type
1522             ///
1523             MultipartForm form = MultipartForm().
1524                 add(formData("Field1", cast(ubyte[])"form field from memory")).
1525                     add(formData("Field2", cast(ubyte[])"file field from memory", ["filename":"data2"])).
1526                     add(formData("Field3", cast(ubyte[])`{"a":"b"}`, ["Content-Type": "application/json"])).
1527                     add(formData("File1", f1, ["filename":"file1", "Content-Type": "application/octet-stream"])).
1528                     add(formData("File2", f2, ["filename":"file2", "Content-Type": "application/octet-stream"]));
1529             /// everything ready, send request
1530             rs = request.post(httpbin_url ~ "post?a=b", form);
1531             /* expected:
1532              {
1533              "args": {
1534              "a": "b"
1535              },
1536              "data": "",
1537              "files": {
1538              "Field2": "file field from memory",
1539              "File1": "file1 content\n",
1540              "File2": "file2 content\n"
1541              },
1542              "form": {
1543              "Field1": "form field from memory",
1544              "Field3": "{\"a\":\"b\"}"
1545              },
1546              "headers": {
1547              "Accept-Encoding": "gzip, deflate",
1548              "Content-Length": "730",
1549              "Content-Type": "multipart/form-data; boundary=d79a383e-7912-4d36-a6db-a6774bf37133",
1550              "Host": "httpbin.org",
1551              "User-Agent": "dlang-requests"
1552              },
1553              "json": null,
1554              "origin": "xxx.xxx.xxx.xxx",
1555              "url": "http://httpbin.org/post?a=b"
1556              }
1557              */
1558             json = parseJSON(cast(string)rs.responseBody);
1559             assert("file field from memory" == cast(string)(json.object["files"].object["Field2"].array.map!(a => cast(ubyte)a.integer).array));
1560             assert("file1 content\n" == cast(string)(json.object["files"].object["File1"].array.map!(a => cast(ubyte)a.integer).array));
1561 
1562             info("httpd Check POST/iterate over multipart form");
1563             form = MultipartForm().
1564                 add(formData("Field1", cast(ubyte[])"form field from memory")).
1565                     add(formData("Field2", cast(ubyte[])"file field from memory", ["filename":"data2"])).
1566                     add(formData("Field3", cast(ubyte[])`{"a":"b"}`, ["Content-Type": "application/json"]));
1567             /// everything ready, send request
1568             rs = request.post(httpbin_url ~ "postIter?a=b", form);
1569             assert(equal(rs.responseBody, "53"));
1570             rs = request.post(httpbin_url ~ "postIter", "0123456789".repeat(1500));
1571             assert(equal(rs.responseBody, "15000"));
1572         }
1573         info("httpd Check cookies");
1574         rs = request.get(httpbin_url ~ "cookies/set?A=abcd&b=cdef");
1575         json = parseJSON(cast(string)rs.responseBody.data).object["cookies"].object;
1576         assert(json["A"].str == "abcd");
1577         assert(json["b"].str == "cdef");
1578     }
1579 }