1 module requests.server.httpd;
2
3 import std.algorithm;
4 import std.array;
5 import std.conv;
6 import std.datetime;
7 import std.exception;
8 import std.experimental.logger;
9 import std.format;
10 import std.parallelism;
11 import std.range;
12 import std.regex;
13 import std.socket;
14 import std.stdio;
15 import std.string;
16 import std.traits;
17 import std.typecons;
18 import core.thread;
19 import requests.utils;
20 import requests.streams;
21 import requests.uri;
22
23 version(vibeD){
24 pragma(msg, "httpd will not compile with vibeD");
25 }
26 else {
27 /*
28 ** This is small http server to run something like httpbin(http://httpbin.org) internally
29 ** for Requests unittest's.
30 */
31
32 enum DSBUFFSIZE = 16*1024;
33
34 class HTTPD_RequestException: Exception {
35 this(string message, string file =__FILE__, size_t line = __LINE__, Throwable next = null) @safe pure nothrow {
36 super(message, file, line, next);
37 }
38 }
39
40 struct HTTPD_Request {
41 private {
42 string _requestLine;
43 string[string] _requestHeaders;
44 Buffer!ubyte _requestBody;
45 bool _keepAlive;
46 URI _uri;
47 string[string] _query; // query in url
48 string _method;
49 string _path;
50 string _json; // json for application/json
51 string[string] _form; // form values for application/x-www-form-urlencoded
52 ubyte[][string] _files;
53 ubyte[] _data; // raw data for unrecognized mime's
54 _DataSource _dataSource;
55 string[string] _cookies;
56 }
57 private mixin(Setter!(string[string])("requestHeaders"));
58 auto ref requestHeaders() inout @property @safe @nogc nothrow {
59 return _requestHeaders;
60 }
61 auto ref cookies() inout @property @safe @nogc nothrow {
62 return _cookies;
63 }
64 private mixin(Setter!(string[string])("query"));
65 auto ref query() inout @property @safe @nogc nothrow {
66 return _query;
67 }
68 auto ref requestBody() inout @property @safe @nogc nothrow {
69 return _requestBody;
70 }
71 private mixin(Setter!string("method"));
72 mixin(Getter("method"));
73 private mixin(Setter!string("requestLine"));
74 mixin(Getter("requestLine"));
75 private mixin(Setter!string("path"));
76 mixin(Getter("path"));
77 private mixin(Setter!bool("keepAlive"));
78 mixin(Getter("keepAlive"));
79 private mixin(Setter!URI("uri"));
80 mixin(Getter("uri"));
81
82 @property string json() {
83 if ( _dataSource._readStarted ) {
84 throw new HTTPD_RequestException("Request read() call already started.");
85 }
86 if ( _dataSource._requestHasBody && !_dataSource._requestBodyProcessed ) {
87 debug(httpd) trace("receiving body on demand for json");
88 loadBodyOnDemand(_dataSource);
89 }
90 return _json;
91 }
92 @property ubyte[] data() {
93 if ( _dataSource._readStarted ) {
94 throw new HTTPD_RequestException("Request body read() already started.");
95 }
96 if ( _dataSource._requestHasBody && !_dataSource._requestBodyProcessed ) {
97 debug(httpd) trace("receiving body on demand for data");
98 loadBodyOnDemand(_dataSource);
99 }
100 return _data;
101 }
102 @property string[string] form() {
103 if ( _dataSource._readStarted ) {
104 throw new HTTPD_RequestException("Request body read() already started.");
105 }
106 if ( _dataSource._requestHasBody && !_dataSource._requestBodyProcessed ) {
107 debug(httpd) trace("receiving body on demand for form");
108 loadBodyOnDemand(_dataSource);
109 }
110 return _form;
111 }
112 @property auto files() {
113 if ( _dataSource._readStarted ) {
114 throw new HTTPD_RequestException("Request body read() already started.");
115 }
116 if ( _dataSource._requestHasBody && !_dataSource._requestBodyProcessed ) {
117 debug(httpd) trace("receiving body on demand for form");
118 loadBodyOnDemand(_dataSource);
119 }
120 return _files;
121 }
122
123 @property bool requestHasBody() pure {
124 if ( "content-length" in _requestHeaders ) {
125 return true;
126 }
127 if ( auto contentTransferEncoding = "transfer-encoding" in _requestHeaders ) {
128 if ( *contentTransferEncoding=="chunked" ) {
129 return true;
130 }
131 }
132 return false;
133 }
134
135 class _DataSource {
136 private {
137 NetworkStream _stream;
138 DataPipe!ubyte _bodyDecoder;
139 DecodeChunked _unChunker;
140 long _contentLength = 0;
141 long _receivedLength = 0;
142 ubyte[] _content;
143 bool _requestHasBody; // request has body
144 bool _requestBodyRecvInProgr; // loading body currently active
145 bool _requestBodyProcessed; // we processed body - this can happens only once
146 bool _requestBodyReceived; // all body data were received from network (we have to close socket if request terminated before all data received)
147 bool _readStarted;
148 }
149 bool empty() {
150 debug(httpd) tracef("datasource empty: %s", _content.length==0);
151 return _content.length==0;
152 }
153 ubyte[] front() {
154 return _content;
155 }
156 void popFront() {
157 debug(httpd) trace("datasource enters popFront");
158 _content.length = 0;
159 if ( !_requestBodyRecvInProgr ) {
160 debug(httpd) trace("popFront called when dataSource is not active anymore");
161 return;
162 }
163 while ( _bodyDecoder.empty && _stream && _stream.isOpen ) {
164 auto b = new ubyte[DSBUFFSIZE];
165 auto read = _stream.receive(b);
166 if ( read == 0 ) {
167 debug(httpd) trace("stream closed when receiving in datasource");
168 _bodyDecoder.flush();
169 _requestBodyRecvInProgr = false;
170 break;
171 }
172 debug(httpd) tracef("place %d bytes to datasource", read);
173 _receivedLength += read;
174 _bodyDecoder.putNoCopy(b[0..read]);
175 if ( (_unChunker && _unChunker.done)
176 || (_contentLength > 0 && _receivedLength >= _contentLength) )
177 {
178 debug(httpd) trace("request body reading complete (due contentLength or due last chunk consumed)");
179 _bodyDecoder.flush();
180 _requestBodyRecvInProgr = false;
181 _requestBodyReceived = true;
182 break;
183 }
184 }
185 _content = _bodyDecoder.getNoCopy().join();
186 debug(httpd) tracef("%d bytes in content after popFront", _content.length);
187 }
188 ///
189 /// raplace current front with another value
190 ///
191 void unPop(ubyte[] data) {
192 assert(data.length > 0);
193 _content = data;
194 }
195 ///
196 /// Scan over input stream,
197 /// can return data from stream
198 /// acc - accumulator for receiving needle
199 /// return empty data if we receiving needle
200 /// if needle found in stream, then acc == needle
201 /// if end of stream happened, then eos = true
202 ///
203 ubyte[] scanUntilR(string needle, ref ubyte[] acc, out bool eos) {
204 auto d = needle.representation;
205 ubyte[] l;
206
207 while (!this.empty) {
208 auto c = this.front;
209 debug(httpd) tracef("on scan: %s", cast(string)c);
210 l = acc ~ c;
211 auto s = l.findSplit(d);
212 if ( s[1].length ) {
213 if ( s[2].length ) {
214 this.unPop(s[2]);
215 } else {
216 this.popFront;
217 }
218 acc = s[1];
219 return s[0];
220 }
221 auto i = min(l.length, d.length);
222 for(;i>0; i--) {
223 if ( l.endsWith(d[0..i]) ) {
224 acc = l[$-i..$];
225 this.popFront;
226 return l[0..$-i];
227 }
228 }
229 if ( i == 0 ) {
230 acc.length = 0;
231 this.popFront;
232 return l;
233 }
234 }
235 eos = true; // end of stream
236 acc.length = 0;
237 return l;
238 }
239 void scanUntil(F)(string needle, F f) {
240 auto d = needle.representation;
241 ubyte[] acc;
242 bool eos; // end of stream
243
244 while( !eos ) {
245 auto l = scanUntilR(needle, acc, eos);
246 debug(httpd) tracef("scanr returned <%s> and <%s>", cast(string)l, cast(string)acc);
247 f(l);
248 if ( acc == needle) {
249 return;
250 }
251 }
252 }
253 void skipUntil(string needle) {
254 auto d = needle.representation;
255 ubyte[] acc;
256 bool eos; // end of stream
257
258 while( !eos ) {
259 auto l = scanUntilR(needle, acc, eos);
260 debug(httpd) tracef("scanr returned <%s> and <%s>", cast(string)l, cast(string)acc);
261 if ( acc == needle) {
262 return;
263 }
264 }
265 }
266 }
267
268 auto createDataSource(string partialBody, NetworkStream stream) {
269
270 if ( !requestHasBody ) {
271 return new _DataSource();
272 }
273
274 auto ds = new _DataSource();
275
276 ds._requestHasBody = true;
277 ds._requestBodyRecvInProgr = true;
278 ds._bodyDecoder = new DataPipe!ubyte;
279 ds._stream = stream;
280
281 if ( auto contentLengthHeader = "content-length" in _requestHeaders ) {
282 ds._contentLength = to!long(*contentLengthHeader);
283 }
284 else if ( auto contentTransferEncoding = "transfer-encoding" in _requestHeaders ) {
285 if ( *contentTransferEncoding=="chunked" ) {
286 ds._unChunker = new DecodeChunked();
287 ds._bodyDecoder.insert(ds._unChunker);
288 }
289 }
290 if ( partialBody.length ) {
291 ds._bodyDecoder.putNoCopy(cast(ubyte[])partialBody);
292 ds._receivedLength = (cast(ubyte[])partialBody).length;
293 }
294 while ( ds._bodyDecoder.empty ) {
295 auto b = new ubyte[DSBUFFSIZE];
296 auto read = stream.receive(b);
297 if ( read == 0 ) {
298 debug(httpd) trace("stream closed when receiving in datasource");
299 ds._requestBodyRecvInProgr = false;
300 return ds;
301 }
302 debug(httpd) tracef("place %d bytes to datasource", read);
303 ds._receivedLength += read;
304 ds._bodyDecoder.putNoCopy(b[0..read]);
305 }
306 ds._content = ds._bodyDecoder.getNoCopy().join();
307 if ( ( ds._contentLength > 0 && ds._receivedLength >= ds._contentLength )
308 || ( ds._unChunker && ds._unChunker.done) ) {
309 // all data received we need not wait any data from network
310 debug(httpd) trace("looks like we received complete request body together with request headers");
311 ds._requestBodyRecvInProgr = false;
312 ds._requestBodyReceived = true;
313 }
314 debug(httpd) tracef("initial content: %d bytes", ds._content.length);
315 return ds;
316 }
317 @property auto contentType() {
318 if ( auto ct = "content-type" in _requestHeaders ) {
319 auto f = (*ct).split(";").map!strip;
320 return f[0];
321 }
322 return null;
323 }
324
325 struct PartData {
326 // handler for each part data stream
327 _DataSource _ds;
328 string _boundary;
329 ubyte[] _content;
330 ubyte[] _acc;
331 bool _done;
332 bool _eos;
333
334 this(_DataSource ds, string boundary) {
335 _ds = ds;
336 _boundary = "\r\n" ~ boundary;
337 _content = _ds.scanUntilR(_boundary, _acc, _eos);
338 }
339 bool empty() {
340 return _content.length == 0;
341 }
342 auto front() {
343 return _content;
344 }
345 void popFront() {
346 _content.length = 0;
347 if ( _done ) {
348 return;
349 }
350 while( _content.length == 0 ) {
351 _content = _ds.scanUntilR(_boundary, _acc, _eos);
352 if ( _eos ) {
353 return;
354 }
355 if (_acc == _boundary) {
356 debug(httpd) tracef("part data done");
357 _ds.skipUntil("\r\n");
358 return;
359 }
360 }
361 }
362 }
363 struct Part {
364 _DataSource _ds;
365 string[string] _headers;
366 string _boundary;
367
368 this(_DataSource ds, string[string] h, string boundary) {
369 _ds = ds;
370 _headers = h;
371 _boundary = boundary;
372 }
373 @property string[string] headers() {
374 return _headers;
375 }
376 @property disposition() {
377 string[string] res;
378 auto d = "content-disposition" in _headers;
379 if ( !d ) {
380 return res;
381 }
382 (*d).split(";").
383 filter!"a.indexOf('=')>0".
384 map! "a.strip.split('=')".
385 each!(p => res[p[0]] = urlDecode(p[1]).strip('"'));
386 return res;
387 }
388 @property data() {
389 return PartData(_ds, _boundary);
390 }
391 }
392 struct MultiPart {
393 string _boundary;
394 _DataSource _ds;
395 Part _part;
396 /*
397 --8a60ded0-ee76-4b6a-a1a0-dccaf93b92e7
398 Content-Disposition: form-data; name=Field1;
399
400 form field from memory
401 --8a60ded0-ee76-4b6a-a1a0-dccaf93b92e7
402 Content-Disposition: form-data; name=Field2; filename=data2
403
404 file field from memory
405 --8a60ded0-ee76-4b6a-a1a0-dccaf93b92e7
406 Content-Disposition: form-data; name=File1; filename=file1
407 Content-Type: application/octet-stream
408
409 file1 content
410
411 --8a60ded0-ee76-4b6a-a1a0-dccaf93b92e7
412 Content-Disposition: form-data; name=File2; filename=file2
413 Content-Type: application/octet-stream
414
415 file2 content
416
417 --8a60ded0-ee76-4b6a-a1a0-dccaf93b92e7--
418 */
419 int opApply(int delegate(Part p) dg) {
420 int result = 0;
421 while(!_ds.empty) {
422 result = dg(_part);
423 if ( result ) {
424 break;
425 }
426 auto headers = skipHeaders();
427 _part = Part(_ds, headers, _boundary);
428 }
429 return result;
430 }
431 auto skipHeaders() {
432 ubyte[] buf;
433 string[string] headers;
434
435 debug(httpd) tracef("Search for headers");
436 _ds.scanUntil("\r\n\r\n", delegate void (ubyte[] data) {
437 buf ~= data;
438 });
439 foreach(h; buf.split('\n').map!"cast(string)a".map!strip.filter!"a.length") {
440 auto parsed = h.findSplit(":");
441 headers[parsed[0].toLower] = parsed[2].strip;
442 }
443 debug(httpd) tracef("Headers: %s ", headers);
444 return headers;
445 }
446 ///
447 /// Find boundary from request headers,
448 /// skip to begin of the first part,
449 /// create first part(read/parse headers, stop on the body begin)
450 ///
451 this(HTTPD_Request rq) {
452 ubyte[] buf, rest;
453 string separator;
454 auto ct = "content-type" in rq._requestHeaders;
455 auto b = (*ct).split(";").map!"a.strip.split(`=`)".filter!"a[0].toLower==`boundary`";
456 if ( b.empty ) {
457 throw new HTTPD_RequestException("Can't find 'boundary' in Content-Type %s".format(*ct));
458 }
459 _boundary = "--" ~ b.front[1];
460 _ds = rq._dataSource;
461 _ds.skipUntil(_boundary~"\r\n");
462 auto headers = skipHeaders();
463 _part = Part(_ds, headers, _boundary);
464 }
465 }
466
467 auto multiPartRead() {
468 return MultiPart(this);
469 }
470
471 auto read() {
472 if ( requestHasBody && _dataSource._requestBodyProcessed ) {
473 throw new HTTPD_RequestException("Request body already consumed by call to data/form/json");
474 }
475 if ( _dataSource._readStarted ) {
476 throw new HTTPD_RequestException("Request body read() already started.");
477 }
478 _dataSource._readStarted = true;
479 return _dataSource;
480 }
481
482 void loadBodyOnDemand(ref _DataSource ds) {
483 ds._requestBodyProcessed = true;
484 debug(httpd) tracef("Process %s onDemand", contentType);
485 switch ( contentType ) {
486 case "application/json":
487 while(!ds.empty) {
488 debug(httpd) tracef("add %d bytes to json from dataSource", ds.front.length);
489 _json ~= cast(string)ds.front;
490 ds.popFront;
491 }
492 break;
493 case "application/x-www-form-urlencoded":
494 string qBody;
495 while(!ds.empty) {
496 debug(httpd) tracef("add %d bytes to json from dataSource", ds.front.length);
497 qBody ~= cast(string)ds.front;
498 ds.popFront;
499 }
500 _form = parseQuery(qBody);
501 break;
502 case "multipart/form-data":
503 debug(httpd) tracef("loading multiPart on demand");
504 auto parts = multiPartRead();
505 foreach(p; parts) {
506 auto disposition = p.disposition;
507 auto data = p.data.joiner.array;
508
509 if ( !("name" in disposition) ) {
510 continue;
511 }
512 if ( auto fn = "filename" in disposition ) {
513 _files[disposition["name"]] = data;
514 } else {
515 _form[disposition["name"]] = cast(string)data;
516 }
517 }
518 break;
519 default:
520 while(!ds.empty) {
521 debug(httpd) tracef("add %d bytes to data from dataSource", ds.front.length);
522 _data ~= ds.front;
523 ds.popFront;
524 }
525 break;
526 }
527 }
528 }
529
530 string[int] codes;
531 shared static this() {
532 codes = [
533 200: "OK",
534 302: "Found",
535 401: "Unauthorized",
536 404: "Not found",
537 405: "Method not allowed",
538 500: "Server error"
539 ];
540 }
541 enum Compression : int {
542 no = 0,
543 gzip = 1,
544 deflate = 2,
545 yes = gzip|deflate,
546 };
547
548 auto response(C)(HTTPD_Request rq, C content, ushort code = 200)
549 if ( isSomeString!C
550 || (__traits(compiles, cast(ubyte[])content))
551 || (__traits(compiles, cast(ubyte[])content.front))
552 )
553 {
554 return new HTTPD_Response!C(rq, content, code);
555 }
556
557 class _Response {
558 abstract void send(NetworkStream);
559 abstract ref string[string] headers();
560 }
561
562 class HTTPD_Response(C) : _Response {
563 ushort _status = 200;
564 string _status_reason = "Unspecified";
565 string[string] _headers;
566 C _content;
567 Compression _compression = Compression.no;
568 HTTPD_Request _request;
569 Cookie[] _cookies;
570
571 mixin(Getter_Setter!ushort("status"));
572 mixin(Getter("compression"));
573 @property void compress(Compression c = Compression.yes) {
574 _compression = c;
575 }
576 this(ref HTTPD_Request request, C content, ushort status = 200) {
577 _status = status;
578 _request = request;
579 _content = content;
580 }
581 override ref string[string] headers() @property {
582 return _headers;
583 }
584 ref Cookie[] cookies() {
585 return _cookies;
586 }
587 void content(C)(C c) @property {
588 _content = makeContent(c);
589 }
590 auto selectCompression(in HTTPD_Request rq, in HTTPD_Response rs) {
591 if ( auto acceptEncodings = "accept-encoding" in rq.requestHeaders) {
592 auto heAccept = (*acceptEncodings).split(",").map!strip;
593 if ( (rs.compression & Compression.gzip) && heAccept.canFind("gzip")) {
594 return "gzip";
595 }
596 if ( (compression & Compression.deflate) && heAccept.canFind("deflate")) {
597 return "deflate";
598 }
599 }
600 return null;
601 }
602 void sendCookies(NetworkStream stream) {
603 if ( _cookies.length ) {
604 foreach(c; _cookies) {
605 auto setCookie = "Set-Cookie: %s=%s; Path=%s\r\n".format(c.attr, c.value, c.path);
606 stream.send(setCookie);
607 }
608 }
609 }
610 final override void send(NetworkStream stream) {
611 import std.zlib;
612 auto statusLine = "HTTP/1.1 " ~ to!string(_status) ~ " " ~ codes.get(_status, _status_reason) ~ " \r\n";
613
614 if ( !stream.isOpen || !stream.isConnected ) {
615 debug(httpd) tracef("Will not send to closed connection");
616 return;
617 }
618 debug(httpd) tracef("sending statusLine: %s", statusLine.stripRight);
619 stream.send(statusLine);
620
621 auto comp = selectCompression(_request, this);
622
623 static if ( isSomeString!C || __traits(compiles, cast(ubyte[])_content) ) {
624 ubyte[] data;
625 if ( comp ) {
626 _headers["content-encoding"] = comp;
627 Compress compressor;
628 final switch (comp) {
629 case "gzip": // gzip
630 compressor = new Compress(6, HeaderFormat.gzip);
631 break;
632 case "deflate": // deflate
633 compressor = new Compress(6, HeaderFormat.deflate);
634 break;
635 }
636 data = cast(ubyte[])compressor.compress(_content);
637 data ~= cast(ubyte[])compressor.flush();
638 }
639 else {
640 data = cast(ubyte[])_content;
641 }
642 _headers["content-length"] = to!string(data.length);
643 foreach(p; _headers.byKeyValue) {
644 stream.send(p.key ~ ": " ~ p.value ~ "\r\n");
645 }
646 if ( _cookies.length ) {
647 sendCookies(stream);
648 }
649 stream.send("\r\n");
650 if (_request.method == "HEAD") {
651 return;
652 }
653 stream.send(data);
654 }
655 else {
656 _headers["transfer-encoding"] = "chunked";
657 Compress compressor;
658 if ( comp !is null ) {
659 _headers["content-encoding"] = comp;
660 final switch (comp) {
661 case "gzip": // gzip
662 compressor = new Compress(6, HeaderFormat.gzip);
663 break;
664 case "deflate": // deflate
665 compressor = new Compress(6, HeaderFormat.deflate);
666 break;
667 }
668 }
669 foreach(p; _headers.byKeyValue) {
670 stream.send(p.key ~ ": " ~ p.value ~ "\r\n");
671 }
672 if ( _cookies.length ) {
673 sendCookies(stream);
674 }
675 stream.send("\r\n");
676 if (_request.method == "HEAD") {
677 return;
678 }
679 ubyte[] data;
680 while(!_content.empty) {
681 auto chunk = cast(ubyte[])_content.front;
682 _content.popFront;
683
684 if ( compressor ) {
685 data ~= cast(ubyte[])compressor.compress(chunk);
686 if ( data.length == 0 ) {
687 continue;
688 }
689 } else {
690 data = chunk;
691 }
692 stream.send("%x\r\n".format(data.length));
693 stream.send(data);
694 stream.send("\r\n");
695 data.length = 0;
696 }
697 if ( compressor ) {
698 data = cast(ubyte[])compressor.flush();
699 stream.send("%x\r\n".format(data.length));
700 stream.send(data);
701 stream.send("\r\n");
702 }
703 stream.send("0\r\n\r\n");
704 }
705 }
706 }
707
708 alias Handler = _Response delegate(in App app, ref HTTPD_Request, RequestArgs);
709
710 struct RequestArgs {
711 private {
712 Captures!string _captures = void;
713 string _string;
714 }
715 this(Captures!string c) @nogc @safe nothrow {
716 _captures = c;
717 }
718 this(string s) @nogc @safe pure nothrow {
719 _string = s;
720 }
721 bool empty() @nogc @safe pure nothrow {
722 return _captures.empty && _string is null;
723 }
724 string opIndex(string s) @safe pure {
725 return _captures[s];
726 }
727 string opIndex(size_t i) @safe pure {
728 if ( _string && i==0 ) {
729 return _string;
730 }
731 return _captures[i];
732 }
733 }
734
735 auto exactRoute(string s, Handler h) @safe pure nothrow {
736 return new ExactRoute(s, h);
737 }
738
739 auto regexRoute(string s, Handler h) @safe {
740 return new RegexRoute(s, h);
741 }
742
743 class Route {
744 Handler _handler;
745 string _origin;
746
747 abstract RequestArgs match(string) {
748 return RequestArgs();
749 };
750 final Handler handler() {
751 return _handler;
752 }
753 final string origin() {
754 return _origin;
755 }
756 }
757
758 class ExactRoute: Route {
759
760 this(string s, Handler h) @safe pure nothrow {
761 _origin = s;
762 _handler = h;
763 }
764 final override RequestArgs match(string input) {
765 if ( input == _origin ) {
766 debug(httpd) tracef("%s matches %s", input, _origin);
767 return RequestArgs(input);
768 }
769 return RequestArgs();
770 }
771 }
772 class RegexRoute: Route {
773 Regex!char _re;
774
775 this(string r, Handler h) @safe {
776 _origin = r;
777 _handler = h;
778 _re = regex(r);
779 }
780 final override RequestArgs match(string input) {
781 auto m = matchFirst(input, _re);
782 debug(httpd) if (!m.empty) {tracef("%s matches %s", input, _origin);}
783 return RequestArgs(m);
784 }
785 }
786
787 struct Router {
788 alias RouteMatch = Tuple!(Handler, "handler", RequestArgs, "args");
789 private Route[] _routes;
790
791 void addRoute(Route r) {
792 _routes ~= r;
793 }
794 auto getRoute(string path) {
795 RouteMatch match;
796 foreach(r; _routes) {
797 auto args = r.match(path);
798 if (!args.empty) {
799 match.handler = r.handler;
800 match.args = args;
801 break;
802 }
803 }
804 return match;
805 }
806 }
807
808 private auto parseQuery(string query) {
809 /// TODO
810 /// switch to return dict of
811 /// struct QueryParam {
812 /// private:
813 /// string name;
814 /// string[] value;
815 /// public:
816 /// uint length() {return value.length;}
817 /// string toString() {return value[0];}
818 /// string[] toArray() {return value;}
819 /// }
820 debug (httpd) tracef("query: %s", query);
821 string[string] q;
822 if ( !query ) {
823 return q;
824 }
825 if ( query[0] == '?') {
826 query = query[1..$];
827 }
828 string[][] parsed = query.splitter("&").
829 map!(s => s.split("=")).
830 filter!"a.length==2".
831 map!(p => [urlDecode(p[0]), urlDecode(p[1])]).
832 array;
833
834 auto grouped = sort!"a[0]<b[0]"(parsed).assumeSorted!"a[0]<b[0]".groupBy();
835 foreach(g; grouped) {
836 string key = g.front[0];
837 string val;
838 auto vals = g.map!"a[1]".array;
839 if (vals.length == 1) {
840 val = vals[0];
841 }
842 if (vals.length > 1) {
843 val = to!string(vals);
844 }
845 q[key] = val;
846 }
847 return q;
848 }
849
850 private bool headersReceived(in ubyte[] data, ref Buffer!ubyte buffer, out string separator) @safe {
851 foreach(s; ["\r\n\r\n", "\n\n"]) {
852 if ( data.canFind(s) || buffer.canFind(s) ) {
853 separator = s;
854 return true;
855 }
856 }
857 return false;
858 }
859
860 private void parseRequestHeaders(in App app, ref HTTPD_Request rq, string buffer) {
861 string lastHeader;
862 auto lines = buffer.splitLines.map!stripRight;
863 rq.requestLine = lines[0];
864 if ( lines.count == 1) {
865 return;
866 }
867 foreach(line; lines[1..$]) {
868 if ( !line.length ) {
869 continue;
870 }
871 if ( line[0] == ' ' || line[0] == '\t' ) {
872 // unfolding https://tools.ietf.org/html/rfc822#section-3.1
873 if ( auto prevValue = lastHeader in rq.requestHeaders) {
874 *prevValue ~= line;
875 }
876 continue;
877 }
878 auto parsed = line.findSplit(":");
879 auto header = parsed[0].toLower;
880 auto value = parsed[2].strip;
881 lastHeader = header;
882 if ( auto h = header in rq.requestHeaders ) {
883 *h ~= "; " ~ value;
884 } else {
885 rq.requestHeaders[header] = value;
886 }
887 debug(httpd) tracef("%s: %s", header, value);
888 }
889 auto rqlFields = rq.requestLine.split(" ");
890 debug (httpd) tracef("rqLine %s", rq.requestLine);
891 rq.method = rqlFields[0];
892 auto scheme = app.useSSL?
893 "https://":
894 "http://";
895 if ( "host" in rq.requestHeaders ) {
896 rq.uri = URI(scheme ~ rq.requestHeaders["host"] ~ rqlFields[1]);
897 } else {
898 rq.uri = URI(scheme ~ app.host ~ rqlFields[1]);
899 }
900 rq.path = rq.uri.path;
901 rq.query = parseQuery(rq.uri.query);
902 debug (httpd) tracef("path: %s", rq.path);
903 debug (httpd) tracef("query: %s", rq.query);
904 //
905 // now analyze what we have
906 //
907 auto header = "connection" in rq.requestHeaders;
908 if ( header && toLower(*header) == "keep-alive") {
909 rq.keepAlive = true;
910 }
911 auto cookies = "cookie" in rq.requestHeaders;
912 if ( cookies ) {
913 (*cookies).split(';').
914 map!"strip(a).split('=')".
915 filter!(kv => kv.length==2).
916 each!(kv => rq._cookies[kv[0]] = kv[1]);
917 }
918 }
919
920 private auto read_request(in App app, NetworkStream stream) {
921 HTTPD_Request rq;
922 Buffer!ubyte input;
923 string separator;
924
925 while( true ) {
926 ubyte[] b = new ubyte[app.bufferSize];
927 auto read = stream.receive(b);
928
929 if ( read == 0 ) {
930 return rq;
931 }
932 debug(httpd) tracef("received %d bytes", read);
933 input.putNoCopy(b[0..read]);
934
935 if ( headersReceived(b, input, separator) ) {
936 break;
937 }
938
939 if ( input.length >= app.maxHeadersSize ) {
940 throw new HTTPD_RequestException("Request headers length %d too large".format(input.length));
941 }
942 }
943 debug(httpd) trace("Headers received");
944 auto s = input.data!(string).findSplit(separator);
945 auto requestHeaders = s[0];
946 debug(httpd) tracef("Headers: %s", cast(string)requestHeaders);
947 parseRequestHeaders(app, rq, requestHeaders);
948 debug(httpd) trace("Headers parsed");
949
950 rq._dataSource = rq.createDataSource(s[2], stream);
951
952 return rq;
953 }
954
955 void processor(in App app, HTTPD httpd, NetworkStream stream) {
956 stream.readTimeout = app.timeout;
957 HTTPD_Request rq;
958 _Response rs;
959 scope (exit) {
960 if ( stream.isOpen ) {
961 stream.close();
962 }
963 }
964 uint rqLimit = max(app.rqLimit, 1);
965 try {
966 while ( rqLimit > 0 ) {
967 rq = read_request(app, stream);
968 if ( !httpd._running || !rq.requestLine.length ) {
969 return;
970 }
971 auto match = httpd._router.getRoute(rq.path);
972 if ( !match.handler ) {
973 // return 404;
974 debug (httpd) tracef("Route not found for %s", rq.path);
975 rs = response(rq, "Requested path %s not found".format(rq.path), 404);
976 break;
977 }
978 auto handler = match.handler;
979 rs = handler(app, rq, match.args);
980 if ( !stream.isOpen ) {
981 debug(httpd) tracef("Request handler closed connection");
982 return;
983 }
984 if ( rq.keepAlive && rqLimit > 1 ) {
985 rs.headers["Connection"] = "Keep-Alive";
986 }
987 if ( rq._dataSource._requestHasBody && !rq._dataSource._requestBodyReceived ) {
988 // for some reason some part of the request body still not received, and it will
989 // stay on the way of next request if this is keep-Alive session,
990 // so we must abort this connection anyway.
991 debug(httpd) trace("Request handler did not consumed whole request body. We have to close connection after sending response.");
992 rs.send(stream);
993 return;
994 }
995 rs.send(stream);
996 --rqLimit;
997 if ( !rq.keepAlive || rqLimit==0 ) {
998 debug(httpd) trace("Finished with that connection");
999 return;
1000 }
1001 debug(httpd) trace("Continue with keepalive request");
1002 rq = rq.init;
1003 }
1004 }
1005 catch (HTTPD_RequestException e) {
1006 debug(httpd) error("Request exception: " ~ e.msg);
1007 rs = response(rq, "Request exception:\n" ~ e.msg, 500);
1008 }
1009 catch (TimeoutException e) {
1010 debug(httpd) {
1011 if ( rq.requestLine ) {
1012 error("Timeout reading/writing to client");
1013 }
1014 }
1015 }
1016 catch (Exception e) {
1017 debug(httpd) error("Unexpected Exception " ~ e.msg);
1018 rs = response(rq, "Unexpected exception:\n" ~ e.msg, 500);
1019 }
1020 catch (Error e) {
1021 error(e.msg, e.info);
1022 rs = response(rq, "Unexpected error:\n" ~ e.msg, 500);
1023 }
1024 try {
1025 if ( stream.isOpen ) {
1026 rs.send(stream);
1027 }
1028 }
1029 catch (Exception e) {
1030 infof("Exception when send %s", e.msg);
1031 }
1032 catch (Error e) {
1033 error("Error sending response: " ~ e.msg);
1034 }
1035 }
1036
1037 class HTTPD
1038 {
1039 private {
1040 TaskPool _server;
1041 __gshared bool _running;
1042 Router _router;
1043 App _app;
1044 }
1045 auto ref addRoute(Route r) {
1046 _router.addRoute(r);
1047 return this;
1048 }
1049 static NetworkStream openStream(in App app) {
1050 auto host = app.host;
1051 auto port = app.port;
1052 Address[] addresses;
1053 SSLOptions _sslOptions;
1054
1055 try {
1056 addresses = getAddress(host, port);
1057 } catch (Exception e) {
1058 throw new ConnectError("Can't resolve name when connect to %s:%d: %s".format(host, port, e.msg));
1059 }
1060 auto tcpStream = app.useSSL?
1061 new SSLStream(_sslOptions):
1062 new TCPStream();
1063 tcpStream.open(addresses[0].addressFamily);
1064 return tcpStream;
1065 }
1066 static void run(in App app, HTTPD httpd) {
1067 Address[] addresses;
1068 try {
1069 addresses = getAddress(app.host, app.port);
1070 } catch (Exception e) {
1071 throw new ConnectError("Can't resolve name when connect to %s:%d: %s".format(app.host, app.port, e.msg));
1072 }
1073 auto tcpStream = openStream(app);
1074 tcpStream.reuseAddr(true);
1075 tcpStream.bind(addresses[0]);
1076 tcpStream.listen(128);
1077 defaultPoolThreads(64);
1078 auto pool = taskPool();
1079 _running = true;
1080 while ( _running ) {
1081 auto stream = tcpStream.accept();
1082 if ( _running ) {
1083 auto connHandler = task!processor(app, httpd, stream);
1084 pool.put(connHandler);
1085 } else {
1086 tcpStream.close();
1087 break;
1088 }
1089 }
1090 }
1091 void app(App a) {
1092 _app = a;
1093 }
1094 void start() {
1095 defaultPoolThreads(64);
1096 _server = taskPool();
1097 auto t = task!run(_app, this);
1098 _server.put(t);
1099 Thread.sleep(500.msecs);
1100 }
1101 void start(App app) {
1102 defaultPoolThreads(64);
1103 _app = app;
1104 _server = taskPool();
1105 auto t = task!run(_app, this);
1106 _server.put(t);
1107 Thread.sleep(500.msecs);
1108 }
1109 void stop() {
1110 if ( !_running ) {
1111 return;
1112 }
1113 _running = false;
1114 try {
1115 auto s = openStream(_app);
1116 s.connect(_app.host, _app.port);
1117 } catch (Exception e) {
1118 }
1119 // _server.stop();
1120 }
1121 }
1122
1123 struct App {
1124 private {
1125 string _name;
1126 string _host;
1127 ushort _port;
1128 Duration _timeout = 30.seconds;
1129 size_t _bufferSize = 16*1024;
1130 size_t _maxHeadersSize = 32*1024;
1131 bool _useSSL = false;
1132 uint _rqLimit = 10; // keepalive requestst per connection
1133 Router _router;
1134 }
1135 mixin(Getter_Setter!string("name"));
1136 mixin(Getter_Setter!string("host"));
1137 mixin(Getter_Setter!ushort("port"));
1138 mixin(Getter_Setter!size_t("bufferSize"));
1139 mixin(Getter_Setter!size_t("maxHeadersSize"));
1140 mixin(Getter_Setter!Duration("timeout"));
1141 mixin(Getter_Setter!bool("useSSL"));
1142 mixin(Getter_Setter!uint("rqLimit"));
1143 this(string name) {
1144 _name = name;
1145 }
1146 }
1147
1148
1149 version(none) private unittest {
1150 import std.json;
1151 import std.conv;
1152 import requests.http: HTTPRequest, TimeoutException, BasicAuthentication, queryParams, MultipartForm, formData;
1153 globalLogLevel(LogLevel.info);
1154
1155 static auto buildReply(ref HTTPD_Request rq) {
1156 auto args = JSONValue(rq.query);
1157 auto headers = JSONValue(rq.requestHeaders);
1158 auto url = JSONValue(rq.uri.uri);
1159 auto json = JSONValue(rq.json);
1160 auto data = JSONValue(rq.data);
1161 auto form = JSONValue(rq.form);
1162 auto files = JSONValue(rq.files);
1163 auto reply = JSONValue(["args":args, "headers": headers, "json": json, "url": url, "data": data, "form": form, "files": files]);
1164 return reply.toString();
1165 }
1166
1167 Router router;
1168 router.addRoute(exactRoute(r"/get", null));
1169 router.addRoute(regexRoute(r"/get/(?P<param>\d+)", null));
1170 auto r = router.getRoute(r"/get");
1171 assert(!r.args.empty);
1172 r = router.getRoute(r"/post");
1173 assert(r.args.empty);
1174
1175 r = router.getRoute(r"/get/333");
1176 assert(!r.args.empty);
1177 assert(r.args["param"]=="333");
1178 r = router.getRoute(r"/get/aaa");
1179 assert(r.args.empty);
1180
1181 HTTPD_Request rq;
1182 string headers = "GET /get?a=b&list[]=1&c=d&list[]=2 HTTP/1.1\n" ~
1183 "Host: host\n" ~
1184 "X-Test: test1\n" ~
1185 " test2\n" ~
1186 "Content-Length: 1\n";
1187 parseRequestHeaders(App(), rq, headers);
1188 assert(rq.requestHeaders["x-test"] == "test1 test2");
1189 assert(rq.requestHeaders["host"] == "host");
1190 assert(rq.path == "/get");
1191 assert(rq.query["a"] == "b");
1192 assert(rq.query["c"] == "d");
1193 assert(rq.query["list[]"] == `["1", "2"]`);
1194 auto root(in App app, ref HTTPD_Request rq, RequestArgs args) {
1195 debug (httpd) trace("handler / called");
1196 auto rs = response(rq, buildReply(rq));
1197 rs.headers["Content-Type"] = "application/json";
1198 return rs;
1199 }
1200 auto get(in App app, ref HTTPD_Request rq, RequestArgs args) {
1201 debug (httpd) trace("handler /get called");
1202 auto rs = response(rq, buildReply(rq));
1203 rs.headers["Content-Type"] = "application/json";
1204 return rs;
1205 }
1206 auto basicAuth(in App app, ref HTTPD_Request rq, RequestArgs args) {
1207 import std.base64;
1208 auto user = args["user"];
1209 auto password= args["password"];
1210 auto auth = cast(string)Base64.decode(rq.requestHeaders["authorization"].split()[1]);
1211 auto up = auth.split(":");
1212 short status;
1213 if ( up[0]==user && up[1]==password) {
1214 status = 200;
1215 } else {
1216 status = 401;
1217 }
1218 auto rs = response(rq, buildReply(rq), status);
1219 rs.headers["Content-Type"] = "application/json";
1220 return rs;
1221 }
1222 auto rredir(in App app, ref HTTPD_Request rq, RequestArgs args) {
1223 auto rs = response(rq, buildReply(rq));
1224 auto redirects = to!long(args["redirects"]);
1225 if ( redirects > 1 ) {
1226 rs.headers["Location"] = "/relative-redirect/%d".format(redirects-1);
1227 } else {
1228 rs.headers["Location"] = "/get";
1229 }
1230 rs.status = 302;
1231 return rs;
1232 }
1233 auto aredir(in App app, ref HTTPD_Request rq, RequestArgs args) {
1234 auto rs = response(rq, buildReply(rq));
1235 auto redirects = to!long(args["redirects"]);
1236 if ( redirects > 1 ) {
1237 rs.headers["Location"] = "http://127.0.0.1:8081/absolute-redirect/%d".format(redirects-1);
1238 } else {
1239 rs.headers["Location"] = "http://127.0.0.1:8081/get";
1240 }
1241 rs.status = 302;
1242 return rs;
1243 }
1244 auto delay(in App app, ref HTTPD_Request rq, RequestArgs args) {
1245 auto delay = dur!"seconds"(to!long(args["delay"]));
1246 Thread.sleep(delay);
1247 auto rs = response(rq, buildReply(rq));
1248 rs.headers["Content-Type"] = "application/json";
1249 return rs;
1250 }
1251 auto gzip(in App app, ref HTTPD_Request rq, RequestArgs args) {
1252 auto rs = response(rq, buildReply(rq));
1253 rs.compress(Compression.gzip);
1254 rs.headers["Content-Type"] = "application/json";
1255 return rs;
1256 }
1257 auto deflate(in App app, ref HTTPD_Request rq, RequestArgs args) {
1258 auto rs = response(rq, buildReply(rq));
1259 rs.compress(Compression.deflate);
1260 return rs;
1261 }
1262 auto range(in App app, ref HTTPD_Request rq, RequestArgs args) {
1263 auto size = to!size_t(args["size"]);
1264 auto rs = response(rq, new ubyte[size].chunks(16));
1265 rs.compress(Compression.yes);
1266 return rs;
1267 }
1268 auto head(in App app, ref HTTPD_Request rq, RequestArgs args) {
1269 if ( rq.method != "HEAD") {
1270 auto rs = response(rq, "Illegal method %s".format(rq.method), 405);
1271 return rs;
1272 }
1273 else {
1274 auto rs = response(rq, buildReply(rq));
1275 rs.compress(Compression.yes);
1276 return rs;
1277 }
1278 }
1279 auto del(in App app, ref HTTPD_Request rq, RequestArgs args) {
1280 if ( rq.method != "DELETE") {
1281 auto rs = response(rq, "Illegal method %s".format(rq.method), 405);
1282 return rs;
1283 }
1284 else {
1285 auto rs = response(rq, buildReply(rq));
1286 return rs;
1287 }
1288 }
1289 auto post(in App app, ref HTTPD_Request rq, RequestArgs args) {
1290 auto rs = response(rq, buildReply(rq));
1291 return rs;
1292 }
1293 auto postIter(in App app, ref HTTPD_Request rq, RequestArgs args) {
1294 int c;
1295
1296 if ( rq.contentType == "multipart/form-data" ) {
1297 auto parts = rq.multiPartRead();
1298 foreach(p; parts) {
1299 auto disposition = p.disposition;
1300 c += p.data.joiner.count;
1301 }
1302 auto rs = response(rq, "%d".format(c));
1303 return rs;
1304 }
1305 else {
1306 auto r = rq.read();
1307 while ( !r.empty ) {
1308 c += r.front.length;
1309 r.popFront;
1310 }
1311 auto rs = response(rq, "%d".format(c));
1312 return rs;
1313 }
1314 }
1315 auto read(in App app, ref HTTPD_Request rq, RequestArgs args) {
1316 auto r = rq.read();
1317 int c;
1318 while ( !r.empty ) {
1319 c += r.front.length;
1320 r.popFront;
1321 }
1322 auto rs = response(rq, "%d".format(c));
1323 return rs;
1324 }
1325 auto readf1(in App app, ref HTTPD_Request rq, RequestArgs args) {
1326 // now call to read must throw exception
1327 auto r = rq.read();
1328 int c;
1329 while ( !r.empty ) {
1330 c += r.front.length;
1331 r.popFront;
1332 break;
1333 }
1334 auto rs = response(rq, "%d".format(c));
1335 return rs;
1336 }
1337 auto cookiesSet(in App app, ref HTTPD_Request rq, RequestArgs args) {
1338 Cookie[] cookies;
1339 foreach(p; rq.query.byKeyValue) {
1340 cookies ~= Cookie("/cookies", rq.requestHeaders["host"], p.key, p.value);
1341 }
1342 auto rs = response(rq, buildReply(rq), 302);
1343 rs.headers["Location"] = "/cookies";
1344 rs.cookies = cookies;
1345 return rs;
1346 }
1347 auto cookies(in App app, ref HTTPD_Request rq, RequestArgs args) {
1348 auto cookies = ["cookies": JSONValue(rq.cookies)];
1349 auto rs = response(rq, JSONValue(cookies).toString);
1350 return rs;
1351 }
1352
1353 auto httpbin = App("httpbin");
1354
1355 httpbin.port = 8081;
1356 httpbin.host = "127.0.0.1";
1357
1358 httpbin.timeout = 10.seconds;
1359 HTTPD server = new HTTPD();
1360
1361 server.addRoute(exactRoute(r"/", &root)).
1362 addRoute(exactRoute(r"/get", &get)).
1363 addRoute(regexRoute(r"/delay/(?P<delay>\d+)", &delay)).
1364 addRoute(regexRoute(r"/relative-redirect/(?P<redirects>\d+)", &rredir)).
1365 addRoute(regexRoute(r"/absolute-redirect/(?P<redirects>\d+)", &aredir)).
1366 addRoute(regexRoute(r"/basic-auth/(?P<user>[^/]+)/(?P<password>[^/]+)", &basicAuth)).
1367 addRoute(exactRoute(r"/gzip", &gzip)).
1368 addRoute(exactRoute(r"/deflate", &deflate)).
1369 addRoute(regexRoute(r"/range/(?P<size>\d+)", &range)).
1370 addRoute(exactRoute(r"/cookies/set", &cookiesSet)).
1371 addRoute(exactRoute(r"/cookies", &cookies)).
1372 addRoute(exactRoute(r"/head", &head)).
1373 addRoute(exactRoute(r"/delete", &del)).
1374 addRoute(exactRoute(r"/read", &read)).
1375 addRoute(exactRoute(r"/readf1", &readf1)).
1376 addRoute(exactRoute(r"/post", &post)).
1377 addRoute(exactRoute(r"/postIter", &postIter));
1378
1379 server.start(httpbin);
1380 scope(exit) {
1381 server.stop();
1382 }
1383 auto request = HTTPRequest();
1384
1385 globalLogLevel(LogLevel.info);
1386 auto httpbin_url = "http://%s:%d/".format(httpbin.host, httpbin.port);
1387 request.timeout = 5.seconds;
1388 request.keepAlive = true;
1389 info("httpd Check GET");
1390 auto rs = request.get(httpbin_url);
1391 assert(rs.code == 200);
1392 assert(rs.responseBody.length > 0);
1393 auto content = rs.responseBody.data!string;
1394 auto json = parseJSON(cast(string)content);
1395 assert(json.object["url"].str == httpbin_url);
1396
1397 info("httpd Check GET with parameters");
1398 rs = request.get(httpbin_url ~ "get", ["c":" d", "a":"b"]);
1399 assert(rs.code == 200);
1400 json = parseJSON(cast(string)rs.responseBody.data).object["args"].object;
1401 assert(json["a"].str == "b");
1402 assert(json["c"].str == " d");
1403
1404 info("httpd Check relative redirect");
1405 rs = request.get(httpbin_url ~ "relative-redirect/2");
1406 assert(rs.history.length == 2);
1407 assert(rs.code==200);
1408
1409 info("httpd Check absolute redirect");
1410 rs = request.get(httpbin_url ~ "absolute-redirect/2");
1411 assert(rs.history.length == 2);
1412 assert(rs.code==200);
1413
1414 info("httpd Check basic auth");
1415 request.authenticator = new BasicAuthentication("user", "password");
1416 rs = request.get(httpbin_url ~ "basic-auth/user/password");
1417 assert(rs.code==200);
1418 request.authenticator = null;
1419
1420 info("httpd Check timeout");
1421 request.timeout = 1.seconds;
1422 assertThrown!TimeoutException(request.get(httpbin_url ~ "delay/2"));
1423 Thread.sleep(1.seconds);
1424 request.timeout = 30.seconds;
1425
1426 info("httpd Check gzip");
1427 rs = request.get(httpbin_url ~ "gzip");
1428 assert(rs.code==200);
1429 json = parseJSON(cast(string)rs.responseBody);
1430 assert(json.object["url"].str == httpbin_url ~ "gzip");
1431
1432 info("httpd Check deflate");
1433 rs = request.get(httpbin_url ~ "deflate");
1434 assert(rs.code==200);
1435 json = parseJSON(cast(string)rs.responseBody);
1436 assert(json.object["url"].str == httpbin_url ~ "deflate");
1437
1438 info("httpd Check range");
1439 rs = request.get(httpbin_url ~ "range/1023");
1440 assert(rs.code==200);
1441 assert(rs.responseBody.length == 1023);
1442
1443 info("httpd Check HEAD");
1444 rs = request.exec!"HEAD"(httpbin_url ~ "head");
1445 assert(rs.code==200);
1446 assert(rs.responseBody.length == 0);
1447
1448 info("httpd Check DELETE");
1449 rs = request.exec!"DELETE"(httpbin_url ~ "delete");
1450 assert(rs.code==200);
1451
1452 info("httpd Check POST json");
1453 rs = request.post(httpbin_url ~ "post?b=x", `{"a":"b", "c":[1,2,3]}`, "application/json");
1454 json = parseJSON(cast(string)rs.responseBody);
1455 auto rqJson = parseJSON(json.object["json"].str);
1456 assert(rqJson.object["a"].str == "b");
1457 assert(equal([1,2,3], rqJson.object["c"].array.map!"a.integer"));
1458
1459 info("httpd Check POST json/chunked body");
1460 rs = request.post(httpbin_url ~ "post?b=x", [`{"a":"b",`,` "c":[1,2,3]}`], "application/json");
1461 json = parseJSON(cast(string)rs.responseBody);
1462 assert(json.object["args"].object["b"].str == "x");
1463 rqJson = parseJSON(json.object["json"].str);
1464 assert(rqJson.object["a"].str == "b");
1465 assert(equal([1,2,3], rqJson.object["c"].array.map!"a.integer"));
1466
1467 rs = request.post(httpbin_url ~ "post", "0123456789".repeat(32));
1468 json = parseJSON(cast(string)rs.responseBody);
1469 assert(equal(json.object["data"].array.map!"a.integer", "0123456789".repeat(32).join));
1470
1471 info("httpd Check POST with params");
1472 rs = request.post(httpbin_url ~ "post", queryParams("b", 2, "a", "A"));
1473 assert(rs.code==200);
1474 auto data = parseJSON(cast(string)rs.responseBody).object["form"].object;
1475 assert((data["a"].str == "A"));
1476 assert((data["b"].str == "2"));
1477
1478 // this is tests for httpd read() interface
1479 info("httpd Check POST/iterating over body");
1480 rs = request.post(httpbin_url ~ "read", "0123456789".repeat(1500));
1481 assert(equal(rs.responseBody, "15000"));
1482
1483 {
1484 request.keepAlive = true;
1485 // this is test on how we can handle keepalive session when previous request leave unread data in socket
1486 try {
1487 rs = request.post(httpbin_url ~ "readf1", "0123456789".repeat(1500));
1488 }
1489 catch (Exception e) {
1490 // this can fail as httpd will close connection prematurely
1491 }
1492 // but next idempotent request must succeed
1493 rs = request.get(httpbin_url ~ "get");
1494 assert(rs.code == 200);
1495 }
1496 //
1497 {
1498 info("httpd Check POST/multipart form");
1499 import std.file;
1500 import std.path;
1501 auto tmpd = tempDir();
1502 auto tmpfname1 = tmpd ~ dirSeparator ~ "request_test1.txt";
1503 auto f = File(tmpfname1, "wb");
1504 f.rawWrite("file1 content\n");
1505 f.close();
1506 auto tmpfname2 = tmpd ~ dirSeparator ~ "request_test2.txt";
1507 f = File(tmpfname2, "wb");
1508 f.rawWrite("file2 content\n");
1509 f.close();
1510 ///
1511 /// Ok, files ready.
1512 /// Now we will prepare Form data
1513 ///
1514 File f1 = File(tmpfname1, "rb");
1515 File f2 = File(tmpfname2, "rb");
1516 scope(exit) {
1517 f1.close();
1518 f2.close();
1519 }
1520 ///
1521 /// for each part we have to set field name, source (ubyte array or opened file) and optional filename and content-type
1522 ///
1523 MultipartForm form = MultipartForm().
1524 add(formData("Field1", cast(ubyte[])"form field from memory")).
1525 add(formData("Field2", cast(ubyte[])"file field from memory", ["filename":"data2"])).
1526 add(formData("Field3", cast(ubyte[])`{"a":"b"}`, ["Content-Type": "application/json"])).
1527 add(formData("File1", f1, ["filename":"file1", "Content-Type": "application/octet-stream"])).
1528 add(formData("File2", f2, ["filename":"file2", "Content-Type": "application/octet-stream"]));
1529 /// everything ready, send request
1530 rs = request.post(httpbin_url ~ "post?a=b", form);
1531 /* expected:
1532 {
1533 "args": {
1534 "a": "b"
1535 },
1536 "data": "",
1537 "files": {
1538 "Field2": "file field from memory",
1539 "File1": "file1 content\n",
1540 "File2": "file2 content\n"
1541 },
1542 "form": {
1543 "Field1": "form field from memory",
1544 "Field3": "{\"a\":\"b\"}"
1545 },
1546 "headers": {
1547 "Accept-Encoding": "gzip, deflate",
1548 "Content-Length": "730",
1549 "Content-Type": "multipart/form-data; boundary=d79a383e-7912-4d36-a6db-a6774bf37133",
1550 "Host": "httpbin.org",
1551 "User-Agent": "dlang-requests"
1552 },
1553 "json": null,
1554 "origin": "xxx.xxx.xxx.xxx",
1555 "url": "http://httpbin.org/post?a=b"
1556 }
1557 */
1558 json = parseJSON(cast(string)rs.responseBody);
1559 assert("file field from memory" == cast(string)(json.object["files"].object["Field2"].array.map!(a => cast(ubyte)a.integer).array));
1560 assert("file1 content\n" == cast(string)(json.object["files"].object["File1"].array.map!(a => cast(ubyte)a.integer).array));
1561
1562 info("httpd Check POST/iterate over multipart form");
1563 form = MultipartForm().
1564 add(formData("Field1", cast(ubyte[])"form field from memory")).
1565 add(formData("Field2", cast(ubyte[])"file field from memory", ["filename":"data2"])).
1566 add(formData("Field3", cast(ubyte[])`{"a":"b"}`, ["Content-Type": "application/json"]));
1567 /// everything ready, send request
1568 rs = request.post(httpbin_url ~ "postIter?a=b", form);
1569 assert(equal(rs.responseBody, "53"));
1570 rs = request.post(httpbin_url ~ "postIter", "0123456789".repeat(1500));
1571 assert(equal(rs.responseBody, "15000"));
1572 }
1573 info("httpd Check cookies");
1574 rs = request.get(httpbin_url ~ "cookies/set?A=abcd&b=cdef");
1575 json = parseJSON(cast(string)rs.responseBody.data).object["cookies"].object;
1576 assert(json["A"].str == "abcd");
1577 assert(json["b"].str == "cdef");
1578 }
1579 }