1 module requests.server.httpd; 2 3 import std.algorithm; 4 import std.array; 5 import std.conv; 6 import std.datetime; 7 import std.exception; 8 import std.experimental.logger; 9 import std.format; 10 import std.parallelism; 11 import std.range; 12 import std.regex; 13 import std.socket; 14 import std.stdio; 15 import std.string; 16 import std.traits; 17 import std.typecons; 18 import core.thread; 19 import requests.utils; 20 import requests.streams; 21 import requests.uri; 22 23 version(vibeD){ 24 pragma(msg, "httpd will not compile with vibeD"); 25 } 26 else { 27 /* 28 ** This is small http server to run something like httpbin(http://httpbin.org) internally 29 ** for Requests unittest's. 30 */ 31 32 enum DSBUFFSIZE = 16*1024; 33 34 class HTTPD_RequestException: Exception { 35 this(string message, string file =__FILE__, size_t line = __LINE__, Throwable next = null) @safe pure nothrow { 36 super(message, file, line, next); 37 } 38 } 39 40 struct HTTPD_Request { 41 private { 42 string _requestLine; 43 string[string] _requestHeaders; 44 Buffer!ubyte _requestBody; 45 bool _keepAlive; 46 URI _uri; 47 string[string] _query; // query in url 48 string _method; 49 string _path; 50 string _json; // json for application/json 51 string[string] _form; // form values for application/x-www-form-urlencoded 52 ubyte[][string] _files; 53 ubyte[] _data; // raw data for unrecognized mime's 54 _DataSource _dataSource; 55 string[string] _cookies; 56 } 57 private mixin(Setter!(string[string])("requestHeaders")); 58 auto ref requestHeaders() inout @property @safe @nogc nothrow { 59 return _requestHeaders; 60 } 61 auto ref cookies() inout @property @safe @nogc nothrow { 62 return _cookies; 63 } 64 private mixin(Setter!(string[string])("query")); 65 auto ref query() inout @property @safe @nogc nothrow { 66 return _query; 67 } 68 auto ref requestBody() inout @property @safe @nogc nothrow { 69 return _requestBody; 70 } 71 private mixin(Setter!string("method")); 72 mixin(Getter("method")); 73 private mixin(Setter!string("requestLine")); 74 mixin(Getter("requestLine")); 75 private mixin(Setter!string("path")); 76 mixin(Getter("path")); 77 private mixin(Setter!bool("keepAlive")); 78 mixin(Getter("keepAlive")); 79 private mixin(Setter!URI("uri")); 80 mixin(Getter("uri")); 81 82 @property string json() { 83 if ( _dataSource._readStarted ) { 84 throw new HTTPD_RequestException("Request read() call already started."); 85 } 86 if ( _dataSource._requestHasBody && !_dataSource._requestBodyProcessed ) { 87 debug(httpd) trace("receiving body on demand for json"); 88 loadBodyOnDemand(_dataSource); 89 } 90 return _json; 91 } 92 @property ubyte[] data() { 93 if ( _dataSource._readStarted ) { 94 throw new HTTPD_RequestException("Request body read() already started."); 95 } 96 if ( _dataSource._requestHasBody && !_dataSource._requestBodyProcessed ) { 97 debug(httpd) trace("receiving body on demand for data"); 98 loadBodyOnDemand(_dataSource); 99 } 100 return _data; 101 } 102 @property string[string] form() { 103 if ( _dataSource._readStarted ) { 104 throw new HTTPD_RequestException("Request body read() already started."); 105 } 106 if ( _dataSource._requestHasBody && !_dataSource._requestBodyProcessed ) { 107 debug(httpd) trace("receiving body on demand for form"); 108 loadBodyOnDemand(_dataSource); 109 } 110 return _form; 111 } 112 @property auto files() { 113 if ( _dataSource._readStarted ) { 114 throw new HTTPD_RequestException("Request body read() already started."); 115 } 116 if ( _dataSource._requestHasBody && !_dataSource._requestBodyProcessed ) { 117 debug(httpd) trace("receiving body on demand for form"); 118 loadBodyOnDemand(_dataSource); 119 } 120 return _files; 121 } 122 123 @property bool requestHasBody() pure { 124 if ( "content-length" in _requestHeaders ) { 125 return true; 126 } 127 if ( auto contentTransferEncoding = "transfer-encoding" in _requestHeaders ) { 128 if ( *contentTransferEncoding=="chunked" ) { 129 return true; 130 } 131 } 132 return false; 133 } 134 135 class _DataSource { 136 private { 137 NetworkStream _stream; 138 DataPipe!ubyte _bodyDecoder; 139 DecodeChunked _unChunker; 140 long _contentLength = 0; 141 long _receivedLength = 0; 142 ubyte[] _content; 143 bool _requestHasBody; // request has body 144 bool _requestBodyRecvInProgr; // loading body currently active 145 bool _requestBodyProcessed; // we processed body - this can happens only once 146 bool _requestBodyReceived; // all body data were received from network (we have to close socket if request terminated before all data received) 147 bool _readStarted; 148 } 149 bool empty() { 150 debug(httpd) tracef("datasource empty: %s", _content.length==0); 151 return _content.length==0; 152 } 153 ubyte[] front() { 154 return _content; 155 } 156 void popFront() { 157 debug(httpd) trace("datasource enters popFront"); 158 _content.length = 0; 159 if ( !_requestBodyRecvInProgr ) { 160 debug(httpd) trace("popFront called when dataSource is not active anymore"); 161 return; 162 } 163 while ( _bodyDecoder.empty && _stream && _stream.isOpen ) { 164 auto b = new ubyte[DSBUFFSIZE]; 165 auto read = _stream.receive(b); 166 if ( read == 0 ) { 167 debug(httpd) trace("stream closed when receiving in datasource"); 168 _bodyDecoder.flush(); 169 _requestBodyRecvInProgr = false; 170 break; 171 } 172 debug(httpd) tracef("place %d bytes to datasource", read); 173 _receivedLength += read; 174 _bodyDecoder.putNoCopy(b[0..read]); 175 if ( (_unChunker && _unChunker.done) 176 || (_contentLength > 0 && _receivedLength >= _contentLength) ) 177 { 178 debug(httpd) trace("request body reading complete (due contentLength or due last chunk consumed)"); 179 _bodyDecoder.flush(); 180 _requestBodyRecvInProgr = false; 181 _requestBodyReceived = true; 182 break; 183 } 184 } 185 _content = _bodyDecoder.getNoCopy().join(); 186 debug(httpd) tracef("%d bytes in content after popFront", _content.length); 187 } 188 /// 189 /// raplace current front with another value 190 /// 191 void unPop(ubyte[] data) { 192 assert(data.length > 0); 193 _content = data; 194 } 195 /// 196 /// Scan over input stream, 197 /// can return data from stream 198 /// acc - accumulator for receiving needle 199 /// return empty data if we receiving needle 200 /// if needle found in stream, then acc == needle 201 /// if end of stream happened, then eos = true 202 /// 203 ubyte[] scanUntilR(string needle, ref ubyte[] acc, out bool eos) { 204 auto d = needle.representation; 205 ubyte[] l; 206 207 while (!this.empty) { 208 auto c = this.front; 209 debug(httpd) tracef("on scan: %s", cast(string)c); 210 l = acc ~ c; 211 auto s = l.findSplit(d); 212 if ( s[1].length ) { 213 if ( s[2].length ) { 214 this.unPop(s[2]); 215 } else { 216 this.popFront; 217 } 218 acc = s[1]; 219 return s[0]; 220 } 221 auto i = min(l.length, d.length); 222 for(;i>0; i--) { 223 if ( l.endsWith(d[0..i]) ) { 224 acc = l[$-i..$]; 225 this.popFront; 226 return l[0..$-i]; 227 } 228 } 229 if ( i == 0 ) { 230 acc.length = 0; 231 this.popFront; 232 return l; 233 } 234 } 235 eos = true; // end of stream 236 acc.length = 0; 237 return l; 238 } 239 void scanUntil(F)(string needle, F f) { 240 auto d = needle.representation; 241 ubyte[] acc; 242 bool eos; // end of stream 243 244 while( !eos ) { 245 auto l = scanUntilR(needle, acc, eos); 246 debug(httpd) tracef("scanr returned <%s> and <%s>", cast(string)l, cast(string)acc); 247 f(l); 248 if ( acc == needle) { 249 return; 250 } 251 } 252 } 253 void skipUntil(string needle) { 254 auto d = needle.representation; 255 ubyte[] acc; 256 bool eos; // end of stream 257 258 while( !eos ) { 259 auto l = scanUntilR(needle, acc, eos); 260 debug(httpd) tracef("scanr returned <%s> and <%s>", cast(string)l, cast(string)acc); 261 if ( acc == needle) { 262 return; 263 } 264 } 265 } 266 } 267 268 auto createDataSource(string partialBody, NetworkStream stream) { 269 270 if ( !requestHasBody ) { 271 return new _DataSource(); 272 } 273 274 auto ds = new _DataSource(); 275 276 ds._requestHasBody = true; 277 ds._requestBodyRecvInProgr = true; 278 ds._bodyDecoder = new DataPipe!ubyte; 279 ds._stream = stream; 280 281 if ( auto contentLengthHeader = "content-length" in _requestHeaders ) { 282 ds._contentLength = to!long(*contentLengthHeader); 283 } 284 else if ( auto contentTransferEncoding = "transfer-encoding" in _requestHeaders ) { 285 if ( *contentTransferEncoding=="chunked" ) { 286 ds._unChunker = new DecodeChunked(); 287 ds._bodyDecoder.insert(ds._unChunker); 288 } 289 } 290 if ( partialBody.length ) { 291 ds._bodyDecoder.putNoCopy(cast(ubyte[])partialBody); 292 ds._receivedLength = (cast(ubyte[])partialBody).length; 293 } 294 while ( ds._bodyDecoder.empty ) { 295 auto b = new ubyte[DSBUFFSIZE]; 296 auto read = stream.receive(b); 297 if ( read == 0 ) { 298 debug(httpd) trace("stream closed when receiving in datasource"); 299 ds._requestBodyRecvInProgr = false; 300 return ds; 301 } 302 debug(httpd) tracef("place %d bytes to datasource", read); 303 ds._receivedLength += read; 304 ds._bodyDecoder.putNoCopy(b[0..read]); 305 } 306 ds._content = ds._bodyDecoder.getNoCopy().join(); 307 if ( ( ds._contentLength > 0 && ds._receivedLength >= ds._contentLength ) 308 || ( ds._unChunker && ds._unChunker.done) ) { 309 // all data received we need not wait any data from network 310 debug(httpd) trace("looks like we received complete request body together with request headers"); 311 ds._requestBodyRecvInProgr = false; 312 ds._requestBodyReceived = true; 313 } 314 debug(httpd) tracef("initial content: %d bytes", ds._content.length); 315 return ds; 316 } 317 @property auto contentType() { 318 if ( auto ct = "content-type" in _requestHeaders ) { 319 auto f = (*ct).split(";").map!strip; 320 return f[0]; 321 } 322 return null; 323 } 324 325 struct PartData { 326 // handler for each part data stream 327 _DataSource _ds; 328 string _boundary; 329 ubyte[] _content; 330 ubyte[] _acc; 331 bool _done; 332 bool _eos; 333 334 this(_DataSource ds, string boundary) { 335 _ds = ds; 336 _boundary = "\r\n" ~ boundary; 337 _content = _ds.scanUntilR(_boundary, _acc, _eos); 338 } 339 bool empty() { 340 return _content.length == 0; 341 } 342 auto front() { 343 return _content; 344 } 345 void popFront() { 346 _content.length = 0; 347 if ( _done ) { 348 return; 349 } 350 while( _content.length == 0 ) { 351 _content = _ds.scanUntilR(_boundary, _acc, _eos); 352 if ( _eos ) { 353 return; 354 } 355 if (_acc == _boundary) { 356 debug(httpd) tracef("part data done"); 357 _ds.skipUntil("\r\n"); 358 return; 359 } 360 } 361 } 362 } 363 struct Part { 364 _DataSource _ds; 365 string[string] _headers; 366 string _boundary; 367 368 this(_DataSource ds, string[string] h, string boundary) { 369 _ds = ds; 370 _headers = h; 371 _boundary = boundary; 372 } 373 @property string[string] headers() { 374 return _headers; 375 } 376 @property disposition() { 377 string[string] res; 378 auto d = "content-disposition" in _headers; 379 if ( !d ) { 380 return res; 381 } 382 (*d).split(";"). 383 filter!"a.indexOf('=')>0". 384 map! "a.strip.split('=')". 385 each!(p => res[p[0]] = urlDecode(p[1]).strip('"')); 386 return res; 387 } 388 @property data() { 389 return PartData(_ds, _boundary); 390 } 391 } 392 struct MultiPart { 393 string _boundary; 394 _DataSource _ds; 395 Part _part; 396 /* 397 --8a60ded0-ee76-4b6a-a1a0-dccaf93b92e7 398 Content-Disposition: form-data; name=Field1; 399 400 form field from memory 401 --8a60ded0-ee76-4b6a-a1a0-dccaf93b92e7 402 Content-Disposition: form-data; name=Field2; filename=data2 403 404 file field from memory 405 --8a60ded0-ee76-4b6a-a1a0-dccaf93b92e7 406 Content-Disposition: form-data; name=File1; filename=file1 407 Content-Type: application/octet-stream 408 409 file1 content 410 411 --8a60ded0-ee76-4b6a-a1a0-dccaf93b92e7 412 Content-Disposition: form-data; name=File2; filename=file2 413 Content-Type: application/octet-stream 414 415 file2 content 416 417 --8a60ded0-ee76-4b6a-a1a0-dccaf93b92e7-- 418 */ 419 int opApply(int delegate(Part p) dg) { 420 int result = 0; 421 while(!_ds.empty) { 422 result = dg(_part); 423 if ( result ) { 424 break; 425 } 426 auto headers = skipHeaders(); 427 _part = Part(_ds, headers, _boundary); 428 } 429 return result; 430 } 431 auto skipHeaders() { 432 ubyte[] buf; 433 string[string] headers; 434 435 debug(httpd) tracef("Search for headers"); 436 _ds.scanUntil("\r\n\r\n", delegate void (ubyte[] data) { 437 buf ~= data; 438 }); 439 foreach(h; buf.split('\n').map!"cast(string)a".map!strip.filter!"a.length") { 440 auto parsed = h.findSplit(":"); 441 headers[parsed[0].toLower] = parsed[2].strip; 442 } 443 debug(httpd) tracef("Headers: %s ", headers); 444 return headers; 445 } 446 /// 447 /// Find boundary from request headers, 448 /// skip to begin of the first part, 449 /// create first part(read/parse headers, stop on the body begin) 450 /// 451 this(HTTPD_Request rq) { 452 ubyte[] buf, rest; 453 string separator; 454 auto ct = "content-type" in rq._requestHeaders; 455 auto b = (*ct).split(";").map!"a.strip.split(`=`)".filter!"a[0].toLower==`boundary`"; 456 if ( b.empty ) { 457 throw new HTTPD_RequestException("Can't find 'boundary' in Content-Type %s".format(*ct)); 458 } 459 _boundary = "--" ~ b.front[1]; 460 _ds = rq._dataSource; 461 _ds.skipUntil(_boundary~"\r\n"); 462 auto headers = skipHeaders(); 463 _part = Part(_ds, headers, _boundary); 464 } 465 } 466 467 auto multiPartRead() { 468 return MultiPart(this); 469 } 470 471 auto read() { 472 if ( requestHasBody && _dataSource._requestBodyProcessed ) { 473 throw new HTTPD_RequestException("Request body already consumed by call to data/form/json"); 474 } 475 if ( _dataSource._readStarted ) { 476 throw new HTTPD_RequestException("Request body read() already started."); 477 } 478 _dataSource._readStarted = true; 479 return _dataSource; 480 } 481 482 void loadBodyOnDemand(ref _DataSource ds) { 483 ds._requestBodyProcessed = true; 484 debug(httpd) tracef("Process %s onDemand", contentType); 485 switch ( contentType ) { 486 case "application/json": 487 while(!ds.empty) { 488 debug(httpd) tracef("add %d bytes to json from dataSource", ds.front.length); 489 _json ~= cast(string)ds.front; 490 ds.popFront; 491 } 492 break; 493 case "application/x-www-form-urlencoded": 494 string qBody; 495 while(!ds.empty) { 496 debug(httpd) tracef("add %d bytes to json from dataSource", ds.front.length); 497 qBody ~= cast(string)ds.front; 498 ds.popFront; 499 } 500 _form = parseQuery(qBody); 501 break; 502 case "multipart/form-data": 503 debug(httpd) tracef("loading multiPart on demand"); 504 auto parts = multiPartRead(); 505 foreach(p; parts) { 506 auto disposition = p.disposition; 507 auto data = p.data.joiner.array; 508 509 if ( !("name" in disposition) ) { 510 continue; 511 } 512 if ( auto fn = "filename" in disposition ) { 513 _files[disposition["name"]] = data; 514 } else { 515 _form[disposition["name"]] = cast(string)data; 516 } 517 } 518 break; 519 default: 520 while(!ds.empty) { 521 debug(httpd) tracef("add %d bytes to data from dataSource", ds.front.length); 522 _data ~= ds.front; 523 ds.popFront; 524 } 525 break; 526 } 527 } 528 } 529 530 string[int] codes; 531 shared static this() { 532 codes = [ 533 200: "OK", 534 302: "Found", 535 401: "Unauthorized", 536 404: "Not found", 537 405: "Method not allowed", 538 500: "Server error" 539 ]; 540 } 541 enum Compression : int { 542 no = 0, 543 gzip = 1, 544 deflate = 2, 545 yes = gzip|deflate, 546 }; 547 548 auto response(C)(HTTPD_Request rq, C content, ushort code = 200) 549 if ( isSomeString!C 550 || (__traits(compiles, cast(ubyte[])content)) 551 || (__traits(compiles, cast(ubyte[])content.front)) 552 ) 553 { 554 return new HTTPD_Response!C(rq, content, code); 555 } 556 557 class _Response { 558 abstract void send(NetworkStream); 559 abstract ref string[string] headers(); 560 } 561 562 class HTTPD_Response(C) : _Response { 563 ushort _status = 200; 564 string _status_reason = "Unspecified"; 565 string[string] _headers; 566 C _content; 567 Compression _compression = Compression.no; 568 HTTPD_Request _request; 569 Cookie[] _cookies; 570 571 mixin(Getter_Setter!ushort("status")); 572 mixin(Getter("compression")); 573 @property void compress(Compression c = Compression.yes) { 574 _compression = c; 575 } 576 this(ref HTTPD_Request request, C content, ushort status = 200) { 577 _status = status; 578 _request = request; 579 _content = content; 580 } 581 override ref string[string] headers() @property { 582 return _headers; 583 } 584 ref Cookie[] cookies() { 585 return _cookies; 586 } 587 void content(C)(C c) @property { 588 _content = makeContent(c); 589 } 590 auto selectCompression(in HTTPD_Request rq, in HTTPD_Response rs) { 591 if ( auto acceptEncodings = "accept-encoding" in rq.requestHeaders) { 592 auto heAccept = (*acceptEncodings).split(",").map!strip; 593 if ( (rs.compression & Compression.gzip) && heAccept.canFind("gzip")) { 594 return "gzip"; 595 } 596 if ( (compression & Compression.deflate) && heAccept.canFind("deflate")) { 597 return "deflate"; 598 } 599 } 600 return null; 601 } 602 void sendCookies(NetworkStream stream) { 603 if ( _cookies.length ) { 604 foreach(c; _cookies) { 605 auto setCookie = "Set-Cookie: %s=%s; Path=%s\r\n".format(c.attr, c.value, c.path); 606 stream.send(setCookie); 607 } 608 } 609 } 610 final override void send(NetworkStream stream) { 611 import std.zlib; 612 auto statusLine = "HTTP/1.1 " ~ to!string(_status) ~ " " ~ codes.get(_status, _status_reason) ~ " \r\n"; 613 614 if ( !stream.isOpen || !stream.isConnected ) { 615 debug(httpd) tracef("Will not send to closed connection"); 616 return; 617 } 618 debug(httpd) tracef("sending statusLine: %s", statusLine.stripRight); 619 stream.send(statusLine); 620 621 auto comp = selectCompression(_request, this); 622 623 static if ( isSomeString!C || __traits(compiles, cast(ubyte[])_content) ) { 624 ubyte[] data; 625 if ( comp ) { 626 _headers["content-encoding"] = comp; 627 Compress compressor; 628 final switch (comp) { 629 case "gzip": // gzip 630 compressor = new Compress(6, HeaderFormat.gzip); 631 break; 632 case "deflate": // deflate 633 compressor = new Compress(6, HeaderFormat.deflate); 634 break; 635 } 636 data = cast(ubyte[])compressor.compress(_content); 637 data ~= cast(ubyte[])compressor.flush(); 638 } 639 else { 640 data = cast(ubyte[])_content; 641 } 642 _headers["content-length"] = to!string(data.length); 643 foreach(p; _headers.byKeyValue) { 644 stream.send(p.key ~ ": " ~ p.value ~ "\r\n"); 645 } 646 if ( _cookies.length ) { 647 sendCookies(stream); 648 } 649 stream.send("\r\n"); 650 if (_request.method == "HEAD") { 651 return; 652 } 653 stream.send(data); 654 } 655 else { 656 _headers["transfer-encoding"] = "chunked"; 657 Compress compressor; 658 if ( comp !is null ) { 659 _headers["content-encoding"] = comp; 660 final switch (comp) { 661 case "gzip": // gzip 662 compressor = new Compress(6, HeaderFormat.gzip); 663 break; 664 case "deflate": // deflate 665 compressor = new Compress(6, HeaderFormat.deflate); 666 break; 667 } 668 } 669 foreach(p; _headers.byKeyValue) { 670 stream.send(p.key ~ ": " ~ p.value ~ "\r\n"); 671 } 672 if ( _cookies.length ) { 673 sendCookies(stream); 674 } 675 stream.send("\r\n"); 676 if (_request.method == "HEAD") { 677 return; 678 } 679 ubyte[] data; 680 while(!_content.empty) { 681 auto chunk = cast(ubyte[])_content.front; 682 _content.popFront; 683 684 if ( compressor ) { 685 data ~= cast(ubyte[])compressor.compress(chunk); 686 if ( data.length == 0 ) { 687 continue; 688 } 689 } else { 690 data = chunk; 691 } 692 stream.send("%x\r\n".format(data.length)); 693 stream.send(data); 694 stream.send("\r\n"); 695 data.length = 0; 696 } 697 if ( compressor ) { 698 data = cast(ubyte[])compressor.flush(); 699 stream.send("%x\r\n".format(data.length)); 700 stream.send(data); 701 stream.send("\r\n"); 702 } 703 stream.send("0\r\n\r\n"); 704 } 705 } 706 } 707 708 alias Handler = _Response delegate(in App app, ref HTTPD_Request, RequestArgs); 709 710 struct RequestArgs { 711 private { 712 Captures!string _captures = void; 713 string _string; 714 } 715 this(Captures!string c) @nogc @safe nothrow { 716 _captures = c; 717 } 718 this(string s) @nogc @safe pure nothrow { 719 _string = s; 720 } 721 bool empty() @nogc @safe pure nothrow { 722 return _captures.empty && _string is null; 723 } 724 string opIndex(string s) @safe pure { 725 return _captures[s]; 726 } 727 string opIndex(size_t i) @safe pure { 728 if ( _string && i==0 ) { 729 return _string; 730 } 731 return _captures[i]; 732 } 733 } 734 735 auto exactRoute(string s, Handler h) @safe pure nothrow { 736 return new ExactRoute(s, h); 737 } 738 739 auto regexRoute(string s, Handler h) @safe { 740 return new RegexRoute(s, h); 741 } 742 743 class Route { 744 Handler _handler; 745 string _origin; 746 747 abstract RequestArgs match(string) { 748 return RequestArgs(); 749 }; 750 final Handler handler() { 751 return _handler; 752 } 753 final string origin() { 754 return _origin; 755 } 756 } 757 758 class ExactRoute: Route { 759 760 this(string s, Handler h) @safe pure nothrow { 761 _origin = s; 762 _handler = h; 763 } 764 final override RequestArgs match(string input) { 765 if ( input == _origin ) { 766 debug(httpd) tracef("%s matches %s", input, _origin); 767 return RequestArgs(input); 768 } 769 return RequestArgs(); 770 } 771 } 772 class RegexRoute: Route { 773 Regex!char _re; 774 775 this(string r, Handler h) @safe { 776 _origin = r; 777 _handler = h; 778 _re = regex(r); 779 } 780 final override RequestArgs match(string input) { 781 auto m = matchFirst(input, _re); 782 debug(httpd) if (!m.empty) {tracef("%s matches %s", input, _origin);} 783 return RequestArgs(m); 784 } 785 } 786 787 struct Router { 788 alias RouteMatch = Tuple!(Handler, "handler", RequestArgs, "args"); 789 private Route[] _routes; 790 791 void addRoute(Route r) { 792 _routes ~= r; 793 } 794 auto getRoute(string path) { 795 RouteMatch match; 796 foreach(r; _routes) { 797 auto args = r.match(path); 798 if (!args.empty) { 799 match.handler = r.handler; 800 match.args = args; 801 break; 802 } 803 } 804 return match; 805 } 806 } 807 808 private auto parseQuery(string query) { 809 /// TODO 810 /// switch to return dict of 811 /// struct QueryParam { 812 /// private: 813 /// string name; 814 /// string[] value; 815 /// public: 816 /// uint length() {return value.length;} 817 /// string toString() {return value[0];} 818 /// string[] toArray() {return value;} 819 /// } 820 debug (httpd) tracef("query: %s", query); 821 string[string] q; 822 if ( !query ) { 823 return q; 824 } 825 if ( query[0] == '?') { 826 query = query[1..$]; 827 } 828 string[][] parsed = query.splitter("&"). 829 map!(s => s.split("=")). 830 filter!"a.length==2". 831 map!(p => [urlDecode(p[0]), urlDecode(p[1])]). 832 array; 833 834 auto grouped = sort!"a[0]<b[0]"(parsed).assumeSorted!"a[0]<b[0]".groupBy(); 835 foreach(g; grouped) { 836 string key = g.front[0]; 837 string val; 838 auto vals = g.map!"a[1]".array; 839 if (vals.length == 1) { 840 val = vals[0]; 841 } 842 if (vals.length > 1) { 843 val = to!string(vals); 844 } 845 q[key] = val; 846 } 847 return q; 848 } 849 850 private bool headersReceived(in ubyte[] data, ref Buffer!ubyte buffer, out string separator) @safe { 851 foreach(s; ["\r\n\r\n", "\n\n"]) { 852 if ( data.canFind(s) || buffer.canFind(s) ) { 853 separator = s; 854 return true; 855 } 856 } 857 return false; 858 } 859 860 private void parseRequestHeaders(in App app, ref HTTPD_Request rq, string buffer) { 861 string lastHeader; 862 auto lines = buffer.splitLines.map!stripRight; 863 rq.requestLine = lines[0]; 864 if ( lines.count == 1) { 865 return; 866 } 867 foreach(line; lines[1..$]) { 868 if ( !line.length ) { 869 continue; 870 } 871 if ( line[0] == ' ' || line[0] == '\t' ) { 872 // unfolding https://tools.ietf.org/html/rfc822#section-3.1 873 if ( auto prevValue = lastHeader in rq.requestHeaders) { 874 *prevValue ~= line; 875 } 876 continue; 877 } 878 auto parsed = line.findSplit(":"); 879 auto header = parsed[0].toLower; 880 auto value = parsed[2].strip; 881 lastHeader = header; 882 if ( auto h = header in rq.requestHeaders ) { 883 *h ~= "; " ~ value; 884 } else { 885 rq.requestHeaders[header] = value; 886 } 887 debug(httpd) tracef("%s: %s", header, value); 888 } 889 auto rqlFields = rq.requestLine.split(" "); 890 debug (httpd) tracef("rqLine %s", rq.requestLine); 891 rq.method = rqlFields[0]; 892 auto scheme = app.useSSL? 893 "https://": 894 "http://"; 895 if ( "host" in rq.requestHeaders ) { 896 rq.uri = URI(scheme ~ rq.requestHeaders["host"] ~ rqlFields[1]); 897 } else { 898 rq.uri = URI(scheme ~ app.host ~ rqlFields[1]); 899 } 900 rq.path = rq.uri.path; 901 rq.query = parseQuery(rq.uri.query); 902 debug (httpd) tracef("path: %s", rq.path); 903 debug (httpd) tracef("query: %s", rq.query); 904 // 905 // now analyze what we have 906 // 907 auto header = "connection" in rq.requestHeaders; 908 if ( header && toLower(*header) == "keep-alive") { 909 rq.keepAlive = true; 910 } 911 auto cookies = "cookie" in rq.requestHeaders; 912 if ( cookies ) { 913 (*cookies).split(';'). 914 map!"strip(a).split('=')". 915 filter!(kv => kv.length==2). 916 each!(kv => rq._cookies[kv[0]] = kv[1]); 917 } 918 } 919 920 private auto read_request(in App app, NetworkStream stream) { 921 HTTPD_Request rq; 922 Buffer!ubyte input; 923 string separator; 924 925 while( true ) { 926 ubyte[] b = new ubyte[app.bufferSize]; 927 auto read = stream.receive(b); 928 929 if ( read == 0 ) { 930 return rq; 931 } 932 debug(httpd) tracef("received %d bytes", read); 933 input.putNoCopy(b[0..read]); 934 935 if ( headersReceived(b, input, separator) ) { 936 break; 937 } 938 939 if ( input.length >= app.maxHeadersSize ) { 940 throw new HTTPD_RequestException("Request headers length %d too large".format(input.length)); 941 } 942 } 943 debug(httpd) trace("Headers received"); 944 auto s = input.data!(string).findSplit(separator); 945 auto requestHeaders = s[0]; 946 debug(httpd) tracef("Headers: %s", cast(string)requestHeaders); 947 parseRequestHeaders(app, rq, requestHeaders); 948 debug(httpd) trace("Headers parsed"); 949 950 rq._dataSource = rq.createDataSource(s[2], stream); 951 952 return rq; 953 } 954 955 void processor(in App app, HTTPD httpd, NetworkStream stream) { 956 stream.readTimeout = app.timeout; 957 HTTPD_Request rq; 958 _Response rs; 959 scope (exit) { 960 if ( stream.isOpen ) { 961 stream.close(); 962 } 963 } 964 uint rqLimit = max(app.rqLimit, 1); 965 try { 966 while ( rqLimit > 0 ) { 967 rq = read_request(app, stream); 968 if ( !httpd._running || !rq.requestLine.length ) { 969 return; 970 } 971 auto match = httpd._router.getRoute(rq.path); 972 if ( !match.handler ) { 973 // return 404; 974 debug (httpd) tracef("Route not found for %s", rq.path); 975 rs = response(rq, "Requested path %s not found".format(rq.path), 404); 976 break; 977 } 978 auto handler = match.handler; 979 rs = handler(app, rq, match.args); 980 if ( !stream.isOpen ) { 981 debug(httpd) tracef("Request handler closed connection"); 982 return; 983 } 984 if ( rq.keepAlive && rqLimit > 1 ) { 985 rs.headers["Connection"] = "Keep-Alive"; 986 } 987 if ( rq._dataSource._requestHasBody && !rq._dataSource._requestBodyReceived ) { 988 // for some reason some part of the request body still not received, and it will 989 // stay on the way of next request if this is keep-Alive session, 990 // so we must abort this connection anyway. 991 debug(httpd) trace("Request handler did not consumed whole request body. We have to close connection after sending response."); 992 rs.send(stream); 993 return; 994 } 995 rs.send(stream); 996 --rqLimit; 997 if ( !rq.keepAlive || rqLimit==0 ) { 998 debug(httpd) trace("Finished with that connection"); 999 return; 1000 } 1001 debug(httpd) trace("Continue with keepalive request"); 1002 rq = rq.init; 1003 } 1004 } 1005 catch (HTTPD_RequestException e) { 1006 debug(httpd) error("Request exception: " ~ e.msg); 1007 rs = response(rq, "Request exception:\n" ~ e.msg, 500); 1008 } 1009 catch (TimeoutException e) { 1010 debug(httpd) { 1011 if ( rq.requestLine ) { 1012 error("Timeout reading/writing to client"); 1013 } 1014 } 1015 } 1016 catch (Exception e) { 1017 debug(httpd) error("Unexpected Exception " ~ e.msg); 1018 rs = response(rq, "Unexpected exception:\n" ~ e.msg, 500); 1019 } 1020 catch (Error e) { 1021 error(e.msg, e.info); 1022 rs = response(rq, "Unexpected error:\n" ~ e.msg, 500); 1023 } 1024 try { 1025 if ( stream.isOpen ) { 1026 rs.send(stream); 1027 } 1028 } 1029 catch (Exception e) { 1030 infof("Exception when send %s", e.msg); 1031 } 1032 catch (Error e) { 1033 error("Error sending response: " ~ e.msg); 1034 } 1035 } 1036 1037 class HTTPD 1038 { 1039 private { 1040 TaskPool _server; 1041 __gshared bool _running; 1042 Router _router; 1043 App _app; 1044 } 1045 auto ref addRoute(Route r) { 1046 _router.addRoute(r); 1047 return this; 1048 } 1049 static NetworkStream openStream(in App app) { 1050 auto host = app.host; 1051 auto port = app.port; 1052 Address[] addresses; 1053 SSLOptions _sslOptions; 1054 1055 try { 1056 addresses = getAddress(host, port); 1057 } catch (Exception e) { 1058 throw new ConnectError("Can't resolve name when connect to %s:%d: %s".format(host, port, e.msg)); 1059 } 1060 auto tcpStream = app.useSSL? 1061 new SSLStream(_sslOptions): 1062 new TCPStream(); 1063 tcpStream.open(addresses[0].addressFamily); 1064 return tcpStream; 1065 } 1066 static void run(in App app, HTTPD httpd) { 1067 Address[] addresses; 1068 try { 1069 addresses = getAddress(app.host, app.port); 1070 } catch (Exception e) { 1071 throw new ConnectError("Can't resolve name when connect to %s:%d: %s".format(app.host, app.port, e.msg)); 1072 } 1073 auto tcpStream = openStream(app); 1074 tcpStream.reuseAddr(true); 1075 tcpStream.bind(addresses[0]); 1076 tcpStream.listen(128); 1077 defaultPoolThreads(64); 1078 auto pool = taskPool(); 1079 _running = true; 1080 while ( _running ) { 1081 auto stream = tcpStream.accept(); 1082 if ( _running ) { 1083 auto connHandler = task!processor(app, httpd, stream); 1084 pool.put(connHandler); 1085 } else { 1086 tcpStream.close(); 1087 break; 1088 } 1089 } 1090 } 1091 void app(App a) { 1092 _app = a; 1093 } 1094 void start() { 1095 defaultPoolThreads(64); 1096 _server = taskPool(); 1097 auto t = task!run(_app, this); 1098 _server.put(t); 1099 Thread.sleep(500.msecs); 1100 } 1101 void start(App app) { 1102 defaultPoolThreads(64); 1103 _app = app; 1104 _server = taskPool(); 1105 auto t = task!run(_app, this); 1106 _server.put(t); 1107 Thread.sleep(500.msecs); 1108 } 1109 void stop() { 1110 if ( !_running ) { 1111 return; 1112 } 1113 _running = false; 1114 try { 1115 auto s = openStream(_app); 1116 s.connect(_app.host, _app.port); 1117 } catch (Exception e) { 1118 } 1119 // _server.stop(); 1120 } 1121 } 1122 1123 struct App { 1124 private { 1125 string _name; 1126 string _host; 1127 ushort _port; 1128 Duration _timeout = 30.seconds; 1129 size_t _bufferSize = 16*1024; 1130 size_t _maxHeadersSize = 32*1024; 1131 bool _useSSL = false; 1132 uint _rqLimit = 10; // keepalive requestst per connection 1133 Router _router; 1134 } 1135 mixin(Getter_Setter!string("name")); 1136 mixin(Getter_Setter!string("host")); 1137 mixin(Getter_Setter!ushort("port")); 1138 mixin(Getter_Setter!size_t("bufferSize")); 1139 mixin(Getter_Setter!size_t("maxHeadersSize")); 1140 mixin(Getter_Setter!Duration("timeout")); 1141 mixin(Getter_Setter!bool("useSSL")); 1142 mixin(Getter_Setter!uint("rqLimit")); 1143 this(string name) { 1144 _name = name; 1145 } 1146 } 1147 1148 1149 version(none) private unittest { 1150 import std.json; 1151 import std.conv; 1152 import requests.http: HTTPRequest, TimeoutException, BasicAuthentication, queryParams, MultipartForm, formData; 1153 globalLogLevel(LogLevel.info); 1154 1155 static auto buildReply(ref HTTPD_Request rq) { 1156 auto args = JSONValue(rq.query); 1157 auto headers = JSONValue(rq.requestHeaders); 1158 auto url = JSONValue(rq.uri.uri); 1159 auto json = JSONValue(rq.json); 1160 auto data = JSONValue(rq.data); 1161 auto form = JSONValue(rq.form); 1162 auto files = JSONValue(rq.files); 1163 auto reply = JSONValue(["args":args, "headers": headers, "json": json, "url": url, "data": data, "form": form, "files": files]); 1164 return reply.toString(); 1165 } 1166 1167 Router router; 1168 router.addRoute(exactRoute(r"/get", null)); 1169 router.addRoute(regexRoute(r"/get/(?P<param>\d+)", null)); 1170 auto r = router.getRoute(r"/get"); 1171 assert(!r.args.empty); 1172 r = router.getRoute(r"/post"); 1173 assert(r.args.empty); 1174 1175 r = router.getRoute(r"/get/333"); 1176 assert(!r.args.empty); 1177 assert(r.args["param"]=="333"); 1178 r = router.getRoute(r"/get/aaa"); 1179 assert(r.args.empty); 1180 1181 HTTPD_Request rq; 1182 string headers = "GET /get?a=b&list[]=1&c=d&list[]=2 HTTP/1.1\n" ~ 1183 "Host: host\n" ~ 1184 "X-Test: test1\n" ~ 1185 " test2\n" ~ 1186 "Content-Length: 1\n"; 1187 parseRequestHeaders(App(), rq, headers); 1188 assert(rq.requestHeaders["x-test"] == "test1 test2"); 1189 assert(rq.requestHeaders["host"] == "host"); 1190 assert(rq.path == "/get"); 1191 assert(rq.query["a"] == "b"); 1192 assert(rq.query["c"] == "d"); 1193 assert(rq.query["list[]"] == `["1", "2"]`); 1194 auto root(in App app, ref HTTPD_Request rq, RequestArgs args) { 1195 debug (httpd) trace("handler / called"); 1196 auto rs = response(rq, buildReply(rq)); 1197 rs.headers["Content-Type"] = "application/json"; 1198 return rs; 1199 } 1200 auto get(in App app, ref HTTPD_Request rq, RequestArgs args) { 1201 debug (httpd) trace("handler /get called"); 1202 auto rs = response(rq, buildReply(rq)); 1203 rs.headers["Content-Type"] = "application/json"; 1204 return rs; 1205 } 1206 auto basicAuth(in App app, ref HTTPD_Request rq, RequestArgs args) { 1207 import std.base64; 1208 auto user = args["user"]; 1209 auto password= args["password"]; 1210 auto auth = cast(string)Base64.decode(rq.requestHeaders["authorization"].split()[1]); 1211 auto up = auth.split(":"); 1212 short status; 1213 if ( up[0]==user && up[1]==password) { 1214 status = 200; 1215 } else { 1216 status = 401; 1217 } 1218 auto rs = response(rq, buildReply(rq), status); 1219 rs.headers["Content-Type"] = "application/json"; 1220 return rs; 1221 } 1222 auto rredir(in App app, ref HTTPD_Request rq, RequestArgs args) { 1223 auto rs = response(rq, buildReply(rq)); 1224 auto redirects = to!long(args["redirects"]); 1225 if ( redirects > 1 ) { 1226 rs.headers["Location"] = "/relative-redirect/%d".format(redirects-1); 1227 } else { 1228 rs.headers["Location"] = "/get"; 1229 } 1230 rs.status = 302; 1231 return rs; 1232 } 1233 auto aredir(in App app, ref HTTPD_Request rq, RequestArgs args) { 1234 auto rs = response(rq, buildReply(rq)); 1235 auto redirects = to!long(args["redirects"]); 1236 if ( redirects > 1 ) { 1237 rs.headers["Location"] = "http://127.0.0.1:8081/absolute-redirect/%d".format(redirects-1); 1238 } else { 1239 rs.headers["Location"] = "http://127.0.0.1:8081/get"; 1240 } 1241 rs.status = 302; 1242 return rs; 1243 } 1244 auto delay(in App app, ref HTTPD_Request rq, RequestArgs args) { 1245 auto delay = dur!"seconds"(to!long(args["delay"])); 1246 Thread.sleep(delay); 1247 auto rs = response(rq, buildReply(rq)); 1248 rs.headers["Content-Type"] = "application/json"; 1249 return rs; 1250 } 1251 auto gzip(in App app, ref HTTPD_Request rq, RequestArgs args) { 1252 auto rs = response(rq, buildReply(rq)); 1253 rs.compress(Compression.gzip); 1254 rs.headers["Content-Type"] = "application/json"; 1255 return rs; 1256 } 1257 auto deflate(in App app, ref HTTPD_Request rq, RequestArgs args) { 1258 auto rs = response(rq, buildReply(rq)); 1259 rs.compress(Compression.deflate); 1260 return rs; 1261 } 1262 auto range(in App app, ref HTTPD_Request rq, RequestArgs args) { 1263 auto size = to!size_t(args["size"]); 1264 auto rs = response(rq, new ubyte[size].chunks(16)); 1265 rs.compress(Compression.yes); 1266 return rs; 1267 } 1268 auto head(in App app, ref HTTPD_Request rq, RequestArgs args) { 1269 if ( rq.method != "HEAD") { 1270 auto rs = response(rq, "Illegal method %s".format(rq.method), 405); 1271 return rs; 1272 } 1273 else { 1274 auto rs = response(rq, buildReply(rq)); 1275 rs.compress(Compression.yes); 1276 return rs; 1277 } 1278 } 1279 auto del(in App app, ref HTTPD_Request rq, RequestArgs args) { 1280 if ( rq.method != "DELETE") { 1281 auto rs = response(rq, "Illegal method %s".format(rq.method), 405); 1282 return rs; 1283 } 1284 else { 1285 auto rs = response(rq, buildReply(rq)); 1286 return rs; 1287 } 1288 } 1289 auto post(in App app, ref HTTPD_Request rq, RequestArgs args) { 1290 auto rs = response(rq, buildReply(rq)); 1291 return rs; 1292 } 1293 auto postIter(in App app, ref HTTPD_Request rq, RequestArgs args) { 1294 int c; 1295 1296 if ( rq.contentType == "multipart/form-data" ) { 1297 auto parts = rq.multiPartRead(); 1298 foreach(p; parts) { 1299 auto disposition = p.disposition; 1300 c += p.data.joiner.count; 1301 } 1302 auto rs = response(rq, "%d".format(c)); 1303 return rs; 1304 } 1305 else { 1306 auto r = rq.read(); 1307 while ( !r.empty ) { 1308 c += r.front.length; 1309 r.popFront; 1310 } 1311 auto rs = response(rq, "%d".format(c)); 1312 return rs; 1313 } 1314 } 1315 auto read(in App app, ref HTTPD_Request rq, RequestArgs args) { 1316 auto r = rq.read(); 1317 int c; 1318 while ( !r.empty ) { 1319 c += r.front.length; 1320 r.popFront; 1321 } 1322 auto rs = response(rq, "%d".format(c)); 1323 return rs; 1324 } 1325 auto readf1(in App app, ref HTTPD_Request rq, RequestArgs args) { 1326 // now call to read must throw exception 1327 auto r = rq.read(); 1328 int c; 1329 while ( !r.empty ) { 1330 c += r.front.length; 1331 r.popFront; 1332 break; 1333 } 1334 auto rs = response(rq, "%d".format(c)); 1335 return rs; 1336 } 1337 auto cookiesSet(in App app, ref HTTPD_Request rq, RequestArgs args) { 1338 Cookie[] cookies; 1339 foreach(p; rq.query.byKeyValue) { 1340 cookies ~= Cookie("/cookies", rq.requestHeaders["host"], p.key, p.value); 1341 } 1342 auto rs = response(rq, buildReply(rq), 302); 1343 rs.headers["Location"] = "/cookies"; 1344 rs.cookies = cookies; 1345 return rs; 1346 } 1347 auto cookies(in App app, ref HTTPD_Request rq, RequestArgs args) { 1348 auto cookies = ["cookies": JSONValue(rq.cookies)]; 1349 auto rs = response(rq, JSONValue(cookies).toString); 1350 return rs; 1351 } 1352 1353 auto httpbin = App("httpbin"); 1354 1355 httpbin.port = 8081; 1356 httpbin.host = "127.0.0.1"; 1357 1358 httpbin.timeout = 10.seconds; 1359 HTTPD server = new HTTPD(); 1360 1361 server.addRoute(exactRoute(r"/", &root)). 1362 addRoute(exactRoute(r"/get", &get)). 1363 addRoute(regexRoute(r"/delay/(?P<delay>\d+)", &delay)). 1364 addRoute(regexRoute(r"/relative-redirect/(?P<redirects>\d+)", &rredir)). 1365 addRoute(regexRoute(r"/absolute-redirect/(?P<redirects>\d+)", &aredir)). 1366 addRoute(regexRoute(r"/basic-auth/(?P<user>[^/]+)/(?P<password>[^/]+)", &basicAuth)). 1367 addRoute(exactRoute(r"/gzip", &gzip)). 1368 addRoute(exactRoute(r"/deflate", &deflate)). 1369 addRoute(regexRoute(r"/range/(?P<size>\d+)", &range)). 1370 addRoute(exactRoute(r"/cookies/set", &cookiesSet)). 1371 addRoute(exactRoute(r"/cookies", &cookies)). 1372 addRoute(exactRoute(r"/head", &head)). 1373 addRoute(exactRoute(r"/delete", &del)). 1374 addRoute(exactRoute(r"/read", &read)). 1375 addRoute(exactRoute(r"/readf1", &readf1)). 1376 addRoute(exactRoute(r"/post", &post)). 1377 addRoute(exactRoute(r"/postIter", &postIter)); 1378 1379 server.start(httpbin); 1380 scope(exit) { 1381 server.stop(); 1382 } 1383 auto request = HTTPRequest(); 1384 1385 globalLogLevel(LogLevel.info); 1386 auto httpbin_url = "http://%s:%d/".format(httpbin.host, httpbin.port); 1387 request.timeout = 5.seconds; 1388 request.keepAlive = true; 1389 info("httpd Check GET"); 1390 auto rs = request.get(httpbin_url); 1391 assert(rs.code == 200); 1392 assert(rs.responseBody.length > 0); 1393 auto content = rs.responseBody.data!string; 1394 auto json = parseJSON(cast(string)content); 1395 assert(json.object["url"].str == httpbin_url); 1396 1397 info("httpd Check GET with parameters"); 1398 rs = request.get(httpbin_url ~ "get", ["c":" d", "a":"b"]); 1399 assert(rs.code == 200); 1400 json = parseJSON(cast(string)rs.responseBody.data).object["args"].object; 1401 assert(json["a"].str == "b"); 1402 assert(json["c"].str == " d"); 1403 1404 info("httpd Check relative redirect"); 1405 rs = request.get(httpbin_url ~ "relative-redirect/2"); 1406 assert(rs.history.length == 2); 1407 assert(rs.code==200); 1408 1409 info("httpd Check absolute redirect"); 1410 rs = request.get(httpbin_url ~ "absolute-redirect/2"); 1411 assert(rs.history.length == 2); 1412 assert(rs.code==200); 1413 1414 info("httpd Check basic auth"); 1415 request.authenticator = new BasicAuthentication("user", "password"); 1416 rs = request.get(httpbin_url ~ "basic-auth/user/password"); 1417 assert(rs.code==200); 1418 request.authenticator = null; 1419 1420 info("httpd Check timeout"); 1421 request.timeout = 1.seconds; 1422 assertThrown!TimeoutException(request.get(httpbin_url ~ "delay/2")); 1423 Thread.sleep(1.seconds); 1424 request.timeout = 30.seconds; 1425 1426 info("httpd Check gzip"); 1427 rs = request.get(httpbin_url ~ "gzip"); 1428 assert(rs.code==200); 1429 json = parseJSON(cast(string)rs.responseBody); 1430 assert(json.object["url"].str == httpbin_url ~ "gzip"); 1431 1432 info("httpd Check deflate"); 1433 rs = request.get(httpbin_url ~ "deflate"); 1434 assert(rs.code==200); 1435 json = parseJSON(cast(string)rs.responseBody); 1436 assert(json.object["url"].str == httpbin_url ~ "deflate"); 1437 1438 info("httpd Check range"); 1439 rs = request.get(httpbin_url ~ "range/1023"); 1440 assert(rs.code==200); 1441 assert(rs.responseBody.length == 1023); 1442 1443 info("httpd Check HEAD"); 1444 rs = request.exec!"HEAD"(httpbin_url ~ "head"); 1445 assert(rs.code==200); 1446 assert(rs.responseBody.length == 0); 1447 1448 info("httpd Check DELETE"); 1449 rs = request.exec!"DELETE"(httpbin_url ~ "delete"); 1450 assert(rs.code==200); 1451 1452 info("httpd Check POST json"); 1453 rs = request.post(httpbin_url ~ "post?b=x", `{"a":"b", "c":[1,2,3]}`, "application/json"); 1454 json = parseJSON(cast(string)rs.responseBody); 1455 auto rqJson = parseJSON(json.object["json"].str); 1456 assert(rqJson.object["a"].str == "b"); 1457 assert(equal([1,2,3], rqJson.object["c"].array.map!"a.integer")); 1458 1459 info("httpd Check POST json/chunked body"); 1460 rs = request.post(httpbin_url ~ "post?b=x", [`{"a":"b",`,` "c":[1,2,3]}`], "application/json"); 1461 json = parseJSON(cast(string)rs.responseBody); 1462 assert(json.object["args"].object["b"].str == "x"); 1463 rqJson = parseJSON(json.object["json"].str); 1464 assert(rqJson.object["a"].str == "b"); 1465 assert(equal([1,2,3], rqJson.object["c"].array.map!"a.integer")); 1466 1467 rs = request.post(httpbin_url ~ "post", "0123456789".repeat(32)); 1468 json = parseJSON(cast(string)rs.responseBody); 1469 assert(equal(json.object["data"].array.map!"a.integer", "0123456789".repeat(32).join)); 1470 1471 info("httpd Check POST with params"); 1472 rs = request.post(httpbin_url ~ "post", queryParams("b", 2, "a", "A")); 1473 assert(rs.code==200); 1474 auto data = parseJSON(cast(string)rs.responseBody).object["form"].object; 1475 assert((data["a"].str == "A")); 1476 assert((data["b"].str == "2")); 1477 1478 // this is tests for httpd read() interface 1479 info("httpd Check POST/iterating over body"); 1480 rs = request.post(httpbin_url ~ "read", "0123456789".repeat(1500)); 1481 assert(equal(rs.responseBody, "15000")); 1482 1483 { 1484 request.keepAlive = true; 1485 // this is test on how we can handle keepalive session when previous request leave unread data in socket 1486 try { 1487 rs = request.post(httpbin_url ~ "readf1", "0123456789".repeat(1500)); 1488 } 1489 catch (Exception e) { 1490 // this can fail as httpd will close connection prematurely 1491 } 1492 // but next idempotent request must succeed 1493 rs = request.get(httpbin_url ~ "get"); 1494 assert(rs.code == 200); 1495 } 1496 // 1497 { 1498 info("httpd Check POST/multipart form"); 1499 import std.file; 1500 import std.path; 1501 auto tmpd = tempDir(); 1502 auto tmpfname1 = tmpd ~ dirSeparator ~ "request_test1.txt"; 1503 auto f = File(tmpfname1, "wb"); 1504 f.rawWrite("file1 content\n"); 1505 f.close(); 1506 auto tmpfname2 = tmpd ~ dirSeparator ~ "request_test2.txt"; 1507 f = File(tmpfname2, "wb"); 1508 f.rawWrite("file2 content\n"); 1509 f.close(); 1510 /// 1511 /// Ok, files ready. 1512 /// Now we will prepare Form data 1513 /// 1514 File f1 = File(tmpfname1, "rb"); 1515 File f2 = File(tmpfname2, "rb"); 1516 scope(exit) { 1517 f1.close(); 1518 f2.close(); 1519 } 1520 /// 1521 /// for each part we have to set field name, source (ubyte array or opened file) and optional filename and content-type 1522 /// 1523 MultipartForm form = MultipartForm(). 1524 add(formData("Field1", cast(ubyte[])"form field from memory")). 1525 add(formData("Field2", cast(ubyte[])"file field from memory", ["filename":"data2"])). 1526 add(formData("Field3", cast(ubyte[])`{"a":"b"}`, ["Content-Type": "application/json"])). 1527 add(formData("File1", f1, ["filename":"file1", "Content-Type": "application/octet-stream"])). 1528 add(formData("File2", f2, ["filename":"file2", "Content-Type": "application/octet-stream"])); 1529 /// everything ready, send request 1530 rs = request.post(httpbin_url ~ "post?a=b", form); 1531 /* expected: 1532 { 1533 "args": { 1534 "a": "b" 1535 }, 1536 "data": "", 1537 "files": { 1538 "Field2": "file field from memory", 1539 "File1": "file1 content\n", 1540 "File2": "file2 content\n" 1541 }, 1542 "form": { 1543 "Field1": "form field from memory", 1544 "Field3": "{\"a\":\"b\"}" 1545 }, 1546 "headers": { 1547 "Accept-Encoding": "gzip, deflate", 1548 "Content-Length": "730", 1549 "Content-Type": "multipart/form-data; boundary=d79a383e-7912-4d36-a6db-a6774bf37133", 1550 "Host": "httpbin.org", 1551 "User-Agent": "dlang-requests" 1552 }, 1553 "json": null, 1554 "origin": "xxx.xxx.xxx.xxx", 1555 "url": "http://httpbin.org/post?a=b" 1556 } 1557 */ 1558 json = parseJSON(cast(string)rs.responseBody); 1559 assert("file field from memory" == cast(string)(json.object["files"].object["Field2"].array.map!(a => cast(ubyte)a.integer).array)); 1560 assert("file1 content\n" == cast(string)(json.object["files"].object["File1"].array.map!(a => cast(ubyte)a.integer).array)); 1561 1562 info("httpd Check POST/iterate over multipart form"); 1563 form = MultipartForm(). 1564 add(formData("Field1", cast(ubyte[])"form field from memory")). 1565 add(formData("Field2", cast(ubyte[])"file field from memory", ["filename":"data2"])). 1566 add(formData("Field3", cast(ubyte[])`{"a":"b"}`, ["Content-Type": "application/json"])); 1567 /// everything ready, send request 1568 rs = request.post(httpbin_url ~ "postIter?a=b", form); 1569 assert(equal(rs.responseBody, "53")); 1570 rs = request.post(httpbin_url ~ "postIter", "0123456789".repeat(1500)); 1571 assert(equal(rs.responseBody, "15000")); 1572 } 1573 info("httpd Check cookies"); 1574 rs = request.get(httpbin_url ~ "cookies/set?A=abcd&b=cdef"); 1575 json = parseJSON(cast(string)rs.responseBody.data).object["cookies"].object; 1576 assert(json["A"].str == "abcd"); 1577 assert(json["b"].str == "cdef"); 1578 } 1579 }