1 module requests.server.httpd; 2 3 import std.algorithm; 4 import std.array; 5 import std.conv; 6 import std.datetime; 7 import std.exception; 8 import std.experimental.logger; 9 import std.format; 10 import std.parallelism; 11 import std.range; 12 import std.regex; 13 import std.socket; 14 import std.stdio; 15 import std.string; 16 import std.traits; 17 import std.typecons; 18 import core.thread; 19 import requests.utils; 20 import requests.streams; 21 import requests.uri; 22 23 version(vibeD){ 24 pragma(msg, "httpd will not compile with vibeD"); 25 } 26 else { 27 /* 28 ** This is small http server to run something like httpbin(http://httpbin.org) internally 29 ** for Requests unittest's. 30 */ 31 32 enum DSBUFFSIZE = 16*1024; 33 34 class HTTPD_RequestException: Exception { 35 this(string message, string file =__FILE__, size_t line = __LINE__, Throwable next = null) @safe pure nothrow { 36 super(message, file, line, next); 37 } 38 } 39 40 struct HTTPD_Request { 41 private { 42 string _requestLine; 43 string[string] _requestHeaders; 44 Buffer!ubyte _requestBody; 45 bool _keepAlive; 46 URI _uri; 47 string[string] _query; // query in url 48 string _method; 49 string _path; 50 string _json; // json for application/json 51 string[string] _form; // form values for application/x-www-form-urlencoded 52 ubyte[][string] _files; 53 ubyte[] _data; // raw data for unrecognized mime's 54 _DataSource _dataSource; 55 string[string] _cookies; 56 } 57 private mixin(Setter!(string[string])("requestHeaders")); 58 auto inout ref requestHeaders() @property @safe @nogc nothrow { 59 return _requestHeaders; 60 } 61 auto inout ref cookies() @property @safe @nogc nothrow { 62 return _cookies; 63 } 64 private mixin(Setter!(string[string])("query")); 65 inout auto ref query() @property @safe @nogc nothrow { 66 return _query; 67 } 68 inout auto ref requestBody() @property @safe @nogc nothrow { 69 return _requestBody; 70 } 71 private mixin(Setter!string("method")); 72 mixin(Getter!string("method")); 73 private mixin(Setter!string("requestLine")); 74 mixin(Getter!string("requestLine")); 75 private mixin(Setter!string("path")); 76 mixin(Getter!string("path")); 77 private mixin(Setter!bool("keepAlive")); 78 mixin(Getter!bool("keepAlive")); 79 private mixin(Setter!URI("uri")); 80 mixin(Getter!URI("uri")); 81 82 @property string json() { 83 if ( _dataSource._readStarted ) { 84 throw new HTTPD_RequestException("Request read() call already started."); 85 } 86 if ( _dataSource._requestHasBody && !_dataSource._requestBodyProcessed ) { 87 debug(httpd) trace("receiving body on demand for json"); 88 loadBodyOnDemand(_dataSource); 89 } 90 return _json; 91 } 92 @property ubyte[] data() { 93 if ( _dataSource._readStarted ) { 94 throw new HTTPD_RequestException("Request body read() already started."); 95 } 96 if ( _dataSource._requestHasBody && !_dataSource._requestBodyProcessed ) { 97 debug(httpd) trace("receiving body on demand for data"); 98 loadBodyOnDemand(_dataSource); 99 } 100 return _data; 101 } 102 @property string[string] form() { 103 if ( _dataSource._readStarted ) { 104 throw new HTTPD_RequestException("Request body read() already started."); 105 } 106 if ( _dataSource._requestHasBody && !_dataSource._requestBodyProcessed ) { 107 debug(httpd) trace("receiving body on demand for form"); 108 loadBodyOnDemand(_dataSource); 109 } 110 return _form; 111 } 112 @property auto files() { 113 if ( _dataSource._readStarted ) { 114 throw new HTTPD_RequestException("Request body read() already started."); 115 } 116 if ( _dataSource._requestHasBody && !_dataSource._requestBodyProcessed ) { 117 debug(httpd) trace("receiving body on demand for form"); 118 loadBodyOnDemand(_dataSource); 119 } 120 return _files; 121 } 122 123 @property bool requestHasBody() pure { 124 if ( "content-length" in _requestHeaders ) { 125 return true; 126 } 127 if ( auto contentTransferEncoding = "transfer-encoding" in _requestHeaders ) { 128 if ( *contentTransferEncoding=="chunked" ) { 129 return true; 130 } 131 } 132 return false; 133 } 134 135 class _DataSource { 136 private { 137 NetworkStream _stream; 138 DataPipe!ubyte _bodyDecoder; 139 DecodeChunked _unChunker; 140 long _contentLength = 0; 141 long _receivedLength = 0; 142 ubyte[] _content; 143 bool _requestHasBody; // request has body 144 bool _requestBodyRecvInProgr; // loading body currently active 145 bool _requestBodyProcessed; // we processed body - this can happens only once 146 bool _requestBodyReceived; // all body data were received from network (we have to close socket if request terminated before all data received) 147 bool _readStarted; 148 } 149 bool empty() { 150 debug(httpd) tracef("datasource empty: %s", _content.length==0); 151 return _content.length==0; 152 } 153 ubyte[] front() { 154 return _content; 155 } 156 void popFront() { 157 debug(httpd) trace("datasource enters popFront"); 158 _content.length = 0; 159 if ( !_requestBodyRecvInProgr ) { 160 debug(httpd) trace("popFront called when dataSource is not active anymore"); 161 return; 162 } 163 while ( _bodyDecoder.empty && _stream && _stream.isOpen ) { 164 auto b = new ubyte[DSBUFFSIZE]; 165 auto read = _stream.receive(b); 166 if ( read == 0 ) { 167 debug(httpd) trace("stream closed when receiving in datasource"); 168 _bodyDecoder.flush(); 169 _requestBodyRecvInProgr = false; 170 break; 171 } 172 debug(httpd) tracef("place %d bytes to datasource", read); 173 _receivedLength += read; 174 _bodyDecoder.putNoCopy(b[0..read]); 175 if ( (_unChunker && _unChunker.done) 176 || (_contentLength > 0 && _receivedLength >= _contentLength) ) 177 { 178 debug(httpd) trace("request body reading complete (due contentLength or due last chunk consumed)"); 179 _bodyDecoder.flush(); 180 _requestBodyRecvInProgr = false; 181 _requestBodyReceived = true; 182 break; 183 } 184 } 185 _content = _bodyDecoder.getNoCopy().join(); 186 debug(httpd) tracef("%d bytes in content after popFront", _content.length); 187 } 188 /// 189 /// raplace current front with another value 190 /// 191 void unPop(ubyte[] data) { 192 assert(data.length > 0); 193 _content = data; 194 } 195 /// 196 /// Scan over input stream, 197 /// can return data from stream 198 /// acc - accumulator for receiving needle 199 /// return empty data if we receiving needle 200 /// if needle found in stream, then acc == needle 201 /// if end of stream happened, then eos = true 202 /// 203 ubyte[] scanUntilR(string needle, ref ubyte[] acc, out bool eos) { 204 auto d = needle.representation; 205 ubyte[] l; 206 207 while (!this.empty) { 208 auto c = this.front; 209 debug(httpd) tracef("on scan: %s", cast(string)c); 210 l = acc ~ c; 211 auto s = l.findSplit(d); 212 if ( s[1].length ) { 213 if ( s[2].length ) { 214 this.unPop(s[2]); 215 } else { 216 this.popFront; 217 } 218 acc = s[1]; 219 return s[0]; 220 } 221 auto i = min(l.length, d.length); 222 for(;i>0; i--) { 223 if ( l.endsWith(d[0..i]) ) { 224 acc = l[$-i..$]; 225 this.popFront; 226 return l[0..$-i]; 227 } 228 } 229 if ( i == 0 ) { 230 acc.length = 0; 231 this.popFront; 232 return l; 233 } 234 } 235 eos = true; // end of stream 236 acc.length = 0; 237 return l; 238 } 239 void scanUntil(F)(string needle, F f) { 240 auto d = needle.representation; 241 ubyte[] acc; 242 bool eos; // end of stream 243 244 while( !eos ) { 245 auto l = scanUntilR(needle, acc, eos); 246 debug(httpd) tracef("scanr returned <%s> and <%s>", cast(string)l, cast(string)acc); 247 f(l); 248 if ( acc == needle) { 249 return; 250 } 251 } 252 } 253 void skipUntil(string needle) { 254 auto d = needle.representation; 255 ubyte[] acc; 256 bool eos; // end of stream 257 258 while( !eos ) { 259 auto l = scanUntilR(needle, acc, eos); 260 debug(httpd) tracef("scanr returned <%s> and <%s>", cast(string)l, cast(string)acc); 261 if ( acc == needle) { 262 return; 263 } 264 } 265 } 266 } 267 268 auto createDataSource(string partialBody, NetworkStream stream) { 269 270 if ( !requestHasBody ) { 271 return new _DataSource(); 272 } 273 274 auto ds = new _DataSource(); 275 276 ds._requestHasBody = true; 277 ds._requestBodyRecvInProgr = true; 278 ds._bodyDecoder = new DataPipe!ubyte; 279 ds._stream = stream; 280 281 if ( auto contentLengthHeader = "content-length" in _requestHeaders ) { 282 ds._contentLength = to!long(*contentLengthHeader); 283 } 284 else if ( auto contentTransferEncoding = "transfer-encoding" in _requestHeaders ) { 285 if ( *contentTransferEncoding=="chunked" ) { 286 ds._unChunker = new DecodeChunked(); 287 ds._bodyDecoder.insert(ds._unChunker); 288 } 289 } 290 if ( partialBody.length ) { 291 ds._bodyDecoder.putNoCopy(cast(ubyte[])partialBody); 292 ds._receivedLength = (cast(ubyte[])partialBody).length; 293 } 294 while ( ds._bodyDecoder.empty ) { 295 auto b = new ubyte[DSBUFFSIZE]; 296 auto read = stream.receive(b); 297 if ( read == 0 ) { 298 debug(httpd) trace("stream closed when receiving in datasource"); 299 ds._requestBodyRecvInProgr = false; 300 return ds; 301 } 302 debug(httpd) tracef("place %d bytes to datasource", read); 303 ds._receivedLength += read; 304 ds._bodyDecoder.putNoCopy(b[0..read]); 305 } 306 ds._content = ds._bodyDecoder.getNoCopy().join(); 307 if ( ( ds._contentLength > 0 && ds._receivedLength >= ds._contentLength ) 308 || ( ds._unChunker && ds._unChunker.done) ) { 309 // all data received we need not wait any data from network 310 debug(httpd) trace("looks like we received complete request body together with request headers"); 311 ds._requestBodyRecvInProgr = false; 312 ds._requestBodyReceived = true; 313 } 314 debug(httpd) tracef("initial content: %d bytes", ds._content.length); 315 return ds; 316 } 317 @property auto contentType() { 318 if ( auto ct = "content-type" in _requestHeaders ) { 319 auto f = (*ct).split(";").map!strip; 320 return f[0]; 321 } 322 return null; 323 } 324 325 struct PartData { 326 // handler for each part data stream 327 _DataSource _ds; 328 string _boundary; 329 ubyte[] _content; 330 ubyte[] _acc; 331 bool _done; 332 bool _eos; 333 334 this(_DataSource ds, string boundary) { 335 _ds = ds; 336 _boundary = "\r\n" ~ boundary; 337 _content = _ds.scanUntilR(_boundary, _acc, _eos); 338 } 339 bool empty() { 340 return _content.length == 0; 341 } 342 auto front() { 343 return _content; 344 } 345 void popFront() { 346 _content.length = 0; 347 if ( _done ) { 348 return; 349 } 350 while( _content.length == 0 ) { 351 _content = _ds.scanUntilR(_boundary, _acc, _eos); 352 if ( _eos ) { 353 return; 354 } 355 if (_acc == _boundary) { 356 debug(httpd) tracef("part data done"); 357 _ds.skipUntil("\r\n"); 358 return; 359 } 360 } 361 } 362 } 363 struct Part { 364 _DataSource _ds; 365 string[string] _headers; 366 string _boundary; 367 368 this(_DataSource ds, string[string] h, string boundary) { 369 _ds = ds; 370 _headers = h; 371 _boundary = boundary; 372 } 373 @property string[string] headers() { 374 return _headers; 375 } 376 @property disposition() { 377 string[string] res; 378 auto d = "content-disposition" in _headers; 379 if ( !d ) { 380 return res; 381 } 382 (*d).split(";"). 383 filter!"a.indexOf('=')>0". 384 map! "a.strip.split('=')". 385 each!(p => res[p[0]] = urlDecode(p[1]).strip('"')); 386 return res; 387 } 388 @property data() { 389 return PartData(_ds, _boundary); 390 } 391 } 392 struct MultiPart { 393 string _boundary; 394 _DataSource _ds; 395 Part _part; 396 /* 397 --8a60ded0-ee76-4b6a-a1a0-dccaf93b92e7 398 Content-Disposition: form-data; name=Field1; 399 400 form field from memory 401 --8a60ded0-ee76-4b6a-a1a0-dccaf93b92e7 402 Content-Disposition: form-data; name=Field2; filename=data2 403 404 file field from memory 405 --8a60ded0-ee76-4b6a-a1a0-dccaf93b92e7 406 Content-Disposition: form-data; name=File1; filename=file1 407 Content-Type: application/octet-stream 408 409 file1 content 410 411 --8a60ded0-ee76-4b6a-a1a0-dccaf93b92e7 412 Content-Disposition: form-data; name=File2; filename=file2 413 Content-Type: application/octet-stream 414 415 file2 content 416 417 --8a60ded0-ee76-4b6a-a1a0-dccaf93b92e7-- 418 */ 419 int opApply(int delegate(Part p) dg) { 420 int result = 0; 421 while(!_ds.empty) { 422 result = dg(_part); 423 if ( result ) { 424 break; 425 } 426 auto headers = skipHeaders(); 427 _part = Part(_ds, headers, _boundary); 428 } 429 return result; 430 } 431 auto skipHeaders() { 432 ubyte[] buf; 433 string[string] headers; 434 435 debug(httpd) tracef("Search for headers"); 436 _ds.scanUntil("\r\n\r\n", delegate void (ubyte[] data) { 437 buf ~= data; 438 }); 439 foreach(h; buf.split('\n').map!"cast(string)a".map!strip.filter!"a.length") { 440 auto parsed = h.findSplit(":"); 441 headers[parsed[0].toLower] = parsed[2].strip; 442 } 443 debug(httpd) tracef("Headers: %s ", headers); 444 return headers; 445 } 446 /// 447 /// Find boundary from request headers, 448 /// skip to begin of the first part, 449 /// create first part(read/parse headers, stop on the body begin) 450 /// 451 this(HTTPD_Request rq) { 452 ubyte[] buf, rest; 453 string separator; 454 auto ct = "content-type" in rq._requestHeaders; 455 auto b = (*ct).split(";").map!"a.strip.split(`=`)".filter!"a[0].toLower==`boundary`"; 456 if ( b.empty ) { 457 throw new HTTPD_RequestException("Can't find 'boundary' in Content-Type %s".format(*ct)); 458 } 459 _boundary = "--" ~ b.front[1]; 460 _ds = rq._dataSource; 461 _ds.skipUntil(_boundary~"\r\n"); 462 auto headers = skipHeaders(); 463 _part = Part(_ds, headers, _boundary); 464 } 465 } 466 467 auto multiPartRead() { 468 return MultiPart(this); 469 } 470 471 auto read() { 472 if ( requestHasBody && _dataSource._requestBodyProcessed ) { 473 throw new HTTPD_RequestException("Request body already consumed by call to data/form/json"); 474 } 475 if ( _dataSource._readStarted ) { 476 throw new HTTPD_RequestException("Request body read() already started."); 477 } 478 _dataSource._readStarted = true; 479 return _dataSource; 480 } 481 482 void loadBodyOnDemand(ref _DataSource ds) { 483 ds._requestBodyProcessed = true; 484 debug(httpd) tracef("Process %s onDemand", contentType); 485 switch ( contentType ) { 486 case "application/json": 487 while(!ds.empty) { 488 debug(httpd) tracef("add %d bytes to json from dataSource", ds.front.length); 489 _json ~= cast(string)ds.front; 490 ds.popFront; 491 } 492 break; 493 case "application/x-www-form-urlencoded": 494 string qBody; 495 while(!ds.empty) { 496 debug(httpd) tracef("add %d bytes to json from dataSource", ds.front.length); 497 qBody ~= cast(string)ds.front; 498 ds.popFront; 499 } 500 _form = parseQuery(qBody); 501 break; 502 case "multipart/form-data": 503 debug(httpd) tracef("loading multiPart on demand"); 504 auto parts = multiPartRead(); 505 foreach(p; parts) { 506 auto disposition = p.disposition; 507 auto data = p.data.joiner.array; 508 509 if ( !("name" in disposition) ) { 510 continue; 511 } 512 if ( auto fn = "filename" in disposition ) { 513 _files[disposition["name"]] = data; 514 } else { 515 _form[disposition["name"]] = cast(string)data; 516 } 517 } 518 break; 519 default: 520 while(!ds.empty) { 521 debug(httpd) tracef("add %d bytes to data from dataSource", ds.front.length); 522 _data ~= ds.front; 523 ds.popFront; 524 } 525 break; 526 } 527 } 528 } 529 530 string[int] codes; 531 static this() { 532 codes = [ 533 200: "OK", 534 302: "Found", 535 401: "Unauthorized", 536 404: "Not found", 537 405: "Method not allowed", 538 500: "Server error" 539 ]; 540 } 541 enum Compression : int { 542 no = 0, 543 gzip = 1, 544 deflate = 2, 545 yes = gzip|deflate, 546 }; 547 548 auto response(C)(HTTPD_Request rq, C content, ushort code = 200) 549 if ( isSomeString!C 550 || (__traits(compiles, cast(ubyte[])content)) 551 || (__traits(compiles, cast(ubyte[])content.front)) 552 ) 553 { 554 return new HTTPD_Response!C(rq, content, code); 555 } 556 557 class _Response { 558 abstract void send(NetworkStream); 559 abstract ref string[string] headers(); 560 } 561 562 class HTTPD_Response(C) : _Response { 563 ushort _status = 200; 564 string[string] _headers; 565 C _content; 566 Compression _compression = Compression.no; 567 HTTPD_Request _request; 568 Cookie[] _cookies; 569 570 mixin(Getter_Setter!ushort("status")); 571 mixin(Getter!Compression("compression")); 572 @property void compress(Compression c = Compression.yes) { 573 _compression = c; 574 } 575 this(ref HTTPD_Request request, C content, ushort status = 200) { 576 _status = status; 577 _request = request; 578 _content = content; 579 } 580 override ref string[string] headers() @property { 581 return _headers; 582 } 583 ref Cookie[] cookies() { 584 return _cookies; 585 } 586 void content(C)(C c) @property { 587 _content = makeContent(c); 588 } 589 auto selectCompression(in HTTPD_Request rq, in HTTPD_Response rs) { 590 if ( auto acceptEncodings = "accept-encoding" in rq.requestHeaders) { 591 auto heAccept = (*acceptEncodings).split(",").map!strip; 592 if ( (rs.compression & Compression.gzip) && heAccept.canFind("gzip")) { 593 return "gzip"; 594 } 595 if ( (compression & Compression.deflate) && heAccept.canFind("deflate")) { 596 return "deflate"; 597 } 598 } 599 return null; 600 } 601 void sendCookies(NetworkStream stream) { 602 if ( _cookies.length ) { 603 foreach(c; _cookies) { 604 auto setCookie = "Set-Cookie: %s=%s; Path=%s\r\n".format(c.attr, c.value, c.path); 605 stream.send(setCookie); 606 } 607 } 608 } 609 final override void send(NetworkStream stream) { 610 import std.zlib; 611 auto statusLine = "HTTP/1.1 " ~ to!string(_status) ~ " " ~ codes.get(_status, "Unspecified") ~ " \r\n"; 612 613 if ( !stream.isOpen || !stream.isConnected ) { 614 debug(httpd) tracef("Will not send to closed connection"); 615 return; 616 } 617 debug(httpd) tracef("sending statusLine: %s", statusLine.stripRight); 618 stream.send(statusLine); 619 620 auto comp = selectCompression(_request, this); 621 622 static if ( isSomeString!C || __traits(compiles, cast(ubyte[])_content) ) { 623 ubyte[] data; 624 if ( comp ) { 625 _headers["content-encoding"] = comp; 626 Compress compressor; 627 final switch (comp) { 628 case "gzip": // gzip 629 compressor = new Compress(6, HeaderFormat.gzip); 630 break; 631 case "deflate": // deflate 632 compressor = new Compress(6, HeaderFormat.deflate); 633 break; 634 } 635 data = cast(ubyte[])compressor.compress(_content); 636 data ~= cast(ubyte[])compressor.flush(); 637 } 638 else { 639 data = cast(ubyte[])_content; 640 } 641 _headers["content-length"] = to!string(data.length); 642 foreach(p; _headers.byKeyValue) { 643 stream.send(p.key ~ ": " ~ p.value ~ "\r\n"); 644 } 645 if ( _cookies.length ) { 646 sendCookies(stream); 647 } 648 stream.send("\r\n"); 649 if (_request.method == "HEAD") { 650 return; 651 } 652 stream.send(data); 653 } 654 else { 655 _headers["transfer-encoding"] = "chunked"; 656 Compress compressor; 657 if ( comp !is null ) { 658 _headers["content-encoding"] = comp; 659 final switch (comp) { 660 case "gzip": // gzip 661 compressor = new Compress(6, HeaderFormat.gzip); 662 break; 663 case "deflate": // deflate 664 compressor = new Compress(6, HeaderFormat.deflate); 665 break; 666 } 667 } 668 foreach(p; _headers.byKeyValue) { 669 stream.send(p.key ~ ": " ~ p.value ~ "\r\n"); 670 } 671 if ( _cookies.length ) { 672 sendCookies(stream); 673 } 674 stream.send("\r\n"); 675 if (_request.method == "HEAD") { 676 return; 677 } 678 ubyte[] data; 679 while(!_content.empty) { 680 auto chunk = cast(ubyte[])_content.front; 681 _content.popFront; 682 683 if ( compressor ) { 684 data ~= cast(ubyte[])compressor.compress(chunk); 685 if ( data.length == 0 ) { 686 continue; 687 } 688 } else { 689 data = chunk; 690 } 691 stream.send("%x\r\n".format(data.length)); 692 stream.send(data); 693 stream.send("\r\n"); 694 data.length = 0; 695 } 696 if ( compressor ) { 697 data = cast(ubyte[])compressor.flush(); 698 stream.send("%x\r\n".format(data.length)); 699 stream.send(data); 700 stream.send("\r\n"); 701 } 702 stream.send("0\r\n\r\n"); 703 } 704 } 705 } 706 707 alias Handler = _Response delegate(in App app, ref HTTPD_Request, RequestArgs); 708 709 struct RequestArgs { 710 private { 711 Captures!string _captures = void; 712 string _string; 713 } 714 this(Captures!string c) @nogc @safe nothrow { 715 _captures = c; 716 } 717 this(string s) @nogc @safe pure nothrow { 718 _string = s; 719 } 720 bool empty() @nogc @safe pure nothrow { 721 return _captures.empty && _string is null; 722 } 723 string opIndex(string s) @safe pure { 724 return _captures[s]; 725 } 726 string opIndex(size_t i) @safe pure { 727 if ( _string && i==0 ) { 728 return _string; 729 } 730 return _captures[i]; 731 } 732 } 733 734 auto exactRoute(string s, Handler h) @safe pure nothrow { 735 return new ExactRoute(s, h); 736 } 737 738 auto regexRoute(string s, Handler h) @safe { 739 return new RegexRoute(s, h); 740 } 741 742 class Route { 743 Handler _handler; 744 string _origin; 745 746 abstract RequestArgs match(string) { 747 return RequestArgs(); 748 }; 749 final Handler handler() { 750 return _handler; 751 } 752 final string origin() { 753 return _origin; 754 } 755 } 756 757 class ExactRoute: Route { 758 759 this(string s, Handler h) @safe pure nothrow { 760 _origin = s; 761 _handler = h; 762 } 763 final override RequestArgs match(string input) { 764 if ( input == _origin ) { 765 debug(httpd) tracef("%s matches %s", input, _origin); 766 return RequestArgs(input); 767 } 768 return RequestArgs(); 769 } 770 } 771 class RegexRoute: Route { 772 Regex!char _re; 773 774 this(string r, Handler h) @safe { 775 _origin = r; 776 _handler = h; 777 _re = regex(r); 778 } 779 final override RequestArgs match(string input) { 780 auto m = matchFirst(input, _re); 781 debug(httpd) if (!m.empty) {tracef("%s matches %s", input, _origin);} 782 return RequestArgs(m); 783 } 784 } 785 786 struct Router { 787 alias RouteMatch = Tuple!(Handler, "handler", RequestArgs, "args"); 788 private Route[] _routes; 789 790 void addRoute(Route r) { 791 _routes ~= r; 792 } 793 auto getRoute(string path) { 794 RouteMatch match; 795 foreach(r; _routes) { 796 auto args = r.match(path); 797 if (!args.empty) { 798 match.handler = r.handler; 799 match.args = args; 800 break; 801 } 802 } 803 return match; 804 } 805 } 806 807 private auto parseQuery(string query) { 808 /// TODO 809 /// switch to return dict of 810 /// struct QueryParam { 811 /// private: 812 /// string name; 813 /// string[] value; 814 /// public: 815 /// uint length() {return value.length;} 816 /// string toString() {return value[0];} 817 /// string[] toArray() {return value;} 818 /// } 819 debug (httpd) tracef("query: %s", query); 820 string[string] q; 821 if ( !query ) { 822 return q; 823 } 824 if ( query[0] == '?') { 825 query = query[1..$]; 826 } 827 string[][] parsed = query.splitter("&"). 828 map!(s => s.split("=")). 829 filter!"a.length==2". 830 map!(p => [urlDecode(p[0]), urlDecode(p[1])]). 831 array; 832 833 auto grouped = sort!"a[0]<b[0]"(parsed).assumeSorted!"a[0]<b[0]".groupBy(); 834 foreach(g; grouped) { 835 string key = g.front[0]; 836 string val; 837 auto vals = g.map!"a[1]".array; 838 if (vals.length == 1) { 839 val = vals[0]; 840 } 841 if (vals.length > 1) { 842 val = to!string(vals); 843 } 844 q[key] = val; 845 } 846 return q; 847 } 848 849 private bool headersReceived(in ubyte[] data, ref Buffer!ubyte buffer, out string separator) @safe { 850 foreach(s; ["\r\n\r\n", "\n\n"]) { 851 if ( data.canFind(s) || buffer.canFind(s) ) { 852 separator = s; 853 return true; 854 } 855 } 856 return false; 857 } 858 859 private void parseRequestHeaders(in App app, ref HTTPD_Request rq, string buffer) { 860 string lastHeader; 861 auto lines = buffer.splitLines.map!stripRight; 862 rq.requestLine = lines[0]; 863 if ( lines.count == 1) { 864 return; 865 } 866 foreach(line; lines[1..$]) { 867 if ( !line.length ) { 868 continue; 869 } 870 if ( line[0] == ' ' || line[0] == '\t' ) { 871 // unfolding https://tools.ietf.org/html/rfc822#section-3.1 872 if ( auto prevValue = lastHeader in rq.requestHeaders) { 873 *prevValue ~= line; 874 } 875 continue; 876 } 877 auto parsed = line.findSplit(":"); 878 auto header = parsed[0].toLower; 879 auto value = parsed[2].strip; 880 lastHeader = header; 881 if ( auto h = header in rq.requestHeaders ) { 882 *h ~= "; " ~ value; 883 } else { 884 rq.requestHeaders[header] = value; 885 } 886 debug(httpd) tracef("%s: %s", header, value); 887 } 888 auto rqlFields = rq.requestLine.split(" "); 889 debug (httpd) tracef("rqLine %s", rq.requestLine); 890 rq.method = rqlFields[0]; 891 auto scheme = app.useSSL? 892 "https://": 893 "http://"; 894 if ( "host" in rq.requestHeaders ) { 895 rq.uri = URI(scheme ~ rq.requestHeaders["host"] ~ rqlFields[1]); 896 } else { 897 rq.uri = URI(scheme ~ app.host ~ rqlFields[1]); 898 } 899 rq.path = rq.uri.path; 900 rq.query = parseQuery(rq.uri.query); 901 debug (httpd) tracef("path: %s", rq.path); 902 debug (httpd) tracef("query: %s", rq.query); 903 // 904 // now analyze what we have 905 // 906 auto header = "connection" in rq.requestHeaders; 907 if ( header && toLower(*header) == "keep-alive") { 908 rq.keepAlive = true; 909 } 910 auto cookies = "cookie" in rq.requestHeaders; 911 if ( cookies ) { 912 (*cookies).split(';'). 913 map!"strip(a).split('=')". 914 filter!(kv => kv.length==2). 915 each!(kv => rq._cookies[kv[0]] = kv[1]); 916 } 917 } 918 919 private auto read_request(in App app, NetworkStream stream) { 920 HTTPD_Request rq; 921 Buffer!ubyte input; 922 string separator; 923 924 while( true ) { 925 ubyte[] b = new ubyte[app.bufferSize]; 926 auto read = stream.receive(b); 927 928 if ( read == 0 ) { 929 return rq; 930 } 931 debug(httpd) tracef("received %d bytes", read); 932 input.putNoCopy(b[0..read]); 933 934 if ( headersReceived(b, input, separator) ) { 935 break; 936 } 937 938 if ( input.length >= app.maxHeadersSize ) { 939 throw new HTTPD_RequestException("Request headers length %d too large".format(input.length)); 940 } 941 } 942 debug(httpd) trace("Headers received"); 943 auto s = input.data!(string).findSplit(separator); 944 auto requestHeaders = s[0]; 945 debug(httpd) tracef("Headers: %s", cast(string)requestHeaders); 946 parseRequestHeaders(app, rq, requestHeaders); 947 debug(httpd) trace("Headers parsed"); 948 949 rq._dataSource = rq.createDataSource(s[2], stream); 950 951 return rq; 952 } 953 954 void processor(in App app, HTTPD httpd, NetworkStream stream) { 955 stream.readTimeout = app.timeout; 956 HTTPD_Request rq; 957 _Response rs; 958 scope (exit) { 959 if ( stream.isOpen ) { 960 stream.close(); 961 } 962 } 963 uint rqLimit = max(app.rqLimit, 1); 964 try { 965 while ( rqLimit > 0 ) { 966 rq = read_request(app, stream); 967 if ( !httpd._running || !rq.requestLine.length ) { 968 return; 969 } 970 auto match = httpd._router.getRoute(rq.path); 971 if ( !match.handler ) { 972 // return 404; 973 debug (httpd) tracef("Route not found for %s", rq.path); 974 rs = response(rq, "Requested path %s not found".format(rq.path), 404); 975 break; 976 } 977 auto handler = match.handler; 978 rs = handler(app, rq, match.args); 979 if ( !stream.isOpen ) { 980 debug(httpd) tracef("Request handler closed connection"); 981 return; 982 } 983 if ( rq.keepAlive && rqLimit > 1 ) { 984 rs.headers["Connection"] = "Keep-Alive"; 985 } 986 if ( rq._dataSource._requestHasBody && !rq._dataSource._requestBodyReceived ) { 987 // for some reason some part of the request body still not received, and it will 988 // stay on the way of next request if this is keep-Alive session, 989 // so we must abort this connection anyway. 990 debug(httpd) trace("Request handler did not consumed whole request body. We have to close connection after sending response."); 991 rs.send(stream); 992 return; 993 } 994 rs.send(stream); 995 --rqLimit; 996 if ( !rq.keepAlive || rqLimit==0 ) { 997 debug(httpd) trace("Finished with that connection"); 998 return; 999 } 1000 debug(httpd) trace("Continue with keepalive request"); 1001 rq = rq.init; 1002 } 1003 } 1004 catch (HTTPD_RequestException e) { 1005 debug(httpd) error("Request exception: " ~ e.msg); 1006 rs = response(rq, "Request exception:\n" ~ e.msg, 500); 1007 } 1008 catch (TimeoutException e) { 1009 debug(httpd) { 1010 if ( rq.requestLine ) { 1011 error("Timeout reading/writing to client"); 1012 } 1013 } 1014 } 1015 catch (Exception e) { 1016 debug(httpd) error("Unexpected Exception " ~ e.msg); 1017 rs = response(rq, "Unexpected exception:\n" ~ e.msg, 500); 1018 } 1019 catch (Error e) { 1020 error(e.msg, e.info); 1021 rs = response(rq, "Unexpected error:\n" ~ e.msg, 500); 1022 } 1023 try { 1024 if ( stream.isOpen ) { 1025 rs.send(stream); 1026 } 1027 } 1028 catch (Exception e) { 1029 infof("Exception when send %s", e.msg); 1030 } 1031 catch (Error e) { 1032 error("Error sending response: " ~ e.msg); 1033 } 1034 } 1035 1036 class HTTPD 1037 { 1038 private { 1039 TaskPool _server; 1040 __gshared bool _running; 1041 Router _router; 1042 App _app; 1043 } 1044 auto ref addRoute(Route r) { 1045 _router.addRoute(r); 1046 return this; 1047 } 1048 static NetworkStream openStream(in App app) { 1049 auto host = app.host; 1050 auto port = app.port; 1051 Address[] addresses; 1052 SSLOptions _sslOptions; 1053 1054 try { 1055 addresses = getAddress(host, port); 1056 } catch (Exception e) { 1057 throw new ConnectError("Can't resolve name when connect to %s:%d: %s".format(host, port, e.msg)); 1058 } 1059 auto tcpStream = app.useSSL? 1060 new SSLStream(_sslOptions): 1061 new TCPStream(); 1062 tcpStream.open(addresses[0].addressFamily); 1063 return tcpStream; 1064 } 1065 static void run(in App app, HTTPD httpd) { 1066 Address[] addresses; 1067 try { 1068 addresses = getAddress(app.host, app.port); 1069 } catch (Exception e) { 1070 throw new ConnectError("Can't resolve name when connect to %s:%d: %s".format(app.host, app.port, e.msg)); 1071 } 1072 auto tcpStream = openStream(app); 1073 tcpStream.reuseAddr(true); 1074 tcpStream.bind(addresses[0]); 1075 tcpStream.listen(128); 1076 defaultPoolThreads(64); 1077 auto pool = taskPool(); 1078 _running = true; 1079 while ( _running ) { 1080 auto stream = tcpStream.accept(); 1081 if ( _running ) { 1082 auto connHandler = task!processor(app, httpd, stream); 1083 pool.put(connHandler); 1084 } else { 1085 tcpStream.close(); 1086 break; 1087 } 1088 } 1089 } 1090 void app(App a) { 1091 _app = a; 1092 } 1093 void start() { 1094 defaultPoolThreads(64); 1095 _server = taskPool(); 1096 auto t = task!run(_app, this); 1097 _server.put(t); 1098 Thread.sleep(500.msecs); 1099 } 1100 void start(App app) { 1101 defaultPoolThreads(64); 1102 _app = app; 1103 _server = taskPool(); 1104 auto t = task!run(_app, this); 1105 _server.put(t); 1106 Thread.sleep(500.msecs); 1107 } 1108 void stop() { 1109 if ( !_running ) { 1110 return; 1111 } 1112 _running = false; 1113 try { 1114 auto s = openStream(_app); 1115 s.connect(_app.host, _app.port); 1116 } catch (Exception e) { 1117 } 1118 // _server.stop(); 1119 } 1120 } 1121 1122 struct App { 1123 private { 1124 string _name; 1125 string _host; 1126 ushort _port; 1127 Duration _timeout = 30.seconds; 1128 size_t _bufferSize = 16*1024; 1129 size_t _maxHeadersSize = 32*1024; 1130 bool _useSSL = false; 1131 uint _rqLimit = 10; // keepalive requestst per connection 1132 Router _router; 1133 } 1134 mixin(Getter_Setter!string("name")); 1135 mixin(Getter_Setter!string("host")); 1136 mixin(Getter_Setter!ushort("port")); 1137 mixin(Getter_Setter!size_t("bufferSize")); 1138 mixin(Getter_Setter!size_t("maxHeadersSize")); 1139 mixin(Getter_Setter!Duration("timeout")); 1140 mixin(Getter_Setter!bool("useSSL")); 1141 mixin(Getter_Setter!uint("rqLimit")); 1142 this(string name) { 1143 _name = name; 1144 } 1145 } 1146 1147 1148 version(none) private unittest { 1149 import std.json; 1150 import std.conv; 1151 import requests.http: HTTPRequest, TimeoutException, BasicAuthentication, queryParams, MultipartForm, formData; 1152 globalLogLevel(LogLevel.info); 1153 1154 static auto buildReply(ref HTTPD_Request rq) { 1155 auto args = JSONValue(rq.query); 1156 auto headers = JSONValue(rq.requestHeaders); 1157 auto url = JSONValue(rq.uri.uri); 1158 auto json = JSONValue(rq.json); 1159 auto data = JSONValue(rq.data); 1160 auto form = JSONValue(rq.form); 1161 auto files = JSONValue(rq.files); 1162 auto reply = JSONValue(["args":args, "headers": headers, "json": json, "url": url, "data": data, "form": form, "files": files]); 1163 return reply.toString(); 1164 } 1165 1166 Router router; 1167 router.addRoute(exactRoute(r"/get", null)); 1168 router.addRoute(regexRoute(r"/get/(?P<param>\d+)", null)); 1169 auto r = router.getRoute(r"/get"); 1170 assert(!r.args.empty); 1171 r = router.getRoute(r"/post"); 1172 assert(r.args.empty); 1173 1174 r = router.getRoute(r"/get/333"); 1175 assert(!r.args.empty); 1176 assert(r.args["param"]=="333"); 1177 r = router.getRoute(r"/get/aaa"); 1178 assert(r.args.empty); 1179 1180 HTTPD_Request rq; 1181 string headers = "GET /get?a=b&list[]=1&c=d&list[]=2 HTTP/1.1\n" ~ 1182 "Host: host\n" ~ 1183 "X-Test: test1\n" ~ 1184 " test2\n" ~ 1185 "Content-Length: 1\n"; 1186 parseRequestHeaders(App(), rq, headers); 1187 assert(rq.requestHeaders["x-test"] == "test1 test2"); 1188 assert(rq.requestHeaders["host"] == "host"); 1189 assert(rq.path == "/get"); 1190 assert(rq.query["a"] == "b"); 1191 assert(rq.query["c"] == "d"); 1192 assert(rq.query["list[]"] == `["1", "2"]`); 1193 auto root(in App app, ref HTTPD_Request rq, RequestArgs args) { 1194 debug (httpd) trace("handler / called"); 1195 auto rs = response(rq, buildReply(rq)); 1196 rs.headers["Content-Type"] = "application/json"; 1197 return rs; 1198 } 1199 auto get(in App app, ref HTTPD_Request rq, RequestArgs args) { 1200 debug (httpd) trace("handler /get called"); 1201 auto rs = response(rq, buildReply(rq)); 1202 rs.headers["Content-Type"] = "application/json"; 1203 return rs; 1204 } 1205 auto basicAuth(in App app, ref HTTPD_Request rq, RequestArgs args) { 1206 import std.base64; 1207 auto user = args["user"]; 1208 auto password= args["password"]; 1209 auto auth = cast(string)Base64.decode(rq.requestHeaders["authorization"].split()[1]); 1210 auto up = auth.split(":"); 1211 short status; 1212 if ( up[0]==user && up[1]==password) { 1213 status = 200; 1214 } else { 1215 status = 401; 1216 } 1217 auto rs = response(rq, buildReply(rq), status); 1218 rs.headers["Content-Type"] = "application/json"; 1219 return rs; 1220 } 1221 auto rredir(in App app, ref HTTPD_Request rq, RequestArgs args) { 1222 auto rs = response(rq, buildReply(rq)); 1223 auto redirects = to!long(args["redirects"]); 1224 if ( redirects > 1 ) { 1225 rs.headers["Location"] = "/relative-redirect/%d".format(redirects-1); 1226 } else { 1227 rs.headers["Location"] = "/get"; 1228 } 1229 rs.status = 302; 1230 return rs; 1231 } 1232 auto aredir(in App app, ref HTTPD_Request rq, RequestArgs args) { 1233 auto rs = response(rq, buildReply(rq)); 1234 auto redirects = to!long(args["redirects"]); 1235 if ( redirects > 1 ) { 1236 rs.headers["Location"] = "http://127.0.0.1:8081/absolute-redirect/%d".format(redirects-1); 1237 } else { 1238 rs.headers["Location"] = "http://127.0.0.1:8081/get"; 1239 } 1240 rs.status = 302; 1241 return rs; 1242 } 1243 auto delay(in App app, ref HTTPD_Request rq, RequestArgs args) { 1244 auto delay = dur!"seconds"(to!long(args["delay"])); 1245 Thread.sleep(delay); 1246 auto rs = response(rq, buildReply(rq)); 1247 rs.headers["Content-Type"] = "application/json"; 1248 return rs; 1249 } 1250 auto gzip(in App app, ref HTTPD_Request rq, RequestArgs args) { 1251 auto rs = response(rq, buildReply(rq)); 1252 rs.compress(Compression.gzip); 1253 rs.headers["Content-Type"] = "application/json"; 1254 return rs; 1255 } 1256 auto deflate(in App app, ref HTTPD_Request rq, RequestArgs args) { 1257 auto rs = response(rq, buildReply(rq)); 1258 rs.compress(Compression.deflate); 1259 return rs; 1260 } 1261 auto range(in App app, ref HTTPD_Request rq, RequestArgs args) { 1262 auto size = to!size_t(args["size"]); 1263 auto rs = response(rq, new ubyte[size].chunks(16)); 1264 rs.compress(Compression.yes); 1265 return rs; 1266 } 1267 auto head(in App app, ref HTTPD_Request rq, RequestArgs args) { 1268 if ( rq.method != "HEAD") { 1269 auto rs = response(rq, "Illegal method %s".format(rq.method), 405); 1270 return rs; 1271 } 1272 else { 1273 auto rs = response(rq, buildReply(rq)); 1274 rs.compress(Compression.yes); 1275 return rs; 1276 } 1277 } 1278 auto del(in App app, ref HTTPD_Request rq, RequestArgs args) { 1279 if ( rq.method != "DELETE") { 1280 auto rs = response(rq, "Illegal method %s".format(rq.method), 405); 1281 return rs; 1282 } 1283 else { 1284 auto rs = response(rq, buildReply(rq)); 1285 return rs; 1286 } 1287 } 1288 auto post(in App app, ref HTTPD_Request rq, RequestArgs args) { 1289 auto rs = response(rq, buildReply(rq)); 1290 return rs; 1291 } 1292 auto postIter(in App app, ref HTTPD_Request rq, RequestArgs args) { 1293 int c; 1294 1295 if ( rq.contentType == "multipart/form-data" ) { 1296 auto parts = rq.multiPartRead(); 1297 foreach(p; parts) { 1298 auto disposition = p.disposition; 1299 c += p.data.joiner.count; 1300 } 1301 auto rs = response(rq, "%d".format(c)); 1302 return rs; 1303 } 1304 else { 1305 auto r = rq.read(); 1306 while ( !r.empty ) { 1307 c += r.front.length; 1308 r.popFront; 1309 } 1310 auto rs = response(rq, "%d".format(c)); 1311 return rs; 1312 } 1313 } 1314 auto read(in App app, ref HTTPD_Request rq, RequestArgs args) { 1315 auto r = rq.read(); 1316 int c; 1317 while ( !r.empty ) { 1318 c += r.front.length; 1319 r.popFront; 1320 } 1321 auto rs = response(rq, "%d".format(c)); 1322 return rs; 1323 } 1324 auto readf1(in App app, ref HTTPD_Request rq, RequestArgs args) { 1325 // now call to read must throw exception 1326 auto r = rq.read(); 1327 int c; 1328 while ( !r.empty ) { 1329 c += r.front.length; 1330 r.popFront; 1331 break; 1332 } 1333 auto rs = response(rq, "%d".format(c)); 1334 return rs; 1335 } 1336 auto cookiesSet(in App app, ref HTTPD_Request rq, RequestArgs args) { 1337 Cookie[] cookies; 1338 foreach(p; rq.query.byKeyValue) { 1339 cookies ~= Cookie("/cookies", rq.requestHeaders["host"], p.key, p.value); 1340 } 1341 auto rs = response(rq, buildReply(rq), 302); 1342 rs.headers["Location"] = "/cookies"; 1343 rs.cookies = cookies; 1344 return rs; 1345 } 1346 auto cookies(in App app, ref HTTPD_Request rq, RequestArgs args) { 1347 auto cookies = ["cookies": JSONValue(rq.cookies)]; 1348 auto rs = response(rq, JSONValue(cookies).toString); 1349 return rs; 1350 } 1351 1352 auto httpbin = App("httpbin"); 1353 1354 httpbin.port = 8081; 1355 httpbin.host = "127.0.0.1"; 1356 1357 httpbin.timeout = 10.seconds; 1358 HTTPD server = new HTTPD(); 1359 1360 server.addRoute(exactRoute(r"/", &root)). 1361 addRoute(exactRoute(r"/get", &get)). 1362 addRoute(regexRoute(r"/delay/(?P<delay>\d+)", &delay)). 1363 addRoute(regexRoute(r"/relative-redirect/(?P<redirects>\d+)", &rredir)). 1364 addRoute(regexRoute(r"/absolute-redirect/(?P<redirects>\d+)", &aredir)). 1365 addRoute(regexRoute(r"/basic-auth/(?P<user>[^/]+)/(?P<password>[^/]+)", &basicAuth)). 1366 addRoute(exactRoute(r"/gzip", &gzip)). 1367 addRoute(exactRoute(r"/deflate", &deflate)). 1368 addRoute(regexRoute(r"/range/(?P<size>\d+)", &range)). 1369 addRoute(exactRoute(r"/cookies/set", &cookiesSet)). 1370 addRoute(exactRoute(r"/cookies", &cookies)). 1371 addRoute(exactRoute(r"/head", &head)). 1372 addRoute(exactRoute(r"/delete", &del)). 1373 addRoute(exactRoute(r"/read", &read)). 1374 addRoute(exactRoute(r"/readf1", &readf1)). 1375 addRoute(exactRoute(r"/post", &post)). 1376 addRoute(exactRoute(r"/postIter", &postIter)); 1377 1378 server.start(httpbin); 1379 scope(exit) { 1380 server.stop(); 1381 } 1382 auto request = HTTPRequest(); 1383 1384 globalLogLevel(LogLevel.info); 1385 auto httpbin_url = "http://%s:%d/".format(httpbin.host, httpbin.port); 1386 request.timeout = 5.seconds; 1387 request.keepAlive = true; 1388 info("httpd Check GET"); 1389 auto rs = request.get(httpbin_url); 1390 assert(rs.code == 200); 1391 assert(rs.responseBody.length > 0); 1392 auto content = rs.responseBody.data!string; 1393 auto json = parseJSON(cast(string)content); 1394 assert(json.object["url"].str == httpbin_url); 1395 1396 info("httpd Check GET with parameters"); 1397 rs = request.get(httpbin_url ~ "get", ["c":" d", "a":"b"]); 1398 assert(rs.code == 200); 1399 json = parseJSON(cast(string)rs.responseBody.data).object["args"].object; 1400 assert(json["a"].str == "b"); 1401 assert(json["c"].str == " d"); 1402 1403 info("httpd Check relative redirect"); 1404 rs = request.get(httpbin_url ~ "relative-redirect/2"); 1405 assert(rs.history.length == 2); 1406 assert(rs.code==200); 1407 1408 info("httpd Check absolute redirect"); 1409 rs = request.get(httpbin_url ~ "absolute-redirect/2"); 1410 assert(rs.history.length == 2); 1411 assert(rs.code==200); 1412 1413 info("httpd Check basic auth"); 1414 request.authenticator = new BasicAuthentication("user", "password"); 1415 rs = request.get(httpbin_url ~ "basic-auth/user/password"); 1416 assert(rs.code==200); 1417 request.authenticator = null; 1418 1419 info("httpd Check timeout"); 1420 request.timeout = 1.seconds; 1421 assertThrown!TimeoutException(request.get(httpbin_url ~ "delay/2")); 1422 Thread.sleep(1.seconds); 1423 request.timeout = 30.seconds; 1424 1425 info("httpd Check gzip"); 1426 rs = request.get(httpbin_url ~ "gzip"); 1427 assert(rs.code==200); 1428 json = parseJSON(cast(string)rs.responseBody); 1429 assert(json.object["url"].str == httpbin_url ~ "gzip"); 1430 1431 info("httpd Check deflate"); 1432 rs = request.get(httpbin_url ~ "deflate"); 1433 assert(rs.code==200); 1434 json = parseJSON(cast(string)rs.responseBody); 1435 assert(json.object["url"].str == httpbin_url ~ "deflate"); 1436 1437 info("httpd Check range"); 1438 rs = request.get(httpbin_url ~ "range/1023"); 1439 assert(rs.code==200); 1440 assert(rs.responseBody.length == 1023); 1441 1442 info("httpd Check HEAD"); 1443 rs = request.exec!"HEAD"(httpbin_url ~ "head"); 1444 assert(rs.code==200); 1445 assert(rs.responseBody.length == 0); 1446 1447 info("httpd Check DELETE"); 1448 rs = request.exec!"DELETE"(httpbin_url ~ "delete"); 1449 assert(rs.code==200); 1450 1451 info("httpd Check POST json"); 1452 rs = request.post(httpbin_url ~ "post?b=x", `{"a":"b", "c":[1,2,3]}`, "application/json"); 1453 json = parseJSON(cast(string)rs.responseBody); 1454 auto rqJson = parseJSON(json.object["json"].str); 1455 assert(rqJson.object["a"].str == "b"); 1456 assert(equal([1,2,3], rqJson.object["c"].array.map!"a.integer")); 1457 1458 info("httpd Check POST json/chunked body"); 1459 rs = request.post(httpbin_url ~ "post?b=x", [`{"a":"b",`,` "c":[1,2,3]}`], "application/json"); 1460 json = parseJSON(cast(string)rs.responseBody); 1461 assert(json.object["args"].object["b"].str == "x"); 1462 rqJson = parseJSON(json.object["json"].str); 1463 assert(rqJson.object["a"].str == "b"); 1464 assert(equal([1,2,3], rqJson.object["c"].array.map!"a.integer")); 1465 1466 rs = request.post(httpbin_url ~ "post", "0123456789".repeat(32)); 1467 json = parseJSON(cast(string)rs.responseBody); 1468 assert(equal(json.object["data"].array.map!"a.integer", "0123456789".repeat(32).join)); 1469 1470 info("httpd Check POST with params"); 1471 rs = request.post(httpbin_url ~ "post", queryParams("b", 2, "a", "A")); 1472 assert(rs.code==200); 1473 auto data = parseJSON(cast(string)rs.responseBody).object["form"].object; 1474 assert((data["a"].str == "A")); 1475 assert((data["b"].str == "2")); 1476 1477 // this is tests for httpd read() interface 1478 info("httpd Check POST/iterating over body"); 1479 rs = request.post(httpbin_url ~ "read", "0123456789".repeat(1500)); 1480 assert(equal(rs.responseBody, "15000")); 1481 1482 { 1483 request.keepAlive = true; 1484 // this is test on how we can handle keepalive session when previous request leave unread data in socket 1485 try { 1486 rs = request.post(httpbin_url ~ "readf1", "0123456789".repeat(1500)); 1487 } 1488 catch (Exception e) { 1489 // this can fail as httpd will close connection prematurely 1490 } 1491 // but next idempotent request must succeed 1492 rs = request.get(httpbin_url ~ "get"); 1493 assert(rs.code == 200); 1494 } 1495 // 1496 { 1497 info("httpd Check POST/multipart form"); 1498 import std.file; 1499 import std.path; 1500 auto tmpd = tempDir(); 1501 auto tmpfname1 = tmpd ~ dirSeparator ~ "request_test1.txt"; 1502 auto f = File(tmpfname1, "wb"); 1503 f.rawWrite("file1 content\n"); 1504 f.close(); 1505 auto tmpfname2 = tmpd ~ dirSeparator ~ "request_test2.txt"; 1506 f = File(tmpfname2, "wb"); 1507 f.rawWrite("file2 content\n"); 1508 f.close(); 1509 /// 1510 /// Ok, files ready. 1511 /// Now we will prepare Form data 1512 /// 1513 File f1 = File(tmpfname1, "rb"); 1514 File f2 = File(tmpfname2, "rb"); 1515 scope(exit) { 1516 f1.close(); 1517 f2.close(); 1518 } 1519 /// 1520 /// for each part we have to set field name, source (ubyte array or opened file) and optional filename and content-type 1521 /// 1522 MultipartForm form = MultipartForm(). 1523 add(formData("Field1", cast(ubyte[])"form field from memory")). 1524 add(formData("Field2", cast(ubyte[])"file field from memory", ["filename":"data2"])). 1525 add(formData("Field3", cast(ubyte[])`{"a":"b"}`, ["Content-Type": "application/json"])). 1526 add(formData("File1", f1, ["filename":"file1", "Content-Type": "application/octet-stream"])). 1527 add(formData("File2", f2, ["filename":"file2", "Content-Type": "application/octet-stream"])); 1528 /// everything ready, send request 1529 rs = request.post(httpbin_url ~ "post?a=b", form); 1530 /* expected: 1531 { 1532 "args": { 1533 "a": "b" 1534 }, 1535 "data": "", 1536 "files": { 1537 "Field2": "file field from memory", 1538 "File1": "file1 content\n", 1539 "File2": "file2 content\n" 1540 }, 1541 "form": { 1542 "Field1": "form field from memory", 1543 "Field3": "{\"a\":\"b\"}" 1544 }, 1545 "headers": { 1546 "Accept-Encoding": "gzip, deflate", 1547 "Content-Length": "730", 1548 "Content-Type": "multipart/form-data; boundary=d79a383e-7912-4d36-a6db-a6774bf37133", 1549 "Host": "httpbin.org", 1550 "User-Agent": "dlang-requests" 1551 }, 1552 "json": null, 1553 "origin": "xxx.xxx.xxx.xxx", 1554 "url": "http://httpbin.org/post?a=b" 1555 } 1556 */ 1557 json = parseJSON(cast(string)rs.responseBody); 1558 assert("file field from memory" == cast(string)(json.object["files"].object["Field2"].array.map!(a => cast(ubyte)a.integer).array)); 1559 assert("file1 content\n" == cast(string)(json.object["files"].object["File1"].array.map!(a => cast(ubyte)a.integer).array)); 1560 1561 info("httpd Check POST/iterate over multipart form"); 1562 form = MultipartForm(). 1563 add(formData("Field1", cast(ubyte[])"form field from memory")). 1564 add(formData("Field2", cast(ubyte[])"file field from memory", ["filename":"data2"])). 1565 add(formData("Field3", cast(ubyte[])`{"a":"b"}`, ["Content-Type": "application/json"])); 1566 /// everything ready, send request 1567 rs = request.post(httpbin_url ~ "postIter?a=b", form); 1568 assert(equal(rs.responseBody, "53")); 1569 rs = request.post(httpbin_url ~ "postIter", "0123456789".repeat(1500)); 1570 assert(equal(rs.responseBody, "15000")); 1571 } 1572 info("httpd Check cookies"); 1573 rs = request.get(httpbin_url ~ "cookies/set?A=abcd&b=cdef"); 1574 json = parseJSON(cast(string)rs.responseBody.data).object["cookies"].object; 1575 assert(json["A"].str == "abcd"); 1576 assert(json["b"].str == "cdef"); 1577 } 1578 }