1 module requests.http; 2 3 private: 4 import std.algorithm; 5 import std.array; 6 import std.conv; 7 import std.datetime; 8 import std.exception; 9 import std.format; 10 import std.stdio; 11 import std.range; 12 import std.string; 13 import std.traits; 14 import std.typecons; 15 import std.experimental.logger; 16 import core.thread; 17 18 import requests.streams; 19 import requests.uri; 20 import requests.utils; 21 import requests.base; 22 23 static immutable ushort[] redirectCodes = [301, 302, 303]; 24 static immutable uint defaultBufferSize = 12*1024; 25 26 public class MaxRedirectsException: Exception { 27 this(string message, string file = __FILE__, size_t line = __LINE__, Throwable next = null) @safe pure nothrow { 28 super(message, file, line, next); 29 } 30 } 31 32 /** 33 * Basic authentication. 34 * Adds $(B Authorization: Basic) header to request. 35 */ 36 public class BasicAuthentication: Auth { 37 private { 38 string _username, _password; 39 string[] _domains; 40 } 41 /// Constructor. 42 /// Params: 43 /// username = username 44 /// password = password 45 /// domains = not used now 46 /// 47 this(string username, string password, string[] domains = []) { 48 _username = username; 49 _password = password; 50 _domains = domains; 51 } 52 override string[string] authHeaders(string domain) { 53 import std.base64; 54 string[string] auth; 55 auth["Authorization"] = "Basic " ~ to!string(Base64.encode(cast(ubyte[])"%s:%s".format(_username, _password))); 56 return auth; 57 } 58 override string userName() { 59 return _username; 60 } 61 override string password() { 62 return _password; 63 } 64 } 65 /// 66 /// 67 /// 68 public auto queryParams(T...)(T params) pure nothrow @safe 69 { 70 static assert (T.length % 2 == 0, "wrong args count"); 71 72 QueryParam[] output; 73 output.reserve = T.length / 2; 74 75 void queryParamsHelper(T...)(T params, ref QueryParam[] output) 76 { 77 static if (T.length > 0) 78 { 79 output ~= QueryParam(params[0].to!string, params[1].to!string); 80 queryParamsHelper(params[2..$], output); 81 } 82 } 83 84 queryParamsHelper(params, output); 85 return output; 86 } 87 88 /// 89 /// Response - result of request execution. 90 /// 91 /// Response.code - response HTTP code. 92 /// Response.status_line - received HTTP status line. 93 /// Response.responseHeaders - received headers. 94 /// Response.responseBody - container for received body 95 /// Response.history - for redirected responses contain all history 96 /// 97 public class HTTPResponse : Response { 98 private { 99 string _status_line; 100 101 HTTPResponse[] _history; // redirects history 102 SysTime _startedAt, _connectedAt, _requestSentAt, _finishedAt; 103 104 mixin(Setter!string("status_line")); 105 } 106 107 ~this() { 108 _responseHeaders = null; 109 _history.length = 0; 110 } 111 112 mixin(Getter!string("status_line")); 113 @property final string[string] responseHeaders() @safe @nogc nothrow { 114 return _responseHeaders; 115 } 116 @property final HTTPResponse[] history() @safe @nogc nothrow { 117 return _history; 118 } 119 120 @property auto getStats() const pure @safe { 121 alias statTuple = Tuple!(Duration, "connectTime", 122 Duration, "sendTime", 123 Duration, "recvTime"); 124 statTuple stat; 125 stat.connectTime = _connectedAt - _startedAt; 126 stat.sendTime = _requestSentAt - _connectedAt; 127 stat.recvTime = _finishedAt - _requestSentAt; 128 return stat; 129 } 130 } 131 /** 132 * Struct to send multiple files in POST request. 133 */ 134 public struct PostFile { 135 /// Path to the file to send. 136 string fileName; 137 /// Name of the field (if empty - send file base name) 138 string fieldName; 139 /// contentType of the file if not empty 140 string contentType; 141 } 142 /// 143 /// This is File-like interface for sending data to multipart fotms 144 /// 145 public interface FiniteReadable { 146 /// size of the content 147 abstract ulong getSize(); 148 /// file-like read() 149 abstract ubyte[] read(); 150 } 151 /// 152 /// Helper to create form elements from File. 153 /// Params: 154 /// name = name of the field in form 155 /// f = opened std.stio.File to send to server 156 /// parameters = optional parameters (most important are "filename" and "Content-Type") 157 /// 158 public auto formData(string name, File f, string[string] parameters = null) { 159 return MultipartForm.FormData(name, new FormDataFile(f), parameters); 160 } 161 /// 162 /// Helper to create form elements from ubyte[]. 163 /// Params: 164 /// name = name of the field in form 165 /// b = data to send to server 166 /// parameters = optional parameters (can be "filename" and "Content-Type") 167 /// 168 public auto formData(string name, ubyte[] b, string[string] parameters = null) { 169 return MultipartForm.FormData(name, new FormDataBytes(b), parameters); 170 } 171 public auto formData(string name, string b, string[string] parameters = null) { 172 return MultipartForm.FormData(name, new FormDataBytes(b.dup.representation), parameters); 173 } 174 public class FormDataBytes : FiniteReadable { 175 private { 176 ulong _size; 177 ubyte[] _data; 178 size_t _offset; 179 bool _exhausted; 180 } 181 this(ubyte[] data) { 182 _data = data; 183 _size = data.length; 184 } 185 final override ulong getSize() { 186 return _size; 187 } 188 final override ubyte[] read() { 189 enforce( !_exhausted, "You can't read froum exhausted source" ); 190 size_t toRead = min(defaultBufferSize, _size - _offset); 191 auto result = _data[_offset.._offset+toRead]; 192 _offset += toRead; 193 if ( toRead == 0 ) { 194 _exhausted = true; 195 } 196 return result; 197 } 198 } 199 public class FormDataFile : FiniteReadable { 200 import std.file; 201 private { 202 File _fileHandle; 203 ulong _fileSize; 204 size_t _processed; 205 bool _exhausted; 206 } 207 this(File file) { 208 import std.file; 209 _fileHandle = file; 210 _fileSize = std.file.getSize(file.name); 211 } 212 final override ulong getSize() pure nothrow @safe { 213 return _fileSize; 214 } 215 final override ubyte[] read() { 216 enforce( !_exhausted, "You can't read froum exhausted source" ); 217 auto b = new ubyte[defaultBufferSize]; 218 auto r = _fileHandle.rawRead(b); 219 auto toRead = min(r.length, _fileSize - _processed); 220 if ( toRead == 0 ) { 221 _exhausted = true; 222 } 223 _processed += toRead; 224 return r[0..toRead]; 225 } 226 } 227 /// 228 /// This struct used to bulld POST's to forms. 229 /// Each part have name and data. data is something that can be read-ed and have size. 230 /// For example this can be string-like object (wrapped for reading) or opened File. 231 /// 232 public struct MultipartForm { 233 package struct FormData { 234 FiniteReadable input; 235 string name; 236 string[string] parameters; 237 this(string name, FiniteReadable i, string[string] parameters = null) { 238 this.input = i; 239 this.name = name; 240 this.parameters = parameters; 241 } 242 } 243 244 private FormData[] _sources; 245 auto add(FormData d) { 246 _sources ~= d; 247 return this; 248 } 249 auto add(string name, FiniteReadable i, string[string]parameters = null) { 250 _sources ~= FormData(name, i, parameters); 251 return this; 252 } 253 } 254 /// 255 256 /// 257 /// Request. 258 /// Configurable parameters: 259 /// $(B method) - string, method to use (GET, POST, ...) 260 /// $(B headers) - string[string], add any additional headers you'd like to send. 261 /// $(B authenticator) - class Auth, class to send auth headers. 262 /// $(B keepAlive) - bool, set true for keepAlive requests. default true. 263 /// $(B maxRedirects) - uint, maximum number of redirects. default 10. 264 /// $(B maxHeadersLength) - size_t, maximum length of server response headers. default = 32KB. 265 /// $(B maxContentLength) - size_t, maximun content length. delault - 0 = unlimited. 266 /// $(B bufferSize) - size_t, send and receive buffer size. default = 16KB. 267 /// $(B verbosity) - uint, level of verbosity(0 - nothing, 1 - headers, 2 - headers and body progress). default = 0. 268 /// $(B proxy) - string, set proxy url if needed. default - null. 269 /// $(B cookie) - Tuple Cookie, Read/Write cookie You can get cookie setted by server, or set cookies before doing request. 270 /// $(B timeout) - Duration, Set timeout value for connect/receive/send. 271 /// 272 public struct HTTPRequest { 273 private { 274 enum _preHeaders = [ 275 "Accept-Encoding": "gzip, deflate", 276 "User-Agent": "dlang-requests" 277 ]; 278 string _method = "GET"; 279 URI _uri; 280 string[string] _headers; 281 string[] _filteredHeaders; 282 Auth _authenticator; 283 bool _keepAlive = true; 284 uint _maxRedirects = 10; 285 size_t _maxHeadersLength = 32 * 1024; // 32 KB 286 size_t _maxContentLength; // 0 - Unlimited 287 string _proxy; 288 uint _verbosity = 0; // 0 - no output, 1 - headers, 2 - headers+body info 289 Duration _timeout = 30.seconds; 290 size_t _bufferSize = defaultBufferSize; // 16k 291 bool _useStreaming; // return iterator instead of completed request 292 293 NetworkStream _stream; 294 HTTPResponse[] _history; // redirects history 295 DataPipe!ubyte _bodyDecoder; 296 DecodeChunked _unChunker; 297 long _contentLength; 298 long _contentReceived; 299 Cookie[] _cookie; 300 SSLOptions _sslOptions; 301 } 302 package HTTPResponse _response; 303 304 mixin(Getter_Setter!string ("method")); 305 mixin(Getter_Setter!bool ("keepAlive")); 306 mixin(Getter_Setter!size_t ("maxContentLength")); 307 mixin(Getter_Setter!size_t ("maxHeadersLength")); 308 mixin(Getter_Setter!size_t ("bufferSize")); 309 mixin(Getter_Setter!uint ("maxRedirects")); 310 mixin(Getter_Setter!uint ("verbosity")); 311 mixin(Getter_Setter!string ("proxy")); 312 mixin(Getter_Setter!Duration ("timeout")); 313 mixin(Setter!Auth ("authenticator")); 314 mixin(Getter_Setter!bool ("useStreaming")); 315 mixin(Getter!long ("contentLength")); 316 mixin(Getter!long ("contentReceived")); 317 mixin(Getter_Setter!SSLOptions ("sslOptions")); 318 319 @property void sslSetVerifyPeer(bool v) pure @safe nothrow @nogc { 320 _sslOptions.setVerifyPeer(v); 321 } 322 @property void sslSetKeyFile(string p, SSLOptions.filetype t = SSLOptions.filetype.pem) pure @safe nothrow @nogc { 323 _sslOptions.setKeyFile(p, t); 324 } 325 @property void sslSetCertFile(string p, SSLOptions.filetype t = SSLOptions.filetype.pem) pure @safe nothrow @nogc { 326 _sslOptions.setCertFile(p, t); 327 } 328 @property void sslSetCaCert(string path) pure @safe nothrow @nogc { 329 _sslOptions.setCaCert(path); 330 } 331 @property final void cookie(Cookie[] s) pure @safe @nogc nothrow { 332 _cookie = s; 333 } 334 335 @property final Cookie[] cookie() pure @safe @nogc nothrow { 336 return _cookie; 337 } 338 339 this(string uri) { 340 _uri = URI(uri); 341 } 342 ~this() { 343 if ( _stream && _stream.isConnected) { 344 _stream.close(); 345 } 346 _stream = null; 347 _headers = null; 348 _authenticator = null; 349 _history = null; 350 _bodyDecoder = null; 351 _unChunker = null; 352 } 353 void clearHeaders() { 354 _headers = null; 355 } 356 @property void uri(in URI newURI) { 357 handleURLChange(_uri, newURI); 358 _uri = newURI; 359 } 360 /// Add headers to request 361 /// Params: 362 /// headers = headers to send. 363 void addHeaders(in string[string] headers) { 364 foreach(pair; headers.byKeyValue) { 365 _headers[pair.key] = pair.value; 366 } 367 } 368 /// Remove headers from request 369 /// Params: 370 /// headers = headers to remove. 371 void removeHeaders(in string[] headers) pure { 372 _filteredHeaders ~= headers; 373 } 374 /// 375 /// compose headers to send 376 /// 377 private string[string] requestHeaders() { 378 string[string] generatedHeaders = _preHeaders; 379 380 if ( _authenticator ) { 381 _authenticator. 382 authHeaders(_uri.host). 383 byKeyValue. 384 each!(pair => generatedHeaders[pair.key] = pair.value); 385 } 386 387 generatedHeaders["Connection"] = _keepAlive?"Keep-Alive":"Close"; 388 generatedHeaders["Host"] = _uri.host; 389 390 if ( _uri.scheme !in standard_ports || _uri.port != standard_ports[_uri.scheme] ) { 391 generatedHeaders["Host"] ~= ":%d".format(_uri.port); 392 } 393 394 _headers.byKey.each!(h => generatedHeaders[h] = _headers[h]); 395 396 if ( _cookie.length ) { 397 auto cs = _cookie. 398 filter!(c => _uri.path.pathMatches(c.path) && _uri.host.domainMatches(c.domain)). 399 map!(c => "%s=%s".format(c.attr, c.value)). 400 joiner(";"); 401 generatedHeaders["Cookie"] = "%s".format(cs); 402 } 403 404 _filteredHeaders.each!(h => generatedHeaders.remove(h)); 405 406 return generatedHeaders; 407 } 408 /// 409 /// Build request string. 410 /// Handle proxy and query parameters. 411 /// 412 private @property string requestString(QueryParam[] params = null) { 413 if ( _proxy ) { 414 return "%s %s HTTP/1.1\r\n".format(_method, _uri.uri); 415 } 416 auto query = _uri.query.dup; 417 if ( params ) { 418 query ~= params2query(params); 419 if ( query[0] != '?' ) { 420 query = "?" ~ query; 421 } 422 } 423 return "%s %s%s HTTP/1.1\r\n".format(_method, _uri.path, query); 424 } 425 /// 426 /// encode parameters and build query part of the url 427 /// 428 private static string params2query(in QueryParam[] params) pure @safe { 429 return params. 430 map!(a => "%s=%s".format(a.key.urlEncoded, a.value.urlEncoded)). 431 join("&"); 432 } 433 // 434 package unittest { 435 assert(params2query(queryParams("a","b", "c", " d "))=="a=b&c=%20d%20"); 436 } 437 /// 438 /// Analyze received headers, take appropriate actions: 439 /// check content length, attach unchunk and uncompress 440 /// 441 private void analyzeHeaders(in string[string] headers) { 442 443 _contentLength = -1; 444 _unChunker = null; 445 auto contentLength = "content-length" in headers; 446 if ( contentLength ) { 447 try { 448 _contentLength = to!long(*contentLength); 449 if ( _maxContentLength && _contentLength > _maxContentLength) { 450 throw new RequestException("ContentLength > maxContentLength (%d>%d)". 451 format(_contentLength, _maxContentLength)); 452 } 453 } catch (ConvException e) { 454 throw new RequestException("Can't convert Content-Length from %s".format(*contentLength)); 455 } 456 } 457 auto transferEncoding = "transfer-encoding" in headers; 458 if ( transferEncoding ) { 459 debug(requests) tracef("transferEncoding: %s", *transferEncoding); 460 if ( *transferEncoding == "chunked") { 461 _unChunker = new DecodeChunked(); 462 _bodyDecoder.insert(_unChunker); 463 } 464 } 465 auto contentEncoding = "content-encoding" in headers; 466 if ( contentEncoding ) switch (*contentEncoding) { 467 default: 468 throw new RequestException("Unknown content-encoding " ~ *contentEncoding); 469 case "gzip": 470 case "deflate": 471 _bodyDecoder.insert(new Decompressor!ubyte); 472 } 473 474 } 475 /// 476 /// Called when we know that all headers already received in buffer. 477 /// This routine does not interpret headers content (see analyzeHeaders). 478 /// 1. Split headers on lines 479 /// 2. store status line, store response code 480 /// 3. unfold headers if needed 481 /// 4. store headers 482 /// 483 private void parseResponseHeaders(in ubyte[] input) { 484 string lastHeader; 485 auto buffer = cast(string)input; 486 487 foreach(line; buffer.split("\n").map!(l => l.stripRight)) { 488 if ( ! _response.status_line.length ) { 489 debug (requests) tracef("statusLine: %s", line); 490 _response.status_line = line; 491 if ( _verbosity >= 1 ) { 492 writefln("< %s", line); 493 } 494 auto parsed = line.split(" "); 495 if ( parsed.length >= 3 ) { 496 _response.code = parsed[1].to!ushort; 497 } 498 continue; 499 } 500 if ( line[0] == ' ' || line[0] == '\t' ) { 501 // unfolding https://tools.ietf.org/html/rfc822#section-3.1 502 if ( auto stored = lastHeader in _response._responseHeaders) { 503 *stored ~= line; 504 } 505 continue; 506 } 507 auto parsed = line.findSplit(":"); 508 auto header = parsed[0].toLower; 509 auto value = parsed[2].strip; 510 511 if ( _verbosity >= 1 ) { 512 writefln("< %s: %s", header, value); 513 } 514 515 lastHeader = header; 516 debug (requests) tracef("Header %s = %s", header, value); 517 518 if ( header != "set-cookie" ) { 519 auto stored = _response.responseHeaders.get(header, null); 520 if ( stored ) { 521 value = stored ~ ", " ~ value; 522 } 523 _response._responseHeaders[header] = value; 524 continue; 525 } 526 _cookie ~= processCookie(value); 527 } 528 } 529 530 /// 531 /// Process Set-Cookie header from server response 532 /// 533 private Cookie[] processCookie(string value ) pure { 534 // cookie processing 535 // 536 // as we can't join several set-cookie lines in single line 537 // < Set-Cookie: idx=7f2800f63c112a65ef5082957bcca24b; expires=Mon, 29-May-2017 00:31:25 GMT; path=/; domain=example.com 538 // < Set-Cookie: idx=7f2800f63c112a65ef5082957bcca24b; expires=Mon, 29-May-2017 00:31:25 GMT; path=/; domain=example.com, cs=ip764-RgKqc-HvSkxRxdQQAKW8LA; path=/; domain=.example.com; HttpOnly 539 // 540 Cookie[] res; 541 string[string] kv; 542 auto fields = value.split(";").map!strip; 543 while(!fields.empty) { 544 auto s = fields.front.findSplit("="); 545 fields.popFront; 546 if ( s[1] != "=" ) { 547 continue; 548 } 549 auto k = s[0]; 550 auto v = s[2]; 551 switch(k.toLower()) { 552 case "domain": 553 k = "domain"; 554 break; 555 case "path": 556 k = "path"; 557 break; 558 case "expires": 559 continue; 560 default: 561 break; 562 } 563 kv[k] = v; 564 } 565 if ( "domain" !in kv ) { 566 kv["domain"] = _uri.host; 567 } 568 if ( "path" !in kv ) { 569 kv["path"] = _uri.path; 570 } 571 auto domain = kv["domain"]; kv.remove("domain"); 572 auto path = kv["path"]; kv.remove("path"); 573 foreach(pair; kv.byKeyValue) { 574 auto _attr = pair.key; 575 auto _value = pair.value; 576 auto cookie = Cookie(path, domain, _attr, _value); 577 res ~= cookie; 578 } 579 return res; 580 } 581 /// 582 /// Do we received \r\n\r\n? 583 /// 584 private bool headersHaveBeenReceived(in ubyte[] data, ref Buffer!ubyte buffer, out string separator) const @safe { 585 foreach(s; ["\r\n\r\n", "\n\n"]) { 586 if ( data.canFind(s) || buffer.canFind(s) ) { 587 separator = s; 588 return true; 589 } 590 } 591 return false; 592 } 593 594 private bool followRedirectResponse() { 595 if ( _history.length >= _maxRedirects ) { 596 throw new MaxRedirectsException("%d redirects reached maxRedirects %d.".format(_history.length, _maxRedirects)); 597 } 598 auto location = "location" in _response.responseHeaders; 599 if ( !location ) { 600 return false; 601 } 602 _history ~= _response; 603 auto connection = "connection" in _response._responseHeaders; 604 if ( !connection || *connection == "close" ) { 605 debug(requests) tracef("Closing connection because of 'Connection: close' or no 'Connection' header"); 606 _stream.close(); 607 } 608 URI oldURI = _uri; 609 URI newURI = oldURI; 610 try { 611 newURI = URI(*location); 612 } catch (UriException e) { 613 debug(requests) trace("Can't parse Location:, try relative uri"); 614 newURI.path = *location; 615 newURI.uri = newURI.recalc_uri; 616 } 617 handleURLChange(oldURI, newURI); 618 oldURI = _response.uri; 619 _uri = newURI; 620 _response = new HTTPResponse; 621 _response.uri = oldURI; 622 _response.finalURI = newURI; 623 return true; 624 } 625 /// 626 /// If uri changed so that we have to change host or port, then we have to close socket stream 627 /// 628 private void handleURLChange(in URI from, in URI to) { 629 if ( _stream !is null && _stream.isConnected && 630 ( from.scheme != to.scheme || from.host != to.host || from.port != to.port) ) { 631 debug tracef("Have to reopen stream, because of URI change"); 632 _stream.close(); 633 } 634 } 635 /// 636 /// if we have new uri, then we need to check if we have to reopen existent connection 637 /// 638 private void checkURL(string url, string file=__FILE__, size_t line=__LINE__) { 639 if (url is null && _uri.uri == "" ) { 640 throw new RequestException("No url configured", file, line); 641 } 642 643 if ( url !is null ) { 644 URI newURI = URI(url); 645 handleURLChange(_uri, newURI); 646 _uri = newURI; 647 } 648 } 649 /// 650 /// Setup connection. Handle proxy and https case 651 /// 652 private void setupConnection() { 653 if ( !_stream || !_stream.isConnected ) { 654 debug(requests) tracef("Set up new connection"); 655 URI uri; 656 if ( _proxy ) { 657 // use proxy uri to connect 658 uri.uri_parse(_proxy); 659 } else { 660 // use original uri 661 uri = _uri; 662 } 663 final switch (uri.scheme) { 664 case "http": 665 _stream = new TCPStream().connect(uri.host, uri.port, _timeout); 666 break; 667 case "https": 668 _stream = new SSLStream(_sslOptions).connect(uri.host, uri.port, _timeout); 669 break; 670 } 671 } else { 672 debug(requests) tracef("Use old connection"); 673 } 674 } 675 /// 676 /// Request sent, now receive response. 677 /// Find headers, split on headers and body, continue to receive body 678 /// 679 private void receiveResponse() { 680 681 _stream.readTimeout = timeout; 682 scope(exit) { 683 if ( _stream && _stream.isOpen ) { 684 _stream.readTimeout = 0.seconds; 685 } 686 } 687 688 _bodyDecoder = new DataPipe!ubyte(); 689 scope(exit) { 690 if ( !_useStreaming ) { 691 _bodyDecoder = null; 692 _unChunker = null; 693 } 694 } 695 696 auto buffer = Buffer!ubyte(); 697 Buffer!ubyte partialBody; 698 ptrdiff_t read; 699 string separator; 700 701 while(true) { 702 703 auto b = new ubyte[_bufferSize]; 704 read = _stream.receive(b); 705 706 debug(requests) tracef("read: %d", read); 707 if ( read == 0 ) { 708 break; 709 } 710 auto data = b[0..read]; 711 buffer.putNoCopy(data); 712 if ( verbosity>=3 ) { 713 writeln(data.dump.join("\n")); 714 } 715 716 if ( buffer.length > maxHeadersLength ) { 717 throw new RequestException("Headers length > maxHeadersLength (%d > %d)".format(buffer.length, maxHeadersLength)); 718 } 719 if ( headersHaveBeenReceived(data, buffer, separator) ) { 720 auto s = buffer.data.findSplit(separator); 721 auto ResponseHeaders = s[0]; 722 partialBody = Buffer!ubyte(s[2]); 723 _contentReceived += partialBody.length; 724 parseResponseHeaders(ResponseHeaders); 725 break; 726 } 727 } 728 729 analyzeHeaders(_response._responseHeaders); 730 731 _bodyDecoder.putNoCopy(partialBody.data); 732 733 if ( _verbosity >= 2 ) writefln("< %d bytes of body received", partialBody.length); 734 735 if ( _method == "HEAD" ) { 736 // HEAD response have ContentLength, but have no body 737 return; 738 } 739 740 while( true ) { 741 if ( _contentLength >= 0 && _contentReceived >= _contentLength ) { 742 debug(requests) trace("Body received."); 743 break; 744 } 745 if ( _unChunker && _unChunker.done ) { 746 break; 747 } 748 if ( _useStreaming && _response._responseBody.length && !redirectCodes.canFind(_response.code) ) { 749 debug(requests) trace("streaming requested"); 750 _response.receiveAsRange.activated = true; 751 _response.receiveAsRange.data = _response._responseBody.data; 752 _response.receiveAsRange.read = delegate ubyte[] () { 753 while(true) { 754 // check if we received everything we need 755 if ( ( _unChunker && _unChunker.done ) 756 || !_stream.isConnected() 757 || (_contentLength > 0 && _contentReceived >= _contentLength) ) 758 { 759 debug(requests) trace("streaming_in receive completed"); 760 _bodyDecoder.flush(); 761 return _bodyDecoder.get(); 762 } 763 // have to continue 764 auto b = new ubyte[_bufferSize]; 765 try { 766 read = _stream.receive(b); 767 } 768 catch (Exception e) { 769 throw new RequestException("streaming_in error reading from socket", __FILE__, __LINE__, e); 770 } 771 debug(requests) tracef("streaming_in received %d bytes", read); 772 773 if ( read == 0 ) { 774 debug(requests) tracef("streaming_in: server closed connection"); 775 _bodyDecoder.flush(); 776 return _bodyDecoder.get(); 777 } 778 779 if ( verbosity>=3 ) { 780 writeln(b[0..read].dump.join("\n")); 781 } 782 783 _contentReceived += read; 784 _bodyDecoder.putNoCopy(b[0..read]); 785 auto res = _bodyDecoder.getNoCopy(); 786 if ( res.length == 0 ) { 787 // there were nothing to produce (beginning of the chunk or no decompressed data) 788 continue; 789 } 790 if (res.length == 1) { 791 return res[0]; 792 } 793 // 794 // I'd like to "return _bodyDecoder.getNoCopy().join;" but if is slower 795 // 796 auto total = res.map!(b=>b.length).sum; 797 // create buffer for joined bytes 798 ubyte[] joined = new ubyte[total]; 799 size_t p; 800 // memcopy 801 foreach(ref _; res) { 802 joined[p .. p + _.length] = _; 803 p += _.length; 804 } 805 return joined; 806 } 807 assert(0); 808 }; 809 // we prepared for streaming 810 return; 811 } 812 813 auto b = new ubyte[_bufferSize]; 814 read = _stream.receive(b); 815 816 if ( read == 0 ) { 817 debug(requests) trace("read done"); 818 break; 819 } 820 if ( _verbosity >= 2 ) { 821 writefln("< %d bytes of body received", read); 822 } 823 824 if ( verbosity>=3 ) { 825 writeln(b[0..read].dump.join("\n")); 826 } 827 828 debug(requests) tracef("read: %d", read); 829 _contentReceived += read; 830 if ( _maxContentLength && _contentReceived > _maxContentLength ) { 831 throw new RequestException("ContentLength > maxContentLength (%d>%d)". 832 format(_contentLength, _maxContentLength)); 833 } 834 835 _bodyDecoder.putNoCopy(b[0..read]); // send buffer to all decoders 836 837 _bodyDecoder.getNoCopy. // fetch result and place to body 838 each!(b => _response._responseBody.putNoCopy(b)); 839 840 debug(requests) tracef("receivedTotal: %d, contentLength: %d, bodyLength: %d", _contentReceived, _contentLength, _response._responseBody.length); 841 842 } 843 _bodyDecoder.flush(); 844 _response._responseBody.putNoCopy(_bodyDecoder.get()); 845 } 846 private bool serverClosedKeepAliveConnection() pure @safe nothrow { 847 return _response._responseHeaders.length == 0 && _keepAlive; 848 } 849 private bool isIdempotent(in string method) pure @safe nothrow { 850 return ["GET", "HEAD"].canFind(method); 851 } 852 /// 853 /// Send multipart for request. 854 /// You would like to use this method for sending large portions of mixed data or uploading files to forms. 855 /// Content of the posted form consist of sources. Each source have at least name and value (can be string-like object or opened file, see more docs for MultipartForm struct) 856 /// Params: 857 /// url = url 858 /// sources = array of sources. 859 HTTPResponse exec(string method="POST")(string url, MultipartForm sources) { 860 import std.uuid; 861 import std.file; 862 // 863 // application/json 864 // 865 bool restartedRequest = false; 866 867 _method = method; 868 869 _response = new HTTPResponse; 870 checkURL(url); 871 _response.uri = _uri; 872 _response.finalURI = _uri; 873 874 connect: 875 _contentReceived = 0; 876 _response._startedAt = Clock.currTime; 877 setupConnection(); 878 879 if ( !_stream.isConnected() ) { 880 return _response; 881 } 882 _response._connectedAt = Clock.currTime; 883 884 Appender!string req; 885 req.put(requestString()); 886 887 string boundary = randomUUID().toString; 888 string[] partHeaders; 889 size_t contentLength; 890 891 foreach(ref part; sources._sources) { 892 string h = "--" ~ boundary ~ "\r\n"; 893 string disposition = `form-data; name="%s"`.format(part.name); 894 string optionals = part. 895 parameters.byKeyValue(). 896 filter!(p => p.key!="Content-Type"). 897 map! (p => "%s=%s".format(p.key, p.value)). 898 join("; "); 899 900 h ~= `Content-Disposition: ` ~ [disposition, optionals].join("; ") ~ "\r\n"; 901 902 auto contentType = "Content-Type" in part.parameters; 903 if ( contentType ) { 904 h ~= "Content-Type: " ~ *contentType ~ "\r\n"; 905 } 906 907 h ~= "\r\n"; 908 partHeaders ~= h; 909 contentLength += h.length + part.input.getSize() + "\r\n".length; 910 } 911 contentLength += "--".length + boundary.length + "--\r\n".length; 912 913 auto h = requestHeaders(); 914 h["Content-Type"] = "multipart/form-data; boundary=" ~ boundary; 915 h["Content-Length"] = to!string(contentLength); 916 h.byKeyValue. 917 map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n"). 918 each!(h => req.put(h)); 919 req.put("\r\n"); 920 921 debug(requests) trace(req.data); 922 if ( _verbosity >= 1 ) req.data.splitLines.each!(a => writeln("> " ~ a)); 923 924 try { 925 _stream.send(req.data()); 926 foreach(ref source; sources._sources) { 927 debug(requests) tracef("sending part headers <%s>", partHeaders.front); 928 _stream.send(partHeaders.front); 929 partHeaders.popFront; 930 while (true) { 931 auto chunk = source.input.read(); 932 if ( chunk.length <= 0 ) { 933 break; 934 } 935 _stream.send(chunk); 936 } 937 _stream.send("\r\n"); 938 } 939 _stream.send("--" ~ boundary ~ "--\r\n"); 940 _response._requestSentAt = Clock.currTime; 941 receiveResponse(); 942 _response._finishedAt = Clock.currTime; 943 } 944 catch (NetworkException e) { 945 errorf("Error sending request: ", e.msg); 946 return _response; 947 } 948 if ( _useStreaming ) { 949 if ( _response._receiveAsRange.activated ) { 950 debug(requests) trace("streaming_in activated"); 951 return _response; 952 } else { 953 _response._receiveAsRange.data = _response.responseBody.data; 954 } 955 } 956 auto connection = "connection" in _response._responseHeaders; 957 if ( !connection || *connection == "close" ) { 958 debug(requests) tracef("Closing connection because of 'Connection: close' or no 'Connection' header"); 959 _stream.close(); 960 } 961 if ( canFind(redirectCodes, _response.code) && followRedirectResponse() ) { 962 if ( _method != "GET" ) { 963 return this.get(); 964 } 965 goto connect; 966 } 967 _response._history = _history; 968 /// 969 return _response; 970 } 971 /// 972 /// POST/PUT/... data from some string(with Content-Length), or from range of strings/bytes (use Transfer-Encoding: chunked). 973 /// When rank 1 (flat array) used as content it must have length. In that case "content" will be sent directly to network, and Content-Length headers will be added. 974 /// If you are goung to send some range and do not know length at the moment when you start to send request, then you can send chunks of chars or ubyte. 975 /// Try not to send too short chunks as this will put additional load on client and server. Chunks of length 2048 or 4096 are ok. 976 /// 977 /// Parameters: 978 /// url = url 979 /// content = string or input range 980 /// contentType = content type 981 /// Returns: 982 /// Response 983 /// Examples: 984 /// --------------------------------------------------------------------------------------------------------- 985 /// rs = rq.exec!"POST"("http://httpbin.org/post", "привiт, свiт!", "application/octet-stream"); 986 /// 987 /// auto s = lineSplitter("one,\ntwo,\nthree."); 988 /// rs = rq.exec!"POST"("http://httpbin.org/post", s, "application/octet-stream"); 989 /// 990 /// auto s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; 991 /// rs = rq.exec!"POST"("http://httpbin.org/post", s.representation.chunks(10), "application/octet-stream"); 992 /// 993 /// auto f = File("tests/test.txt", "rb"); 994 /// rs = rq.exec!"POST"("http://httpbin.org/post", f.byChunk(3), "application/octet-stream"); 995 /// -------------------------------------------------------------------------------------------------------- 996 HTTPResponse exec(string method="POST", R)(string url, R content, string contentType="application/octet-stream") 997 if ( (rank!R == 1) 998 || (rank!R == 2 && isSomeChar!(Unqual!(typeof(content.front.front)))) 999 || (rank!R == 2 && (is(Unqual!(typeof(content.front.front)) == ubyte))) 1000 ) { 1001 // 1002 // application/json 1003 // 1004 bool restartedRequest = false; 1005 1006 _method = method; 1007 1008 _response = new HTTPResponse; 1009 checkURL(url); 1010 _response.uri = _uri; 1011 _response.finalURI = _uri; 1012 1013 connect: 1014 _contentReceived = 0; 1015 _response._startedAt = Clock.currTime; 1016 setupConnection(); 1017 1018 if ( !_stream.isConnected() ) { 1019 return _response; 1020 } 1021 _response._connectedAt = Clock.currTime; 1022 1023 Appender!string req; 1024 req.put(requestString()); 1025 1026 auto h = requestHeaders; 1027 if ( contentType ) { 1028 h["Content-Type"] = contentType; 1029 } 1030 static if ( rank!R == 1 ) { 1031 h["Content-Length"] = to!string(content.length); 1032 } else { 1033 h["Transfer-Encoding"] = "chunked"; 1034 } 1035 h.byKeyValue. 1036 map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n"). 1037 each!(h => req.put(h)); 1038 req.put("\r\n"); 1039 1040 debug(requests) trace(req.data); 1041 if ( _verbosity >= 1 ) { 1042 req.data.splitLines.each!(a => writeln("> " ~ a)); 1043 } 1044 1045 try { 1046 // send headers 1047 _stream.send(req.data()); 1048 // send body 1049 static if ( rank!R == 1) { 1050 _stream.send(content); 1051 } else { 1052 while ( !content.empty ) { 1053 auto chunk = content.front; 1054 auto chunkHeader = "%x\r\n".format(chunk.length); 1055 debug(requests) tracef("sending %s%s", chunkHeader, chunk); 1056 _stream.send(chunkHeader); 1057 _stream.send(chunk); 1058 _stream.send("\r\n"); 1059 content.popFront; 1060 } 1061 debug(requests) tracef("sent"); 1062 _stream.send("0\r\n\r\n"); 1063 } 1064 _response._requestSentAt = Clock.currTime; 1065 receiveResponse(); 1066 _response._finishedAt = Clock.currTime; 1067 } catch (NetworkException e) { 1068 _stream.close(); 1069 throw new RequestException("Network error during data exchange"); 1070 } 1071 1072 if ( _useStreaming ) { 1073 if ( _response._receiveAsRange.activated ) { 1074 debug(requests) trace("streaming_in activated"); 1075 return _response; 1076 } else { 1077 _response._receiveAsRange.data = _response.responseBody.data; 1078 } 1079 } 1080 auto connection = "connection" in _response._responseHeaders; 1081 if ( !connection || *connection == "close" ) { 1082 debug(requests) tracef("Closing connection because of 'Connection: close' or no 'Connection' header"); 1083 _stream.close(); 1084 } 1085 if ( canFind(redirectCodes, _response.code) && followRedirectResponse() ) { 1086 if ( _method != "GET" ) { 1087 return this.get(); 1088 } 1089 goto connect; 1090 } 1091 /// 1092 _response._history = _history; 1093 return _response; 1094 } 1095 /// 1096 /// Send request with pameters. 1097 /// If used for POST or PUT requests then application/x-www-form-urlencoded used. 1098 /// Request parameters will be encoded into request string or placed in request body for POST/PUT 1099 /// requests. 1100 /// Parameters: 1101 /// url = url 1102 /// params = request parameters 1103 /// Returns: 1104 /// Response 1105 /// Examples: 1106 /// --------------------------------------------------------------------------------- 1107 /// rs = Request().exec!"GET"("http://httpbin.org/get", ["c":"d", "a":"b"]); 1108 /// --------------------------------------------------------------------------------- 1109 /// 1110 HTTPResponse exec(string method="GET")(string url = null, QueryParam[] params = null) { 1111 1112 _method = method; 1113 _response = new HTTPResponse; 1114 _history.length = 0; 1115 bool restartedRequest = false; // True if this is restarted keepAlive request 1116 string encoded; 1117 1118 checkURL(url); 1119 _response.uri = _uri; 1120 _response.finalURI = _uri; 1121 1122 connect: 1123 _contentReceived = 0; 1124 _response._startedAt = Clock.currTime; 1125 setupConnection(); 1126 1127 if ( !_stream.isConnected() ) { 1128 return _response; 1129 } 1130 _response._connectedAt = Clock.currTime; 1131 1132 auto h = requestHeaders(); 1133 1134 Appender!string req; 1135 1136 switch (_method) { 1137 case "POST","PUT": 1138 encoded = params2query(params); 1139 h["Content-Type"] = "application/x-www-form-urlencoded"; 1140 h["Content-Length"] = to!string(encoded.length); 1141 req.put(requestString()); 1142 break; 1143 default: 1144 req.put(requestString(params)); 1145 } 1146 1147 h.byKeyValue. 1148 map!(kv => kv.key ~ ": " ~ kv.value ~ "\r\n"). 1149 each!(h => req.put(h)); 1150 req.put("\r\n"); 1151 if ( encoded ) { 1152 req.put(encoded); 1153 } 1154 1155 debug(requests) trace(req.data); 1156 1157 if ( _verbosity >= 1 ) req.data.splitLines.each!(a => writeln("> " ~ a)); 1158 // 1159 // Now send request and receive response 1160 // 1161 try { 1162 _stream.send(req.data()); 1163 _response._requestSentAt = Clock.currTime; 1164 receiveResponse(); 1165 _response._finishedAt = Clock.currTime; 1166 } 1167 catch (NetworkException e) { 1168 // On SEND this can means: 1169 // we started to send request to the server, but it closed connection because of keepalive timeout. 1170 // We have to restart request if possible. 1171 1172 // On RECEIVE - if we received something - then this exception is real and unexpected error. 1173 // If we didn't receive anything - we can restart request again as it can be 1174 if ( _response._responseHeaders.length != 0 ) { 1175 _stream.close(); 1176 throw new RequestException("Unexpected network error"); 1177 } 1178 } 1179 1180 if ( serverClosedKeepAliveConnection() 1181 && !restartedRequest 1182 && isIdempotent(_method) 1183 ) { 1184 /// 1185 /// We didn't receive any data (keepalive connectioin closed?) 1186 /// and we can restart this request. 1187 /// Go ahead. 1188 /// 1189 debug(requests) tracef("Server closed keepalive connection"); 1190 _stream.close(); 1191 restartedRequest = true; 1192 goto connect; 1193 } 1194 1195 if ( _useStreaming ) { 1196 if ( _response._receiveAsRange.activated ) { 1197 debug(requests) trace("streaming_in activated"); 1198 return _response; 1199 } else { 1200 // this can happen if whole response body received together with headers 1201 _response._receiveAsRange.data = _response.responseBody.data; 1202 } 1203 } 1204 1205 auto connection = "connection" in _response._responseHeaders; 1206 if ( !connection || *connection == "close" ) { 1207 debug(requests) tracef("Closing connection because of 'Connection: close' or no 'Connection' header"); 1208 _stream.close(); 1209 } 1210 if ( _verbosity >= 1 ) { 1211 writeln(">> Connect time: ", _response._connectedAt - _response._startedAt); 1212 writeln(">> Request send time: ", _response._requestSentAt - _response._connectedAt); 1213 writeln(">> Response recv time: ", _response._finishedAt - _response._requestSentAt); 1214 } 1215 if ( canFind(redirectCodes, _response.code) && followRedirectResponse() ) { 1216 if ( _method != "GET" ) { 1217 return this.get(); 1218 } 1219 goto connect; 1220 } 1221 /// 1222 _response._history = _history; 1223 return _response; 1224 } 1225 1226 /// WRAPPERS 1227 /// 1228 /// send file(s) using POST and multipart form. 1229 /// This wrapper will be deprecated, use post with MultipartForm - it is more general and clear. 1230 /// Parameters: 1231 /// url = url 1232 /// files = array of PostFile structures 1233 /// Returns: 1234 /// Response 1235 /// Each PostFile structure contain path to file, and optional field name and content type. 1236 /// If no field name provided, then basename of the file will be used. 1237 /// application/octet-stream is default when no content type provided. 1238 /// Example: 1239 /// --------------------------------------------------------------- 1240 /// PostFile[] files = [ 1241 /// {fileName:"tests/abc.txt", fieldName:"abc", contentType:"application/octet-stream"}, 1242 /// {fileName:"tests/test.txt"} 1243 /// ]; 1244 /// rs = rq.exec!"POST"("http://httpbin.org/post", files); 1245 /// --------------------------------------------------------------- 1246 /// 1247 HTTPResponse exec(string method="POST")(string url, PostFile[] files) if (method=="POST") { 1248 MultipartForm multipart; 1249 File[] toClose; 1250 foreach(ref f; files) { 1251 File file = File(f.fileName, "rb"); 1252 toClose ~= file; 1253 string fileName = f.fileName ? f.fileName : f.fieldName; 1254 string contentType = f.contentType ? f.contentType : "application/octetstream"; 1255 multipart.add(f.fieldName, new FormDataFile(file), ["filename":fileName, "Content-Type": contentType]); 1256 } 1257 auto res = exec!"POST"(url, multipart); 1258 toClose.each!"a.close"; 1259 return res; 1260 } 1261 /// 1262 /// exec request with parameters when you can use dictionary (when you have no duplicates in parameter names) 1263 /// Consider switch to exec(url, QueryParams) as it more generic and clear. 1264 /// Parameters: 1265 /// url = url 1266 /// params = dictionary with field names as keys and field values as values. 1267 /// Returns: 1268 /// Response 1269 HTTPResponse exec(string method="GET")(string url, string[string] params) { 1270 return exec!method(url, params.byKeyValue.map!(p => QueryParam(p.key, p.value)).array); 1271 } 1272 /// 1273 /// GET request. Simple wrapper over exec!"GET" 1274 /// Params: 1275 /// args = request parameters. see exec docs. 1276 /// 1277 HTTPResponse get(A...)(A args) { 1278 return exec!"GET"(args); 1279 } 1280 /// 1281 /// POST request. Simple wrapper over exec!"POST" 1282 /// Params: 1283 /// uri = endpoint uri 1284 /// args = request parameters. see exec docs. 1285 /// 1286 HTTPResponse post(A...)(string uri, A args) { 1287 return exec!"POST"(uri, args); 1288 } 1289 } 1290 1291 version(vibeD) { 1292 import std.json; 1293 package string httpTestServer() { 1294 return "http://httpbin.org/"; 1295 } 1296 package string fromJsonArrayToStr(JSONValue v) { 1297 return v.str; 1298 } 1299 } 1300 else { 1301 import std.json; 1302 package string httpTestServer() { 1303 return "http://127.0.0.1:8081/"; 1304 } 1305 package string fromJsonArrayToStr(JSONValue v) { 1306 return cast(string)(v.array.map!"cast(ubyte)a.integer".array); 1307 } 1308 } 1309 1310 1311 package unittest { 1312 import std.json; 1313 import std.array; 1314 1315 globalLogLevel(LogLevel.info); 1316 1317 string httpbinUrl = httpTestServer(); 1318 version(vibeD) { 1319 } 1320 else { 1321 import httpbin; 1322 auto server = httpbinApp(); 1323 server.start(); 1324 scope(exit) { 1325 server.stop(); 1326 } 1327 } 1328 HTTPRequest rq; 1329 HTTPResponse rs; 1330 info("Check GET"); 1331 rs = rq.get(httpbinUrl); 1332 assert(rs.code==200); 1333 assert(rs.responseBody.length > 0); 1334 1335 info("Check GET with AA params"); 1336 { 1337 rs = HTTPRequest().get(httpbinUrl ~ "get", ["c":" d", "a":"b"]); 1338 assert(rs.code == 200); 1339 auto json = parseJSON(rs.responseBody.data).object["args"].object; 1340 assert(json["c"].str == " d"); 1341 assert(json["a"].str == "b"); 1342 } 1343 info("Check POST files"); 1344 { 1345 import std.file; 1346 import std.path; 1347 auto tmpd = tempDir(); 1348 auto tmpfname = tmpd ~ dirSeparator ~ "request_test.txt"; 1349 auto f = File(tmpfname, "wb"); 1350 f.rawWrite("abcdefgh\n12345678\n"); 1351 f.close(); 1352 // files 1353 PostFile[] files = [ 1354 {fileName: tmpfname, fieldName:"abc", contentType:"application/octet-stream"}, 1355 {fileName: tmpfname} 1356 ]; 1357 rs = rq.post(httpbinUrl ~ "post", files); 1358 assert(rs.code==200); 1359 } 1360 info("Check POST chunked from file.byChunk"); 1361 { 1362 import std.file; 1363 import std.path; 1364 auto tmpd = tempDir(); 1365 auto tmpfname = tmpd ~ dirSeparator ~ "request_test.txt"; 1366 auto f = File(tmpfname, "wb"); 1367 f.rawWrite("abcdefgh\n12345678\n"); 1368 f.close(); 1369 f = File(tmpfname, "rb"); 1370 rs = rq.post(httpbinUrl ~ "post", f.byChunk(3), "application/octet-stream"); 1371 assert(rs.code==200); 1372 auto data = fromJsonArrayToStr(parseJSON(rs.responseBody).object["data"]); 1373 assert(data=="abcdefgh\n12345678\n"); 1374 f.close(); 1375 } 1376 info("Check POST chunked from lineSplitter"); 1377 { 1378 auto s = lineSplitter("one,\ntwo,\nthree."); 1379 rs = rq.exec!"POST"(httpbinUrl ~ "post", s, "application/octet-stream"); 1380 assert(rs.code==200); 1381 auto data = fromJsonArrayToStr(parseJSON(rs.responseBody).object["data"]); 1382 assert(data=="one,two,three."); 1383 } 1384 info("Check POST chunked from array"); 1385 { 1386 auto s = ["one,", "two,", "three."]; 1387 rs = rq.post(httpbinUrl ~ "post", s, "application/octet-stream"); 1388 assert(rs.code==200); 1389 auto data = fromJsonArrayToStr(parseJSON(rs.responseBody).object["data"]); 1390 assert(data=="one,two,three."); 1391 } 1392 info("Check POST chunked using std.range.chunks()"); 1393 { 1394 auto s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; 1395 rs = rq.post(httpbinUrl ~ "post", s.representation.chunks(10), "application/octet-stream"); 1396 assert(rs.code==200); 1397 auto data = fromJsonArrayToStr(parseJSON(rs.responseBody.data).object["data"]); 1398 assert(data==s); 1399 } 1400 info("Check POST from QueryParams"); 1401 { 1402 rs = rq.post(httpbinUrl ~ "post", queryParams("name[]", "first", "name[]", 2)); 1403 assert(rs.code==200); 1404 auto data = parseJSON(rs.responseBody).object["form"].object; 1405 string[] a; 1406 try { 1407 a = to!(string[])(data["name[]"].str); 1408 } 1409 catch (JSONException e) { 1410 a = data["name[]"].array.map!"a.str".array; 1411 } 1412 assert(equal(["first", "2"], a)); 1413 } 1414 info("Check POST from AA"); 1415 { 1416 rs = rq.post(httpbinUrl ~ "post", ["a":"b ", "c":"d"]); 1417 assert(rs.code==200); 1418 auto form = parseJSON(rs.responseBody.data).object["form"].object; 1419 assert(form["a"].str == "b "); 1420 assert(form["c"].str == "d"); 1421 } 1422 info("Check POST json"); 1423 { 1424 rs = rq.post(httpbinUrl ~ "post?b=x", `{"a":"a b", "c":[1,2,3]}`, "application/json"); 1425 assert(rs.code==200); 1426 auto json = parseJSON(rs.responseBody).object["args"].object; 1427 assert(json["b"].str == "x"); 1428 json = parseJSON(rs.responseBody).object["json"].object; 1429 assert(json["a"].str == "a b"); 1430 assert(json["c"].array.map!(a=>a.integer).array == [1,2,3]); 1431 } 1432 info("Check HEAD"); 1433 rs = rq.exec!"HEAD"(httpbinUrl); 1434 assert(rs.code==200); 1435 info("Check DELETE"); 1436 rs = rq.exec!"DELETE"(httpbinUrl ~ "delete"); 1437 assert(rs.code==200); 1438 info("Check PUT"); 1439 rs = rq.exec!"PUT"(httpbinUrl ~ "put", `{"a":"b", "c":[1,2,3]}`, "application/json"); 1440 assert(rs.code==200); 1441 assert(parseJSON(rs.responseBody).object["json"].object["a"].str=="b"); 1442 info("Check PATCH"); 1443 rs = rq.exec!"PATCH"(httpbinUrl ~ "patch", "привiт, свiт!", "application/octet-stream"); 1444 assert(rs.code==200); 1445 info("Check compressed content"); 1446 rs = rq.get(httpbinUrl ~ "gzip"); 1447 assert(rs.code==200); 1448 bool gzipped = parseJSON(rs.responseBody).object["gzipped"].type == JSON_TYPE.TRUE; 1449 assert(gzipped); 1450 info("gzip - ok"); 1451 rs = rq.get(httpbinUrl ~ "deflate"); 1452 assert(rs.code==200); 1453 bool deflated = parseJSON(rs.responseBody).object["deflated"].type == JSON_TYPE.TRUE; 1454 assert(deflated); 1455 info("deflate - ok"); 1456 1457 info("Check redirects"); 1458 rs = rq.get(httpbinUrl ~ "relative-redirect/2"); 1459 assert(rs.history.length == 2); 1460 assert(rs.code==200); 1461 rs = rq.get(httpbinUrl ~ "absolute-redirect/2"); 1462 assert(rs.history.length == 2); 1463 assert(rs.code==200); 1464 1465 rq.maxRedirects = 2; 1466 assertThrown!MaxRedirectsException(rq.get(httpbinUrl ~ "absolute-redirect/3")); 1467 1468 info("Check cookie"); 1469 { 1470 rs = rq.get(httpbinUrl ~ "cookies/set?A=abcd&b=cdef"); 1471 assert(rs.code == 200); 1472 auto json = parseJSON(rs.responseBody.data).object["cookies"].object; 1473 assert(json["A"].str == "abcd"); 1474 assert(json["b"].str == "cdef"); 1475 foreach(c; rq.cookie) { 1476 final switch(c.attr) { 1477 case "A": 1478 assert(c.value == "abcd"); 1479 break; 1480 case "b": 1481 assert(c.value == "cdef"); 1482 break; 1483 } 1484 } 1485 } 1486 info("Check chunked content"); 1487 rs = rq.get(httpbinUrl ~ "range/1024"); 1488 assert(rs.code==200); 1489 assert(rs.responseBody.length==1024); 1490 1491 info("Check basic auth"); 1492 rq.authenticator = new BasicAuthentication("user", "passwd"); 1493 rs = rq.get(httpbinUrl ~ "basic-auth/user/passwd"); 1494 assert(rs.code==200); 1495 1496 info("Check limits"); 1497 rq = HTTPRequest(); 1498 rq.maxContentLength = 1; 1499 assertThrown!RequestException(rq.get(httpbinUrl)); 1500 rq = HTTPRequest(); 1501 rq.maxHeadersLength = 1; 1502 assertThrown!RequestException(rq.get(httpbinUrl)); 1503 rq = HTTPRequest(); 1504 info("Check POST multiPartForm"); 1505 { 1506 /// This is example on usage files with MultipartForm data. 1507 /// For this example we have to create files which will be sent. 1508 import std.file; 1509 import std.path; 1510 /// preapare files 1511 auto tmpd = tempDir(); 1512 auto tmpfname1 = tmpd ~ dirSeparator ~ "request_test1.txt"; 1513 auto f = File(tmpfname1, "wb"); 1514 f.rawWrite("file1 content\n"); 1515 f.close(); 1516 auto tmpfname2 = tmpd ~ dirSeparator ~ "request_test2.txt"; 1517 f = File(tmpfname2, "wb"); 1518 f.rawWrite("file2 content\n"); 1519 f.close(); 1520 /// 1521 /// Ok, files ready. 1522 /// Now we will prepare Form data 1523 /// 1524 File f1 = File(tmpfname1, "rb"); 1525 File f2 = File(tmpfname2, "rb"); 1526 scope(exit) { 1527 f1.close(); 1528 f2.close(); 1529 } 1530 /// 1531 /// for each part we have to set field name, source (ubyte array or opened file) and optional filename and content-type 1532 /// 1533 MultipartForm mForm = MultipartForm(). 1534 add(formData("Field1", cast(ubyte[])"form field from memory")). 1535 add(formData("Field2", cast(ubyte[])"file field from memory", ["filename":"data2"])). 1536 add(formData("File1", f1, ["filename":"file1", "Content-Type": "application/octet-stream"])). 1537 add(formData("File2", f2, ["filename":"file2", "Content-Type": "application/octet-stream"])); 1538 /// everything ready, send request 1539 rs = rq.post(httpbinUrl ~ "post", mForm); 1540 } 1541 info("Check exception handling, error messages and timeous are OK"); 1542 rq.timeout = 1.seconds; 1543 assertThrown!TimeoutException(rq.get(httpbinUrl ~ "delay/3")); 1544 // assertThrown!ConnectError(rq.get("http://0.0.0.0:65000/")); 1545 // assertThrown!ConnectError(rq.get("http://1.1.1.1/")); 1546 // assertThrown!ConnectError(rq.get("http://gkhgkhgkjhgjhgfjhgfjhgf/")); 1547 }