1 module requests; 2 3 public import requests.http; 4 public import requests.ftp; 5 public import requests.streams; 6 public import requests.base; 7 public import requests.uri; 8 9 import std.datetime; 10 import std.conv; 11 import std.experimental.logger; 12 import requests.uri; 13 14 /*********************************** 15 * This is simplest interface to both http and ftp protocols. 16 * Request has methods get, post and exec which routed to proper concrete handler (http or ftp, etc). 17 * To enable some protocol-specific featutes you have to use protocol interface directly (see docs for HTTPRequest or FTPRequest) 18 */ 19 struct Request { 20 private { 21 URI _uri; 22 HTTPRequest _http; // route all http/https requests here 23 FTPRequest _ftp; // route all ftp requests here 24 } 25 /// Set timeout on IO operation. 26 /// $(B v) - timeout value 27 /// 28 @property void timeout(Duration v) pure @nogc nothrow { 29 _http.timeout = v; 30 _ftp.timeout = v; 31 } 32 /// Set http keepAlive value 33 /// $(B v) - use keepalive requests - $(B true), or not - $(B false) 34 @property void keepAlive(bool v) pure @nogc nothrow { 35 _http.keepAlive = v; 36 } 37 /// Set limit on HTTP redirects 38 /// $(B v) - limit on redirect depth 39 @property void maxRedirects(uint v) pure @nogc nothrow { 40 _http.maxRedirects = v; 41 } 42 /// Set maximum content lenth both for http and ftp requests 43 /// $(B v) - maximum content length in bytes. When limit reached - throw RequestException 44 @property void maxContentLength(size_t v) pure @nogc nothrow { 45 _http.maxContentLength = v; 46 _ftp.maxContentLength = v; 47 } 48 /// Set maximum length for HTTP headers 49 /// $(B v) - maximum length of the HTTP response. When limit reached - throw RequestException 50 @property void maxHeadersLength(size_t v) pure @nogc nothrow { 51 _http.maxHeadersLength = v; 52 } 53 /// Set IO buffer size for http and ftp requests 54 /// $(B v) - buffer size in bytes. 55 @property void bufferSize(size_t v) { 56 _http.bufferSize = v; 57 _ftp.bufferSize = v; 58 } 59 /// Set verbosity for HTTP or FTP requests. 60 /// $(B v) - verbosity level (0 - no output, 1 - headers to stdout, 2 - headers and body progress to stdout). default = 0. 61 @property void verbosity(uint v) { 62 _http.verbosity = v; 63 _ftp.verbosity = v; 64 } 65 /// Set authenticator for http requests. 66 /// $(B v) - Auth instance. 67 @property void authenticator(Auth v) { 68 _http.authenticator = v; 69 } 70 /// Set Cookie for http requests. 71 /// $(B v) - array of cookie. 72 @property void cookie(Cookie[] v) pure @nogc nothrow { 73 _http.cookie = v; 74 } 75 /// Get Cookie for http requests. 76 /// $(B v) - array of cookie. 77 @property Cookie[] cookie() pure @nogc nothrow { 78 return _http.cookie; 79 } 80 @property void useStreaming(bool v) pure @nogc nothrow { 81 _http.useStreaming = v; 82 _ftp.useStreaming = v; 83 } 84 @property long contentReceived() pure @nogc nothrow { 85 final switch ( _uri.scheme ) { 86 case "http", "https": 87 return _http.contentReceived; 88 case "ftp": 89 return _ftp.contentReceived; 90 } 91 } 92 @property long contentLength() pure @nogc nothrow { 93 final switch ( _uri.scheme ) { 94 case "http", "https": 95 return _http.contentLength; 96 case "ftp": 97 return _ftp.contentLength; 98 } 99 } 100 /// Execute GET for http and retrieve file for FTP. 101 /// You have to provide at least $(B uri). All other arguments should conform to HTTPRequest.get or FTPRequest.get depending on the URI scheme. 102 /// When arguments do not conform scheme (for example you try to call get("ftp://somehost.net/pub/README", {"a":"b"}) which doesn't make sense) 103 /// you will receive Exception("Operation not supported for ftp") 104 /// 105 Response get(A...)(string uri, A args) { 106 if ( uri ) { 107 _uri = URI(uri); 108 } 109 final switch ( _uri.scheme ) { 110 case "http", "https": 111 _http.uri = _uri; 112 static if (__traits(compiles, _http.get(null, args))) { 113 return _http.get(null, args); 114 } else { 115 throw new Exception("Operation not supported for http"); 116 } 117 case "ftp": 118 static if (args.length == 0) { 119 return _ftp.get(uri); 120 } else { 121 throw new Exception("Operation not supported for ftp"); 122 } 123 } 124 } 125 /// Execute POST for http and STOR file for FTP. 126 /// You have to provide $(B uri) and data. Data should conform to HTTPRequest.post or FTPRequest.post depending on the URI scheme. 127 /// When arguments do not conform scheme you will receive Exception("Operation not supported for ftp") 128 /// 129 Response post(A...)(string uri, A args) { 130 if ( uri ) { 131 _uri = URI(uri); 132 } 133 final switch ( _uri.scheme ) { 134 case "http", "https": 135 _http.uri = _uri; 136 static if (__traits(compiles, _http.post(null, args))) { 137 return _http.post(null, args); 138 } else { 139 throw new Exception("Operation not supported for http"); 140 } 141 case "ftp": 142 static if (__traits(compiles, _ftp.post(uri, args))) { 143 return _ftp.post(uri, args); 144 } else { 145 throw new Exception("Operation not supported for ftp"); 146 } 147 } 148 } 149 Response exec(string method="GET", A...)(A args) { 150 return _http.exec!(method)(args); 151 } 152 } 153 /// 154 package unittest { 155 import std.algorithm; 156 import std.range; 157 import std.array; 158 import std.json; 159 import std.stdio; 160 import std.string; 161 import std.exception; 162 163 globalLogLevel(LogLevel.info); 164 165 infof("testing Request"); 166 Request rq; 167 Response rs; 168 // 169 rs = rq.get("https://httpbin.org/"); 170 assert(rs.code==200); 171 assert(rs.responseBody.length > 0); 172 rs = rq.get("http://httpbin.org/get", ["c":" d", "a":"b"]); 173 assert(rs.code == 200); 174 auto json = parseJSON(rs.responseBody.data).object["args"].object; 175 assert(json["c"].str == " d"); 176 assert(json["a"].str == "b"); 177 178 globalLogLevel(LogLevel.info); 179 rq = Request(); 180 rq.keepAlive = true; 181 // handmade json 182 info("Check POST json"); 183 rs = rq.post("http://httpbin.org/post?b=x", `{"a":"☺ ", "c":[1,2,3]}`, "application/json"); 184 assert(rs.code==200); 185 json = parseJSON(rs.responseBody.data).object["args"].object; 186 assert(json["b"].str == "x"); 187 json = parseJSON(rs.responseBody.data).object["json"].object; 188 assert(json["a"].str == "☺ "); 189 assert(json["c"].array.map!(a=>a.integer).array == [1,2,3]); 190 { 191 import std.file; 192 import std.path; 193 auto tmpd = tempDir(); 194 auto tmpfname = tmpd ~ dirSeparator ~ "request_test.txt"; 195 auto f = File(tmpfname, "wb"); 196 f.rawWrite("abcdefgh\n12345678\n"); 197 f.close(); 198 // files 199 globalLogLevel(LogLevel.info); 200 info("Check POST files"); 201 PostFile[] files = [ 202 {fileName: tmpfname, fieldName:"abc", contentType:"application/octet-stream"}, 203 {fileName: tmpfname} 204 ]; 205 rs = rq.post("http://httpbin.org/post", files); 206 assert(rs.code==200); 207 info("Check POST chunked from file.byChunk"); 208 f = File(tmpfname, "rb"); 209 rs = rq.post("http://httpbin.org/post", f.byChunk(3), "application/octet-stream"); 210 assert(rs.code==200); 211 auto data = parseJSON(rs.responseBody.data).object["data"].str; 212 assert(data=="abcdefgh\n12345678\n"); 213 f.close(); 214 } 215 { 216 // string 217 info("Check POST utf8 string"); 218 rs = rq.post("http://httpbin.org/post", "привiт, свiт!", "application/octet-stream"); 219 assert(rs.code==200); 220 auto data = parseJSON(rs.responseBody.data).object["data"].str; 221 assert(data=="привiт, свiт!"); 222 } 223 // ranges 224 { 225 info("Check POST chunked from lineSplitter"); 226 auto s = lineSplitter("one,\ntwo,\nthree."); 227 rs = rq.exec!"POST"("http://httpbin.org/post", s, "application/octet-stream"); 228 assert(rs.code==200); 229 auto data = parseJSON(rs.responseBody.toString).object["data"].str; 230 assert(data=="one,two,three."); 231 } 232 { 233 info("Check POST chunked from array"); 234 auto s = ["one,", "two,", "three."]; 235 rs = rq.post("http://httpbin.org/post", s, "application/octet-stream"); 236 assert(rs.code==200); 237 auto data = parseJSON(rs.responseBody.data).object["data"].str; 238 assert(data=="one,two,three."); 239 } 240 { 241 info("Check POST chunked using std.range.chunks()"); 242 auto s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; 243 rs = rq.post("http://httpbin.org/post", s.representation.chunks(10), "application/octet-stream"); 244 assert(rs.code==200); 245 auto data = parseJSON(rs.responseBody.data).object["data"].str; 246 assert(data==s); 247 } 248 // associative array 249 rs = rq.post("http://httpbin.org/post", ["a":"b ", "c":"d"]); 250 assert(rs.code==200); 251 auto form = parseJSON(rs.responseBody.data).object["form"].object; 252 assert(form["a"].str == "b "); 253 assert(form["c"].str == "d"); 254 info("Check HEAD"); 255 rs = rq.exec!"HEAD"("http://httpbin.org/"); 256 assert(rs.code==200); 257 info("Check DELETE"); 258 rs = rq.exec!"DELETE"("http://httpbin.org/delete"); 259 assert(rs.code==200); 260 info("Check PUT"); 261 rs = rq.exec!"PUT"("http://httpbin.org/put", `{"a":"b", "c":[1,2,3]}`, "application/json"); 262 assert(rs.code==200); 263 info("Check PATCH"); 264 rs = rq.exec!"PATCH"("http://httpbin.org/patch", "привiт, свiт!", "application/octet-stream"); 265 assert(rs.code==200); 266 267 info("Check compressed content"); 268 globalLogLevel(LogLevel.info); 269 rq = Request(); 270 rq.keepAlive = true; 271 rs = rq.get("http://httpbin.org/gzip"); 272 assert(rs.code==200); 273 info("gzip - ok"); 274 rs = rq.get("http://httpbin.org/deflate"); 275 assert(rs.code==200); 276 info("deflate - ok"); 277 278 info("Check redirects"); 279 globalLogLevel(LogLevel.info); 280 rq = Request(); 281 rq.keepAlive = true; 282 rs = rq.get("http://httpbin.org/relative-redirect/2"); 283 assert((cast(HTTPResponse)rs).history.length == 2); 284 assert((cast(HTTPResponse)rs).code==200); 285 286 info("Check cookie"); 287 rq = Request(); 288 rs = rq.get("http://httpbin.org/cookies/set?A=abcd&b=cdef"); 289 assert(rs.code == 200); 290 json = parseJSON(rs.responseBody.data).object["cookies"].object; 291 assert(json["A"].str == "abcd"); 292 assert(json["b"].str == "cdef"); 293 auto cookie = rq.cookie(); 294 foreach(c; rq.cookie) { 295 final switch(c.attr) { 296 case "A": 297 assert(c.value == "abcd"); 298 break; 299 case "b": 300 assert(c.value == "cdef"); 301 break; 302 } 303 } 304 rs = rq.get("http://httpbin.org/absolute-redirect/2"); 305 assert((cast(HTTPResponse)rs).history.length == 2); 306 assert((cast(HTTPResponse)rs).code==200); 307 // rq = Request(); 308 rq.maxRedirects = 2; 309 rq.keepAlive = false; 310 assertThrown!MaxRedirectsException(rq.get("https://httpbin.org/absolute-redirect/3")); 311 312 info("Check utf8 content"); 313 globalLogLevel(LogLevel.info); 314 rq = Request(); 315 rs = rq.get("http://httpbin.org/encoding/utf8"); 316 assert(rs.code==200); 317 318 info("Check chunked content"); 319 globalLogLevel(LogLevel.info); 320 rq = Request(); 321 rq.keepAlive = true; 322 rq.bufferSize = 16*1024; 323 rs = rq.get("http://httpbin.org/range/1024"); 324 assert(rs.code==200); 325 assert(rs.responseBody.length==1024); 326 327 info("Check basic auth"); 328 globalLogLevel(LogLevel.info); 329 rq = Request(); 330 rq.authenticator = new BasicAuthentication("user", "passwd"); 331 rs = rq.get("http://httpbin.org/basic-auth/user/passwd"); 332 assert(rs.code==200); 333 334 globalLogLevel(LogLevel.info); 335 info("Check limits"); 336 rq = Request(); 337 rq.maxContentLength = 1; 338 assertThrown!RequestException(rq.get("http://httpbin.org/")); 339 rq = Request(); 340 rq.maxHeadersLength = 1; 341 assertThrown!RequestException(rq.get("http://httpbin.org/")); 342 // 343 info("ftp post ", "ftp://speedtest.tele2.net/upload/TEST.TXT"); 344 rs = rq.post("ftp://speedtest.tele2.net/upload/TEST.TXT", "test, ignore please\n".representation); 345 assert(rs.code == 226); 346 info("ftp get ", "ftp://speedtest.tele2.net/nonexistent", ", in same session."); 347 rs = rq.get("ftp://speedtest.tele2.net/nonexistent"); 348 assert(rs.code != 226); 349 info("ftp get ", "ftp://speedtest.tele2.net/1KB.zip", ", in same session."); 350 rs = rq.get("ftp://speedtest.tele2.net/1KB.zip"); 351 assert(rs.code == 226); 352 assert(rs.responseBody.length == 1024); 353 info("ftp post ", "ftp://speedtest.tele2.net/upload/TEST.TXT"); 354 rs = rq.post("ftp://speedtest.tele2.net/upload/TEST.TXT", "another test, ignore please\n".representation); 355 assert(rs.code == 226); 356 info("ftp get ", "ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT"); 357 rs = rq.get("ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT"); 358 assert(rs.code == 226); 359 info("testing ftp - done."); 360 } 361 362 auto queryParams(A...)(A args) pure @safe nothrow { 363 QueryParam[] res; 364 static if ( args.length >= 2 ) { 365 res = QueryParam(args[0].to!string, args[1].to!string) ~ queryParams(args[2..$]); 366 } 367 return res; 368 } 369 /** 370 * Call GET, and return response content. 371 * This is the simplest case, when all you need is the response body and have no parameters. 372 * Returns: 373 * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method. 374 */ 375 public auto ref getContent(A...)(string url) { 376 auto rq = Request(); 377 auto rs = rq.get(url); 378 return rs.responseBody; 379 } 380 /** 381 * Call GET, and return response content. 382 * args = string[string] fo query parameters. 383 * Returns: 384 * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method. 385 */ 386 public auto ref getContent(A...)(string url, string[string] args) { 387 auto rq = Request(); 388 auto rs = rq.get(url, args); 389 return rs.responseBody; 390 } 391 /** 392 * Call GET, and return response content. 393 * args = QueryParam[] of parameters. 394 * Returns: 395 * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method. 396 */ 397 public auto ref getContent(A...)(string url, QueryParam[] args) { 398 auto rq = Request(); 399 auto rs = rq.get(url, args); 400 return rs.responseBody; 401 } 402 /** 403 * Call GET, and return response content. 404 * args = variadic args to supply parameter names and values. 405 * Returns: 406 * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method. 407 */ 408 public auto ref getContent(A...)(string url, A args) if (args.length > 1 && args.length % 2 == 0 ) { 409 return Request(). 410 get(url, queryParams(args)). 411 responseBody; 412 } 413 414 /// 415 package unittest { 416 import std.algorithm; 417 import std.stdio; 418 import std.json; 419 globalLogLevel(LogLevel.info); 420 info("Test getContent"); 421 auto r = getContent("https://httpbin.org/stream/20"); 422 assert(r.splitter('\n').filter!("a.length>0").count == 20); 423 r = getContent("ftp://speedtest.tele2.net/1KB.zip"); 424 assert(r.length == 1024); 425 r = getContent("https://httpbin.org/get", ["a":"b", "c":"d"]); 426 427 string name = "user", sex = "male"; 428 int age = 42; 429 r = getContent("https://httpbin.org/get", "name", name, "age", age, "sex", sex); 430 431 info("Test receiveAsRange with GET"); 432 auto rq = Request(); 433 rq.useStreaming = true; 434 rq.bufferSize = 16; 435 auto rs = rq.get("http://httpbin.org/stream/20"); 436 auto stream = rs.receiveAsRange(); 437 ubyte[] streamedContent; 438 while( !stream.empty() ) { 439 streamedContent ~= stream.front; 440 stream.popFront(); 441 } 442 rq = Request(); 443 rs = rq.get("http://httpbin.org/stream/20"); 444 assert(streamedContent == rs.responseBody.data); 445 rq.useStreaming = true; 446 streamedContent.length = 0; 447 rs = rq.get("ftp://speedtest.tele2.net/1KB.zip"); 448 stream = rs.receiveAsRange; 449 while( !stream.empty() ) { 450 streamedContent ~= stream.front; 451 stream.popFront(); 452 } 453 assert(streamedContent.length == 1024); 454 } 455 /// 456 package unittest { 457 globalLogLevel(LogLevel.info); 458 info("Test get in parallel"); 459 import std.stdio; 460 import std.parallelism; 461 import std.algorithm; 462 import std.string; 463 import core.atomic; 464 465 immutable auto urls = [ 466 "http://httpbin.org/stream/10", 467 "https://httpbin.org/stream/20", 468 "http://httpbin.org/stream/30", 469 "https://httpbin.org/stream/40", 470 "http://httpbin.org/stream/50", 471 "https://httpbin.org/stream/60", 472 "http://httpbin.org/stream/70", 473 ]; 474 475 defaultPoolThreads(3); 476 477 shared short lines; 478 479 foreach(url; parallel(urls)) { 480 atomicOp!"+="(lines, getContent(url).splitter("\n").count); 481 } 482 assert(lines == 287); 483 } 484 485 /** 486 * Call post and return response content. 487 */ 488 public auto postContent(A...)(string url, A args) { 489 auto rq = Request(); 490 auto rs = rq.post(url, args); 491 return rs.responseBody; 492 } 493 494 /// 495 package unittest { 496 import std.json; 497 import std.string; 498 import std.stdio; 499 import std.range; 500 501 globalLogLevel(LogLevel.info); 502 info("Test postContent"); 503 auto r = postContent("http://httpbin.org/post", `{"a":"b", "c":1}`, "application/json"); 504 assert(parseJSON(r.data).object["json"].object["c"].integer == 1); 505 506 /// ftp upload from range 507 info("Test postContent ftp"); 508 r = postContent("ftp://speedtest.tele2.net/upload/TEST.TXT", "test, ignore please\n".representation); 509 assert(r.length == 0); 510 511 /// Posting to forms (for small data) 512 /// 513 /// posting query parameters using "application/x-www-form-urlencoded" 514 info("Test postContent using query params"); 515 postContent("http://httpbin.org/post", queryParams("first", "a", "second", 2)); 516 517 /// posting using multipart/form-data (large data and files). See docs fot HTTPRequest 518 info("Test postContent form"); 519 MultipartForm form; 520 form.add(formData(/* field name */ "greeting", /* content */ cast(ubyte[])"hello")); 521 postContent("http://httpbin.org/post", form); 522 523 /// you can do this using Request struct to access response details 524 info("Test postContent form via Request()"); 525 auto rq = Request(); 526 form = MultipartForm().add(formData(/* field name */ "greeting", /* content */ cast(ubyte[])"hello")); 527 auto rs = rq.post("http://httpbin.org/post", form); 528 assert(rs.code == 200); 529 530 info("Test receiveAsRange with POST"); 531 rq = Request(); 532 rq.useStreaming = true; 533 rq.bufferSize = 16; 534 string s = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; 535 rs = rq.post("http://httpbin.org/post", s.representation.chunks(10), "application/octet-stream"); 536 auto stream = rs.receiveAsRange(); 537 ubyte[] streamedContent; 538 while( !stream.empty() ) { 539 streamedContent ~= stream.front; 540 stream.popFront(); 541 } 542 rq = Request(); 543 rs = rq.post("http://httpbin.org/post", s.representation.chunks(10), "application/octet-stream"); 544 assert(streamedContent == rs.responseBody.data); 545 546 } 547