1 module requests; 2 3 public import requests.http; 4 public import requests.ftp; 5 public import requests.streams; 6 public import requests.base; 7 public import requests.uri; 8 9 import std.datetime; 10 import std.conv; 11 import std.experimental.logger; 12 import requests.uri; 13 14 /*********************************** 15 * This is simplest interface to both http and ftp protocols. 16 * Request has methods get, post and exec which routed to proper concrete handler (http or ftp, etc). 17 * To enable some protocol-specific featutes you have to use protocol interface directly (see docs for HTTPRequest or FTPRequest) 18 */ 19 struct Request { 20 private { 21 URI _uri; 22 HTTPRequest _http; // route all http/https requests here 23 FTPRequest _ftp; // route all ftp requests here 24 } 25 /// Set timeout on IO operation. 26 /// $(B v) - timeout value 27 /// 28 @property void timeout(Duration v) pure @nogc nothrow { 29 _http.timeout = v; 30 _ftp.timeout = v; 31 } 32 /// Set http keepAlive value 33 /// $(B v) - use keepalive requests - $(B true), or not - $(B false) 34 @property void keepAlive(bool v) pure @nogc nothrow { 35 _http.keepAlive = v; 36 } 37 /// Set limit on HTTP redirects 38 /// $(B v) - limit on redirect depth 39 @property void maxRedirects(uint v) pure @nogc nothrow { 40 _http.maxRedirects = v; 41 } 42 /// Set maximum content lenth both for http and ftp requests 43 /// $(B v) - maximum content length in bytes. When limit reached - throw RequestException 44 @property void maxContentLength(size_t v) pure @nogc nothrow { 45 _http.maxContentLength = v; 46 _ftp.maxContentLength = v; 47 } 48 /// Set maximum length for HTTP headers 49 /// $(B v) - maximum length of the HTTP response. When limit reached - throw RequestException 50 @property void maxHeadersLength(size_t v) pure @nogc nothrow { 51 _http.maxHeadersLength = v; 52 } 53 /// Set IO buffer size for http and ftp requests 54 /// $(B v) - buffer size in bytes. 55 @property void bufferSize(size_t v) { 56 _http.bufferSize = v; 57 _ftp.bufferSize = v; 58 } 59 /// Set verbosity for HTTP or FTP requests. 60 /// $(B v) - verbosity level (0 - no output, 1 - headers to stdout, 2 - headers and body progress to stdout). default = 0. 61 @property void verbosity(uint v) { 62 _http.verbosity = v; 63 _ftp.verbosity = v; 64 } 65 /// Set authenticator for http requests. 66 /// $(B v) - Auth instance. 67 @property void authenticator(Auth v) { 68 _http.authenticator = v; 69 } 70 /// Set Cookie for http requests. 71 /// $(B v) - array of cookie. 72 @property void cookie(Cookie[] v) pure @nogc nothrow { 73 _http.cookie = v; 74 } 75 /// Get Cookie for http requests. 76 /// $(B v) - array of cookie. 77 @property Cookie[] cookie() pure @nogc nothrow { 78 return _http.cookie; 79 } 80 @property void useStreaming(bool v) pure @nogc nothrow { 81 _http.useStreaming = v; 82 _ftp.useStreaming = v; 83 } 84 @property long contentReceived() pure @nogc nothrow { 85 final switch ( _uri.scheme ) { 86 case "http", "https": 87 return _http.contentReceived; 88 case "ftp": 89 return _ftp.contentReceived; 90 } 91 } 92 @property long contentLength() pure @nogc nothrow { 93 final switch ( _uri.scheme ) { 94 case "http", "https": 95 return _http.contentLength; 96 case "ftp": 97 return _ftp.contentLength; 98 } 99 } 100 /// Execute GET for http and retrieve file for FTP. 101 /// You have to provide at least $(B uri). All other arguments should conform to HTTPRequest.get or FTPRequest.get depending on the URI scheme. 102 /// When arguments do not conform scheme (for example you try to call get("ftp://somehost.net/pub/README", {"a":"b"}) which doesn't make sense) 103 /// you will receive Exception("Operation not supported for ftp") 104 /// 105 Response get(A...)(string uri, A args) { 106 if ( uri ) { 107 _uri = URI(uri); 108 } 109 final switch ( _uri.scheme ) { 110 case "http", "https": 111 _http.uri = _uri; 112 static if (__traits(compiles, _http.get(null, args))) { 113 return _http.get(null, args); 114 } else { 115 throw new Exception("Operation not supported for http"); 116 } 117 case "ftp": 118 static if (args.length == 0) { 119 return _ftp.get(uri); 120 } else { 121 throw new Exception("Operation not supported for ftp"); 122 } 123 } 124 } 125 /// Execute POST for http and STOR file for FTP. 126 /// You have to provide $(B uri) and data. Data should conform to HTTPRequest.post or FTPRequest.post depending on the URI scheme. 127 /// When arguments do not conform scheme you will receive Exception("Operation not supported for ftp") 128 /// 129 Response post(A...)(string uri, A args) { 130 if ( uri ) { 131 _uri = URI(uri); 132 } 133 final switch ( _uri.scheme ) { 134 case "http", "https": 135 _http.uri = _uri; 136 static if (__traits(compiles, _http.post(null, args))) { 137 return _http.post(null, args); 138 } else { 139 throw new Exception("Operation not supported for http"); 140 } 141 case "ftp": 142 static if (__traits(compiles, _ftp.post(uri, args))) { 143 return _ftp.post(uri, args); 144 } else { 145 throw new Exception("Operation not supported for ftp"); 146 } 147 } 148 } 149 Response exec(string method="GET", A...)(A args) { 150 return _http.exec!(method)(args); 151 } 152 } 153 /// 154 package unittest { 155 import std.algorithm; 156 import std.range; 157 import std.array; 158 import std.json; 159 import std.stdio; 160 import std.string; 161 import std.exception; 162 163 globalLogLevel(LogLevel.info); 164 165 infof("testing Request"); 166 Request rq; 167 Response rs; 168 // 169 rs = rq.get("https://httpbin.org/"); 170 assert(rs.code==200); 171 assert(rs.responseBody.length > 0); 172 rs = rq.get("http://httpbin.org/get", ["c":" d", "a":"b"]); 173 assert(rs.code == 200); 174 auto json = parseJSON(rs.responseBody.data).object["args"].object; 175 assert(json["c"].str == " d"); 176 assert(json["a"].str == "b"); 177 178 globalLogLevel(LogLevel.info); 179 rq = Request(); 180 rq.keepAlive = true; 181 // handmade json 182 info("Check POST json"); 183 rs = rq.post("http://httpbin.org/post?b=x", `{"a":"☺ ", "c":[1,2,3]}`, "application/json"); 184 assert(rs.code==200); 185 json = parseJSON(rs.responseBody.data).object["args"].object; 186 assert(json["b"].str == "x"); 187 json = parseJSON(rs.responseBody.data).object["json"].object; 188 assert(json["a"].str == "☺ "); 189 assert(json["c"].array.map!(a=>a.integer).array == [1,2,3]); 190 { 191 import std.file; 192 import std.path; 193 auto tmpd = tempDir(); 194 auto tmpfname = tmpd ~ dirSeparator ~ "request_test.txt"; 195 auto f = File(tmpfname, "wb"); 196 f.rawWrite("abcdefgh\n12345678\n"); 197 f.close(); 198 // files 199 globalLogLevel(LogLevel.info); 200 info("Check POST files"); 201 PostFile[] files = [ 202 {fileName: tmpfname, fieldName:"abc", contentType:"application/octet-stream"}, 203 {fileName: tmpfname} 204 ]; 205 rs = rq.post("http://httpbin.org/post", files); 206 assert(rs.code==200); 207 info("Check POST chunked from file.byChunk"); 208 f = File(tmpfname, "rb"); 209 rs = rq.post("http://httpbin.org/post", f.byChunk(3), "application/octet-stream"); 210 assert(rs.code==200); 211 auto data = parseJSON(rs.responseBody.data).object["data"].str; 212 assert(data=="abcdefgh\n12345678\n"); 213 f.close(); 214 } 215 { 216 // string 217 info("Check POST utf8 string"); 218 rs = rq.post("http://httpbin.org/post", "привiт, свiт!", "application/octet-stream"); 219 assert(rs.code==200); 220 auto data = parseJSON(rs.responseBody.data).object["data"].str; 221 assert(data=="привiт, свiт!"); 222 } 223 // ranges 224 { 225 info("Check POST chunked from lineSplitter"); 226 auto s = lineSplitter("one,\ntwo,\nthree."); 227 rs = rq.exec!"POST"("http://httpbin.org/post", s, "application/octet-stream"); 228 assert(rs.code==200); 229 auto data = parseJSON(rs.responseBody.toString).object["data"].str; 230 assert(data=="one,two,three."); 231 } 232 { 233 info("Check POST chunked from array"); 234 auto s = ["one,", "two,", "three."]; 235 rs = rq.post("http://httpbin.org/post", s, "application/octet-stream"); 236 assert(rs.code==200); 237 auto data = parseJSON(rs.responseBody.data).object["data"].str; 238 assert(data=="one,two,three."); 239 } 240 { 241 info("Check POST chunked using std.range.chunks()"); 242 auto s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; 243 rs = rq.post("http://httpbin.org/post", s.representation.chunks(10), "application/octet-stream"); 244 assert(rs.code==200); 245 auto data = parseJSON(rs.responseBody.data).object["data"].str; 246 assert(data==s); 247 } 248 // associative array 249 rs = rq.post("http://httpbin.org/post", ["a":"b ", "c":"d"]); 250 assert(rs.code==200); 251 auto form = parseJSON(rs.responseBody.data).object["form"].object; 252 assert(form["a"].str == "b "); 253 assert(form["c"].str == "d"); 254 info("Check HEAD"); 255 rs = rq.exec!"HEAD"("http://httpbin.org/"); 256 assert(rs.code==200); 257 info("Check DELETE"); 258 rs = rq.exec!"DELETE"("http://httpbin.org/delete"); 259 assert(rs.code==200); 260 info("Check PUT"); 261 rs = rq.exec!"PUT"("http://httpbin.org/put", `{"a":"b", "c":[1,2,3]}`, "application/json"); 262 assert(rs.code==200); 263 info("Check PATCH"); 264 rs = rq.exec!"PATCH"("http://httpbin.org/patch", "привiт, свiт!", "application/octet-stream"); 265 assert(rs.code==200); 266 267 info("Check compressed content"); 268 globalLogLevel(LogLevel.info); 269 rq = Request(); 270 rq.keepAlive = true; 271 rs = rq.get("http://httpbin.org/gzip"); 272 assert(rs.code==200); 273 info("gzip - ok"); 274 rs = rq.get("http://httpbin.org/deflate"); 275 assert(rs.code==200); 276 info("deflate - ok"); 277 278 info("Check redirects"); 279 globalLogLevel(LogLevel.info); 280 rq = Request(); 281 rq.keepAlive = true; 282 rs = rq.get("http://httpbin.org/relative-redirect/2"); 283 assert((cast(HTTPResponse)rs).history.length == 2); 284 assert((cast(HTTPResponse)rs).code==200); 285 286 info("Check cookie"); 287 rq = Request(); 288 rs = rq.get("http://httpbin.org/cookies/set?A=abcd&b=cdef"); 289 assert(rs.code == 200); 290 json = parseJSON(rs.responseBody.data).object["cookies"].object; 291 assert(json["A"].str == "abcd"); 292 assert(json["b"].str == "cdef"); 293 auto cookie = rq.cookie(); 294 foreach(c; rq.cookie) { 295 final switch(c.attr) { 296 case "A": 297 assert(c.value == "abcd"); 298 break; 299 case "b": 300 assert(c.value == "cdef"); 301 break; 302 } 303 } 304 rs = rq.get("http://httpbin.org/absolute-redirect/2"); 305 assert((cast(HTTPResponse)rs).history.length == 2); 306 assert((cast(HTTPResponse)rs).code==200); 307 // rq = Request(); 308 rq.maxRedirects = 2; 309 rq.keepAlive = false; 310 rs = rq.get("https://httpbin.org/absolute-redirect/3"); 311 assert((cast(HTTPResponse)rs).history.length == 2); 312 assert((cast(HTTPResponse)rs).code==302); 313 314 info("Check utf8 content"); 315 globalLogLevel(LogLevel.info); 316 rq = Request(); 317 rs = rq.get("http://httpbin.org/encoding/utf8"); 318 assert(rs.code==200); 319 320 info("Check chunked content"); 321 globalLogLevel(LogLevel.info); 322 rq = Request(); 323 rq.keepAlive = true; 324 rq.bufferSize = 16*1024; 325 rs = rq.get("http://httpbin.org/range/1024"); 326 assert(rs.code==200); 327 assert(rs.responseBody.length==1024); 328 329 info("Check basic auth"); 330 globalLogLevel(LogLevel.info); 331 rq = Request(); 332 rq.authenticator = new BasicAuthentication("user", "passwd"); 333 rs = rq.get("http://httpbin.org/basic-auth/user/passwd"); 334 assert(rs.code==200); 335 336 globalLogLevel(LogLevel.info); 337 info("Check limits"); 338 rq = Request(); 339 rq.maxContentLength = 1; 340 assertThrown!RequestException(rq.get("http://httpbin.org/")); 341 rq = Request(); 342 rq.maxHeadersLength = 1; 343 assertThrown!RequestException(rq.get("http://httpbin.org/")); 344 // 345 info("ftp post ", "ftp://speedtest.tele2.net/upload/TEST.TXT"); 346 rs = rq.post("ftp://speedtest.tele2.net/upload/TEST.TXT", "test, ignore please\n".representation); 347 assert(rs.code == 226); 348 info("ftp get ", "ftp://speedtest.tele2.net/nonexistent", ", in same session."); 349 rs = rq.get("ftp://speedtest.tele2.net/nonexistent"); 350 assert(rs.code != 226); 351 info("ftp get ", "ftp://speedtest.tele2.net/1KB.zip", ", in same session."); 352 rs = rq.get("ftp://speedtest.tele2.net/1KB.zip"); 353 assert(rs.code == 226); 354 assert(rs.responseBody.length == 1024); 355 info("ftp get ", "ftp://ftp.uni-bayreuth.de/README"); 356 rs = rq.get("ftp://ftp.uni-bayreuth.de/README"); 357 assert(rs.code == 226); 358 info("ftp post ", "ftp://speedtest.tele2.net/upload/TEST.TXT"); 359 rs = rq.post("ftp://speedtest.tele2.net/upload/TEST.TXT", "another test, ignore please\n".representation); 360 assert(rs.code == 226); 361 info("ftp get ", "ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT"); 362 rs = rq.get("ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT"); 363 assert(rs.code == 226); 364 info("testing ftp - done."); 365 } 366 367 auto queryParams(A...)(A args) pure @safe nothrow { 368 QueryParam[] res; 369 static if ( args.length >= 2 ) { 370 res = QueryParam(args[0].to!string, args[1].to!string) ~ queryParams(args[2..$]); 371 } 372 return res; 373 } 374 /** 375 * Call GET, and return response content. 376 * This is the simplest case, when all you need is the response body and have no parameters. 377 * Returns: 378 * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method. 379 */ 380 public auto getContent(A...)(string url) { 381 auto rq = Request(); 382 auto rs = rq.get(url); 383 return rs.responseBody; 384 } 385 /** 386 * Call GET, and return response content. 387 * args = string[string] fo query parameters. 388 * Returns: 389 * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method. 390 */ 391 public auto getContent(A...)(string url, string[string] args) { 392 auto rq = Request(); 393 auto rs = rq.get(url, args); 394 return rs.responseBody; 395 } 396 /** 397 * Call GET, and return response content. 398 * args = QueryParam[] of parameters. 399 * Returns: 400 * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method. 401 */ 402 public auto getContent(A...)(string url, QueryParam[] args) { 403 auto rq = Request(); 404 auto rs = rq.get(url, args); 405 return rs.responseBody; 406 } 407 /** 408 * Call GET, and return response content. 409 * args = variadic args to supply parameter names and values. 410 * Returns: 411 * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method. 412 */ 413 public auto getContent(A...)(string url, A args) if (args.length > 1 && args.length % 2 == 0 ) { 414 return Request(). 415 get(url, queryParams(args)). 416 responseBody; 417 } 418 419 /// 420 package unittest { 421 import std.algorithm; 422 import std.stdio; 423 import std.json; 424 globalLogLevel(LogLevel.info); 425 info("Test getContent"); 426 auto r = getContent("https://httpbin.org/stream/20"); 427 assert(r.splitter('\n').filter!("a.length>0").count == 20); 428 r = getContent("ftp://speedtest.tele2.net/1KB.zip"); 429 assert(r.length == 1024); 430 r = getContent("https://httpbin.org/get", ["a":"b", "c":"d"]); 431 432 string name = "user", sex = "male"; 433 int age = 42; 434 r = getContent("https://httpbin.org/get", "name", name, "age", age, "sex", sex); 435 436 info("Test receiveAsRange with GET"); 437 auto rq = Request(); 438 rq.useStreaming = true; 439 rq.bufferSize = 16; 440 auto rs = rq.get("http://httpbin.org/get"); 441 auto stream = rs.receiveAsRange(); 442 ubyte[] streamedContent; 443 while( !stream.empty() ) { 444 streamedContent ~= stream.front; 445 stream.popFront(); 446 } 447 rq = Request(); 448 rs = rq.get("http://httpbin.org/get"); 449 assert(streamedContent == rs.responseBody.data); 450 rq.useStreaming = true; 451 streamedContent.length = 0; 452 rs = rq.get("ftp://speedtest.tele2.net/1KB.zip"); 453 stream = rs.receiveAsRange; 454 while( !stream.empty() ) { 455 streamedContent ~= stream.front; 456 stream.popFront(); 457 } 458 459 } 460 /// 461 package unittest { 462 globalLogLevel(LogLevel.info); 463 info("Test get in parallel"); 464 import std.stdio; 465 import std.parallelism; 466 import std.algorithm; 467 import std.string; 468 import core.atomic; 469 470 immutable auto urls = [ 471 "http://httpbin.org/stream/10", 472 "https://httpbin.org/stream/20", 473 "http://httpbin.org/stream/30", 474 "https://httpbin.org/stream/40", 475 "http://httpbin.org/stream/50", 476 "https://httpbin.org/stream/60", 477 "http://httpbin.org/stream/70", 478 ]; 479 480 defaultPoolThreads(3); 481 482 shared short lines; 483 484 foreach(url; parallel(urls)) { 485 atomicOp!"+="(lines, getContent(url).splitter("\n").count); 486 } 487 assert(lines == 287); 488 } 489 490 /** 491 * Call post and return response content. 492 */ 493 public auto postContent(A...)(string url, A args) { 494 auto rq = Request(); 495 auto rs = rq.post(url, args); 496 return rs.responseBody; 497 } 498 499 /// 500 package unittest { 501 import std.json; 502 import std.string; 503 import std.stdio; 504 import std.range; 505 506 globalLogLevel(LogLevel.info); 507 info("Test postContent"); 508 auto r = postContent("http://httpbin.org/post", `{"a":"b", "c":1}`, "application/json"); 509 assert(parseJSON(r.data).object["json"].object["c"].integer == 1); 510 511 /// ftp upload from range 512 info("Test postContent ftp"); 513 r = postContent("ftp://speedtest.tele2.net/upload/TEST.TXT", "test, ignore please\n".representation); 514 assert(r.length == 0); 515 516 /// Posting to forms (for small data) 517 /// 518 /// posting query parameters using "application/x-www-form-urlencoded" 519 info("Test postContent using query params"); 520 postContent("http://httpbin.org/post", queryParams("first", "a", "second", 2)); 521 522 /// posting using multipart/form-data (large data and files). See docs fot HTTPRequest 523 info("Test postContent form"); 524 MultipartForm form; 525 form.add(formData(/* field name */ "greeting", /* content */ cast(ubyte[])"hello")); 526 postContent("http://httpbin.org/post", form); 527 528 /// you can do this using Request struct to access response details 529 info("Test postContent form via Request()"); 530 auto rq = Request(); 531 form = MultipartForm().add(formData(/* field name */ "greeting", /* content */ cast(ubyte[])"hello")); 532 auto rs = rq.post("http://httpbin.org/post", form); 533 assert(rs.code == 200); 534 535 info("Test receiveAsRange with POST"); 536 rq = Request(); 537 rq.useStreaming = true; 538 rq.bufferSize = 16; 539 string s = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; 540 rs = rq.post("http://httpbin.org/post", s.representation.chunks(10), "application/octet-stream"); 541 auto stream = rs.receiveAsRange(); 542 ubyte[] streamedContent; 543 while( !stream.empty() ) { 544 streamedContent ~= stream.front; 545 stream.popFront(); 546 } 547 rq = Request(); 548 rs = rq.post("http://httpbin.org/post", s.representation.chunks(10), "application/octet-stream"); 549 assert(streamedContent == rs.responseBody.data); 550 551 } 552