1 module requests; 2 3 public import requests.http; 4 public import requests.ftp; 5 public import requests.streams; 6 public import requests.base; 7 public import requests.uri; 8 9 import std.datetime; 10 import std.conv; 11 import std.experimental.logger; 12 import requests.uri; 13 14 /*********************************** 15 * This is simplest interface to both http and ftp protocols. 16 * Request has methods get, post and exec which routed to proper concrete handler (http or ftp, etc). 17 * To enable some protocol-specific featutes you have to use protocol interface directly (see docs for HTTPRequest or FTPRequest) 18 */ 19 struct Request { 20 private { 21 URI _uri; 22 HTTPRequest _http; // route all http/https requests here 23 FTPRequest _ftp; // route all ftp requests here 24 } 25 /// Set timeout on IO operation. 26 /// $(B v) - timeout value 27 /// 28 @property void timeout(Duration v) pure @nogc nothrow { 29 _http.timeout = v; 30 _ftp.timeout = v; 31 } 32 /// Set http keepAlive value 33 /// $(B v) - use keepalive requests - $(B true), or not - $(B false) 34 @property void keepAlive(bool v) pure @nogc nothrow { 35 _http.keepAlive = v; 36 } 37 /// Set limit on HTTP redirects 38 /// $(B v) - limit on redirect depth 39 @property void maxRedirects(uint v) pure @nogc nothrow { 40 _http.maxRedirects = v; 41 } 42 /// Set maximum content lenth both for http and ftp requests 43 /// $(B v) - maximum content length in bytes. When limit reached - throw RequestException 44 @property void maxContentLength(size_t v) pure @nogc nothrow { 45 _http.maxContentLength = v; 46 _ftp.maxContentLength = v; 47 } 48 /// Set maximum length for HTTP headers 49 /// $(B v) - maximum length of the HTTP response. When limit reached - throw RequestException 50 @property void maxHeadersLength(size_t v) pure @nogc nothrow { 51 _http.maxHeadersLength = v; 52 } 53 /// Set IO buffer size for http and ftp requests 54 /// $(B v) - buffer size in bytes. 55 @property void bufferSize(size_t v) { 56 _http.bufferSize = v; 57 _ftp.bufferSize = v; 58 } 59 /// Set verbosity for HTTP or FTP requests. 60 /// $(B v) - verbosity level (0 - no output, 1 - headers to stdout, 2 - headers and body progress to stdout). default = 0. 61 @property void verbosity(uint v) { 62 _http.verbosity = v; 63 _ftp.verbosity = v; 64 } 65 /// Set authenticator for http requests. 66 /// $(B v) - Auth instance. 67 @property void authenticator(Auth v) { 68 _http.authenticator = v; 69 } 70 /// Set Cookie for http requests. 71 /// $(B v) - array of cookie. 72 @property void cookie(Cookie[] v) pure @nogc nothrow { 73 _http.cookie = v; 74 } 75 /// Get Cookie for http requests. 76 /// $(B v) - array of cookie. 77 @property Cookie[] cookie() pure @nogc nothrow { 78 return _http.cookie; 79 } 80 /// Execute GET for http and retrieve file for FTP. 81 /// You have to provide at least $(B uri). All other arguments should conform to HTTPRequest.get or FTPRequest.get depending on the URI scheme. 82 /// When arguments do not conform scheme (for example you try to call get("ftp://somehost.net/pub/README", {"a":"b"}) which doesn't make sense) 83 /// you will receive Exception("Operation not supported for ftp") 84 /// 85 Response get(A...)(string uri, A args) { 86 if ( uri ) { 87 _uri = URI(uri); 88 } 89 final switch ( _uri.scheme ) { 90 case "http", "https": 91 _http.uri = _uri; 92 static if (__traits(compiles, _http.get(null, args))) { 93 return _http.get(null, args); 94 } else { 95 throw new Exception("Operation not supported for http"); 96 } 97 case "ftp": 98 static if (args.length == 0) { 99 return _ftp.get(uri); 100 } else { 101 throw new Exception("Operation not supported for ftp"); 102 } 103 } 104 } 105 /// Execute POST for http and STOR file for FTP. 106 /// You have to provide $(B uri) and data. Data should conform to HTTPRequest.post or FTPRequest.post depending on the URI scheme. 107 /// When arguments do not conform scheme you will receive Exception("Operation not supported for ftp") 108 /// 109 Response post(A...)(string uri, A args) { 110 if ( uri ) { 111 _uri = URI(uri); 112 } 113 final switch ( _uri.scheme ) { 114 case "http", "https": 115 _http.uri = _uri; 116 static if (__traits(compiles, _http.post(null, args))) { 117 return _http.post(null, args); 118 } else { 119 throw new Exception("Operation not supported for http"); 120 } 121 case "ftp": 122 static if (__traits(compiles, _ftp.post(uri, args))) { 123 return _ftp.post(uri, args); 124 } else { 125 throw new Exception("Operation not supported for ftp"); 126 } 127 } 128 } 129 Response exec(string method="GET", A...)(A args) { 130 return _http.exec!(method)(args); 131 } 132 } 133 /// 134 package unittest { 135 import std.algorithm; 136 import std.range; 137 import std.array; 138 import std.json; 139 import std.stdio; 140 import std.string; 141 import std.exception; 142 143 globalLogLevel(LogLevel.info); 144 145 infof("testing Request"); 146 Request rq; 147 Response rs; 148 // 149 rs = rq.get("https://httpbin.org/"); 150 assert(rs.code==200); 151 assert(rs.responseBody.length > 0); 152 rs = rq.get("http://httpbin.org/get", ["c":" d", "a":"b"]); 153 assert(rs.code == 200); 154 auto json = parseJSON(rs.responseBody.data).object["args"].object; 155 assert(json["c"].str == " d"); 156 assert(json["a"].str == "b"); 157 158 globalLogLevel(LogLevel.info); 159 rq = Request(); 160 rq.keepAlive = true; 161 // handmade json 162 info("Check POST json"); 163 rs = rq.post("http://httpbin.org/post?b=x", `{"a":"☺ ", "c":[1,2,3]}`, "application/json"); 164 assert(rs.code==200); 165 json = parseJSON(rs.responseBody.data).object["args"].object; 166 assert(json["b"].str == "x"); 167 json = parseJSON(rs.responseBody.data).object["json"].object; 168 assert(json["a"].str == "☺ "); 169 assert(json["c"].array.map!(a=>a.integer).array == [1,2,3]); 170 { 171 import std.file; 172 import std.path; 173 auto tmpd = tempDir(); 174 auto tmpfname = tmpd ~ dirSeparator ~ "request_test.txt"; 175 auto f = File(tmpfname, "wb"); 176 f.rawWrite("abcdefgh\n12345678\n"); 177 f.close(); 178 // files 179 globalLogLevel(LogLevel.info); 180 info("Check POST files"); 181 PostFile[] files = [ 182 {fileName: tmpfname, fieldName:"abc", contentType:"application/octet-stream"}, 183 {fileName: tmpfname} 184 ]; 185 rs = rq.post("http://httpbin.org/post", files); 186 assert(rs.code==200); 187 info("Check POST chunked from file.byChunk"); 188 f = File(tmpfname, "rb"); 189 rs = rq.post("http://httpbin.org/post", f.byChunk(3), "application/octet-stream"); 190 assert(rs.code==200); 191 auto data = parseJSON(rs.responseBody.data).object["data"].str; 192 assert(data=="abcdefgh\n12345678\n"); 193 f.close(); 194 } 195 { 196 // string 197 info("Check POST utf8 string"); 198 rs = rq.post("http://httpbin.org/post", "привiт, свiт!", "application/octet-stream"); 199 assert(rs.code==200); 200 auto data = parseJSON(rs.responseBody.data).object["data"].str; 201 assert(data=="привiт, свiт!"); 202 } 203 // ranges 204 { 205 info("Check POST chunked from lineSplitter"); 206 auto s = lineSplitter("one,\ntwo,\nthree."); 207 rs = rq.exec!"POST"("http://httpbin.org/post", s, "application/octet-stream"); 208 assert(rs.code==200); 209 auto data = parseJSON(rs.responseBody.toString).object["data"].str; 210 assert(data=="one,two,three."); 211 } 212 { 213 info("Check POST chunked from array"); 214 auto s = ["one,", "two,", "three."]; 215 rs = rq.post("http://httpbin.org/post", s, "application/octet-stream"); 216 assert(rs.code==200); 217 auto data = parseJSON(rs.responseBody.data).object["data"].str; 218 assert(data=="one,two,three."); 219 } 220 { 221 info("Check POST chunked using std.range.chunks()"); 222 auto s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; 223 rs = rq.post("http://httpbin.org/post", s.representation.chunks(10), "application/octet-stream"); 224 assert(rs.code==200); 225 auto data = parseJSON(rs.responseBody.data).object["data"].str; 226 assert(data==s); 227 } 228 // associative array 229 rs = rq.post("http://httpbin.org/post", ["a":"b ", "c":"d"]); 230 assert(rs.code==200); 231 auto form = parseJSON(rs.responseBody.data).object["form"].object; 232 assert(form["a"].str == "b "); 233 assert(form["c"].str == "d"); 234 info("Check HEAD"); 235 rs = rq.exec!"HEAD"("http://httpbin.org/"); 236 assert(rs.code==200); 237 info("Check DELETE"); 238 rs = rq.exec!"DELETE"("http://httpbin.org/delete"); 239 assert(rs.code==200); 240 info("Check PUT"); 241 rs = rq.exec!"PUT"("http://httpbin.org/put", `{"a":"b", "c":[1,2,3]}`, "application/json"); 242 assert(rs.code==200); 243 info("Check PATCH"); 244 rs = rq.exec!"PATCH"("http://httpbin.org/patch", "привiт, свiт!", "application/octet-stream"); 245 assert(rs.code==200); 246 247 info("Check compressed content"); 248 globalLogLevel(LogLevel.info); 249 rq = Request(); 250 rq.keepAlive = true; 251 rs = rq.get("http://httpbin.org/gzip"); 252 assert(rs.code==200); 253 info("gzip - ok"); 254 rs = rq.get("http://httpbin.org/deflate"); 255 assert(rs.code==200); 256 info("deflate - ok"); 257 258 info("Check redirects"); 259 globalLogLevel(LogLevel.info); 260 rq = Request(); 261 rq.keepAlive = true; 262 rs = rq.get("http://httpbin.org/relative-redirect/2"); 263 assert((cast(HTTPResponse)rs).history.length == 2); 264 assert((cast(HTTPResponse)rs).code==200); 265 266 info("Check cookie"); 267 rq = Request(); 268 rs = rq.get("http://httpbin.org/cookies/set?A=abcd&b=cdef"); 269 assert(rs.code == 200); 270 json = parseJSON(rs.responseBody.data).object["cookies"].object; 271 assert(json["A"].str == "abcd"); 272 assert(json["b"].str == "cdef"); 273 auto cookie = rq.cookie(); 274 foreach(c; rq.cookie) { 275 final switch(c.attr) { 276 case "A": 277 assert(c.value == "abcd"); 278 break; 279 case "b": 280 assert(c.value == "cdef"); 281 break; 282 } 283 } 284 rs = rq.get("http://httpbin.org/absolute-redirect/2"); 285 assert((cast(HTTPResponse)rs).history.length == 2); 286 assert((cast(HTTPResponse)rs).code==200); 287 // rq = Request(); 288 rq.maxRedirects = 2; 289 rq.keepAlive = false; 290 rs = rq.get("https://httpbin.org/absolute-redirect/3"); 291 assert((cast(HTTPResponse)rs).history.length == 2); 292 assert((cast(HTTPResponse)rs).code==302); 293 294 info("Check utf8 content"); 295 globalLogLevel(LogLevel.info); 296 rq = Request(); 297 rs = rq.get("http://httpbin.org/encoding/utf8"); 298 assert(rs.code==200); 299 300 info("Check chunked content"); 301 globalLogLevel(LogLevel.info); 302 rq = Request(); 303 rq.keepAlive = true; 304 rq.bufferSize = 16*1024; 305 rs = rq.get("http://httpbin.org/range/1024"); 306 assert(rs.code==200); 307 assert(rs.responseBody.length==1024); 308 309 info("Check basic auth"); 310 globalLogLevel(LogLevel.info); 311 rq = Request(); 312 rq.authenticator = new BasicAuthentication("user", "passwd"); 313 rs = rq.get("http://httpbin.org/basic-auth/user/passwd"); 314 assert(rs.code==200); 315 316 globalLogLevel(LogLevel.info); 317 info("Check limits"); 318 rq = Request(); 319 rq.maxContentLength = 1; 320 assertThrown!RequestException(rq.get("http://httpbin.org/")); 321 rq = Request(); 322 rq.maxHeadersLength = 1; 323 assertThrown!RequestException(rq.get("http://httpbin.org/")); 324 // 325 info("ftp post ", "ftp://speedtest.tele2.net/upload/TEST.TXT"); 326 rs = rq.post("ftp://speedtest.tele2.net/upload/TEST.TXT", "test, ignore please\n".representation); 327 assert(rs.code == 226); 328 info("ftp get ", "ftp://speedtest.tele2.net/nonexistent", ", in same session."); 329 rs = rq.get("ftp://speedtest.tele2.net/nonexistent"); 330 assert(rs.code != 226); 331 info("ftp get ", "ftp://speedtest.tele2.net/1KB.zip", ", in same session."); 332 rs = rq.get("ftp://speedtest.tele2.net/1KB.zip"); 333 assert(rs.code == 226); 334 assert(rs.responseBody.length == 1024); 335 info("ftp get ", "ftp://ftp.uni-bayreuth.de/README"); 336 rs = rq.get("ftp://ftp.uni-bayreuth.de/README"); 337 assert(rs.code == 226); 338 info("ftp post ", "ftp://speedtest.tele2.net/upload/TEST.TXT"); 339 rs = rq.post("ftp://speedtest.tele2.net/upload/TEST.TXT", "another test, ignore please\n".representation); 340 assert(rs.code == 226); 341 info("ftp get ", "ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT"); 342 rs = rq.get("ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT"); 343 assert(rs.code == 226); 344 info("testing ftp - done."); 345 } 346 347 auto queryParams(A...)(A args) pure @safe nothrow { 348 QueryParam[] res; 349 static if ( args.length >= 2 ) { 350 res = QueryParam(args[0].to!string, args[1].to!string) ~ queryParams(args[2..$]); 351 } 352 return res; 353 } 354 /** 355 * Call GET, and return response content. 356 * This is the simplest case, when all you need is the response body and have no parameters. 357 * Returns: 358 * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method. 359 */ 360 public auto getContent(A...)(string url) { 361 auto rq = Request(); 362 auto rs = rq.get(url); 363 return rs.responseBody; 364 } 365 /** 366 * Call GET, and return response content. 367 * args = string[string] fo query parameters. 368 * Returns: 369 * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method. 370 */ 371 public auto getContent(A...)(string url, string[string] args) { 372 auto rq = Request(); 373 auto rs = rq.get(url, args); 374 return rs.responseBody; 375 } 376 /** 377 * Call GET, and return response content. 378 * args = QueryParam[] of parameters. 379 * Returns: 380 * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method. 381 */ 382 public auto getContent(A...)(string url, QueryParam[] args) { 383 auto rq = Request(); 384 auto rs = rq.get(url, args); 385 return rs.responseBody; 386 } 387 /** 388 * Call GET, and return response content. 389 * args = variadic args to supply parameter names and values. 390 * Returns: 391 * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method. 392 */ 393 public auto getContent(A...)(string url, A args) if (args.length > 1 && args.length % 2 == 0 ) { 394 return Request(). 395 get(url, queryParams(args)). 396 responseBody; 397 } 398 399 /// 400 package unittest { 401 import std.algorithm; 402 import std.stdio; 403 import std.json; 404 globalLogLevel(LogLevel.info); 405 info("Test getContent"); 406 auto r = getContent("https://httpbin.org/stream/20"); 407 assert(r.splitter('\n').filter!("a.length>0").count == 20); 408 r = getContent("ftp://speedtest.tele2.net/1KB.zip"); 409 assert(r.length == 1024); 410 r = getContent("https://httpbin.org/get", ["a":"b", "c":"d"]); 411 412 string name = "user", sex = "male"; 413 int age = 42; 414 r = getContent("https://httpbin.org/get", "name", name, "age", age, "sex", sex); 415 } 416 /// 417 package unittest { 418 globalLogLevel(LogLevel.info); 419 info("Test get in parallel"); 420 import std.stdio; 421 import std.parallelism; 422 import std.algorithm; 423 import std.string; 424 import core.atomic; 425 426 immutable auto urls = [ 427 "http://httpbin.org/stream/10", 428 "https://httpbin.org/stream/20", 429 "http://httpbin.org/stream/30", 430 "https://httpbin.org/stream/40", 431 "http://httpbin.org/stream/50", 432 "https://httpbin.org/stream/60", 433 "http://httpbin.org/stream/70", 434 ]; 435 436 defaultPoolThreads(5); 437 438 shared short lines; 439 440 foreach(url; parallel(urls)) { 441 atomicOp!"+="(lines, getContent(url).splitter("\n").count); 442 } 443 assert(lines == 287); 444 } 445 446 /** 447 * Call post and return response content. 448 */ 449 public auto postContent(A...)(string url, A args) { 450 auto rq = Request(); 451 auto rs = rq.post(url, args); 452 return rs.responseBody; 453 } 454 455 /// 456 package unittest { 457 import std.json; 458 import std.string; 459 import std.stdio; 460 globalLogLevel(LogLevel.info); 461 info("Test postContent"); 462 auto r = postContent("http://httpbin.org/post", `{"a":"b", "c":1}`, "application/json"); 463 assert(parseJSON(r.data).object["json"].object["c"].integer == 1); 464 465 /// ftp upload from range 466 info("Test postContent ftp"); 467 r = postContent("ftp://speedtest.tele2.net/upload/TEST.TXT", "test, ignore please\n".representation); 468 assert(r.length == 0); 469 470 /// Posting to forms (for small data) 471 /// 472 /// posting query parameters using "application/x-www-form-urlencoded" 473 info("Test postContent using query params"); 474 postContent("http://httpbin.org/post", queryParams("first", "a", "second", 2)); 475 476 /// posting using multipart/form-data (large data and files). See docs fot HTTPRequest 477 info("Test postContent form"); 478 MultipartForm form; 479 form.add(formData(/* field name */ "greeting", /* content */ cast(ubyte[])"hello")); 480 postContent("http://httpbin.org/post", form); 481 482 /// you can do this using Request struct to access response details 483 info("Test postContent form via Request()"); 484 auto rq = Request(); 485 form = MultipartForm().add(formData(/* field name */ "greeting", /* content */ cast(ubyte[])"hello")); 486 auto rs = rq.post("http://httpbin.org/post", form); 487 assert(rs.code == 200); 488 } 489