1 module requests; 2 3 public import requests.http; 4 public import requests.ftp; 5 public import requests.streams; 6 public import requests.base; 7 public import requests.uri; 8 public import requests.request; 9 public import requests.pool; 10 11 import std.datetime; 12 import std.conv; 13 import std.experimental.logger; 14 import requests.utils; 15 16 /// 17 package unittest { 18 import std.algorithm; 19 import std.range; 20 import std.array; 21 import std.json; 22 import std.stdio; 23 import std.string; 24 import std.exception; 25 26 string httpbinUrl = httpTestServer(); 27 28 version(vibeD) { 29 } 30 else { 31 import httpbin; 32 auto server = httpbinApp(); 33 server.start(); 34 scope(exit) { 35 server.stop(); 36 } 37 } 38 39 globalLogLevel(LogLevel.info); 40 41 infof("testing Request"); 42 Request rq; 43 Response rs; 44 // 45 rs = rq.get(httpbinUrl); 46 assert(rs.code==200); 47 assert(rs.responseBody.length > 0); 48 rs = rq.get(httpbinUrl ~ "get", ["c":" d", "a":"b"]); 49 assert(rs.code == 200); 50 auto json = parseJSON(cast(string)rs.responseBody.data).object["args"].object; 51 assert(json["c"].str == " d"); 52 assert(json["a"].str == "b"); 53 54 55 rq = Request(); 56 rq.keepAlive = true; 57 { 58 info("Check handling incomplete status line"); 59 rs = rq.get(httpbinUrl ~ "incomplete"); 60 if (httpbinUrl != "http://httpbin.org/") { 61 assert(rs.code==600); 62 } 63 } 64 // handmade json 65 info("Check POST json"); 66 rs = rq.post(httpbinUrl ~ "post?b=x", `{"a":"b ", "c":[1,2,3]}`, "application/json"); 67 assert(rs.code==200); 68 json = parseJSON(cast(string)rs.responseBody.data).object["args"].object; 69 assert(json["b"].str == "x"); 70 json = parseJSON(cast(string)rs.responseBody.data).object["json"].object; 71 assert(json["a"].str == "b "); 72 assert(json["c"].array.map!(a=>a.integer).array == [1,2,3]); 73 { 74 import std.file; 75 import std.path; 76 auto tmpd = tempDir(); 77 auto tmpfname = tmpd ~ dirSeparator ~ "request_test.txt"; 78 auto f = File(tmpfname, "wb"); 79 f.rawWrite("abcdefgh\n12345678\n"); 80 f.close(); 81 // files 82 info("Check POST files"); 83 PostFile[] files = [ 84 {fileName: tmpfname, fieldName:"abc", contentType:"application/octet-stream"}, 85 {fileName: tmpfname} 86 ]; 87 rs = rq.post(httpbinUrl ~ "post", files); 88 assert(rs.code==200); 89 info("Check POST chunked from file.byChunk"); 90 f = File(tmpfname, "rb"); 91 rs = rq.post(httpbinUrl ~ "post", f.byChunk(3), "application/octet-stream"); 92 if (httpbinUrl != "http://httpbin.org/" ) { 93 assert(rs.code==200); 94 auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]); 95 assert(data=="abcdefgh\n12345678\n"); 96 } 97 f.close(); 98 } 99 // ranges 100 { 101 info("Check POST chunked from lineSplitter"); 102 auto s = lineSplitter("one,\ntwo,\nthree."); 103 rs = rq.exec!"POST"(httpbinUrl ~ "post", s, "application/octet-stream"); 104 if (httpbinUrl != "http://httpbin.org/" ) { 105 assert(rs.code==200); 106 auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]); 107 assert(data=="one,two,three."); 108 } 109 } 110 { 111 info("Check POST chunked from array"); 112 auto s = ["one,", "two,", "three."]; 113 rs = rq.post(httpbinUrl ~ "post", s, "application/octet-stream"); 114 if (httpbinUrl != "http://httpbin.org/" ) { 115 assert(rs.code==200); 116 auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]); 117 assert(data=="one,two,three."); 118 } 119 } 120 { 121 info("Check POST chunked using std.range.chunks()"); 122 auto s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; 123 rs = rq.post(httpbinUrl ~ "post", s.representation.chunks(10), "application/octet-stream"); 124 if (httpbinUrl != "http://httpbin.org/") { 125 assert(rs.code==200); 126 auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]); 127 assert(data==s); 128 } 129 } 130 // associative array 131 rs = rq.post(httpbinUrl ~ "post", ["a":"b ", "c":"d"]); 132 assert(rs.code==200); 133 auto form = parseJSON(cast(string)rs.responseBody.data).object["form"].object; 134 assert(form["a"].str == "b "); 135 assert(form["c"].str == "d"); 136 info("Check HEAD"); 137 rs = rq.exec!"HEAD"(httpbinUrl); 138 assert(rs.code==200); 139 info("Check DELETE"); 140 rs = rq.exec!"DELETE"(httpbinUrl ~ "delete"); 141 assert(rs.code==200); 142 info("Check PUT"); 143 rs = rq.exec!"PUT"(httpbinUrl ~ "put", `{"a":"b", "c":[1,2,3]}`, "application/json"); 144 assert(rs.code==200); 145 info("Check PATCH"); 146 rs = rq.exec!"PATCH"(httpbinUrl ~ "patch", "привiт, свiт!", "application/octet-stream"); 147 assert(rs.code==200); 148 149 info("Check compressed content"); 150 rq = Request(); 151 rq.keepAlive = true; 152 rq.addHeaders(["X-Header": "test"]); 153 rs = rq.get(httpbinUrl ~ "gzip"); 154 assert(rs.code==200); 155 info("gzip - ok"); 156 rs = rq.get(httpbinUrl ~ "deflate"); 157 assert(rs.code==200); 158 info("deflate - ok"); 159 160 info("Check redirects"); 161 rq = Request(); 162 rq.keepAlive = true; 163 rs = rq.get(httpbinUrl ~ "relative-redirect/2"); 164 assert((cast(HTTPResponse)rs).history.length == 2); 165 assert((cast(HTTPResponse)rs).code==200); 166 167 info("Check cookie"); 168 rq = Request(); 169 rs = rq.get(httpbinUrl ~ "cookies/set?A=abcd&b=cdef"); 170 assert(rs.code == 200); 171 json = parseJSON(cast(string)rs.responseBody.data).object["cookies"].object; 172 assert(json["A"].str == "abcd"); 173 assert(json["b"].str == "cdef"); 174 auto cookie = rq.cookie(); 175 foreach(c; rq.cookie) { 176 final switch(c.attr) { 177 case "A": 178 assert(c.value == "abcd"); 179 break; 180 case "b": 181 assert(c.value == "cdef"); 182 break; 183 } 184 } 185 rs = rq.get(httpbinUrl ~ "absolute-redirect/2"); 186 assert((cast(HTTPResponse)rs).history.length == 2); 187 assert((cast(HTTPResponse)rs).code==200); 188 // rq = Request(); 189 rq.maxRedirects = 2; 190 rq.keepAlive = false; 191 assertThrown!MaxRedirectsException(rq.get(httpbinUrl ~ "absolute-redirect/3")); 192 193 info("Check chunked content"); 194 rq = Request(); 195 rq.keepAlive = true; 196 rq.bufferSize = 16*1024; 197 rs = rq.get(httpbinUrl ~ "range/1024"); 198 assert(rs.code==200); 199 assert(rs.responseBody.length==1024); 200 201 info("Check basic auth"); 202 rq = Request(); 203 rq.authenticator = new BasicAuthentication("user", "passwd"); 204 rs = rq.get(httpbinUrl ~ "basic-auth/user/passwd"); 205 assert(rs.code==200); 206 207 info("Check limits"); 208 rq = Request(); 209 rq.maxContentLength = 1; 210 assertThrown!RequestException(rq.get(httpbinUrl)); 211 rq = Request(); 212 rq.maxHeadersLength = 1; 213 assertThrown!RequestException(rq.get(httpbinUrl)); 214 215 info("Test getContent"); 216 auto r = getContent(httpbinUrl ~ "stream/20"); 217 assert(r.splitter('\n').filter!("a.length>0").count == 20); 218 r = getContent(httpbinUrl ~ "get", ["a":"b", "c":"d"]); 219 string name = "user", sex = "male"; 220 int age = 42; 221 r = getContent(httpbinUrl ~ "get", "name", name, "age", age, "sex", sex); 222 223 info("Test receiveAsRange with GET"); 224 rq = Request(); 225 rq.useStreaming = true; 226 rq.bufferSize = 16; 227 rs = rq.get(httpbinUrl ~ "stream/20"); 228 auto stream = rs.receiveAsRange(); 229 ubyte[] streamedContent; 230 while( !stream.empty() ) { 231 streamedContent ~= stream.front; 232 stream.popFront(); 233 } 234 rq = Request(); 235 rs = rq.get(httpbinUrl ~ "stream/20"); 236 assert(streamedContent.length == rs.responseBody.data.length, 237 "streamedContent.length(%d) == rs.responseBody.data.length(%d)".format(streamedContent.length, rs.responseBody.data.length)); 238 info("Test postContent"); 239 r = postContent(httpbinUrl ~ "post", `{"a":"b", "c":1}`, "application/json"); 240 assert(parseJSON(cast(string)r).object["json"].object["c"].integer == 1); 241 242 /// Posting to forms (for small data) 243 /// 244 /// posting query parameters using "application/x-www-form-urlencoded" 245 info("Test postContent using query params"); 246 postContent(httpbinUrl ~ "post", queryParams("first", "a", "second", 2)); 247 248 /// posting using multipart/form-data (large data and files). See docs fot HTTPRequest 249 info("Test postContent form"); 250 MultipartForm mpform; 251 mpform.add(formData(/* field name */ "greeting", /* content */ cast(ubyte[])"hello")); 252 postContent(httpbinUrl ~ "post", mpform); 253 254 /// you can do this using Request struct to access response details 255 info("Test postContent form via Request()"); 256 rq = Request(); 257 mpform = MultipartForm().add(formData(/* field name */ "greeting", /* content */ cast(ubyte[])"hello")); 258 rs = rq.post(httpbinUrl ~ "post", mpform); 259 assert(rs.code == 200); 260 261 info("Test receiveAsRange with POST"); 262 streamedContent.length = 0; 263 rq = Request(); 264 rq.useStreaming = true; 265 rq.bufferSize = 16; 266 string s = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; 267 rs = rq.post(httpbinUrl ~ "post", s.representation, "application/octet-stream"); 268 stream = rs.receiveAsRange(); 269 while( !stream.empty() ) { 270 streamedContent ~= stream.front; 271 stream.popFront(); 272 } 273 rq = Request(); 274 rs = rq.post(httpbinUrl ~ "post", s.representation, "application/octet-stream"); 275 assert(streamedContent == rs.responseBody.data); 276 277 278 info("Test POST'ing from Rank(2) range with user-provided Content-Length"); 279 rq = Request(); 280 rq.addHeaders(["content-length": to!string(s.length)]); 281 rs = rq.post(httpbinUrl ~ "post", s.representation.chunks(10), "application/octet-stream"); 282 auto flat_content = parseJSON(cast(string)rs.responseBody().data).object["data"]; 283 if ( flat_content.type == JSON_TYPE.STRING ) { 284 // httpbin.org returns string in "data" 285 assert(s == flat_content.str); 286 } else { 287 // internal httpbin server return array ob bytes 288 assert(s.representation == flat_content.array.map!(i => i.integer).array); 289 } 290 291 info("Test get in parallel"); 292 { 293 import std.stdio; 294 import std.parallelism; 295 import std.algorithm; 296 import std.string; 297 import core.atomic; 298 299 immutable auto urls = [ 300 "stream/10", 301 "stream/20", 302 "stream/30", 303 "stream/40", 304 "stream/50", 305 "stream/60", 306 "stream/70", 307 ].map!(a => httpbinUrl ~ a).array.idup; 308 309 defaultPoolThreads(4); 310 311 shared short lines; 312 313 foreach(url; parallel(urls)) { 314 atomicOp!"+="(lines, getContent(url).splitter("\n").count); 315 } 316 assert(lines == 287); 317 318 } 319 } 320 321 auto queryParams(A...)(A args) pure @safe nothrow { 322 QueryParam[] res; 323 static if ( args.length >= 2 ) { 324 res = QueryParam(args[0].to!string, args[1].to!string) ~ queryParams(args[2..$]); 325 } 326 return res; 327 } 328 /** 329 * Call GET, and return response content. 330 * This is the simplest case, when all you need is the response body and have no parameters. 331 * Returns: 332 * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method. 333 */ 334 public auto ref getContent(A...)(string url) { 335 auto rq = Request(); 336 auto rs = rq.get(url); 337 return rs.responseBody; 338 } 339 /** 340 * Call GET, and return response content. 341 * args = string[string] fo query parameters. 342 * Returns: 343 * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method. 344 */ 345 public auto ref getContent(A...)(string url, string[string] args) { 346 auto rq = Request(); 347 auto rs = rq.get(url, args); 348 return rs.responseBody; 349 } 350 /** 351 * Call GET, and return response content. 352 * args = QueryParam[] of parameters. 353 * Returns: 354 * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method. 355 */ 356 public auto ref getContent(A...)(string url, QueryParam[] args) { 357 auto rq = Request(); 358 auto rs = rq.get(url, args); 359 return rs.responseBody; 360 } 361 /** 362 * Call GET, and return response content. 363 * args = variadic args to supply parameter names and values. 364 * Returns: 365 * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method. 366 */ 367 public auto ref getContent(A...)(string url, A args) if (args.length > 1 && args.length % 2 == 0 ) { 368 return Request(). 369 get(url, queryParams(args)). 370 responseBody; 371 } 372 373 /// 374 /// Call post and return response content. 375 /// 376 public auto postContent(A...)(string url, A args) { 377 auto rq = Request(); 378 auto rs = rq.post(url, args); 379 return rs.responseBody; 380 } 381 382 /// 383 package unittest { 384 import std.json; 385 import std.string; 386 import std.stdio; 387 import std.range; 388 import std.process; 389 390 globalLogLevel(LogLevel.info); 391 // while we have no internal ftp server we can run tests in non-reloable networking environment 392 bool unreliable_network = environment.get("UNRELIABLENETWORK", "false") == "true"; 393 394 /// ftp upload from range 395 info("Test getContent(ftp)"); 396 auto r = getContent("ftp://speedtest.tele2.net/1KB.zip"); 397 assert(unreliable_network || r.length == 1024); 398 399 info("Test postContent ftp"); 400 r = postContent("ftp://speedtest.tele2.net/upload/TEST.TXT", "test, ignore please\n".representation); 401 assert(unreliable_network || r.length == 0); 402 403 info("Test receiveAsRange with GET(ftp)"); 404 ubyte[] streamedContent; 405 auto rq = Request(); 406 rq.useStreaming = true; 407 streamedContent.length = 0; 408 auto rs = rq.get("ftp://speedtest.tele2.net/1KB.zip"); 409 auto stream = rs.receiveAsRange; 410 while( !stream.empty() ) { 411 streamedContent ~= stream.front; 412 stream.popFront(); 413 } 414 assert(unreliable_network || streamedContent.length == 1024); 415 // 416 info("ftp post ", "ftp://speedtest.tele2.net/upload/TEST.TXT"); 417 rs = rq.post("ftp://speedtest.tele2.net/upload/TEST.TXT", "test, ignore please\n".representation); 418 assert(unreliable_network || rs.code == 226); 419 info("ftp get ", "ftp://speedtest.tele2.net/nonexistent", ", in same session."); 420 rs = rq.get("ftp://speedtest.tele2.net/nonexistent"); 421 assert(unreliable_network || rs.code != 226); 422 rq.useStreaming = false; 423 info("ftp get ", "ftp://speedtest.tele2.net/1KB.zip", ", in same session."); 424 rs = rq.get("ftp://speedtest.tele2.net/1KB.zip"); 425 assert(unreliable_network || rs.code == 226); 426 assert(unreliable_network || rs.responseBody.length == 1024); 427 info("ftp post ", "ftp://speedtest.tele2.net/upload/TEST.TXT"); 428 rs = rq.post("ftp://speedtest.tele2.net/upload/TEST.TXT", "another test, ignore please\n".representation); 429 assert(unreliable_network || rs.code == 226); 430 info("ftp get ", "ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT"); 431 try { 432 rs = rq.get("ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT"); 433 } catch (ConnectError e) 434 { 435 } 436 assert(unreliable_network || rs.code == 226); 437 rq.authenticator = new BasicAuthentication("anonymous", "request@"); 438 try { 439 rs = rq.get("ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT"); 440 } catch (ConnectError e) 441 { 442 } 443 assert(unreliable_network || rs.code == 226); 444 info("testing ftp - done."); 445 } 446