1 module requests; 2 3 public import requests.http; 4 public import requests.ftp; 5 public import requests.streams; 6 public import requests.base; 7 public import requests.uri; 8 public import requests.request; 9 public import requests.pool; 10 11 import std.datetime; 12 import std.conv; 13 import std.experimental.logger; 14 import requests.utils; 15 16 /// 17 package unittest { 18 import std.algorithm; 19 import std.range; 20 import std.array; 21 import std.json; 22 import std.stdio; 23 import std.string; 24 import std.exception; 25 26 string httpbinUrl = httpTestServer(); 27 28 version(vibeD) { 29 } 30 else { 31 import httpbin; 32 auto server = httpbinApp(); 33 server.start(); 34 scope(exit) { 35 server.stop(); 36 } 37 } 38 39 globalLogLevel(LogLevel.info); 40 41 infof("testing Request"); 42 Request rq; 43 Response rs; 44 // 45 rs = rq.get(httpbinUrl); 46 assert(rs.code==200); 47 assert(rs.responseBody.length > 0); 48 rs = rq.get(httpbinUrl ~ "get", ["c":" d", "a":"b"]); 49 assert(rs.code == 200); 50 auto json = parseJSON(cast(string)rs.responseBody.data).object["args"].object; 51 assert(json["c"].str == " d"); 52 assert(json["a"].str == "b"); 53 54 rq = Request(); 55 rq.keepAlive = true; 56 // handmade json 57 info("Check POST json"); 58 rs = rq.post(httpbinUrl ~ "post?b=x", `{"a":"b ", "c":[1,2,3]}`, "application/json"); 59 assert(rs.code==200); 60 json = parseJSON(cast(string)rs.responseBody.data).object["args"].object; 61 assert(json["b"].str == "x"); 62 json = parseJSON(cast(string)rs.responseBody.data).object["json"].object; 63 assert(json["a"].str == "b "); 64 assert(json["c"].array.map!(a=>a.integer).array == [1,2,3]); 65 { 66 import std.file; 67 import std.path; 68 auto tmpd = tempDir(); 69 auto tmpfname = tmpd ~ dirSeparator ~ "request_test.txt"; 70 auto f = File(tmpfname, "wb"); 71 f.rawWrite("abcdefgh\n12345678\n"); 72 f.close(); 73 // files 74 info("Check POST files"); 75 PostFile[] files = [ 76 {fileName: tmpfname, fieldName:"abc", contentType:"application/octet-stream"}, 77 {fileName: tmpfname} 78 ]; 79 rs = rq.post(httpbinUrl ~ "post", files); 80 assert(rs.code==200); 81 info("Check POST chunked from file.byChunk"); 82 f = File(tmpfname, "rb"); 83 rs = rq.post(httpbinUrl ~ "post", f.byChunk(3), "application/octet-stream"); 84 if (httpbinUrl != "http://httpbin.org/" ) { 85 assert(rs.code==200); 86 auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]); 87 assert(data=="abcdefgh\n12345678\n"); 88 } 89 f.close(); 90 } 91 // ranges 92 { 93 info("Check POST chunked from lineSplitter"); 94 auto s = lineSplitter("one,\ntwo,\nthree."); 95 rs = rq.exec!"POST"(httpbinUrl ~ "post", s, "application/octet-stream"); 96 if (httpbinUrl != "http://httpbin.org/" ) { 97 assert(rs.code==200); 98 auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]); 99 assert(data=="one,two,three."); 100 } 101 } 102 { 103 info("Check POST chunked from array"); 104 auto s = ["one,", "two,", "three."]; 105 rs = rq.post(httpbinUrl ~ "post", s, "application/octet-stream"); 106 if (httpbinUrl != "http://httpbin.org/" ) { 107 assert(rs.code==200); 108 auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]); 109 assert(data=="one,two,three."); 110 } 111 } 112 { 113 info("Check POST chunked using std.range.chunks()"); 114 auto s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; 115 rs = rq.post(httpbinUrl ~ "post", s.representation.chunks(10), "application/octet-stream"); 116 if (httpbinUrl != "http://httpbin.org/") { 117 assert(rs.code==200); 118 auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]); 119 assert(data==s); 120 } 121 } 122 // associative array 123 rs = rq.post(httpbinUrl ~ "post", ["a":"b ", "c":"d"]); 124 assert(rs.code==200); 125 auto form = parseJSON(cast(string)rs.responseBody.data).object["form"].object; 126 assert(form["a"].str == "b "); 127 assert(form["c"].str == "d"); 128 info("Check HEAD"); 129 rs = rq.exec!"HEAD"(httpbinUrl); 130 assert(rs.code==200); 131 info("Check DELETE"); 132 rs = rq.exec!"DELETE"(httpbinUrl ~ "delete"); 133 assert(rs.code==200); 134 info("Check PUT"); 135 rs = rq.exec!"PUT"(httpbinUrl ~ "put", `{"a":"b", "c":[1,2,3]}`, "application/json"); 136 assert(rs.code==200); 137 info("Check PATCH"); 138 rs = rq.exec!"PATCH"(httpbinUrl ~ "patch", "привiт, свiт!", "application/octet-stream"); 139 assert(rs.code==200); 140 141 info("Check compressed content"); 142 rq = Request(); 143 rq.keepAlive = true; 144 rq.addHeaders(["X-Header": "test"]); 145 rs = rq.get(httpbinUrl ~ "gzip"); 146 assert(rs.code==200); 147 info("gzip - ok"); 148 rs = rq.get(httpbinUrl ~ "deflate"); 149 assert(rs.code==200); 150 info("deflate - ok"); 151 152 info("Check redirects"); 153 rq = Request(); 154 rq.keepAlive = true; 155 rs = rq.get(httpbinUrl ~ "relative-redirect/2"); 156 assert((cast(HTTPResponse)rs).history.length == 2); 157 assert((cast(HTTPResponse)rs).code==200); 158 159 info("Check cookie"); 160 rq = Request(); 161 rs = rq.get(httpbinUrl ~ "cookies/set?A=abcd&b=cdef"); 162 assert(rs.code == 200); 163 json = parseJSON(cast(string)rs.responseBody.data).object["cookies"].object; 164 assert(json["A"].str == "abcd"); 165 assert(json["b"].str == "cdef"); 166 auto cookie = rq.cookie(); 167 foreach(c; rq.cookie) { 168 final switch(c.attr) { 169 case "A": 170 assert(c.value == "abcd"); 171 break; 172 case "b": 173 assert(c.value == "cdef"); 174 break; 175 } 176 } 177 rs = rq.get(httpbinUrl ~ "absolute-redirect/2"); 178 assert((cast(HTTPResponse)rs).history.length == 2); 179 assert((cast(HTTPResponse)rs).code==200); 180 // rq = Request(); 181 rq.maxRedirects = 2; 182 rq.keepAlive = false; 183 assertThrown!MaxRedirectsException(rq.get(httpbinUrl ~ "absolute-redirect/3")); 184 185 info("Check chunked content"); 186 rq = Request(); 187 rq.keepAlive = true; 188 rq.bufferSize = 16*1024; 189 rs = rq.get(httpbinUrl ~ "range/1024"); 190 assert(rs.code==200); 191 assert(rs.responseBody.length==1024); 192 193 info("Check basic auth"); 194 rq = Request(); 195 rq.authenticator = new BasicAuthentication("user", "passwd"); 196 rs = rq.get(httpbinUrl ~ "basic-auth/user/passwd"); 197 assert(rs.code==200); 198 199 info("Check limits"); 200 rq = Request(); 201 rq.maxContentLength = 1; 202 assertThrown!RequestException(rq.get(httpbinUrl)); 203 rq = Request(); 204 rq.maxHeadersLength = 1; 205 assertThrown!RequestException(rq.get(httpbinUrl)); 206 207 info("Test getContent"); 208 auto r = getContent(httpbinUrl ~ "stream/20"); 209 assert(r.splitter('\n').filter!("a.length>0").count == 20); 210 r = getContent(httpbinUrl ~ "get", ["a":"b", "c":"d"]); 211 string name = "user", sex = "male"; 212 int age = 42; 213 r = getContent(httpbinUrl ~ "get", "name", name, "age", age, "sex", sex); 214 215 info("Test receiveAsRange with GET"); 216 rq = Request(); 217 rq.useStreaming = true; 218 rq.bufferSize = 16; 219 rs = rq.get(httpbinUrl ~ "stream/20"); 220 auto stream = rs.receiveAsRange(); 221 ubyte[] streamedContent; 222 while( !stream.empty() ) { 223 streamedContent ~= stream.front; 224 stream.popFront(); 225 } 226 rq = Request(); 227 rs = rq.get(httpbinUrl ~ "stream/20"); 228 assert(streamedContent.length == rs.responseBody.data.length); 229 info("Test postContent"); 230 r = postContent(httpbinUrl ~ "post", `{"a":"b", "c":1}`, "application/json"); 231 assert(parseJSON(cast(string)r).object["json"].object["c"].integer == 1); 232 233 /// Posting to forms (for small data) 234 /// 235 /// posting query parameters using "application/x-www-form-urlencoded" 236 info("Test postContent using query params"); 237 postContent(httpbinUrl ~ "post", queryParams("first", "a", "second", 2)); 238 239 /// posting using multipart/form-data (large data and files). See docs fot HTTPRequest 240 info("Test postContent form"); 241 MultipartForm mpform; 242 mpform.add(formData(/* field name */ "greeting", /* content */ cast(ubyte[])"hello")); 243 postContent(httpbinUrl ~ "post", mpform); 244 245 /// you can do this using Request struct to access response details 246 info("Test postContent form via Request()"); 247 rq = Request(); 248 mpform = MultipartForm().add(formData(/* field name */ "greeting", /* content */ cast(ubyte[])"hello")); 249 rs = rq.post(httpbinUrl ~ "post", mpform); 250 assert(rs.code == 200); 251 252 info("Test receiveAsRange with POST"); 253 streamedContent.length = 0; 254 rq = Request(); 255 rq.useStreaming = true; 256 rq.bufferSize = 16; 257 string s = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; 258 rs = rq.post(httpbinUrl ~ "post", s.representation.chunks(10), "application/octet-stream"); 259 stream = rs.receiveAsRange(); 260 while( !stream.empty() ) { 261 streamedContent ~= stream.front; 262 stream.popFront(); 263 } 264 rq = Request(); 265 rs = rq.post(httpbinUrl ~ "post", s.representation.chunks(10), "application/octet-stream"); 266 assert(streamedContent == rs.responseBody.data); 267 info("Test get in parallel"); 268 { 269 import std.stdio; 270 import std.parallelism; 271 import std.algorithm; 272 import std.string; 273 import core.atomic; 274 275 immutable auto urls = [ 276 "stream/10", 277 "stream/20", 278 "stream/30", 279 "stream/40", 280 "stream/50", 281 "stream/60", 282 "stream/70", 283 ].map!(a => httpbinUrl ~ a).array.idup; 284 285 defaultPoolThreads(4); 286 287 shared short lines; 288 289 foreach(url; parallel(urls)) { 290 atomicOp!"+="(lines, getContent(url).splitter("\n").count); 291 } 292 assert(lines == 287); 293 294 } 295 } 296 297 auto queryParams(A...)(A args) pure @safe nothrow { 298 QueryParam[] res; 299 static if ( args.length >= 2 ) { 300 res = QueryParam(args[0].to!string, args[1].to!string) ~ queryParams(args[2..$]); 301 } 302 return res; 303 } 304 /** 305 * Call GET, and return response content. 306 * This is the simplest case, when all you need is the response body and have no parameters. 307 * Returns: 308 * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method. 309 */ 310 public auto ref getContent(A...)(string url) { 311 auto rq = Request(); 312 auto rs = rq.get(url); 313 return rs.responseBody; 314 } 315 /** 316 * Call GET, and return response content. 317 * args = string[string] fo query parameters. 318 * Returns: 319 * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method. 320 */ 321 public auto ref getContent(A...)(string url, string[string] args) { 322 auto rq = Request(); 323 auto rs = rq.get(url, args); 324 return rs.responseBody; 325 } 326 /** 327 * Call GET, and return response content. 328 * args = QueryParam[] of parameters. 329 * Returns: 330 * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method. 331 */ 332 public auto ref getContent(A...)(string url, QueryParam[] args) { 333 auto rq = Request(); 334 auto rs = rq.get(url, args); 335 return rs.responseBody; 336 } 337 /** 338 * Call GET, and return response content. 339 * args = variadic args to supply parameter names and values. 340 * Returns: 341 * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method. 342 */ 343 public auto ref getContent(A...)(string url, A args) if (args.length > 1 && args.length % 2 == 0 ) { 344 return Request(). 345 get(url, queryParams(args)). 346 responseBody; 347 } 348 349 /// 350 /// Call post and return response content. 351 /// 352 public auto postContent(A...)(string url, A args) { 353 auto rq = Request(); 354 auto rs = rq.post(url, args); 355 return rs.responseBody; 356 } 357 358 /// 359 package unittest { 360 import std.json; 361 import std.string; 362 import std.stdio; 363 import std.range; 364 365 globalLogLevel(LogLevel.info); 366 367 /// ftp upload from range 368 info("Test getContent(ftp)"); 369 auto r = getContent("ftp://speedtest.tele2.net/1KB.zip"); 370 assert(r.length == 1024); 371 372 info("Test postContent ftp"); 373 r = postContent("ftp://speedtest.tele2.net/upload/TEST.TXT", "test, ignore please\n".representation); 374 assert(r.length == 0); 375 376 info("Test receiveAsRange with GET(ftp)"); 377 ubyte[] streamedContent; 378 auto rq = Request(); 379 rq.useStreaming = true; 380 streamedContent.length = 0; 381 auto rs = rq.get("ftp://speedtest.tele2.net/1KB.zip"); 382 auto stream = rs.receiveAsRange; 383 while( !stream.empty() ) { 384 streamedContent ~= stream.front; 385 stream.popFront(); 386 } 387 assert(streamedContent.length == 1024); 388 // 389 info("ftp post ", "ftp://speedtest.tele2.net/upload/TEST.TXT"); 390 rs = rq.post("ftp://speedtest.tele2.net/upload/TEST.TXT", "test, ignore please\n".representation); 391 assert(rs.code == 226); 392 info("ftp get ", "ftp://speedtest.tele2.net/nonexistent", ", in same session."); 393 rs = rq.get("ftp://speedtest.tele2.net/nonexistent"); 394 assert(rs.code != 226); 395 rq.useStreaming = false; 396 info("ftp get ", "ftp://speedtest.tele2.net/1KB.zip", ", in same session."); 397 rs = rq.get("ftp://speedtest.tele2.net/1KB.zip"); 398 assert(rs.code == 226); 399 assert(rs.responseBody.length == 1024); 400 info("ftp post ", "ftp://speedtest.tele2.net/upload/TEST.TXT"); 401 rs = rq.post("ftp://speedtest.tele2.net/upload/TEST.TXT", "another test, ignore please\n".representation); 402 assert(rs.code == 226); 403 info("ftp get ", "ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT"); 404 rs = rq.get("ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT"); 405 assert(rs.code == 226); 406 rq.authenticator = new BasicAuthentication("anonymous", "request@"); 407 rs = rq.get("ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT"); 408 assert(rs.code == 226); 409 info("testing ftp - done."); 410 } 411