1 module requests; 2 3 public import requests.http; 4 public import requests.ftp; 5 public import requests.streams; 6 public import requests.base; 7 public import requests.uri; 8 public import requests.request; 9 public import requests.pool; 10 11 import std.datetime; 12 import std.conv; 13 import std.experimental.logger; 14 import requests.utils; 15 16 /// 17 package unittest { 18 import std.algorithm; 19 import std.range; 20 import std.array; 21 import std.json; 22 import std.stdio; 23 import std.string; 24 import std.exception; 25 26 string httpbinUrl = httpTestServer(); 27 28 version(vibeD) { 29 } 30 else { 31 import httpbin; 32 auto server = httpbinApp(); 33 server.start(); 34 scope(exit) { 35 server.stop(); 36 } 37 } 38 39 globalLogLevel(LogLevel.info); 40 41 infof("testing Request"); 42 Request rq; 43 Response rs; 44 // 45 rs = rq.get(httpbinUrl); 46 assert(rs.code==200); 47 assert(rs.responseBody.length > 0); 48 rs = rq.get(httpbinUrl ~ "get", ["c":" d", "a":"b"]); 49 assert(rs.code == 200); 50 auto json = parseJSON(rs.responseBody.data).object["args"].object; 51 assert(json["c"].str == " d"); 52 assert(json["a"].str == "b"); 53 54 rq = Request(); 55 rq.keepAlive = true; 56 // handmade json 57 info("Check POST json"); 58 rs = rq.post(httpbinUrl ~ "post?b=x", `{"a":"b ", "c":[1,2,3]}`, "application/json"); 59 assert(rs.code==200); 60 json = parseJSON(rs.responseBody.data).object["args"].object; 61 assert(json["b"].str == "x"); 62 json = parseJSON(rs.responseBody.data).object["json"].object; 63 assert(json["a"].str == "b "); 64 assert(json["c"].array.map!(a=>a.integer).array == [1,2,3]); 65 { 66 import std.file; 67 import std.path; 68 auto tmpd = tempDir(); 69 auto tmpfname = tmpd ~ dirSeparator ~ "request_test.txt"; 70 auto f = File(tmpfname, "wb"); 71 f.rawWrite("abcdefgh\n12345678\n"); 72 f.close(); 73 // files 74 info("Check POST files"); 75 PostFile[] files = [ 76 {fileName: tmpfname, fieldName:"abc", contentType:"application/octet-stream"}, 77 {fileName: tmpfname} 78 ]; 79 rs = rq.post(httpbinUrl ~ "post", files); 80 assert(rs.code==200); 81 info("Check POST chunked from file.byChunk"); 82 f = File(tmpfname, "rb"); 83 rs = rq.post(httpbinUrl ~ "post", f.byChunk(3), "application/octet-stream"); 84 assert(rs.code==200); 85 auto data = fromJsonArrayToStr(parseJSON(rs.responseBody).object["data"]); 86 assert(data=="abcdefgh\n12345678\n"); 87 f.close(); 88 } 89 // ranges 90 { 91 info("Check POST chunked from lineSplitter"); 92 auto s = lineSplitter("one,\ntwo,\nthree."); 93 rs = rq.exec!"POST"(httpbinUrl ~ "post", s, "application/octet-stream"); 94 assert(rs.code==200); 95 auto data = fromJsonArrayToStr(parseJSON(rs.responseBody).object["data"]); 96 assert(data=="one,two,three."); 97 } 98 { 99 info("Check POST chunked from array"); 100 auto s = ["one,", "two,", "three."]; 101 rs = rq.post(httpbinUrl ~ "post", s, "application/octet-stream"); 102 assert(rs.code==200); 103 auto data = fromJsonArrayToStr(parseJSON(rs.responseBody).object["data"]); 104 assert(data=="one,two,three."); 105 } 106 { 107 info("Check POST chunked using std.range.chunks()"); 108 auto s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; 109 rs = rq.post(httpbinUrl ~ "post", s.representation.chunks(10), "application/octet-stream"); 110 assert(rs.code==200); 111 auto data = fromJsonArrayToStr(parseJSON(rs.responseBody).object["data"]); 112 assert(data==s); 113 } 114 // associative array 115 rs = rq.post(httpbinUrl ~ "post", ["a":"b ", "c":"d"]); 116 assert(rs.code==200); 117 auto form = parseJSON(rs.responseBody.data).object["form"].object; 118 assert(form["a"].str == "b "); 119 assert(form["c"].str == "d"); 120 info("Check HEAD"); 121 rs = rq.exec!"HEAD"(httpbinUrl); 122 assert(rs.code==200); 123 info("Check DELETE"); 124 rs = rq.exec!"DELETE"(httpbinUrl ~ "delete"); 125 assert(rs.code==200); 126 info("Check PUT"); 127 rs = rq.exec!"PUT"(httpbinUrl ~ "put", `{"a":"b", "c":[1,2,3]}`, "application/json"); 128 assert(rs.code==200); 129 info("Check PATCH"); 130 rs = rq.exec!"PATCH"(httpbinUrl ~ "patch", "привiт, свiт!", "application/octet-stream"); 131 assert(rs.code==200); 132 133 info("Check compressed content"); 134 rq = Request(); 135 rq.keepAlive = true; 136 rq.addHeaders(["X-Header": "test"]); 137 rs = rq.get(httpbinUrl ~ "gzip"); 138 assert(rs.code==200); 139 info("gzip - ok"); 140 rs = rq.get(httpbinUrl ~ "deflate"); 141 assert(rs.code==200); 142 info("deflate - ok"); 143 144 info("Check redirects"); 145 rq = Request(); 146 rq.keepAlive = true; 147 rs = rq.get(httpbinUrl ~ "relative-redirect/2"); 148 assert((cast(HTTPResponse)rs).history.length == 2); 149 assert((cast(HTTPResponse)rs).code==200); 150 151 info("Check cookie"); 152 rq = Request(); 153 rs = rq.get(httpbinUrl ~ "cookies/set?A=abcd&b=cdef"); 154 assert(rs.code == 200); 155 json = parseJSON(rs.responseBody.data).object["cookies"].object; 156 assert(json["A"].str == "abcd"); 157 assert(json["b"].str == "cdef"); 158 auto cookie = rq.cookie(); 159 foreach(c; rq.cookie) { 160 final switch(c.attr) { 161 case "A": 162 assert(c.value == "abcd"); 163 break; 164 case "b": 165 assert(c.value == "cdef"); 166 break; 167 } 168 } 169 rs = rq.get(httpbinUrl ~ "absolute-redirect/2"); 170 assert((cast(HTTPResponse)rs).history.length == 2); 171 assert((cast(HTTPResponse)rs).code==200); 172 // rq = Request(); 173 rq.maxRedirects = 2; 174 rq.keepAlive = false; 175 assertThrown!MaxRedirectsException(rq.get(httpbinUrl ~ "absolute-redirect/3")); 176 177 info("Check chunked content"); 178 rq = Request(); 179 rq.keepAlive = true; 180 rq.bufferSize = 16*1024; 181 rs = rq.get(httpbinUrl ~ "range/1024"); 182 assert(rs.code==200); 183 assert(rs.responseBody.length==1024); 184 185 info("Check basic auth"); 186 rq = Request(); 187 rq.authenticator = new BasicAuthentication("user", "passwd"); 188 rs = rq.get(httpbinUrl ~ "basic-auth/user/passwd"); 189 assert(rs.code==200); 190 191 info("Check limits"); 192 rq = Request(); 193 rq.maxContentLength = 1; 194 assertThrown!RequestException(rq.get(httpbinUrl)); 195 rq = Request(); 196 rq.maxHeadersLength = 1; 197 assertThrown!RequestException(rq.get(httpbinUrl)); 198 199 info("Test getContent"); 200 auto r = getContent(httpbinUrl ~ "stream/20"); 201 assert(r.splitter('\n').filter!("a.length>0").count == 20); 202 r = getContent(httpbinUrl ~ "get", ["a":"b", "c":"d"]); 203 string name = "user", sex = "male"; 204 int age = 42; 205 r = getContent(httpbinUrl ~ "get", "name", name, "age", age, "sex", sex); 206 207 info("Test receiveAsRange with GET"); 208 rq = Request(); 209 rq.useStreaming = true; 210 rq.bufferSize = 16; 211 rs = rq.get(httpbinUrl ~ "stream/20"); 212 auto stream = rs.receiveAsRange(); 213 ubyte[] streamedContent; 214 while( !stream.empty() ) { 215 streamedContent ~= stream.front; 216 stream.popFront(); 217 } 218 rq = Request(); 219 rs = rq.get(httpbinUrl ~ "stream/20"); 220 assert(streamedContent == rs.responseBody.data); 221 info("Test postContent"); 222 r = postContent(httpbinUrl ~ "post", `{"a":"b", "c":1}`, "application/json"); 223 assert(parseJSON(r).object["json"].object["c"].integer == 1); 224 225 /// Posting to forms (for small data) 226 /// 227 /// posting query parameters using "application/x-www-form-urlencoded" 228 info("Test postContent using query params"); 229 postContent(httpbinUrl ~ "post", queryParams("first", "a", "second", 2)); 230 231 /// posting using multipart/form-data (large data and files). See docs fot HTTPRequest 232 info("Test postContent form"); 233 MultipartForm mpform; 234 mpform.add(formData(/* field name */ "greeting", /* content */ cast(ubyte[])"hello")); 235 postContent(httpbinUrl ~ "post", mpform); 236 237 /// you can do this using Request struct to access response details 238 info("Test postContent form via Request()"); 239 rq = Request(); 240 mpform = MultipartForm().add(formData(/* field name */ "greeting", /* content */ cast(ubyte[])"hello")); 241 rs = rq.post(httpbinUrl ~ "post", mpform); 242 assert(rs.code == 200); 243 244 info("Test receiveAsRange with POST"); 245 streamedContent.length = 0; 246 rq = Request(); 247 rq.useStreaming = true; 248 rq.bufferSize = 16; 249 string s = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; 250 rs = rq.post(httpbinUrl ~ "post", s.representation.chunks(10), "application/octet-stream"); 251 stream = rs.receiveAsRange(); 252 while( !stream.empty() ) { 253 streamedContent ~= stream.front; 254 stream.popFront(); 255 } 256 rq = Request(); 257 rs = rq.post(httpbinUrl ~ "post", s.representation.chunks(10), "application/octet-stream"); 258 assert(streamedContent == rs.responseBody.data); 259 info("Test get in parallel"); 260 { 261 import std.stdio; 262 import std.parallelism; 263 import std.algorithm; 264 import std.string; 265 import core.atomic; 266 267 immutable auto urls = [ 268 "stream/10", 269 "stream/20", 270 "stream/30", 271 "stream/40", 272 "stream/50", 273 "stream/60", 274 "stream/70", 275 ].map!(a => httpbinUrl ~ a).array.idup; 276 277 defaultPoolThreads(4); 278 279 shared short lines; 280 281 foreach(url; parallel(urls)) { 282 atomicOp!"+="(lines, getContent(url).splitter("\n").count); 283 } 284 assert(lines == 287); 285 286 } 287 } 288 289 auto queryParams(A...)(A args) pure @safe nothrow { 290 QueryParam[] res; 291 static if ( args.length >= 2 ) { 292 res = QueryParam(args[0].to!string, args[1].to!string) ~ queryParams(args[2..$]); 293 } 294 return res; 295 } 296 /** 297 * Call GET, and return response content. 298 * This is the simplest case, when all you need is the response body and have no parameters. 299 * Returns: 300 * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method. 301 */ 302 public auto ref getContent(A...)(string url) { 303 auto rq = Request(); 304 auto rs = rq.get(url); 305 return rs.responseBody; 306 } 307 /** 308 * Call GET, and return response content. 309 * args = string[string] fo query parameters. 310 * Returns: 311 * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method. 312 */ 313 public auto ref getContent(A...)(string url, string[string] args) { 314 auto rq = Request(); 315 auto rs = rq.get(url, args); 316 return rs.responseBody; 317 } 318 /** 319 * Call GET, and return response content. 320 * args = QueryParam[] of parameters. 321 * Returns: 322 * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method. 323 */ 324 public auto ref getContent(A...)(string url, QueryParam[] args) { 325 auto rq = Request(); 326 auto rs = rq.get(url, args); 327 return rs.responseBody; 328 } 329 /** 330 * Call GET, and return response content. 331 * args = variadic args to supply parameter names and values. 332 * Returns: 333 * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method. 334 */ 335 public auto ref getContent(A...)(string url, A args) if (args.length > 1 && args.length % 2 == 0 ) { 336 return Request(). 337 get(url, queryParams(args)). 338 responseBody; 339 } 340 341 /// 342 /// Call post and return response content. 343 /// 344 public auto postContent(A...)(string url, A args) { 345 auto rq = Request(); 346 auto rs = rq.post(url, args); 347 return rs.responseBody; 348 } 349 350 /// 351 package unittest { 352 import std.json; 353 import std.string; 354 import std.stdio; 355 import std.range; 356 357 globalLogLevel(LogLevel.info); 358 359 /// ftp upload from range 360 info("Test getContent(ftp)"); 361 auto r = getContent("ftp://speedtest.tele2.net/1KB.zip"); 362 assert(r.length == 1024); 363 364 info("Test postContent ftp"); 365 r = postContent("ftp://speedtest.tele2.net/upload/TEST.TXT", "test, ignore please\n".representation); 366 assert(r.length == 0); 367 368 info("Test receiveAsRange with GET(ftp)"); 369 ubyte[] streamedContent; 370 auto rq = Request(); 371 rq.useStreaming = true; 372 streamedContent.length = 0; 373 auto rs = rq.get("ftp://speedtest.tele2.net/1KB.zip"); 374 auto stream = rs.receiveAsRange; 375 while( !stream.empty() ) { 376 streamedContent ~= stream.front; 377 stream.popFront(); 378 } 379 assert(streamedContent.length == 1024); 380 // 381 info("ftp post ", "ftp://speedtest.tele2.net/upload/TEST.TXT"); 382 rs = rq.post("ftp://speedtest.tele2.net/upload/TEST.TXT", "test, ignore please\n".representation); 383 assert(rs.code == 226); 384 info("ftp get ", "ftp://speedtest.tele2.net/nonexistent", ", in same session."); 385 rs = rq.get("ftp://speedtest.tele2.net/nonexistent"); 386 assert(rs.code != 226); 387 rq.useStreaming = false; 388 info("ftp get ", "ftp://speedtest.tele2.net/1KB.zip", ", in same session."); 389 rs = rq.get("ftp://speedtest.tele2.net/1KB.zip"); 390 assert(rs.code == 226); 391 assert(rs.responseBody.length == 1024); 392 info("ftp post ", "ftp://speedtest.tele2.net/upload/TEST.TXT"); 393 rs = rq.post("ftp://speedtest.tele2.net/upload/TEST.TXT", "another test, ignore please\n".representation); 394 assert(rs.code == 226); 395 info("ftp get ", "ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT"); 396 rs = rq.get("ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT"); 397 assert(rs.code == 226); 398 rq.authenticator = new BasicAuthentication("anonymous", "request@"); 399 rs = rq.get("ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT"); 400 assert(rs.code == 226); 401 info("testing ftp - done."); 402 } 403