requests source code

1 module requests;
2 
3 public import requests.http;
4 public import requests.ftp;
5 public import requests.streams;
6 public import requests.base;
7 public import requests.uri;
8 public import requests.request;
9 public import requests.pool;
10 public import requests.utils;
11 
12 import std.datetime;
13 import std.conv;
14 import std.experimental.logger;
15 import requests.utils;
16 
17 ///
18 package unittest {
19     import std.algorithm;
20     import std.range;
21     import std.array;
22     import std.json;
23     import std.stdio;
24     import std.string;
25     import std.exception;
26 
27     string httpbinUrl = httpTestServer();
28     globalLogLevel(LogLevel.info);
29 
30     version(vibeD) {
31     }
32     else {
33         import httpbin;
34         auto server = httpbinApp();
35         server.start();
36         scope(exit) {
37             server.stop();
38         }
39     }
40 
41 
42     infof("testing Request");
43     Request rq;
44     Response rs;
45     // moved from http.d
46 
47     URI uri = URI(httpbinUrl);
48     rs = rq.get(httpbinUrl);
49     assert(rs.code==200);
50     assert(rs.responseBody.length > 0);
51     assert(rq.format("%m|%h|%p|%P|%q|%U") ==
52     "GET|%s|%d|%s||%s"
53     .format(uri.host, uri.port, uri.path, httpbinUrl));
54 
55     //
56     rs = rq.get(httpbinUrl);
57     assert(rs.code==200);
58     assert(rs.responseBody.length > 0);
59     rs = rq.get(httpbinUrl ~ "get", ["c":" d", "a":"b"]);
60     assert(rs.code == 200);
61     auto json = parseJSON(cast(string)rs.responseBody.data).object["args"].object;
62     assert(json["c"].str == " d");
63     assert(json["a"].str == "b");
64 
65     
66     rq = Request();
67     rq.keepAlive = false; // disable keepalive on idempotents requests
68     {
69         info("Check handling incomplete status line");
70         rs = rq.get(httpbinUrl ~ "incomplete");
71         if (httpbinUrl != "http://httpbin.org/") {
72             // 600 when test direct server respond, or 502 if test through squid
73             assert(rs.code==600 || rs.code == 502);
74         }
75     }
76     // handmade json
77     info("Check POST json");
78     rs = rq.post(httpbinUrl ~ "post?b=x", `{"a":"b ", "c":[1,2,3]}`, "application/json");
79     assert(rs.code==200);
80     json = parseJSON(cast(string)rs.responseBody.data).object["args"].object;
81     assert(json["b"].str == "x");
82     json = parseJSON(cast(string)rs.responseBody.data).object["json"].object;
83     assert(json["a"].str == "b ");
84     assert(json["c"].array.map!(a=>a.integer).array == [1,2,3]);
85     {
86         import std.file;
87         import std.path;
88         auto tmpd = tempDir();
89         auto tmpfname = tmpd ~ dirSeparator ~ "request_test.txt";
90         auto f = File(tmpfname, "wb");
91         f.rawWrite("abcdefgh\n12345678\n");
92         f.close();
93         // files
94         info("Check POST files");
95         PostFile[] files = [
96             {fileName: tmpfname, fieldName:"abc", contentType:"application/octet-stream"}, 
97             {fileName: tmpfname}
98         ];
99         rs = rq.post(httpbinUrl ~ "post", files);
100         assert(rs.code==200);
101         info("Check POST chunked from file.byChunk");
102         f = File(tmpfname, "rb");
103         rs = rq.post(httpbinUrl ~ "post", f.byChunk(3), "application/octet-stream");
104         if (httpbinUrl != "http://httpbin.org/" ) {
105             assert(rs.code==200);
106             auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]);
107             assert(data=="abcdefgh\n12345678\n");
108         }
109         f.close();
110     }
111     // ranges
112     {
113         info("Check POST chunked from lineSplitter");
114         auto s = lineSplitter("one,\ntwo,\nthree.");
115         rs = rq.exec!"POST"(httpbinUrl ~ "post", s, "application/octet-stream");
116         if (httpbinUrl != "http://httpbin.org/" ) {
117             assert(rs.code==200);
118             auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]);
119             assert(data=="one,two,three.");
120         }
121     }
122     {
123         info("Check POST chunked from array");
124         auto s = ["one,", "two,", "three."];
125         rs = rq.post(httpbinUrl ~ "post", s, "application/octet-stream");
126         if (httpbinUrl != "http://httpbin.org/" ) {
127             assert(rs.code==200);
128             auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]);
129             assert(data=="one,two,three.");
130         }
131     }
132     {
133         info("Check POST chunked using std.range.chunks()");
134         auto s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
135         rs = rq.post(httpbinUrl ~ "post", s.representation.chunks(10), "application/octet-stream");
136         if (httpbinUrl != "http://httpbin.org/") {
137             assert(rs.code==200);
138             auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]);
139             assert(data==s);
140         }
141     }
142     info("Check POST from QueryParams");
143     {
144         rs = rq.post(httpbinUrl ~ "post", queryParams("name[]", "first", "name[]", 2));
145         assert(rs.code==200);
146         auto data = parseJSON(cast(string)rs.responseBody).object["form"].object;
147         string[] a;
148         try {
149             a = to!(string[])(data["name[]"].str);
150         }
151         catch (JSONException e) {
152             a = data["name[]"].array.map!"a.str".array;
153         }
154         assert(equal(["first", "2"], a));
155     }
156     info("Check POST json");
157     {
158         rs = rq.post(httpbinUrl ~ "post?b=x", `{"a":"a b", "c":[1,2,3]}`, "application/json");
159         assert(rs.code==200);
160         auto j = parseJSON(cast(string)rs.responseBody).object["args"].object;
161         assert(j["b"].str == "x");
162         j = parseJSON(cast(string)rs.responseBody).object["json"].object;
163         assert(j["a"].str == "a b");
164         assert(j["c"].array.map!(a=>a.integer).array == [1,2,3]);
165     }
166     // associative array
167     info("Check POST from AA");
168     rs = rq.post(httpbinUrl ~ "post", ["a":"b ", "c":"d"]);
169     assert(rs.code==200);
170     auto form = parseJSON(cast(string)rs.responseBody.data).object["form"].object;
171     assert(form["a"].str == "b ");
172     assert(form["c"].str == "d");
173     info("Check HEAD");
174     rs = rq.exec!"HEAD"(httpbinUrl);
175     assert(rs.code==200);
176     rs = rq.execute("HEAD", httpbinUrl);
177     assert(rs.code==200);
178     info("Check DELETE");
179     rs = rq.exec!"DELETE"(httpbinUrl ~ "delete");
180     assert(rs.code==200);
181     rs = rq.execute("DELETE", httpbinUrl ~ "delete");
182     assert(rs.code==200);
183     info("Check PUT");
184     rs = rq.exec!"PUT"(httpbinUrl ~ "put",  `{"a":"b", "c":[1,2,3]}`, "application/json");
185     assert(rs.code==200);
186     rs = rq.execute("PUT", httpbinUrl ~ "put",  `{"a":"b", "c":[1,2,3]}`, "application/json");
187     assert(rs.code==200);
188     info("Check PATCH");
189     rs = rq.exec!"PATCH"(httpbinUrl ~ "patch", "привiт, свiт!", "application/octet-stream");
190     assert(rs.code==200);
191     rs = rq.execute("PATCH", httpbinUrl ~ "patch", "привiт, свiт!", "application/octet-stream");
192     assert(rs.code==200);
193 
194     info("Check compressed content");
195     rq = Request();
196     rq.keepAlive = true;
197     rq.addHeaders(["X-Header": "test"]);
198     rs = rq.get(httpbinUrl ~ "gzip");
199     assert(rs.code==200);
200     info("gzip - ok");
201     rs = rq.get(httpbinUrl ~ "deflate");
202     assert(rs.code==200);
203     info("deflate - ok");
204     
205     info("Check cookie");
206     rq = Request();
207     rs = rq.get(httpbinUrl ~ "cookies/set?A=abcd&b=cdef");
208     assert(rs.code == 200);
209     json = parseJSON(cast(string)rs.responseBody.data).object["cookies"].object;
210     assert(json["A"].str == "abcd");
211     assert(json["b"].str == "cdef");
212     foreach(c; rq.cookie._array) {
213         final switch(c.attr) {
214             case "A":
215                 assert(c.value == "abcd");
216                 break;
217             case "b":
218                 assert(c.value == "cdef");
219                 break;
220         }
221     }
222 
223     info("Check redirects");
224     rq = Request();
225     rq.keepAlive = true;
226     rs = rq.get(httpbinUrl ~ "relative-redirect/2");
227     assert((cast(HTTPResponse)rs).history.length == 2);
228     assert((cast(HTTPResponse)rs).code==200);
229 
230     rs = rq.get(httpbinUrl ~ "absolute-redirect/2");
231     assert((cast(HTTPResponse)rs).history.length == 2);
232     assert((cast(HTTPResponse)rs).code==200);
233     //    rq = Request();
234     info("Check maxredirects");
235     rq.maxRedirects = 2;
236     rq.keepAlive = false;
237     assertThrown!MaxRedirectsException(rq.get(httpbinUrl ~ "absolute-redirect/3"));
238 
239     rq.maxRedirects = 0;
240     rs = rq.get(httpbinUrl ~ "absolute-redirect/1");
241     assert(rs.code==302);
242     rq.maxRedirects = 10;
243 
244     info("Check chunked content");
245     rq = Request();
246     rq.keepAlive = true;
247     rq.bufferSize = 16*1024;
248     rs = rq.get(httpbinUrl ~ "range/1024");
249     assert(rs.code==200);
250     assert(rs.responseBody.length==1024);
251     
252     info("Check basic auth");
253     rq = Request();
254     rq.authenticator = new BasicAuthentication("user", "passwd");
255     rs = rq.get(httpbinUrl ~ "basic-auth/user/passwd");
256     assert(rs.code==200);
257     
258     info("Check limits");
259     rq = Request();
260     rq.maxContentLength = 1;
261     assertThrown!RequestException(rq.get(httpbinUrl));
262     rq = Request();
263     rq.maxHeadersLength = 1;
264     assertThrown!RequestException(rq.get(httpbinUrl));
265 
266     info("Test getContent");
267     auto r = getContent(httpbinUrl ~ "stream/20");
268     assert(r.splitter('\n').filter!("a.length>0").count == 20);
269     r = getContent(httpbinUrl ~ "get", ["a":"b", "c":"d"]);
270     string name = "user", sex = "male";
271     immutable age = 42;
272     r = getContent(httpbinUrl ~ "get", "name", name, "age", age, "sex", sex);
273 
274     info("Test receiveAsRange with GET");
275     rq = Request();
276     rq.useStreaming = true;
277     rq.bufferSize = 16;
278     rs = rq.get(httpbinUrl ~ "stream/20");
279     auto stream = rs.receiveAsRange();
280     ubyte[] streamedContent;
281     while( !stream.empty() ) {
282         streamedContent ~= stream.front;
283         stream.popFront();
284     }
285     rq = Request();
286     rs = rq.get(httpbinUrl ~ "stream/20");
287     assert(streamedContent.length == rs.responseBody.data.length,
288             "streamedContent.length(%d) == rs.responseBody.data.length(%d)".
289             format(streamedContent.length, rs.responseBody.data.length));
290     info("Test postContent");
291     r = postContent(httpbinUrl ~ "post", `{"a":"b", "c":1}`, "application/json");
292     assert(parseJSON(cast(string)r).object["json"].object["c"].integer == 1);
293 
294     /// Posting to forms (for small data)
295     ///
296     /// posting query parameters using "application/x-www-form-urlencoded"
297     info("Test postContent using query params");
298     r = postContent(httpbinUrl ~ "post", queryParams("first", "a", "second", 2));
299     assert(parseJSON(cast(string)r).object["form"].object["first"].str == "a");
300     
301     /// posting using multipart/form-data (large data and files). See docs fot HTTPRequest
302     info("Test postContent form");
303     MultipartForm mpform;
304     mpform.add(formData(/* field name */ "greeting", /* content */ cast(ubyte[])"hello"));
305     r = postContent(httpbinUrl ~ "post", mpform);
306     assert(parseJSON(cast(string)r).object["form"].object["greeting"].str == "hello");
307     
308     /// you can do this using Request struct to access response details
309     info("Test postContent form via Request()");
310     rq = Request();
311     mpform = MultipartForm().add(formData(/* field name */ "greeting", /* content */ cast(ubyte[])"hello"));
312     rs = rq.post(httpbinUrl ~ "post", mpform);
313     assert(rs.code == 200);
314     assert(parseJSON(cast(string)(rs.responseBody().data)).object["form"].object["greeting"].str == "hello");
315 
316     info("Test receiveAsRange with POST");
317     streamedContent.length = 0;
318     rq = Request();
319     rq.useStreaming = true;
320     rq.bufferSize = 16;
321     string s = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
322     rs = rq.post(httpbinUrl ~ "post", s.representation, "application/octet-stream");
323     stream = rs.receiveAsRange();
324     while( !stream.empty() ) {
325         streamedContent ~= stream.front;
326         stream.popFront();
327     }
328     rq = Request();
329     rs = rq.post(httpbinUrl ~ "post", s.representation, "application/octet-stream");
330     assert(streamedContent == rs.responseBody.data);
331 
332     streamedContent.length = 0;
333     rq = Request();
334     rq.useStreaming = true;
335     rq.bufferSize = 16;
336     mpform = MultipartForm().add(formData(/* field name */ "greeting", /* content */ cast(ubyte[])"hello"));
337     rs = rq.post(httpbinUrl ~ "post", mpform);
338     stream = rs.receiveAsRange();
339     while( !stream.empty() ) {
340         streamedContent ~= stream.front;
341         stream.popFront();
342     }
343     info("Check POST files using multiPartForm");
344     {
345         /// This is example on usage files with MultipartForm data.
346         /// For this example we have to create files which will be sent.
347         import std.file;
348         import std.path;
349         /// preapare files
350         auto tmpd = tempDir();
351         auto tmpfname1 = tmpd ~ dirSeparator ~ "request_test1.txt";
352         auto f = File(tmpfname1, "wb");
353         f.rawWrite("file1 content\n");
354         f.close();
355         auto tmpfname2 = tmpd ~ dirSeparator ~ "request_test2.txt";
356         f = File(tmpfname2, "wb");
357         f.rawWrite("file2 content\n");
358         f.close();
359         ///
360         /// Ok, files ready.
361         /// Now we will prepare Form data
362         ///
363         File f1 = File(tmpfname1, "rb");
364         File f2 = File(tmpfname2, "rb");
365         scope(exit) {
366             f1.close();
367             f2.close();
368         }
369         ///
370         /// for each part we have to set field name, source (ubyte array or opened file) and optional filename and content-type
371         ///
372         MultipartForm mForm = MultipartForm().
373         add(formData("Field1", cast(ubyte[])"form field from memory")).
374         add(formData("Field2", cast(ubyte[])"file field from memory", ["filename":"data2"])).
375         add(formData("File1", f1, ["filename":"file1", "Content-Type": "application/octet-stream"])).
376         add(formData("File2", f2, ["filename":"file2", "Content-Type": "application/octet-stream"]));
377         /// everything ready, send request
378         rs = rq.post(httpbinUrl ~ "post", mForm);
379     }
380 
381     rq = Request();
382     mpform = MultipartForm().add(formData(/* field name */ "greeting", /* content */ cast(ubyte[])"hello"));
383     rs = rq.post(httpbinUrl ~ "post", mpform);
384     assert(parseJSON(cast(string)(rs.responseBody().data)).object["form"].object["greeting"].str ==
385            parseJSON(cast(string)streamedContent).object["form"].object["greeting"].str);
386 
387 
388     info("Test POST'ing from Rank(2) range with user-provided Content-Length");
389     rq.addHeaders(["content-length": to!string(s.length)]);
390     rs = rq.post(httpbinUrl ~ "post", s.representation.chunks(10), "application/octet-stream");
391     auto flat_content = parseJSON(cast(string)rs.responseBody().data).object["data"];
392     if ( flat_content.type == JSON_TYPE.STRING ) {
393         // httpbin.org returns string in "data"
394         assert(s == flat_content.str);
395     } else {
396         // internal httpbin server return array ob bytes
397         assert(s.representation == flat_content.array.map!(i => i.integer).array);
398     }
399 
400     info("Check exception handling, error messages and timeouts are OK");
401     rq.clearHeaders();
402     rq.timeout = 1.seconds;
403     assertThrown!TimeoutException(rq.get(httpbinUrl ~ "delay/3"));
404 
405     info("Test get in parallel");
406     {
407         import std.stdio;
408         import std.parallelism;
409         import std.algorithm;
410         import std.string;
411         import core.atomic;
412         
413         immutable auto urls = [
414             "stream/10",
415             "stream/20",
416             "stream/30",
417             "stream/40",
418             "stream/50",
419             "stream/60",
420             "stream/70",
421         ].map!(a => httpbinUrl ~ a).array.idup;
422         
423         defaultPoolThreads(4);
424         
425         shared short lines;
426         
427         foreach(url; parallel(urls)) {
428             atomicOp!"+="(lines, getContent(url).splitter("\n").count);
429         }
430         assert(lines == 287);
431         
432     }
433 }
434 ///
435 /// Create array of query params from args
436 ///
437 auto queryParams(A...)(A args) pure @safe nothrow {
438     QueryParam[] res;
439     static if ( args.length >= 2 ) {
440         res = [QueryParam(args[0].to!string, args[1].to!string)] ~ queryParams(args[2..$]);
441     }
442     return res;
443 }
444 /**
445  * Call GET, and return response content.
446  * This is the simplest case, when all you need is the response body and have no parameters.
447  * Returns:
448  * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
449  */
450 public auto ref getContent(A...)(string url) {
451     auto rq = Request();
452     auto rs = rq.get(url);
453     return rs.responseBody;
454 }
455 /**
456  * Call GET, and return response content.
457  * args = string[string] fo query parameters.
458  * Returns:
459  * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
460  */
461 public auto ref getContent(A...)(string url, string[string] args) {
462     auto rq = Request();
463     auto rs = rq.get(url, args);
464     return rs.responseBody;
465 }
466 /**
467  * Call GET, and return response content.
468  * args = QueryParam[] of parameters.
469  * Returns:
470  * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
471  */
472 public auto ref getContent(A...)(string url, QueryParam[] args) {
473     auto rq = Request();
474     auto rs = rq.get(url, args);
475     return rs.responseBody;
476 }
477 /**
478  * Call GET, and return response content.
479  * args = variadic args to supply parameter names and values.
480  * Returns:
481  * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
482  */
483 public auto ref getContent(A...)(string url, A args) if (args.length > 1 && args.length % 2 == 0 ) {
484     return Request().
485             get(url, queryParams(args)).
486             responseBody;
487 }
488 
489 ///
490 /// Call post and return response content.
491 ///
492 public auto postContent(A...)(string url, A args) {
493     auto rq = Request();
494     auto rs = rq.post(url, args);
495     return rs.responseBody;
496 }
497 
498 ///
499 package unittest {
500     import std.json;
501     import std.string;
502     import std.stdio;
503     import std.range;
504     import std.process;
505 
506     globalLogLevel(LogLevel.info);
507     // while we have no internal ftp server we can run tests in non-reloable networking environment
508     immutable unreliable_network = environment.get("UNRELIABLENETWORK", "false") == "true";
509 
510     /// ftp upload from range
511     info("Test getContent(ftp)");
512     auto r = getContent("ftp://speedtest.tele2.net/1KB.zip");
513     assert(unreliable_network || r.length == 1024);
514 
515     info("Test receiveAsRange with GET(ftp)");
516     ubyte[] streamedContent;
517     auto rq = Request();
518     rq.useStreaming = true;
519     streamedContent.length = 0;
520     auto rs = rq.get("ftp://speedtest.tele2.net/1KB.zip");
521     auto stream = rs.receiveAsRange;
522     while( !stream.empty() ) {
523         streamedContent ~= stream.front;
524         stream.popFront();
525     }
526     assert(unreliable_network || streamedContent.length == 1024);
527     info("Test postContent ftp");
528     r = postContent("ftp://speedtest.tele2.net/upload/TEST.TXT", "test, ignore please\n".representation);
529     assert(unreliable_network || r.length == 0);
530 
531     //
532     info("ftp post ", "ftp://speedtest.tele2.net/upload/TEST.TXT");
533     rs = rq.post("ftp://speedtest.tele2.net/upload/TEST.TXT", "test, ignore please\n".representation);
534     assert(unreliable_network || rs.code == 226);
535     info("ftp get  ", "ftp://speedtest.tele2.net/nonexistent", ", in same session.");
536     rs = rq.get("ftp://speedtest.tele2.net/nonexistent");
537     assert(unreliable_network || rs.code != 226);
538     rq.useStreaming = false;
539     info("ftp get  ", "ftp://speedtest.tele2.net/1KB.zip", ", in same session.");
540     rs = rq.get("ftp://speedtest.tele2.net/1KB.zip");
541     assert(unreliable_network || rs.code == 226);
542     assert(unreliable_network || rs.responseBody.length == 1024);
543     info("ftp post ", "ftp://speedtest.tele2.net/upload/TEST.TXT");
544     rs = rq.post("ftp://speedtest.tele2.net/upload/TEST.TXT", "another test, ignore please\n".representation);
545     assert(unreliable_network || rs.code == 226);
546     info("ftp get  ", "ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT");
547     try {
548         rs = rq.get("ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT");
549     } catch (ConnectError e)
550     {
551     }
552     assert(unreliable_network || rs.code == 226);
553     rq.authenticator = new BasicAuthentication("anonymous", "request@");
554     try {
555         rs = rq.get("ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT");
556     } catch (ConnectError e)
557     {
558     }
559     assert(unreliable_network || rs.code == 226);
560     info("testing ftp - done.");
561 }
562