requests source code

1 /**********************************************************************************
2 
3 HTTP client library, inspired by python-requests with goals:
4 
5  $(UL
6    $(LI small memory footprint)
7    $(LI performance)
8    $(LI simple, high level API)
9    $(LI native D implementation)
10  )
11 */
12 module requests;
13 
14 public import requests.http;
15 public import requests.ftp;
16 public import requests.streams;
17 public import requests.base;
18 public import requests.uri;
19 public import requests.request;
20 public import requests.pool;
21 public import requests.utils;
22 
23 import std.datetime;
24 import std.conv;
25 import std.experimental.logger;
26 import requests.utils;
27 
28 ///
29 package unittest {
30     import std.algorithm;
31     import std.range;
32     import std.array;
33     import std.json;
34     import std.stdio;
35     import std.string;
36     import std.exception;
37 
38     string httpbinUrl = httpTestServer();
39     globalLogLevel(LogLevel.info);
40 
41     version(vibeD) {
42     }
43     else {
44         import httpbin;
45         auto server = httpbinApp();
46         server.start();
47         scope(exit) {
48             server.stop();
49         }
50     }
51 
52 
53     infof("testing Request");
54     Request rq;
55     Response rs;
56     // moved from http.d
57 
58     URI uri = URI(httpbinUrl);
59     rs = rq.get(httpbinUrl);
60     assert(rs.code==200);
61     assert(rs.responseBody.length > 0);
62     assert(rq.format("%m|%h|%p|%P|%q|%U") ==
63     "GET|%s|%d|%s||%s"
64     .format(uri.host, uri.port, uri.path, httpbinUrl));
65 
66     //
67     rs = rq.get(httpbinUrl);
68     assert(rs.code==200);
69     assert(rs.responseBody.length > 0);
70     rs = rq.get(httpbinUrl ~ "get", ["c":" d", "a":"b"]);
71     assert(rs.code == 200);
72     auto json = parseJSON(cast(string)rs.responseBody.data).object["args"].object;
73     assert(json["c"].str == " d");
74     assert(json["a"].str == "b");
75 
76     
77     rq = Request();
78     rq.keepAlive = false; // disable keepalive on idempotents requests
79     {
80         info("Check handling incomplete status line");
81         rs = rq.get(httpbinUrl ~ "incomplete");
82         if (httpbinUrl != "http://httpbin.org/") {
83             // 600 when test direct server respond, or 502 if test through squid
84             assert(rs.code==600 || rs.code == 502);
85         }
86     }
87     // handmade json
88     info("Check POST json");
89     rs = rq.post(httpbinUrl ~ "post?b=x", `{"a":"b ", "c":[1,2,3]}`, "application/json");
90     assert(rs.code==200);
91     json = parseJSON(cast(string)rs.responseBody.data).object["args"].object;
92     assert(json["b"].str == "x");
93     json = parseJSON(cast(string)rs.responseBody.data).object["json"].object;
94     assert(json["a"].str == "b ");
95     assert(json["c"].array.map!(a=>a.integer).array == [1,2,3]);
96     {
97         import std.file;
98         import std.path;
99         auto tmpd = tempDir();
100         auto tmpfname = tmpd ~ dirSeparator ~ "request_test.txt";
101         auto f = File(tmpfname, "wb");
102         f.rawWrite("abcdefgh\n12345678\n");
103         f.close();
104         // files
105         info("Check POST files");
106         PostFile[] files = [
107             {fileName: tmpfname, fieldName:"abc", contentType:"application/octet-stream"}, 
108             {fileName: tmpfname}
109         ];
110         rs = rq.post(httpbinUrl ~ "post", files);
111         assert(rs.code==200);
112         info("Check POST chunked from file.byChunk");
113         f = File(tmpfname, "rb");
114         rs = rq.post(httpbinUrl ~ "post", f.byChunk(3), "application/octet-stream");
115         if (httpbinUrl != "http://httpbin.org/" ) {
116             assert(rs.code==200);
117             auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]);
118             assert(data=="abcdefgh\n12345678\n");
119         }
120         f.close();
121     }
122     // ranges
123     {
124         info("Check POST chunked from lineSplitter");
125         auto s = lineSplitter("one,\ntwo,\nthree.");
126         rs = rq.exec!"POST"(httpbinUrl ~ "post", s, "application/octet-stream");
127         if (httpbinUrl != "http://httpbin.org/" ) {
128             assert(rs.code==200);
129             auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]);
130             assert(data=="one,two,three.");
131         }
132     }
133     {
134         info("Check POST chunked from array");
135         auto s = ["one,", "two,", "three."];
136         rs = rq.post(httpbinUrl ~ "post", s, "application/octet-stream");
137         if (httpbinUrl != "http://httpbin.org/" ) {
138             assert(rs.code==200);
139             auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]);
140             assert(data=="one,two,three.");
141         }
142     }
143     {
144         info("Check POST chunked using std.range.chunks()");
145         auto s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
146         rs = rq.post(httpbinUrl ~ "post", s.representation.chunks(10), "application/octet-stream");
147         if (httpbinUrl != "http://httpbin.org/") {
148             assert(rs.code==200);
149             auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]);
150             assert(data==s);
151         }
152     }
153     info("Check POST from QueryParams");
154     {
155         rs = rq.post(httpbinUrl ~ "post", queryParams("name[]", "first", "name[]", 2));
156         assert(rs.code==200);
157         auto data = parseJSON(cast(string)rs.responseBody).object["form"].object;
158         string[] a;
159         try {
160             a = to!(string[])(data["name[]"].str);
161         }
162         catch (JSONException e) {
163             a = data["name[]"].array.map!"a.str".array;
164         }
165         assert(equal(["first", "2"], a));
166     }
167     info("Check POST json");
168     {
169         rs = rq.post(httpbinUrl ~ "post?b=x", `{"a":"a b", "c":[1,2,3]}`, "application/json");
170         assert(rs.code==200);
171         auto j = parseJSON(cast(string)rs.responseBody).object["args"].object;
172         assert(j["b"].str == "x");
173         j = parseJSON(cast(string)rs.responseBody).object["json"].object;
174         assert(j["a"].str == "a b");
175         assert(j["c"].array.map!(a=>a.integer).array == [1,2,3]);
176     }
177     // associative array
178     info("Check POST from AA");
179     rs = rq.post(httpbinUrl ~ "post", ["a":"b ", "c":"d"]);
180     assert(rs.code==200);
181     auto form = parseJSON(cast(string)rs.responseBody.data).object["form"].object;
182     assert(form["a"].str == "b ");
183     assert(form["c"].str == "d");
184     info("Check HEAD");
185     rs = rq.exec!"HEAD"(httpbinUrl);
186     assert(rs.code==200);
187     rs = rq.execute("HEAD", httpbinUrl);
188     assert(rs.code==200);
189     info("Check DELETE");
190     rs = rq.exec!"DELETE"(httpbinUrl ~ "delete");
191     assert(rs.code==200);
192     rs = rq.execute("DELETE", httpbinUrl ~ "delete");
193     assert(rs.code==200);
194     info("Check PUT");
195     rs = rq.exec!"PUT"(httpbinUrl ~ "put",  `{"a":"b", "c":[1,2,3]}`, "application/json");
196     assert(rs.code==200);
197     rs = rq.execute("PUT", httpbinUrl ~ "put",  `{"a":"b", "c":[1,2,3]}`, "application/json");
198     assert(rs.code==200);
199     info("Check PATCH");
200     rs = rq.exec!"PATCH"(httpbinUrl ~ "patch", "привiт, свiт!", "application/octet-stream");
201     assert(rs.code==200);
202     rs = rq.execute("PATCH", httpbinUrl ~ "patch", "привiт, свiт!", "application/octet-stream");
203     assert(rs.code==200);
204 
205     info("Check compressed content");
206     rq = Request();
207     rq.keepAlive = true;
208     rq.addHeaders(["X-Header": "test"]);
209     rs = rq.get(httpbinUrl ~ "gzip");
210     assert(rs.code==200);
211     info("gzip - ok");
212     rs = rq.get(httpbinUrl ~ "deflate");
213     assert(rs.code==200);
214     info("deflate - ok");
215     
216     info("Check cookie");
217     rq = Request();
218     rs = rq.get(httpbinUrl ~ "cookies/set?A=abcd&b=cdef");
219     assert(rs.code == 200);
220     json = parseJSON(cast(string)rs.responseBody.data).object["cookies"].object;
221     assert(json["A"].str == "abcd");
222     assert(json["b"].str == "cdef");
223     foreach(c; rq.cookie._array) {
224         final switch(c.attr) {
225             case "A":
226                 assert(c.value == "abcd");
227                 break;
228             case "b":
229                 assert(c.value == "cdef");
230                 break;
231         }
232     }
233 
234     info("Check redirects");
235     rq = Request();
236     rq.keepAlive = true;
237     rs = rq.get(httpbinUrl ~ "relative-redirect/2");
238     assert((cast(HTTPResponse)rs).history.length == 2);
239     assert((cast(HTTPResponse)rs).code==200);
240 
241     rs = rq.get(httpbinUrl ~ "absolute-redirect/2");
242     assert((cast(HTTPResponse)rs).history.length == 2);
243     assert((cast(HTTPResponse)rs).code==200);
244     //    rq = Request();
245     info("Check maxredirects");
246     rq.maxRedirects = 2;
247     rq.keepAlive = false;
248     assertThrown!MaxRedirectsException(rq.get(httpbinUrl ~ "absolute-redirect/3"));
249 
250     rq.maxRedirects = 0;
251     rs = rq.get(httpbinUrl ~ "absolute-redirect/1");
252     assert(rs.code==302);
253     rq.maxRedirects = 10;
254 
255     info("Check chunked content");
256     rq = Request();
257     rq.keepAlive = true;
258     rq.bufferSize = 16*1024;
259     rs = rq.get(httpbinUrl ~ "range/1024");
260     assert(rs.code==200);
261     assert(rs.responseBody.length==1024);
262     
263     info("Check basic auth");
264     rq = Request();
265     rq.authenticator = new BasicAuthentication("user", "passwd");
266     rs = rq.get(httpbinUrl ~ "basic-auth/user/passwd");
267     assert(rs.code==200);
268     
269     info("Check limits");
270     rq = Request();
271     rq.maxContentLength = 1;
272     assertThrown!RequestException(rq.get(httpbinUrl));
273     rq = Request();
274     rq.maxHeadersLength = 1;
275     assertThrown!RequestException(rq.get(httpbinUrl));
276 
277     info("Test getContent");
278     auto r = getContent(httpbinUrl ~ "stream/20");
279     assert(r.splitter('\n').filter!("a.length>0").count == 20);
280     r = getContent(httpbinUrl ~ "get", ["a":"b", "c":"d"]);
281     string name = "user", sex = "male";
282     immutable age = 42;
283     r = getContent(httpbinUrl ~ "get", "name", name, "age", age, "sex", sex);
284 
285     info("Test receiveAsRange with GET");
286     rq = Request();
287     rq.useStreaming = true;
288     rq.bufferSize = 16;
289     rs = rq.get(httpbinUrl ~ "stream/20");
290     auto stream = rs.receiveAsRange();
291     ubyte[] streamedContent;
292     while( !stream.empty() ) {
293         streamedContent ~= stream.front;
294         stream.popFront();
295     }
296     rq = Request();
297     rs = rq.get(httpbinUrl ~ "stream/20");
298     assert(streamedContent.length == rs.responseBody.data.length,
299             "streamedContent.length(%d) == rs.responseBody.data.length(%d)".
300             format(streamedContent.length, rs.responseBody.data.length));
301     info("Test postContent");
302     r = postContent(httpbinUrl ~ "post", `{"a":"b", "c":1}`, "application/json");
303     assert(parseJSON(cast(string)r).object["json"].object["c"].integer == 1);
304 
305     /// Posting to forms (for small data)
306     ///
307     /// posting query parameters using "application/x-www-form-urlencoded"
308     info("Test postContent using query params");
309     r = postContent(httpbinUrl ~ "post", queryParams("first", "a", "second", 2));
310     assert(parseJSON(cast(string)r).object["form"].object["first"].str == "a");
311     
312     /// posting using multipart/form-data (large data and files). See docs fot HTTPRequest
313     info("Test postContent form");
314     MultipartForm mpform;
315     mpform.add(formData(/* field name */ "greeting", /* content */ cast(ubyte[])"hello"));
316     r = postContent(httpbinUrl ~ "post", mpform);
317     assert(parseJSON(cast(string)r).object["form"].object["greeting"].str == "hello");
318     
319     /// you can do this using Request struct to access response details
320     info("Test postContent form via Request()");
321     rq = Request();
322     mpform = MultipartForm().add(formData(/* field name */ "greeting", /* content */ cast(ubyte[])"hello"));
323     rs = rq.post(httpbinUrl ~ "post", mpform);
324     assert(rs.code == 200);
325     assert(parseJSON(cast(string)(rs.responseBody().data)).object["form"].object["greeting"].str == "hello");
326 
327     info("Test receiveAsRange with POST");
328     streamedContent.length = 0;
329     rq = Request();
330     rq.useStreaming = true;
331     rq.bufferSize = 16;
332     string s = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
333     rs = rq.post(httpbinUrl ~ "post", s.representation, "application/octet-stream");
334     stream = rs.receiveAsRange();
335     while( !stream.empty() ) {
336         streamedContent ~= stream.front;
337         stream.popFront();
338     }
339     rq = Request();
340     rs = rq.post(httpbinUrl ~ "post", s.representation, "application/octet-stream");
341     assert(streamedContent == rs.responseBody.data);
342 
343     streamedContent.length = 0;
344     rq = Request();
345     rq.useStreaming = true;
346     rq.bufferSize = 16;
347     mpform = MultipartForm().add(formData(/* field name */ "greeting", /* content */ cast(ubyte[])"hello"));
348     rs = rq.post(httpbinUrl ~ "post", mpform);
349     stream = rs.receiveAsRange();
350     while( !stream.empty() ) {
351         streamedContent ~= stream.front;
352         stream.popFront();
353     }
354     info("Check POST files using multiPartForm");
355     {
356         /// This is example on usage files with MultipartForm data.
357         /// For this example we have to create files which will be sent.
358         import std.file;
359         import std.path;
360         /// preapare files
361         auto tmpd = tempDir();
362         auto tmpfname1 = tmpd ~ dirSeparator ~ "request_test1.txt";
363         auto f = File(tmpfname1, "wb");
364         f.rawWrite("file1 content\n");
365         f.close();
366         auto tmpfname2 = tmpd ~ dirSeparator ~ "request_test2.txt";
367         f = File(tmpfname2, "wb");
368         f.rawWrite("file2 content\n");
369         f.close();
370         ///
371         /// Ok, files ready.
372         /// Now we will prepare Form data
373         ///
374         File f1 = File(tmpfname1, "rb");
375         File f2 = File(tmpfname2, "rb");
376         scope(exit) {
377             f1.close();
378             f2.close();
379         }
380         ///
381         /// for each part we have to set field name, source (ubyte array or opened file) and optional filename and content-type
382         ///
383         MultipartForm mForm = MultipartForm().
384         add(formData("Field1", cast(ubyte[])"form field from memory")).
385         add(formData("Field2", cast(ubyte[])"file field from memory", ["filename":"data2"])).
386         add(formData("File1", f1, ["filename":"file1", "Content-Type": "application/octet-stream"])).
387         add(formData("File2", f2, ["filename":"file2", "Content-Type": "application/octet-stream"]));
388         /// everything ready, send request
389         rs = rq.post(httpbinUrl ~ "post", mForm);
390     }
391 
392     rq = Request();
393     mpform = MultipartForm().add(formData(/* field name */ "greeting", /* content */ cast(ubyte[])"hello"));
394     rs = rq.post(httpbinUrl ~ "post", mpform);
395     assert(parseJSON(cast(string)(rs.responseBody().data)).object["form"].object["greeting"].str ==
396            parseJSON(cast(string)streamedContent).object["form"].object["greeting"].str);
397 
398 
399     info("Test POST'ing from Rank(2) range with user-provided Content-Length");
400     rq.addHeaders(["content-length": to!string(s.length)]);
401     rs = rq.post(httpbinUrl ~ "post", s.representation.chunks(10), "application/octet-stream");
402     auto flat_content = parseJSON(cast(string)rs.responseBody().data).object["data"];
403     if ( flat_content.type == JSON_TYPE.STRING ) {
404         // httpbin.org returns string in "data"
405         assert(s == flat_content.str);
406     } else {
407         // internal httpbin server return array ob bytes
408         assert(s.representation == flat_content.array.map!(i => i.integer).array);
409     }
410 
411     info("Check exception handling, error messages and timeouts are OK");
412     rq.clearHeaders();
413     rq.timeout = 1.seconds;
414     assertThrown!TimeoutException(rq.get(httpbinUrl ~ "delay/3"));
415 
416     info("Test get in parallel");
417     {
418         import std.stdio;
419         import std.parallelism;
420         import std.algorithm;
421         import std.string;
422         import core.atomic;
423         
424         immutable auto urls = [
425             "stream/10",
426             "stream/20",
427             "stream/30",
428             "stream/40",
429             "stream/50",
430             "stream/60",
431             "stream/70",
432         ].map!(a => httpbinUrl ~ a).array.idup;
433         
434         defaultPoolThreads(4);
435         
436         shared short lines;
437         
438         foreach(url; parallel(urls)) {
439             atomicOp!"+="(lines, getContent(url).splitter("\n").count);
440         }
441         assert(lines == 287);
442         
443     }
444 }
445 /**
446  * Call GET, and return response content.
447  *
448  * This is the simplest case, when all you need is the response body and have no parameters.
449  * Returns:
450  * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
451  */
452 public auto ref getContent(string url) {
453     auto rq = Request();
454     auto rs = rq.get(url);
455     return rs.responseBody;
456 }
457 /**
458  * Call GET, with parameters, and return response content.
459  *
460  * args = string[string] fo query parameters.
461  * Returns:
462  * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
463  */
464 public auto ref getContent(string url, string[string] args) {
465     auto rq = Request();
466     auto rs = rq.get(url, args);
467     return rs.responseBody;
468 }
469 /**
470  * Call GET, with parameters, and return response content.
471  * 
472  * args = QueryParam[] of parameters.
473  * Returns:
474  * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
475  */
476 public auto ref getContent(string url, QueryParam[] args) {
477     auto rq = Request();
478     auto rs = rq.get(url, args);
479     return rs.responseBody;
480 }
481 /**
482  * Call GET, and return response content.
483  * args = variadic args to supply parameter names and values.
484  * Returns:
485  * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
486  */
487 public auto ref getContent(A...)(string url, A args) if (args.length > 1 && args.length % 2 == 0 ) {
488     return Request().
489             get(url, queryParams(args)).
490             responseBody;
491 }
492 
493 ///
494 /// Call post and return response content.
495 ///
496 public auto postContent(A...)(string url, A args) {
497     auto rq = Request();
498     auto rs = rq.post(url, args);
499     return rs.responseBody;
500 }
501 
502 ///
503 package unittest {
504     import std.json;
505     import std.string;
506     import std.stdio;
507     import std.range;
508     import std.process;
509 
510     //globalLogLevel(LogLevel.info);
511     //// while we have no internal ftp server we can run tests in non-reloable networking environment
512     //immutable unreliable_network = environment.get("UNRELIABLENETWORK", "false") == "true";
513     //
514     ///// ftp upload from range
515     //info("Test getContent(ftp)");
516     //auto r = getContent("ftp://speedtest.tele2.net/1KB.zip");
517     //assert(unreliable_network || r.length == 1024);
518     //
519     //info("Test receiveAsRange with GET(ftp)");
520     //ubyte[] streamedContent;
521     //auto rq = Request();
522     //rq.useStreaming = true;
523     //streamedContent.length = 0;
524     //auto rs = rq.get("ftp://speedtest.tele2.net/1KB.zip");
525     //auto stream = rs.receiveAsRange;
526     //while( !stream.empty() ) {
527     //    streamedContent ~= stream.front;
528     //    stream.popFront();
529     //}
530     //assert(unreliable_network || streamedContent.length == 1024);
531     //info("Test postContent ftp");
532     //r = postContent("ftp://speedtest.tele2.net/upload/TEST.TXT", "test, ignore please\n".representation);
533     //assert(unreliable_network || r.length == 0);
534     //
535     ////
536     //info("ftp post ", "ftp://speedtest.tele2.net/upload/TEST.TXT");
537     //rs = rq.post("ftp://speedtest.tele2.net/upload/TEST.TXT", "test, ignore please\n".representation);
538     //assert(unreliable_network || rs.code == 226);
539     //info("ftp get  ", "ftp://speedtest.tele2.net/nonexistent", ", in same session.");
540     //rs = rq.get("ftp://speedtest.tele2.net/nonexistent");
541     //assert(unreliable_network || rs.code != 226);
542     //rq.useStreaming = false;
543     //info("ftp get  ", "ftp://speedtest.tele2.net/1KB.zip", ", in same session.");
544     //rs = rq.get("ftp://speedtest.tele2.net/1KB.zip");
545     //assert(unreliable_network || rs.code == 226);
546     //assert(unreliable_network || rs.responseBody.length == 1024);
547     //info("ftp post ", "ftp://speedtest.tele2.net/upload/TEST.TXT");
548     //rs = rq.post("ftp://speedtest.tele2.net/upload/TEST.TXT", "another test, ignore please\n".representation);
549     //assert(unreliable_network || rs.code == 226);
550     //info("ftp get  ", "ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT");
551     //try {
552     //    rs = rq.get("ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT");
553     //} catch (ConnectError e)
554     //{
555     //}
556     //assert(unreliable_network || rs.code == 226);
557     //rq.authenticator = new BasicAuthentication("anonymous", "request@");
558     //try {
559     //    rs = rq.get("ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT");
560     //} catch (ConnectError e)
561     //{
562     //}
563     //assert(unreliable_network || rs.code == 226);
564     //info("testing ftp - done.");
565 }
566