requests source code

1 module requests;
2 
3 public import requests.http;
4 public import requests.ftp;
5 public import requests.streams;
6 public import requests.base;
7 public import requests.uri;
8 
9 import std.datetime;
10 import std.conv;
11 import std.experimental.logger;
12 import requests.uri;
13 
14 /***********************************
15  * This is simplest interface to both http and ftp protocols.
16  * Request has methods get, post and exec which routed to proper concrete handler (http or ftp, etc).
17  * To enable some protocol-specific featutes you have to use protocol interface directly (see docs for HTTPRequest or FTPRequest)
18  */
19 struct Request {
20     private {
21         URI         _uri;
22         HTTPRequest _http;  // route all http/https requests here
23         FTPRequest  _ftp;   // route all ftp requests here
24     }
25     /// Set timeout on IO operation.
26     /// $(B v) - timeout value
27     /// 
28     @property void timeout(Duration v) pure @nogc nothrow {
29         _http.timeout = v;
30         _ftp.timeout = v;
31     }
32     /// Set http keepAlive value
33     /// $(B v) - use keepalive requests - $(B true), or not - $(B false)
34     @property void keepAlive(bool v) pure @nogc nothrow {
35         _http.keepAlive = v;
36     }
37     /// Set limit on HTTP redirects
38     /// $(B v) - limit on redirect depth
39     @property void maxRedirects(uint v) pure @nogc nothrow {
40         _http.maxRedirects = v;
41     }
42     /// Set maximum content lenth both for http and ftp requests
43     /// $(B v) - maximum content length in bytes. When limit reached - throw RequestException
44     @property void maxContentLength(size_t v) pure @nogc nothrow {
45         _http.maxContentLength = v;
46         _ftp.maxContentLength = v;
47     }
48     /// Set maximum length for HTTP headers
49     /// $(B v) - maximum length of the HTTP response. When limit reached - throw RequestException
50     @property void maxHeadersLength(size_t v) pure @nogc nothrow {
51         _http.maxHeadersLength = v;
52     }
53     /// Set IO buffer size for http and ftp requests
54     /// $(B v) - buffer size in bytes.
55     @property void bufferSize(size_t v) {
56         _http.bufferSize = v;
57         _ftp.bufferSize = v;
58     }
59     /// Set verbosity for HTTP or FTP requests.
60     /// $(B v) - verbosity level (0 - no output, 1 - headers to stdout, 2 - headers and body progress to stdout). default = 0.
61     @property void verbosity(uint v) {
62         _http.verbosity = v;
63         _ftp.verbosity = v;
64     }
65     /// Set authenticator for http requests.
66     /// $(B v) - Auth instance.
67     @property void authenticator(Auth v) {
68         _http.authenticator = v;
69     }
70     /// Set Cookie for http requests.
71     /// $(B v) - array of cookie.
72     @property void cookie(Cookie[] v) pure @nogc nothrow {
73         _http.cookie = v;
74     }
75     /// Get Cookie for http requests.
76     /// $(B v) - array of cookie.
77     @property Cookie[] cookie()  pure @nogc nothrow {
78         return _http.cookie;
79     }
80     @property void useStreaming(bool v) pure @nogc nothrow {
81         _http.useStreaming = v;
82         _ftp.useStreaming = v;
83     }
84     @property long contentReceived() pure @nogc nothrow {
85         final switch ( _uri.scheme ) {
86             case "http", "https":
87                 return _http.contentReceived;
88             case "ftp":
89                 return _ftp.contentReceived;
90         }
91     }
92     @property long contentLength() pure @nogc nothrow {
93         final switch ( _uri.scheme ) {
94             case "http", "https":
95                 return _http.contentLength;
96             case "ftp":
97                 return _ftp.contentLength;
98         }
99     }
100     /// Execute GET for http and retrieve file for FTP.
101     /// You have to provide at least $(B uri). All other arguments should conform to HTTPRequest.get or FTPRequest.get depending on the URI scheme.
102     /// When arguments do not conform scheme (for example you try to call get("ftp://somehost.net/pub/README", {"a":"b"}) which doesn't make sense)
103     /// you will receive Exception("Operation not supported for ftp")
104     ///
105     Response get(A...)(string uri, A args) {
106         if ( uri ) {
107             _uri = URI(uri);
108         }
109         final switch ( _uri.scheme ) {
110             case "http", "https":
111                 _http.uri = _uri;
112                 static if (__traits(compiles, _http.get(null, args))) {
113                     return _http.get(null, args);
114                 } else {
115                     throw new Exception("Operation not supported for http");
116                 }
117             case "ftp":
118                 static if (args.length == 0) {
119                     return _ftp.get(uri);
120                 } else {
121                     throw new Exception("Operation not supported for ftp");
122                 }
123         }
124     }
125     /// Execute POST for http and STOR file for FTP.
126     /// You have to provide  $(B uri) and data. Data should conform to HTTPRequest.post or FTPRequest.post depending on the URI scheme.
127     /// When arguments do not conform scheme you will receive Exception("Operation not supported for ftp")
128     ///
129     Response post(A...)(string uri, A args) {
130         if ( uri ) {
131             _uri = URI(uri);
132         }
133         final switch ( _uri.scheme ) {
134             case "http", "https":
135                 _http.uri = _uri;
136                 static if (__traits(compiles, _http.post(null, args))) {
137                     return _http.post(null, args);
138                 } else {
139                     throw new Exception("Operation not supported for http");
140                 }
141             case "ftp":
142                 static if (__traits(compiles, _ftp.post(uri, args))) {
143                     return _ftp.post(uri, args);
144                 } else {
145                     throw new Exception("Operation not supported for ftp");
146                 }
147         }
148     }
149     Response exec(string method="GET", A...)(A args) {
150         return _http.exec!(method)(args);
151     }
152 }
153 ///
154 package unittest {
155     import std.algorithm;
156     import std.range;
157     import std.array;
158     import std.json;
159     import std.stdio;
160     import std.string;
161     import std.exception;
162 
163     globalLogLevel(LogLevel.info);
164 
165     infof("testing Request");
166     Request rq;
167     Response rs;
168     //
169     rs = rq.get("https://httpbin.org/");
170     assert(rs.code==200);
171     assert(rs.responseBody.length > 0);
172     rs = rq.get("http://httpbin.org/get", ["c":" d", "a":"b"]);
173     assert(rs.code == 200);
174     auto json = parseJSON(rs.responseBody.data).object["args"].object;
175     assert(json["c"].str == " d");
176     assert(json["a"].str == "b");
177     
178     globalLogLevel(LogLevel.info);
179     rq = Request();
180     rq.keepAlive = true;
181     // handmade json
182     info("Check POST json");
183     rs = rq.post("http://httpbin.org/post?b=x", `{"a":"☺ ", "c":[1,2,3]}`, "application/json");
184     assert(rs.code==200);
185     json = parseJSON(rs.responseBody.data).object["args"].object;
186     assert(json["b"].str == "x");
187     json = parseJSON(rs.responseBody.data).object["json"].object;
188     assert(json["a"].str == "☺ ");
189     assert(json["c"].array.map!(a=>a.integer).array == [1,2,3]);
190     {
191         import std.file;
192         import std.path;
193         auto tmpd = tempDir();
194         auto tmpfname = tmpd ~ dirSeparator ~ "request_test.txt";
195         auto f = File(tmpfname, "wb");
196         f.rawWrite("abcdefgh\n12345678\n");
197         f.close();
198         // files
199         globalLogLevel(LogLevel.info);
200         info("Check POST files");
201         PostFile[] files = [
202         {fileName: tmpfname, fieldName:"abc", contentType:"application/octet-stream"}, 
203         {fileName: tmpfname}
204         ];
205         rs = rq.post("http://httpbin.org/post", files);
206         assert(rs.code==200);
207         info("Check POST chunked from file.byChunk");
208         f = File(tmpfname, "rb");
209         rs = rq.post("http://httpbin.org/post", f.byChunk(3), "application/octet-stream");
210         assert(rs.code==200);
211         auto data = parseJSON(rs.responseBody.data).object["data"].str;
212         assert(data=="abcdefgh\n12345678\n");
213         f.close();
214     }
215     {
216         // string
217         info("Check POST utf8 string");
218         rs = rq.post("http://httpbin.org/post", "привiт, свiт!", "application/octet-stream");
219         assert(rs.code==200);
220         auto data = parseJSON(rs.responseBody.data).object["data"].str;
221         assert(data=="привiт, свiт!");
222     }
223     // ranges
224     {
225         info("Check POST chunked from lineSplitter");
226         auto s = lineSplitter("one,\ntwo,\nthree.");
227         rs = rq.exec!"POST"("http://httpbin.org/post", s, "application/octet-stream");
228         assert(rs.code==200);
229         auto data = parseJSON(rs.responseBody.toString).object["data"].str;
230         assert(data=="one,two,three.");
231     }
232     {
233         info("Check POST chunked from array");
234         auto s = ["one,", "two,", "three."];
235         rs = rq.post("http://httpbin.org/post", s, "application/octet-stream");
236         assert(rs.code==200);
237         auto data = parseJSON(rs.responseBody.data).object["data"].str;
238         assert(data=="one,two,three.");
239     }
240     {
241         info("Check POST chunked using std.range.chunks()");
242         auto s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
243         rs = rq.post("http://httpbin.org/post", s.representation.chunks(10), "application/octet-stream");
244         assert(rs.code==200);
245         auto data = parseJSON(rs.responseBody.data).object["data"].str;
246         assert(data==s);
247     }
248     // associative array
249     rs = rq.post("http://httpbin.org/post", ["a":"b ", "c":"d"]);
250     assert(rs.code==200);
251     auto form = parseJSON(rs.responseBody.data).object["form"].object;
252     assert(form["a"].str == "b ");
253     assert(form["c"].str == "d");
254     info("Check HEAD");
255     rs = rq.exec!"HEAD"("http://httpbin.org/");
256     assert(rs.code==200);
257     info("Check DELETE");
258     rs = rq.exec!"DELETE"("http://httpbin.org/delete");
259     assert(rs.code==200);
260     info("Check PUT");
261     rs = rq.exec!"PUT"("http://httpbin.org/put",  `{"a":"b", "c":[1,2,3]}`, "application/json");
262     assert(rs.code==200);
263     info("Check PATCH");
264     rs = rq.exec!"PATCH"("http://httpbin.org/patch", "привiт, свiт!", "application/octet-stream");
265     assert(rs.code==200);
266     
267     info("Check compressed content");
268     globalLogLevel(LogLevel.info);
269     rq = Request();
270     rq.keepAlive = true;
271     rs = rq.get("http://httpbin.org/gzip");
272     assert(rs.code==200);
273     info("gzip - ok");
274     rs = rq.get("http://httpbin.org/deflate");
275     assert(rs.code==200);
276     info("deflate - ok");
277     
278     info("Check redirects");
279     globalLogLevel(LogLevel.info);
280     rq = Request();
281     rq.keepAlive = true;
282     rs = rq.get("http://httpbin.org/relative-redirect/2");
283     assert((cast(HTTPResponse)rs).history.length == 2);
284     assert((cast(HTTPResponse)rs).code==200);
285 
286     info("Check cookie");
287     rq = Request();
288     rs = rq.get("http://httpbin.org/cookies/set?A=abcd&b=cdef");
289     assert(rs.code == 200);
290     json = parseJSON(rs.responseBody.data).object["cookies"].object;
291     assert(json["A"].str == "abcd");
292     assert(json["b"].str == "cdef");
293     auto cookie = rq.cookie();
294     foreach(c; rq.cookie) {
295         final switch(c.attr) {
296             case "A":
297                 assert(c.value == "abcd");
298                 break;
299             case "b":
300                 assert(c.value == "cdef");
301                 break;
302         }
303     }
304     rs = rq.get("http://httpbin.org/absolute-redirect/2");
305     assert((cast(HTTPResponse)rs).history.length == 2);
306     assert((cast(HTTPResponse)rs).code==200);
307     //    rq = Request();
308     rq.maxRedirects = 2;
309     rq.keepAlive = false;
310     rs = rq.get("https://httpbin.org/absolute-redirect/3");
311     assert((cast(HTTPResponse)rs).history.length == 2);
312     assert((cast(HTTPResponse)rs).code==302);
313     
314     info("Check utf8 content");
315     globalLogLevel(LogLevel.info);
316     rq = Request();
317     rs = rq.get("http://httpbin.org/encoding/utf8");
318     assert(rs.code==200);
319     
320     info("Check chunked content");
321     globalLogLevel(LogLevel.info);
322     rq = Request();
323     rq.keepAlive = true;
324     rq.bufferSize = 16*1024;
325     rs = rq.get("http://httpbin.org/range/1024");
326     assert(rs.code==200);
327     assert(rs.responseBody.length==1024);
328     
329     info("Check basic auth");
330     globalLogLevel(LogLevel.info);
331     rq = Request();
332     rq.authenticator = new BasicAuthentication("user", "passwd");
333     rs = rq.get("http://httpbin.org/basic-auth/user/passwd");
334     assert(rs.code==200);
335     
336     globalLogLevel(LogLevel.info);
337     info("Check limits");
338     rq = Request();
339     rq.maxContentLength = 1;
340     assertThrown!RequestException(rq.get("http://httpbin.org/"));
341     rq = Request();
342     rq.maxHeadersLength = 1;
343     assertThrown!RequestException(rq.get("http://httpbin.org/"));
344     //
345     info("ftp post ", "ftp://speedtest.tele2.net/upload/TEST.TXT");
346     rs = rq.post("ftp://speedtest.tele2.net/upload/TEST.TXT", "test, ignore please\n".representation);
347     assert(rs.code == 226);
348     info("ftp get  ", "ftp://speedtest.tele2.net/nonexistent", ", in same session.");
349     rs = rq.get("ftp://speedtest.tele2.net/nonexistent");
350     assert(rs.code != 226);
351     info("ftp get  ", "ftp://speedtest.tele2.net/1KB.zip", ", in same session.");
352     rs = rq.get("ftp://speedtest.tele2.net/1KB.zip");
353     assert(rs.code == 226);
354     assert(rs.responseBody.length == 1024);
355     info("ftp get  ", "ftp://ftp.uni-bayreuth.de/README");
356     rs = rq.get("ftp://ftp.uni-bayreuth.de/README");
357     assert(rs.code == 226);
358     info("ftp post ", "ftp://speedtest.tele2.net/upload/TEST.TXT");
359     rs = rq.post("ftp://speedtest.tele2.net/upload/TEST.TXT", "another test, ignore please\n".representation);
360     assert(rs.code == 226);
361     info("ftp get  ", "ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT");
362     rs = rq.get("ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT");
363     assert(rs.code == 226);
364     info("testing ftp - done.");
365 }
366 
367 auto queryParams(A...)(A args) pure @safe nothrow {
368     QueryParam[] res;
369     static if ( args.length >= 2 ) {
370         res = QueryParam(args[0].to!string, args[1].to!string) ~ queryParams(args[2..$]);
371     }
372     return res;
373 }
374 /**
375  * Call GET, and return response content.
376  * This is the simplest case, when all you need is the response body and have no parameters.
377  * Returns:
378  * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
379  */
380 public auto getContent(A...)(string url) {
381     auto rq = Request();
382     auto rs = rq.get(url);
383     return rs.responseBody;
384 }
385 /**
386  * Call GET, and return response content.
387  * args = string[string] fo query parameters.
388  * Returns:
389  * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
390  */
391 public auto getContent(A...)(string url, string[string] args) {
392     auto rq = Request();
393     auto rs = rq.get(url, args);
394     return rs.responseBody;
395 }
396 /**
397  * Call GET, and return response content.
398  * args = QueryParam[] of parameters.
399  * Returns:
400  * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
401  */
402 public auto getContent(A...)(string url, QueryParam[] args) {
403     auto rq = Request();
404     auto rs = rq.get(url, args);
405     return rs.responseBody;
406 }
407 /**
408  * Call GET, and return response content.
409  * args = variadic args to supply parameter names and values.
410  * Returns:
411  * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
412  */
413 public auto getContent(A...)(string url, A args) if (args.length > 1 && args.length % 2 == 0 ) {
414     return Request().
415             get(url, queryParams(args)).
416             responseBody;
417 }
418 
419 ///
420 package unittest {
421     import std.algorithm;
422     import std.stdio;
423     import std.json;
424     globalLogLevel(LogLevel.info);
425     info("Test getContent");
426     auto r = getContent("https://httpbin.org/stream/20");
427     assert(r.splitter('\n').filter!("a.length>0").count == 20);
428     r = getContent("ftp://speedtest.tele2.net/1KB.zip");
429     assert(r.length == 1024);
430     r = getContent("https://httpbin.org/get", ["a":"b", "c":"d"]);
431 
432     string name = "user", sex = "male";
433     int    age = 42;
434     r = getContent("https://httpbin.org/get", "name", name, "age", age, "sex", sex);
435 
436     info("Test receiveAsRange with GET");
437     auto rq = Request();
438     rq.useStreaming = true;
439     rq.bufferSize = 16;
440     auto rs = rq.get("http://httpbin.org/get");
441     auto stream = rs.receiveAsRange();
442     ubyte[] streamedContent;
443     while( !stream.empty() ) {
444         streamedContent ~= stream.front;
445         stream.popFront();
446     }
447     rq = Request();
448     rs = rq.get("http://httpbin.org/get");
449     assert(streamedContent == rs.responseBody.data);
450     rq.useStreaming = true;
451     streamedContent.length = 0;
452     rs = rq.get("ftp://speedtest.tele2.net/1KB.zip");
453     stream = rs.receiveAsRange;
454     while( !stream.empty() ) {
455         streamedContent ~= stream.front;
456         stream.popFront();
457     }
458 
459 }
460 ///
461 package unittest {
462     globalLogLevel(LogLevel.info);
463     info("Test get in parallel");
464     import std.stdio;
465     import std.parallelism;
466     import std.algorithm;
467     import std.string;
468     import core.atomic;
469     
470     immutable auto urls = [
471         "http://httpbin.org/stream/10",
472         "https://httpbin.org/stream/20",
473         "http://httpbin.org/stream/30",
474         "https://httpbin.org/stream/40",
475         "http://httpbin.org/stream/50",
476         "https://httpbin.org/stream/60",
477         "http://httpbin.org/stream/70",
478     ];
479     
480     defaultPoolThreads(5);
481     
482     shared short lines;
483     
484     foreach(url; parallel(urls)) {
485         atomicOp!"+="(lines, getContent(url).splitter("\n").count);
486     }
487     assert(lines == 287);
488 }
489 
490 /**
491  * Call post and return response content.
492  */
493 public auto postContent(A...)(string url, A args) {
494     auto rq = Request();
495     auto rs = rq.post(url, args);
496     return rs.responseBody;
497 }
498 
499 ///
500 package unittest {
501     import std.json;
502     import std.string;
503     import std.stdio;
504     import std.range;
505 
506     globalLogLevel(LogLevel.info);
507     info("Test postContent");
508     auto r = postContent("http://httpbin.org/post", `{"a":"b", "c":1}`, "application/json");
509     assert(parseJSON(r.data).object["json"].object["c"].integer == 1);
510 
511     /// ftp upload from range
512     info("Test postContent ftp");
513     r = postContent("ftp://speedtest.tele2.net/upload/TEST.TXT", "test, ignore please\n".representation);
514     assert(r.length == 0);
515 
516     /// Posting to forms (for small data)
517     ///
518     /// posting query parameters using "application/x-www-form-urlencoded"
519     info("Test postContent using query params");
520     postContent("http://httpbin.org/post", queryParams("first", "a", "second", 2));
521 
522     /// posting using multipart/form-data (large data and files). See docs fot HTTPRequest
523     info("Test postContent form");
524     MultipartForm form;
525     form.add(formData(/* field name */ "greeting", /* content */ cast(ubyte[])"hello"));
526     postContent("http://httpbin.org/post", form);
527 
528     /// you can do this using Request struct to access response details
529     info("Test postContent form via Request()");
530     auto rq = Request();
531     form = MultipartForm().add(formData(/* field name */ "greeting", /* content */ cast(ubyte[])"hello"));
532     auto rs = rq.post("http://httpbin.org/post", form);
533     assert(rs.code == 200);
534 
535     info("Test receiveAsRange with POST");
536     rq = Request();
537     rq.useStreaming = true;
538     rq.bufferSize = 16;
539     string s = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
540     rs = rq.post("http://httpbin.org/post", s.representation.chunks(10), "application/octet-stream");
541     auto stream = rs.receiveAsRange();
542     ubyte[] streamedContent;
543     while( !stream.empty() ) {
544         streamedContent ~= stream.front;
545         stream.popFront();
546     }
547     rq = Request();
548     rs = rq.post("http://httpbin.org/post", s.representation.chunks(10), "application/octet-stream");
549     assert(streamedContent == rs.responseBody.data);
550 
551 }
552