requests source code

1 module requests;
2 
3 public import requests.http;
4 public import requests.ftp;
5 public import requests.streams;
6 public import requests.base;
7 public import requests.uri;
8 
9 import std.datetime;
10 import std.conv;
11 import std.experimental.logger;
12 import requests.uri;
13 
14 /***********************************
15  * This is simplest interface to both http and ftp protocols.
16  * Request has methods get, post and exec which routed to proper concrete handler (http or ftp, etc).
17  * To enable some protocol-specific featutes you have to use protocol interface directly (see docs for HTTPRequest or FTPRequest)
18  */
19 struct Request {
20     private {
21         URI         _uri;
22         HTTPRequest _http;  // route all http/https requests here
23         FTPRequest  _ftp;   // route all ftp requests here
24     }
25     /// Set timeout on IO operation.
26     /// $(B v) - timeout value
27     /// 
28     @property void timeout(Duration v) pure @nogc nothrow {
29         _http.timeout = v;
30         _ftp.timeout = v;
31     }
32     /// Set http keepAlive value
33     /// $(B v) - use keepalive requests - $(B true), or not - $(B false)
34     @property void keepAlive(bool v) pure @nogc nothrow {
35         _http.keepAlive = v;
36     }
37     /// Set limit on HTTP redirects
38     /// $(B v) - limit on redirect depth
39     @property void maxRedirects(uint v) pure @nogc nothrow {
40         _http.maxRedirects = v;
41     }
42     /// Set maximum content lenth both for http and ftp requests
43     /// $(B v) - maximum content length in bytes. When limit reached - throw RequestException
44     @property void maxContentLength(size_t v) pure @nogc nothrow {
45         _http.maxContentLength = v;
46         _ftp.maxContentLength = v;
47     }
48     /// Set maximum length for HTTP headers
49     /// $(B v) - maximum length of the HTTP response. When limit reached - throw RequestException
50     @property void maxHeadersLength(size_t v) pure @nogc nothrow {
51         _http.maxHeadersLength = v;
52     }
53     /// Set IO buffer size for http and ftp requests
54     /// $(B v) - buffer size in bytes.
55     @property void bufferSize(size_t v) {
56         _http.bufferSize = v;
57         _ftp.bufferSize = v;
58     }
59     /// Set verbosity for HTTP or FTP requests.
60     /// $(B v) - verbosity level (0 - no output, 1 - headers to stdout, 2 - headers and body progress to stdout). default = 0.
61     @property void verbosity(uint v) {
62         _http.verbosity = v;
63         _ftp.verbosity = v;
64     }
65     /// Set authenticator for http requests.
66     /// $(B v) - Auth instance.
67     @property void authenticator(Auth v) {
68         _http.authenticator = v;
69     }
70     /// Set Cookie for http requests.
71     /// $(B v) - array of cookie.
72     @property void cookie(Cookie[] v) pure @nogc nothrow {
73         _http.cookie = v;
74     }
75     /// Get Cookie for http requests.
76     /// $(B v) - array of cookie.
77     @property Cookie[] cookie()  pure @nogc nothrow {
78         return _http.cookie;
79     }
80     @property void useStreaming(bool v) pure @nogc nothrow {
81         _http.useStreaming = v;
82         _ftp.useStreaming = v;
83     }
84     @property long contentReceived() pure @nogc nothrow {
85         final switch ( _uri.scheme ) {
86             case "http", "https":
87                 return _http.contentReceived;
88             case "ftp":
89                 return _ftp.contentReceived;
90         }
91     }
92     @property long contentLength() pure @nogc nothrow {
93         final switch ( _uri.scheme ) {
94             case "http", "https":
95                 return _http.contentLength;
96             case "ftp":
97                 return _ftp.contentLength;
98         }
99     }
100     /// Execute GET for http and retrieve file for FTP.
101     /// You have to provide at least $(B uri). All other arguments should conform to HTTPRequest.get or FTPRequest.get depending on the URI scheme.
102     /// When arguments do not conform scheme (for example you try to call get("ftp://somehost.net/pub/README", {"a":"b"}) which doesn't make sense)
103     /// you will receive Exception("Operation not supported for ftp")
104     ///
105     Response get(A...)(string uri, A args) {
106         if ( uri ) {
107             _uri = URI(uri);
108         }
109         final switch ( _uri.scheme ) {
110             case "http", "https":
111                 _http.uri = _uri;
112                 static if (__traits(compiles, _http.get(null, args))) {
113                     return _http.get(null, args);
114                 } else {
115                     throw new Exception("Operation not supported for http");
116                 }
117             case "ftp":
118                 static if (args.length == 0) {
119                     return _ftp.get(uri);
120                 } else {
121                     throw new Exception("Operation not supported for ftp");
122                 }
123         }
124     }
125     /// Execute POST for http and STOR file for FTP.
126     /// You have to provide  $(B uri) and data. Data should conform to HTTPRequest.post or FTPRequest.post depending on the URI scheme.
127     /// When arguments do not conform scheme you will receive Exception("Operation not supported for ftp")
128     ///
129     Response post(A...)(string uri, A args) {
130         if ( uri ) {
131             _uri = URI(uri);
132         }
133         final switch ( _uri.scheme ) {
134             case "http", "https":
135                 _http.uri = _uri;
136                 static if (__traits(compiles, _http.post(null, args))) {
137                     return _http.post(null, args);
138                 } else {
139                     throw new Exception("Operation not supported for http");
140                 }
141             case "ftp":
142                 static if (__traits(compiles, _ftp.post(uri, args))) {
143                     return _ftp.post(uri, args);
144                 } else {
145                     throw new Exception("Operation not supported for ftp");
146                 }
147         }
148     }
149     Response exec(string method="GET", A...)(A args) {
150         return _http.exec!(method)(args);
151     }
152 }
153 ///
154 package unittest {
155     import std.algorithm;
156     import std.range;
157     import std.array;
158     import std.json;
159     import std.stdio;
160     import std.string;
161     import std.exception;
162 
163     globalLogLevel(LogLevel.info);
164 
165     infof("testing Request");
166     Request rq;
167     Response rs;
168     //
169     rs = rq.get("https://httpbin.org/");
170     assert(rs.code==200);
171     assert(rs.responseBody.length > 0);
172     rs = rq.get("http://httpbin.org/get", ["c":" d", "a":"b"]);
173     assert(rs.code == 200);
174     auto json = parseJSON(rs.responseBody.data).object["args"].object;
175     assert(json["c"].str == " d");
176     assert(json["a"].str == "b");
177     
178     globalLogLevel(LogLevel.info);
179     rq = Request();
180     rq.keepAlive = true;
181     // handmade json
182     info("Check POST json");
183     rs = rq.post("http://httpbin.org/post?b=x", `{"a":"☺ ", "c":[1,2,3]}`, "application/json");
184     assert(rs.code==200);
185     json = parseJSON(rs.responseBody.data).object["args"].object;
186     assert(json["b"].str == "x");
187     json = parseJSON(rs.responseBody.data).object["json"].object;
188     assert(json["a"].str == "☺ ");
189     assert(json["c"].array.map!(a=>a.integer).array == [1,2,3]);
190     {
191         import std.file;
192         import std.path;
193         auto tmpd = tempDir();
194         auto tmpfname = tmpd ~ dirSeparator ~ "request_test.txt";
195         auto f = File(tmpfname, "wb");
196         f.rawWrite("abcdefgh\n12345678\n");
197         f.close();
198         // files
199         globalLogLevel(LogLevel.info);
200         info("Check POST files");
201         PostFile[] files = [
202         {fileName: tmpfname, fieldName:"abc", contentType:"application/octet-stream"}, 
203         {fileName: tmpfname}
204         ];
205         rs = rq.post("http://httpbin.org/post", files);
206         assert(rs.code==200);
207         info("Check POST chunked from file.byChunk");
208         f = File(tmpfname, "rb");
209         rs = rq.post("http://httpbin.org/post", f.byChunk(3), "application/octet-stream");
210         assert(rs.code==200);
211         auto data = parseJSON(rs.responseBody.data).object["data"].str;
212         assert(data=="abcdefgh\n12345678\n");
213         f.close();
214     }
215     {
216         // string
217         info("Check POST utf8 string");
218         rs = rq.post("http://httpbin.org/post", "привiт, свiт!", "application/octet-stream");
219         assert(rs.code==200);
220         auto data = parseJSON(rs.responseBody.data).object["data"].str;
221         assert(data=="привiт, свiт!");
222     }
223     // ranges
224     {
225         info("Check POST chunked from lineSplitter");
226         auto s = lineSplitter("one,\ntwo,\nthree.");
227         rs = rq.exec!"POST"("http://httpbin.org/post", s, "application/octet-stream");
228         assert(rs.code==200);
229         auto data = parseJSON(rs.responseBody.toString).object["data"].str;
230         assert(data=="one,two,three.");
231     }
232     {
233         info("Check POST chunked from array");
234         auto s = ["one,", "two,", "three."];
235         rs = rq.post("http://httpbin.org/post", s, "application/octet-stream");
236         assert(rs.code==200);
237         auto data = parseJSON(rs.responseBody.data).object["data"].str;
238         assert(data=="one,two,three.");
239     }
240     {
241         info("Check POST chunked using std.range.chunks()");
242         auto s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
243         rs = rq.post("http://httpbin.org/post", s.representation.chunks(10), "application/octet-stream");
244         assert(rs.code==200);
245         auto data = parseJSON(rs.responseBody.data).object["data"].str;
246         assert(data==s);
247     }
248     // associative array
249     rs = rq.post("http://httpbin.org/post", ["a":"b ", "c":"d"]);
250     assert(rs.code==200);
251     auto form = parseJSON(rs.responseBody.data).object["form"].object;
252     assert(form["a"].str == "b ");
253     assert(form["c"].str == "d");
254     info("Check HEAD");
255     rs = rq.exec!"HEAD"("http://httpbin.org/");
256     assert(rs.code==200);
257     info("Check DELETE");
258     rs = rq.exec!"DELETE"("http://httpbin.org/delete");
259     assert(rs.code==200);
260     info("Check PUT");
261     rs = rq.exec!"PUT"("http://httpbin.org/put",  `{"a":"b", "c":[1,2,3]}`, "application/json");
262     assert(rs.code==200);
263     info("Check PATCH");
264     rs = rq.exec!"PATCH"("http://httpbin.org/patch", "привiт, свiт!", "application/octet-stream");
265     assert(rs.code==200);
266     
267     info("Check compressed content");
268     globalLogLevel(LogLevel.info);
269     rq = Request();
270     rq.keepAlive = true;
271     rs = rq.get("http://httpbin.org/gzip");
272     assert(rs.code==200);
273     info("gzip - ok");
274     rs = rq.get("http://httpbin.org/deflate");
275     assert(rs.code==200);
276     info("deflate - ok");
277     
278     info("Check redirects");
279     globalLogLevel(LogLevel.info);
280     rq = Request();
281     rq.keepAlive = true;
282     rs = rq.get("http://httpbin.org/relative-redirect/2");
283     assert((cast(HTTPResponse)rs).history.length == 2);
284     assert((cast(HTTPResponse)rs).code==200);
285 
286     info("Check cookie");
287     rq = Request();
288     rs = rq.get("http://httpbin.org/cookies/set?A=abcd&b=cdef");
289     assert(rs.code == 200);
290     json = parseJSON(rs.responseBody.data).object["cookies"].object;
291     assert(json["A"].str == "abcd");
292     assert(json["b"].str == "cdef");
293     auto cookie = rq.cookie();
294     foreach(c; rq.cookie) {
295         final switch(c.attr) {
296             case "A":
297                 assert(c.value == "abcd");
298                 break;
299             case "b":
300                 assert(c.value == "cdef");
301                 break;
302         }
303     }
304     rs = rq.get("http://httpbin.org/absolute-redirect/2");
305     assert((cast(HTTPResponse)rs).history.length == 2);
306     assert((cast(HTTPResponse)rs).code==200);
307     //    rq = Request();
308     rq.maxRedirects = 2;
309     rq.keepAlive = false;
310     assertThrown!MaxRedirectsException(rq.get("https://httpbin.org/absolute-redirect/3"));
311 
312     info("Check utf8 content");
313     globalLogLevel(LogLevel.info);
314     rq = Request();
315     rs = rq.get("http://httpbin.org/encoding/utf8");
316     assert(rs.code==200);
317     
318     info("Check chunked content");
319     globalLogLevel(LogLevel.info);
320     rq = Request();
321     rq.keepAlive = true;
322     rq.bufferSize = 16*1024;
323     rs = rq.get("http://httpbin.org/range/1024");
324     assert(rs.code==200);
325     assert(rs.responseBody.length==1024);
326     
327     info("Check basic auth");
328     globalLogLevel(LogLevel.info);
329     rq = Request();
330     rq.authenticator = new BasicAuthentication("user", "passwd");
331     rs = rq.get("http://httpbin.org/basic-auth/user/passwd");
332     assert(rs.code==200);
333     
334     globalLogLevel(LogLevel.info);
335     info("Check limits");
336     rq = Request();
337     rq.maxContentLength = 1;
338     assertThrown!RequestException(rq.get("http://httpbin.org/"));
339     rq = Request();
340     rq.maxHeadersLength = 1;
341     assertThrown!RequestException(rq.get("http://httpbin.org/"));
342     //
343     info("ftp post ", "ftp://speedtest.tele2.net/upload/TEST.TXT");
344     rs = rq.post("ftp://speedtest.tele2.net/upload/TEST.TXT", "test, ignore please\n".representation);
345     assert(rs.code == 226);
346     info("ftp get  ", "ftp://speedtest.tele2.net/nonexistent", ", in same session.");
347     rs = rq.get("ftp://speedtest.tele2.net/nonexistent");
348     assert(rs.code != 226);
349     info("ftp get  ", "ftp://speedtest.tele2.net/1KB.zip", ", in same session.");
350     rs = rq.get("ftp://speedtest.tele2.net/1KB.zip");
351     assert(rs.code == 226);
352     assert(rs.responseBody.length == 1024);
353     info("ftp post ", "ftp://speedtest.tele2.net/upload/TEST.TXT");
354     rs = rq.post("ftp://speedtest.tele2.net/upload/TEST.TXT", "another test, ignore please\n".representation);
355     assert(rs.code == 226);
356     info("ftp get  ", "ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT");
357     rs = rq.get("ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT");
358     assert(rs.code == 226);
359     info("testing ftp - done.");
360 }
361 
362 auto queryParams(A...)(A args) pure @safe nothrow {
363     QueryParam[] res;
364     static if ( args.length >= 2 ) {
365         res = QueryParam(args[0].to!string, args[1].to!string) ~ queryParams(args[2..$]);
366     }
367     return res;
368 }
369 /**
370  * Call GET, and return response content.
371  * This is the simplest case, when all you need is the response body and have no parameters.
372  * Returns:
373  * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
374  */
375 public auto ref getContent(A...)(string url) {
376     auto rq = Request();
377     auto rs = rq.get(url);
378     return rs.responseBody;
379 }
380 /**
381  * Call GET, and return response content.
382  * args = string[string] fo query parameters.
383  * Returns:
384  * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
385  */
386 public auto ref getContent(A...)(string url, string[string] args) {
387     auto rq = Request();
388     auto rs = rq.get(url, args);
389     return rs.responseBody;
390 }
391 /**
392  * Call GET, and return response content.
393  * args = QueryParam[] of parameters.
394  * Returns:
395  * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
396  */
397 public auto ref getContent(A...)(string url, QueryParam[] args) {
398     auto rq = Request();
399     auto rs = rq.get(url, args);
400     return rs.responseBody;
401 }
402 /**
403  * Call GET, and return response content.
404  * args = variadic args to supply parameter names and values.
405  * Returns:
406  * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
407  */
408 public auto ref getContent(A...)(string url, A args) if (args.length > 1 && args.length % 2 == 0 ) {
409     return Request().
410             get(url, queryParams(args)).
411             responseBody;
412 }
413 
414 ///
415 package unittest {
416     import std.algorithm;
417     import std.stdio;
418     import std.json;
419     globalLogLevel(LogLevel.info);
420     info("Test getContent");
421     auto r = getContent("https://httpbin.org/stream/20");
422     assert(r.splitter('\n').filter!("a.length>0").count == 20);
423     r = getContent("ftp://speedtest.tele2.net/1KB.zip");
424     assert(r.length == 1024);
425     r = getContent("https://httpbin.org/get", ["a":"b", "c":"d"]);
426 
427     string name = "user", sex = "male";
428     int    age = 42;
429     r = getContent("https://httpbin.org/get", "name", name, "age", age, "sex", sex);
430 
431     info("Test receiveAsRange with GET");
432     auto rq = Request();
433     rq.useStreaming = true;
434     rq.bufferSize = 16;
435     auto rs = rq.get("http://httpbin.org/stream/20");
436     auto stream = rs.receiveAsRange();
437     ubyte[] streamedContent;
438     while( !stream.empty() ) {
439         streamedContent ~= stream.front;
440         stream.popFront();
441     }
442     rq = Request();
443     rs = rq.get("http://httpbin.org/stream/20");
444     assert(streamedContent == rs.responseBody.data);
445     rq.useStreaming = true;
446     streamedContent.length = 0;
447     rs = rq.get("ftp://speedtest.tele2.net/1KB.zip");
448     stream = rs.receiveAsRange;
449     while( !stream.empty() ) {
450         streamedContent ~= stream.front;
451         stream.popFront();
452     }
453     assert(streamedContent.length == 1024);
454 }
455 ///
456 package unittest {
457     globalLogLevel(LogLevel.info);
458     info("Test get in parallel");
459     import std.stdio;
460     import std.parallelism;
461     import std.algorithm;
462     import std.string;
463     import core.atomic;
464     
465     immutable auto urls = [
466         "http://httpbin.org/stream/10",
467         "https://httpbin.org/stream/20",
468         "http://httpbin.org/stream/30",
469         "https://httpbin.org/stream/40",
470         "http://httpbin.org/stream/50",
471         "https://httpbin.org/stream/60",
472         "http://httpbin.org/stream/70",
473     ];
474     
475     defaultPoolThreads(3);
476     
477     shared short lines;
478     
479     foreach(url; parallel(urls)) {
480         atomicOp!"+="(lines, getContent(url).splitter("\n").count);
481     }
482     assert(lines == 287);
483 }
484 
485 /**
486  * Call post and return response content.
487  */
488 public auto postContent(A...)(string url, A args) {
489     auto rq = Request();
490     auto rs = rq.post(url, args);
491     return rs.responseBody;
492 }
493 
494 ///
495 package unittest {
496     import std.json;
497     import std.string;
498     import std.stdio;
499     import std.range;
500 
501     globalLogLevel(LogLevel.info);
502     info("Test postContent");
503     auto r = postContent("http://httpbin.org/post", `{"a":"b", "c":1}`, "application/json");
504     assert(parseJSON(r.data).object["json"].object["c"].integer == 1);
505 
506     /// ftp upload from range
507     info("Test postContent ftp");
508     r = postContent("ftp://speedtest.tele2.net/upload/TEST.TXT", "test, ignore please\n".representation);
509     assert(r.length == 0);
510 
511     /// Posting to forms (for small data)
512     ///
513     /// posting query parameters using "application/x-www-form-urlencoded"
514     info("Test postContent using query params");
515     postContent("http://httpbin.org/post", queryParams("first", "a", "second", 2));
516 
517     /// posting using multipart/form-data (large data and files). See docs fot HTTPRequest
518     info("Test postContent form");
519     MultipartForm form;
520     form.add(formData(/* field name */ "greeting", /* content */ cast(ubyte[])"hello"));
521     postContent("http://httpbin.org/post", form);
522 
523     /// you can do this using Request struct to access response details
524     info("Test postContent form via Request()");
525     auto rq = Request();
526     form = MultipartForm().add(formData(/* field name */ "greeting", /* content */ cast(ubyte[])"hello"));
527     auto rs = rq.post("http://httpbin.org/post", form);
528     assert(rs.code == 200);
529 
530     info("Test receiveAsRange with POST");
531     rq = Request();
532     rq.useStreaming = true;
533     rq.bufferSize = 16;
534     string s = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
535     rs = rq.post("http://httpbin.org/post", s.representation.chunks(10), "application/octet-stream");
536     auto stream = rs.receiveAsRange();
537     ubyte[] streamedContent;
538     while( !stream.empty() ) {
539         streamedContent ~= stream.front;
540         stream.popFront();
541     }
542     rq = Request();
543     rs = rq.post("http://httpbin.org/post", s.representation.chunks(10), "application/octet-stream");
544     assert(streamedContent == rs.responseBody.data);
545 
546 }
547