requests source code

1 module requests;
2 
3 public import requests.http;
4 public import requests.ftp;
5 public import requests.streams;
6 public import requests.base;
7 public import requests.uri;
8 
9 import std.datetime;
10 import std.conv;
11 import std.experimental.logger;
12 import requests.utils;
13 
14 /**
15    This is simplest interface to both http and ftp protocols.
16    Request has methods get, post and exec which routed to proper concrete handler (http or ftp, etc).
17    To enable some protocol-specific featutes you have to use protocol interface directly (see docs for HTTPRequest or FTPRequest)
18 */
19 public struct Request {
20     private {
21         URI         _uri;
22         HTTPRequest _http;  // route all http/https requests here
23         FTPRequest  _ftp;   // route all ftp requests here
24     }
25     /// Set timeout on IO operation.
26     /// $(B v) - timeout value
27     /// 
28     public @property void timeout(Duration v) pure @nogc nothrow {
29         _http.timeout = v;
30         _ftp.timeout = v;
31     }
32     /// Set http keepAlive value
33     /// $(B v) - use keepalive requests - $(B true), or not - $(B false)
34     @property void keepAlive(bool v) pure @nogc nothrow {
35         _http.keepAlive = v;
36     }
37     /// Set limit on HTTP redirects
38     /// $(B v) - limit on redirect depth
39     @property void maxRedirects(uint v) pure @nogc nothrow {
40         _http.maxRedirects = v;
41     }
42     /// Set maximum content lenth both for http and ftp requests
43     /// $(B v) - maximum content length in bytes. When limit reached - throw RequestException
44     @property void maxContentLength(size_t v) pure @nogc nothrow {
45         _http.maxContentLength = v;
46         _ftp.maxContentLength = v;
47     }
48     /// Set maximum length for HTTP headers
49     /// $(B v) - maximum length of the HTTP response. When limit reached - throw RequestException
50     @property void maxHeadersLength(size_t v) pure @nogc nothrow {
51         _http.maxHeadersLength = v;
52     }
53     /// Set IO buffer size for http and ftp requests
54     /// $(B v) - buffer size in bytes.
55     @property void bufferSize(size_t v) {
56         _http.bufferSize = v;
57         _ftp.bufferSize = v;
58     }
59     /// Set verbosity for HTTP or FTP requests.
60     /// $(B v) - verbosity level (0 - no output, 1 - headers to stdout, 2 - headers and body progress to stdout). default = 0.
61     @property void verbosity(uint v) {
62         _http.verbosity = v;
63         _ftp.verbosity = v;
64     }
65     /// Set authenticator for http requests.
66     /// $(B v) - Auth instance.
67     @property void authenticator(Auth v) {
68         _http.authenticator = v;
69     }
70     /// Set Cookie for http requests.
71     /// $(B v) - array of cookie.
72     @property void cookie(Cookie[] v) pure @nogc nothrow {
73         _http.cookie = v;
74     }
75     /// Get Cookie for http requests.
76     /// $(B v) - array of cookie.
77     @property Cookie[] cookie()  pure @nogc nothrow {
78         return _http.cookie;
79     }
80     ///
81     /// set "streaming" property
82     /// Params:
83     /// v = value to set (true - use streaming)
84     /// 
85     @property void useStreaming(bool v) pure @nogc nothrow {
86         _http.useStreaming = v;
87         _ftp.useStreaming = v;
88     }
89     ///
90     /// get length og actually received content.
91     /// this value increase over time, while we receive data
92     /// 
93     @property long contentReceived() pure @nogc nothrow {
94         final switch ( _uri.scheme ) {
95             case "http", "https":
96                 return _http.contentReceived;
97             case "ftp":
98                 return _ftp.contentReceived;
99         }
100     }
101     /// get contentLength of the responce
102     @property long contentLength() pure @nogc nothrow {
103         final switch ( _uri.scheme ) {
104             case "http", "https":
105                 return _http.contentLength;
106             case "ftp":
107                 return _ftp.contentLength;
108         }
109     }
110     @property void sslSetVerifyPeer(bool v) {
111         _http.sslSetVerifyPeer(v);
112     }
113     @property void sslSetKeyFile(string path, SSLOptions.filetype type = SSLOptions.filetype.pem) pure @safe nothrow @nogc {
114         _http.sslSetKeyFile(path, type);
115     }
116     @property void sslSetCertFile(string path, SSLOptions.filetype type = SSLOptions.filetype.pem) pure @safe nothrow @nogc {
117         _http.sslSetCertFile(path, type);
118     }
119     @property void sslSetCaCert(string path) pure @safe nothrow @nogc {
120         _http.sslSetCaCert(path);
121     }
122     @property auto sslOptions() {
123         return _http.sslOptions();
124     }
125     /// Add headers to request
126     /// Params:
127     /// headers = headers to send.
128     void addHeaders(in string[string] headers) {
129         _http.addHeaders(headers);
130     }
131     /// Execute GET for http and retrieve file for FTP.
132     /// You have to provide at least $(B uri). All other arguments should conform to HTTPRequest.get or FTPRequest.get depending on the URI scheme.
133     /// When arguments do not conform scheme (for example you try to call get("ftp://somehost.net/pub/README", {"a":"b"}) which doesn't make sense)
134     /// you will receive Exception("Operation not supported for ftp")
135     ///
136     Response get(A...)(string uri, A args) {
137         if ( uri ) {
138             _uri = URI(uri);
139         }
140         final switch ( _uri.scheme ) {
141             case "http", "https":
142                 _http.uri = _uri;
143                 static if (__traits(compiles, _http.get(null, args))) {
144                     return _http.get(null, args);
145                 } else {
146                     throw new Exception("Operation not supported for http");
147                 }
148             case "ftp":
149                 static if (args.length == 0) {
150                     return _ftp.get(uri);
151                 } else {
152                     throw new Exception("Operation not supported for ftp");
153                 }
154         }
155     }
156     /// Execute POST for http and STOR file for FTP.
157     /// You have to provide  $(B uri) and data. Data should conform to HTTPRequest.post or FTPRequest.post depending on the URI scheme.
158     /// When arguments do not conform scheme you will receive Exception("Operation not supported for ftp")
159     ///
160     Response post(A...)(string uri, A args) {
161         if ( uri ) {
162             _uri = URI(uri);
163         }
164         final switch ( _uri.scheme ) {
165             case "http", "https":
166                 _http.uri = _uri;
167                 static if (__traits(compiles, _http.post(null, args))) {
168                     return _http.post(null, args);
169                 } else {
170                     throw new Exception("Operation not supported for http");
171                 }
172             case "ftp":
173                 static if (__traits(compiles, _ftp.post(uri, args))) {
174                     return _ftp.post(uri, args);
175                 } else {
176                     throw new Exception("Operation not supported for ftp");
177                 }
178         }
179     }
180     Response exec(string method="GET", A...)(A args) {
181         return _http.exec!(method)(args);
182     }
183 }
184 ///
185 package unittest {
186     import std.algorithm;
187     import std.range;
188     import std.array;
189     import std.json;
190     import std.stdio;
191     import std.string;
192     import std.exception;
193 
194     string httpbinUrl = httpTestServer();
195 
196     version(vibeD) {
197     }
198     else {
199         import httpbin;
200         auto server = httpbinApp();
201         server.start();
202         scope(exit) {
203             server.stop();
204         }
205     }
206 
207     globalLogLevel(LogLevel.info);
208 
209     infof("testing Request");
210     Request rq;
211     Response rs;
212     //
213     rs = rq.get(httpbinUrl);
214     assert(rs.code==200);
215     assert(rs.responseBody.length > 0);
216     rs = rq.get(httpbinUrl ~ "get", ["c":" d", "a":"b"]);
217     assert(rs.code == 200);
218     auto json = parseJSON(rs.responseBody.data).object["args"].object;
219     assert(json["c"].str == " d");
220     assert(json["a"].str == "b");
221     
222     rq = Request();
223     rq.keepAlive = true;
224     // handmade json
225     info("Check POST json");
226     rs = rq.post(httpbinUrl ~ "post?b=x", `{"a":"b ", "c":[1,2,3]}`, "application/json");
227     assert(rs.code==200);
228     json = parseJSON(rs.responseBody.data).object["args"].object;
229     assert(json["b"].str == "x");
230     json = parseJSON(rs.responseBody.data).object["json"].object;
231     assert(json["a"].str == "b ");
232     assert(json["c"].array.map!(a=>a.integer).array == [1,2,3]);
233     {
234         import std.file;
235         import std.path;
236         auto tmpd = tempDir();
237         auto tmpfname = tmpd ~ dirSeparator ~ "request_test.txt";
238         auto f = File(tmpfname, "wb");
239         f.rawWrite("abcdefgh\n12345678\n");
240         f.close();
241         // files
242         info("Check POST files");
243         PostFile[] files = [
244             {fileName: tmpfname, fieldName:"abc", contentType:"application/octet-stream"}, 
245             {fileName: tmpfname}
246         ];
247         rs = rq.post(httpbinUrl ~ "post", files);
248         assert(rs.code==200);
249         info("Check POST chunked from file.byChunk");
250         f = File(tmpfname, "rb");
251         rs = rq.post(httpbinUrl ~ "post", f.byChunk(3), "application/octet-stream");
252         assert(rs.code==200);
253         auto data = fromJsonArrayToStr(parseJSON(rs.responseBody).object["data"]);
254         assert(data=="abcdefgh\n12345678\n");
255         f.close();
256     }
257     // ranges
258     {
259         info("Check POST chunked from lineSplitter");
260         auto s = lineSplitter("one,\ntwo,\nthree.");
261         rs = rq.exec!"POST"(httpbinUrl ~ "post", s, "application/octet-stream");
262         assert(rs.code==200);
263         auto data = fromJsonArrayToStr(parseJSON(rs.responseBody).object["data"]);
264         assert(data=="one,two,three.");
265     }
266     {
267         info("Check POST chunked from array");
268         auto s = ["one,", "two,", "three."];
269         rs = rq.post(httpbinUrl ~ "post", s, "application/octet-stream");
270         assert(rs.code==200);
271         auto data = fromJsonArrayToStr(parseJSON(rs.responseBody).object["data"]);
272         assert(data=="one,two,three.");
273     }
274     {
275         info("Check POST chunked using std.range.chunks()");
276         auto s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
277         rs = rq.post(httpbinUrl ~ "post", s.representation.chunks(10), "application/octet-stream");
278         assert(rs.code==200);
279         auto data = fromJsonArrayToStr(parseJSON(rs.responseBody).object["data"]);
280         assert(data==s);
281     }
282     // associative array
283     rs = rq.post(httpbinUrl ~ "post", ["a":"b ", "c":"d"]);
284     assert(rs.code==200);
285     auto form = parseJSON(rs.responseBody.data).object["form"].object;
286     assert(form["a"].str == "b ");
287     assert(form["c"].str == "d");
288     info("Check HEAD");
289     rs = rq.exec!"HEAD"(httpbinUrl);
290     assert(rs.code==200);
291     info("Check DELETE");
292     rs = rq.exec!"DELETE"(httpbinUrl ~ "delete");
293     assert(rs.code==200);
294     info("Check PUT");
295     rs = rq.exec!"PUT"(httpbinUrl ~ "put",  `{"a":"b", "c":[1,2,3]}`, "application/json");
296     assert(rs.code==200);
297     info("Check PATCH");
298     rs = rq.exec!"PATCH"(httpbinUrl ~ "patch", "привiт, свiт!", "application/octet-stream");
299     assert(rs.code==200);
300     
301     info("Check compressed content");
302     rq = Request();
303     rq.keepAlive = true;
304     rq.addHeaders(["X-Header": "test"]);
305     rs = rq.get(httpbinUrl ~ "gzip");
306     assert(rs.code==200);
307     info("gzip - ok");
308     rs = rq.get(httpbinUrl ~ "deflate");
309     assert(rs.code==200);
310     info("deflate - ok");
311     
312     info("Check redirects");
313     rq = Request();
314     rq.keepAlive = true;
315     rs = rq.get(httpbinUrl ~ "relative-redirect/2");
316     assert((cast(HTTPResponse)rs).history.length == 2);
317     assert((cast(HTTPResponse)rs).code==200);
318 
319     info("Check cookie");
320     rq = Request();
321     rs = rq.get(httpbinUrl ~ "cookies/set?A=abcd&b=cdef");
322     assert(rs.code == 200);
323     json = parseJSON(rs.responseBody.data).object["cookies"].object;
324     assert(json["A"].str == "abcd");
325     assert(json["b"].str == "cdef");
326     auto cookie = rq.cookie();
327     foreach(c; rq.cookie) {
328         final switch(c.attr) {
329             case "A":
330                 assert(c.value == "abcd");
331                 break;
332             case "b":
333                 assert(c.value == "cdef");
334                 break;
335         }
336     }
337     rs = rq.get(httpbinUrl ~ "absolute-redirect/2");
338     assert((cast(HTTPResponse)rs).history.length == 2);
339     assert((cast(HTTPResponse)rs).code==200);
340     //    rq = Request();
341     rq.maxRedirects = 2;
342     rq.keepAlive = false;
343     assertThrown!MaxRedirectsException(rq.get(httpbinUrl ~ "absolute-redirect/3"));
344 
345     info("Check chunked content");
346     rq = Request();
347     rq.keepAlive = true;
348     rq.bufferSize = 16*1024;
349     rs = rq.get(httpbinUrl ~ "range/1024");
350     assert(rs.code==200);
351     assert(rs.responseBody.length==1024);
352     
353     info("Check basic auth");
354     rq = Request();
355     rq.authenticator = new BasicAuthentication("user", "passwd");
356     rs = rq.get(httpbinUrl ~ "basic-auth/user/passwd");
357     assert(rs.code==200);
358     
359     info("Check limits");
360     rq = Request();
361     rq.maxContentLength = 1;
362     assertThrown!RequestException(rq.get(httpbinUrl));
363     rq = Request();
364     rq.maxHeadersLength = 1;
365     assertThrown!RequestException(rq.get(httpbinUrl));
366 
367     info("Test getContent");
368     auto r = getContent(httpbinUrl ~ "stream/20");
369     assert(r.splitter('\n').filter!("a.length>0").count == 20);
370     r = getContent(httpbinUrl ~ "get", ["a":"b", "c":"d"]);
371     string name = "user", sex = "male";
372     int    age = 42;
373     r = getContent(httpbinUrl ~ "get", "name", name, "age", age, "sex", sex);
374 
375     info("Test receiveAsRange with GET");
376     rq = Request();
377     rq.useStreaming = true;
378     rq.bufferSize = 16;
379     rs = rq.get(httpbinUrl ~ "stream/20");
380     auto stream = rs.receiveAsRange();
381     ubyte[] streamedContent;
382     while( !stream.empty() ) {
383         streamedContent ~= stream.front;
384         stream.popFront();
385     }
386     rq = Request();
387     rs = rq.get(httpbinUrl ~ "stream/20");
388     assert(streamedContent == rs.responseBody.data);
389     info("Test postContent");
390     r = postContent(httpbinUrl ~ "post", `{"a":"b", "c":1}`, "application/json");
391     assert(parseJSON(r).object["json"].object["c"].integer == 1);
392 
393     /// Posting to forms (for small data)
394     ///
395     /// posting query parameters using "application/x-www-form-urlencoded"
396     info("Test postContent using query params");
397     postContent(httpbinUrl ~ "post", queryParams("first", "a", "second", 2));
398     
399     /// posting using multipart/form-data (large data and files). See docs fot HTTPRequest
400     info("Test postContent form");
401     MultipartForm mpform;
402     mpform.add(formData(/* field name */ "greeting", /* content */ cast(ubyte[])"hello"));
403     postContent(httpbinUrl ~ "post", mpform);
404     
405     /// you can do this using Request struct to access response details
406     info("Test postContent form via Request()");
407     rq = Request();
408     mpform = MultipartForm().add(formData(/* field name */ "greeting", /* content */ cast(ubyte[])"hello"));
409     rs = rq.post(httpbinUrl ~ "post", mpform);
410     assert(rs.code == 200);
411     
412     info("Test receiveAsRange with POST");
413     streamedContent.length = 0;
414     rq = Request();
415     rq.useStreaming = true;
416     rq.bufferSize = 16;
417     string s = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
418     rs = rq.post(httpbinUrl ~ "post", s.representation.chunks(10), "application/octet-stream");
419     stream = rs.receiveAsRange();
420     while( !stream.empty() ) {
421         streamedContent ~= stream.front;
422         stream.popFront();
423     }
424     rq = Request();
425     rs = rq.post(httpbinUrl ~ "post", s.representation.chunks(10), "application/octet-stream");
426     assert(streamedContent == rs.responseBody.data);
427     info("Test get in parallel");
428     {
429         import std.stdio;
430         import std.parallelism;
431         import std.algorithm;
432         import std.string;
433         import core.atomic;
434         
435         immutable auto urls = [
436             "stream/10",
437             "stream/20",
438             "stream/30",
439             "stream/40",
440             "stream/50",
441             "stream/60",
442             "stream/70",
443         ].map!(a => httpbinUrl ~ a).array.idup;
444         
445         defaultPoolThreads(4);
446         
447         shared short lines;
448         
449         foreach(url; parallel(urls)) {
450             atomicOp!"+="(lines, getContent(url).splitter("\n").count);
451         }
452         assert(lines == 287);
453         
454     }
455 }
456 
457 auto queryParams(A...)(A args) pure @safe nothrow {
458     QueryParam[] res;
459     static if ( args.length >= 2 ) {
460         res = QueryParam(args[0].to!string, args[1].to!string) ~ queryParams(args[2..$]);
461     }
462     return res;
463 }
464 /**
465  * Call GET, and return response content.
466  * This is the simplest case, when all you need is the response body and have no parameters.
467  * Returns:
468  * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
469  */
470 public auto ref getContent(A...)(string url) {
471     auto rq = Request();
472     auto rs = rq.get(url);
473     return rs.responseBody;
474 }
475 /**
476  * Call GET, and return response content.
477  * args = string[string] fo query parameters.
478  * Returns:
479  * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
480  */
481 public auto ref getContent(A...)(string url, string[string] args) {
482     auto rq = Request();
483     auto rs = rq.get(url, args);
484     return rs.responseBody;
485 }
486 /**
487  * Call GET, and return response content.
488  * args = QueryParam[] of parameters.
489  * Returns:
490  * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
491  */
492 public auto ref getContent(A...)(string url, QueryParam[] args) {
493     auto rq = Request();
494     auto rs = rq.get(url, args);
495     return rs.responseBody;
496 }
497 /**
498  * Call GET, and return response content.
499  * args = variadic args to supply parameter names and values.
500  * Returns:
501  * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
502  */
503 public auto ref getContent(A...)(string url, A args) if (args.length > 1 && args.length % 2 == 0 ) {
504     return Request().
505             get(url, queryParams(args)).
506             responseBody;
507 }
508 
509 ///
510 /// Call post and return response content.
511 ///
512 public auto postContent(A...)(string url, A args) {
513     auto rq = Request();
514     auto rs = rq.post(url, args);
515     return rs.responseBody;
516 }
517 
518 ///
519 package unittest {
520     import std.json;
521     import std.string;
522     import std.stdio;
523     import std.range;
524 
525     globalLogLevel(LogLevel.info);
526 
527     /// ftp upload from range
528     info("Test postContent ftp");
529     auto r = postContent("ftp://speedtest.tele2.net/upload/TEST.TXT", "test, ignore please\n".representation);
530     assert(r.length == 0);
531 
532     info("Test getContent(ftp)");
533     r = getContent("ftp://speedtest.tele2.net/1KB.zip");
534     assert(r.length == 1024);
535     
536     info("Test receiveAsRange with GET(ftp)");
537     ubyte[] streamedContent;
538     auto rq = Request();
539     rq.useStreaming = true;
540     streamedContent.length = 0;
541     auto rs = rq.get("ftp://speedtest.tele2.net/1KB.zip");
542     auto stream = rs.receiveAsRange;
543     while( !stream.empty() ) {
544         streamedContent ~= stream.front;
545         stream.popFront();
546     }
547     assert(streamedContent.length == 1024);
548     //
549     info("ftp post ", "ftp://speedtest.tele2.net/upload/TEST.TXT");
550     rs = rq.post("ftp://speedtest.tele2.net/upload/TEST.TXT", "test, ignore please\n".representation);
551     assert(rs.code == 226);
552     info("ftp get  ", "ftp://speedtest.tele2.net/nonexistent", ", in same session.");
553     rs = rq.get("ftp://speedtest.tele2.net/nonexistent");
554     assert(rs.code != 226);
555     rq.useStreaming = false;
556     info("ftp get  ", "ftp://speedtest.tele2.net/1KB.zip", ", in same session.");
557     rs = rq.get("ftp://speedtest.tele2.net/1KB.zip");
558     assert(rs.code == 226);
559     assert(rs.responseBody.length == 1024);
560     info("ftp post ", "ftp://speedtest.tele2.net/upload/TEST.TXT");
561     rs = rq.post("ftp://speedtest.tele2.net/upload/TEST.TXT", "another test, ignore please\n".representation);
562     assert(rs.code == 226);
563     info("ftp get  ", "ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT");
564     rs = rq.get("ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT");
565     assert(rs.code == 226);
566     info("testing ftp - done.");
567 }
568