requests source code

1 module requests;
2 
3 public import requests.http;
4 public import requests.ftp;
5 public import requests.streams;
6 public import requests.base;
7 
8 import std.datetime;
9 import std.experimental.logger;
10 import requests.uri;
11 
12 /***********************************
13  * This is simplest interface to both http and ftp protocols.
14  * Request has methods get, post and exec which routed to proper concrete handler (http or ftp, etc).
15  * To enable some protocol-specific featutes you have to use protocol interface directly (see docs for HTTPRequest or FTPRequest)
16  */
17 struct Request {
18     private {
19         URI         _uri;
20         HTTPRequest _http;  // route all http/https requests here
21         FTPRequest  _ftp;   // route all ftp requests here
22     }
23     /// Set timeout on IO operation.
24     /// $(B v) - timeout value
25     /// 
26     @property void timeout(Duration v) pure @nogc nothrow {
27         _http.timeout = v;
28         _ftp.timeout = v;
29     }
30     /// Set http keepAlive value
31     /// $(B v) - use keepalive requests - $(B true), or not - $(B false)
32     @property void keepAlive(bool v) pure @nogc nothrow {
33         _http.keepAlive = v;
34     }
35     /// Set limit on HTTP redirects
36     /// $(B v) - limit on redirect depth
37     @property void maxRedirects(uint v) pure @nogc nothrow {
38         _http.maxRedirects = v;
39     }
40     /// Set maximum content lenth both for http and ftp requests
41     /// $(B v) - maximum content length in bytes. When limit reached - throw RequestException
42     @property void maxContentLength(size_t v) pure @nogc nothrow {
43         _http.maxContentLength = v;
44         _ftp.maxContentLength = v;
45     }
46     /// Set maximum length for HTTP headers
47     /// $(B v) - maximum length of the HTTP response. When limit reached - throw RequestException
48     @property void maxHeadersLength(size_t v) pure @nogc nothrow {
49         _http.maxHeadersLength = v;
50     }
51     /// Set IO buffer size for http and ftp requests
52     /// $(B v) - buffer size in bytes.
53     @property void bufferSize(size_t v) {
54         _http.bufferSize = v;
55         _ftp.bufferSize = v;
56     }
57     /// Set verbosity for HTTP or FTP requests.
58     /// $(B v) - verbosity level (0 - no output, 1 - headers to stdout, 2 - headers and body progress to stdout). default = 0.
59     @property void verbosity(uint v) {
60         _http.verbosity = v;
61         _ftp.verbosity = v;
62     }
63     /// Set authenticator for http requests.
64     /// $(B v) - Auth instance.
65     @property void authenticator(Auth v) {
66         _http.authenticator = v;
67     }
68     /// Execute GET for http and retrieve file for FTP.
69     /// You have to provide at least $(B uri). All other arguments should conform to HTTPRequest.get or FTPRequest.get depending on the URI scheme.
70     /// When arguments do not conform scheme (for example you try to call get("ftp://somehost.net/pub/README", {"a":"b"}) which doesn't make sense)
71     /// you will receive Exception("Operation not supported for ftp")
72     ///
73     Response get(A...)(string uri, A args) {
74         if ( uri ) {
75             _uri = URI(uri);
76         }
77         final switch ( _uri.scheme ) {
78             case "http", "https":
79                 _http.uri = _uri;
80                 static if (__traits(compiles, _http.get(null, args))) {
81                     return _http.get(null, args);
82                 } else {
83                     throw new Exception("Operation not supported for http");
84                 }
85             case "ftp":
86                 return _ftp.get(uri);
87         }
88     }
89     /// Execute POST for http and STOR file for FTP.
90     /// You have to provide  $(B uri) and data. Data should conform to HTTPRequest.post or FTPRequest.post depending on the URI scheme.
91     /// When arguments do not conform scheme you will receive Exception("Operation not supported for ftp")
92     ///
93     Response post(A...)(string uri, A args) {
94         if ( uri ) {
95             _uri = URI(uri);
96         }
97         final switch ( _uri.scheme ) {
98             case "http", "https":
99                 _http.uri = _uri;
100                 static if (__traits(compiles, _http.post(null, args))) {
101                     return _http.post(null, args);
102                 } else {
103                     throw new Exception("Operation not supported for http");
104                 }
105             case "ftp":
106                 static if (__traits(compiles, _ftp.post(uri, args))) {
107                     return _ftp.post(uri, args);
108                 } else {
109                     throw new Exception("Operation not supported for ftp");
110                 }
111         }
112     }
113     Response exec(string method="GET", A...)(A args) {
114         return _http.exec!(method)(args);
115     }
116 }
117 ///
118 unittest {
119     import std.algorithm;
120     import std.range;
121     import std.array;
122     import std.json;
123     import std.stdio;
124     import std.string;
125     import std.exception;
126 
127     globalLogLevel(LogLevel.info);
128 
129     infof("testing Request");
130     Request rq;
131     Response rs;
132     //
133     rs = rq.get("https://httpbin.org/");
134     assert(rs.code==200);
135     assert(rs.responseBody.length > 0);
136     rs = rq.get("http://httpbin.org/get", ["c":" d", "a":"b"]);
137     assert(rs.code == 200);
138     auto json = parseJSON(rs.responseBody.data).object["args"].object;
139     assert(json["c"].str == " d");
140     assert(json["a"].str == "b");
141     
142     globalLogLevel(LogLevel.info);
143     rq = Request();
144     rq.keepAlive = true;
145     // handmade json
146     info("Check POST json");
147     rs = rq.post("http://httpbin.org/post?b=x", `{"a":"☺ ", "c":[1,2,3]}`, "application/json");
148     assert(rs.code==200);
149     json = parseJSON(rs.responseBody.data).object["args"].object;
150     assert(json["b"].str == "x");
151     json = parseJSON(rs.responseBody.data).object["json"].object;
152     assert(json["a"].str == "☺ ");
153     assert(json["c"].array.map!(a=>a.integer).array == [1,2,3]);
154     {
155         import std.file;
156         import std.path;
157         auto tmpd = tempDir();
158         auto tmpfname = tmpd ~ dirSeparator ~ "request_test.txt";
159         auto f = File(tmpfname, "wb");
160         f.rawWrite("abcdefgh\n12345678\n");
161         f.close();
162         // files
163         globalLogLevel(LogLevel.info);
164         info("Check POST files");
165         PostFile[] files = [
166         {fileName: tmpfname, fieldName:"abc", contentType:"application/octet-stream"}, 
167         {fileName: tmpfname}
168         ];
169         rs = rq.post("http://httpbin.org/post", files);
170         assert(rs.code==200);
171         info("Check POST chunked from file.byChunk");
172         f = File(tmpfname, "rb");
173         rs = rq.post("http://httpbin.org/post", f.byChunk(3), "application/octet-stream");
174         assert(rs.code==200);
175         auto data = parseJSON(rs.responseBody.data).object["data"].str;
176         assert(data=="abcdefgh\n12345678\n");
177         f.close();
178     }
179     {
180         // string
181         info("Check POST utf8 string");
182         rs = rq.post("http://httpbin.org/post", "привiт, свiт!", "application/octet-stream");
183         assert(rs.code==200);
184         auto data = parseJSON(rs.responseBody.data).object["data"].str;
185         assert(data=="привiт, свiт!");
186     }
187     // ranges
188     {
189         info("Check POST chunked from lineSplitter");
190         auto s = lineSplitter("one,\ntwo,\nthree.");
191         rs = rq.exec!"POST"("http://httpbin.org/post", s, "application/octet-stream");
192         assert(rs.code==200);
193         auto data = parseJSON(rs.responseBody.toString).object["data"].str;
194         assert(data=="one,two,three.");
195     }
196     {
197         info("Check POST chunked from array");
198         auto s = ["one,", "two,", "three."];
199         rs = rq.post("http://httpbin.org/post", s, "application/octet-stream");
200         assert(rs.code==200);
201         auto data = parseJSON(rs.responseBody.data).object["data"].str;
202         assert(data=="one,two,three.");
203     }
204     {
205         info("Check POST chunked using std.range.chunks()");
206         auto s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
207         rs = rq.post("http://httpbin.org/post", s.representation.chunks(10), "application/octet-stream");
208         assert(rs.code==200);
209         auto data = parseJSON(rs.responseBody.data).object["data"].str;
210         assert(data==s);
211     }
212     // associative array
213     rs = rq.post("http://httpbin.org/post", ["a":"b ", "c":"d"]);
214     assert(rs.code==200);
215     auto form = parseJSON(rs.responseBody.data).object["form"].object;
216     assert(form["a"].str == "b ");
217     assert(form["c"].str == "d");
218     info("Check HEAD");
219     rs = rq.exec!"HEAD"("http://httpbin.org/");
220     assert(rs.code==200);
221     info("Check DELETE");
222     rs = rq.exec!"DELETE"("http://httpbin.org/delete");
223     assert(rs.code==200);
224     info("Check PUT");
225     rs = rq.exec!"PUT"("http://httpbin.org/put",  `{"a":"b", "c":[1,2,3]}`, "application/json");
226     assert(rs.code==200);
227     info("Check PATCH");
228     rs = rq.exec!"PATCH"("http://httpbin.org/patch", "привiт, свiт!", "application/octet-stream");
229     assert(rs.code==200);
230     
231     info("Check compressed content");
232     globalLogLevel(LogLevel.info);
233     rq = Request();
234     rq.keepAlive = true;
235     rs = rq.get("http://httpbin.org/gzip");
236     assert(rs.code==200);
237     info("gzip - ok");
238     rs = rq.get("http://httpbin.org/deflate");
239     assert(rs.code==200);
240     info("deflate - ok");
241     
242     info("Check redirects");
243     globalLogLevel(LogLevel.info);
244     rq = Request();
245     rq.keepAlive = true;
246     rs = rq.get("http://httpbin.org/relative-redirect/2");
247     assert((cast(HTTPResponse)rs).history.length == 2);
248     assert((cast(HTTPResponse)rs).code==200);
249     //    rq = Request();
250     //    rq.keepAlive = true;
251     //    rq.proxy = "http://localhost:8888/";
252     rs = rq.get("http://httpbin.org/absolute-redirect/2");
253     assert((cast(HTTPResponse)rs).history.length == 2);
254     assert((cast(HTTPResponse)rs).code==200);
255     //    rq = Request();
256     rq.maxRedirects = 2;
257     rq.keepAlive = false;
258     rs = rq.get("https://httpbin.org/absolute-redirect/3");
259     assert((cast(HTTPResponse)rs).history.length == 2);
260     assert((cast(HTTPResponse)rs).code==302);
261     
262     info("Check utf8 content");
263     globalLogLevel(LogLevel.info);
264     rq = Request();
265     rs = rq.get("http://httpbin.org/encoding/utf8");
266     assert(rs.code==200);
267     
268     info("Check chunked content");
269     globalLogLevel(LogLevel.info);
270     rq = Request();
271     rq.keepAlive = true;
272     rq.bufferSize = 16*1024;
273     rs = rq.get("http://httpbin.org/range/1024");
274     assert(rs.code==200);
275     assert(rs.responseBody.length==1024);
276     
277     info("Check basic auth");
278     globalLogLevel(LogLevel.info);
279     rq = Request();
280     rq.authenticator = new BasicAuthentication("user", "passwd");
281     rs = rq.get("http://httpbin.org/basic-auth/user/passwd");
282     assert(rs.code==200);
283     
284     globalLogLevel(LogLevel.info);
285     info("Check exception handling, error messages are OK");
286     rq = Request();
287     rq.timeout = 1.seconds;
288     assertThrown!TimeoutException(rq.get("http://httpbin.org/delay/3"));
289     assertThrown!ConnectError(rq.get("http://0.0.0.0:65000/"));
290     assertThrown!ConnectError(rq.get("http://1.1.1.1/"));
291     //assertThrown!ConnectError(rq.get("http://gkhgkhgkjhgjhgfjhgfjhgf/"));
292     
293     globalLogLevel(LogLevel.info);
294     info("Check limits");
295     rq = Request();
296     rq.maxContentLength = 1;
297     assertThrown!RequestException(rq.get("http://httpbin.org/"));
298     rq = Request();
299     rq.maxHeadersLength = 1;
300     assertThrown!RequestException(rq.get("http://httpbin.org/"));
301     //
302     info("ftp post ", "ftp://speedtest.tele2.net/upload/TEST.TXT");
303     rs = rq.post("ftp://speedtest.tele2.net/upload/TEST.TXT", "test, ignore please\n".representation);
304     assert(rs.code == 226);
305     info("ftp get  ", "ftp://speedtest.tele2.net/nonexistent", ", in same session.");
306     rs = rq.get("ftp://speedtest.tele2.net/nonexistent");
307     assert(rs.code != 226);
308     info("ftp get  ", "ftp://speedtest.tele2.net/1KB.zip", ", in same session.");
309     rs = rq.get("ftp://speedtest.tele2.net/1KB.zip");
310     assert(rs.code == 226);
311     assert(rs.responseBody.length == 1024);
312     info("ftp get  ", "ftp://ftp.uni-bayreuth.de/README");
313     rs = rq.get("ftp://ftp.uni-bayreuth.de/README");
314     assert(rs.code == 226);
315     info("ftp post ", "ftp://speedtest.tele2.net/upload/TEST.TXT");
316     rs = rq.post("ftp://speedtest.tele2.net/upload/TEST.TXT", "another test, ignore please\n".representation);
317     assert(rs.code == 226);
318     info("ftp get  ", "ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT");
319     rs = rq.get("ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT");
320     assert(rs.code == 226);
321     info("testing ftp - done.");
322 }
323 
324 /**
325  * Call GET, and return response content.
326  * This is the simplest case, when all you need is the response body.
327  * Returns:
328  * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
329  */
330 public auto getContent(A...)(string url, A args) {
331     auto rq = Request();
332     auto rs = rq.get(url, args);
333     return rs.responseBody;
334 }
335 ///
336 public unittest {
337     import std.algorithm;
338     globalLogLevel(LogLevel.info);
339     info("Test getContent");
340     auto r = getContent("https://httpbin.org/stream/20");
341     assert(r.splitter('\n').filter!("a.length>0").count == 20);
342     r = getContent("ftp://speedtest.tele2.net/1KB.zip");
343     assert(r.length == 1024);
344 }
345 
346 /**
347  * Call post and return response content.
348  */
349 public auto postContent(A...)(string url, A args) {
350     auto rq = Request();
351     auto rs = rq.post(url, args);
352     return rs.responseBody;
353 }
354 ///
355 public unittest {
356     import std.json;
357     import std.string;
358     globalLogLevel(LogLevel.info);
359     info("Test postContent");
360     auto r = postContent("http://httpbin.org/post", `{"a":"b", "c":1}`, "application/json");
361     assert(parseJSON(r.data).object["json"].object["c"].integer == 1);
362     r = postContent("ftp://speedtest.tele2.net/upload/TEST.TXT", "test, ignore please\n".representation);
363     assert(r.length == 0);
364 }