requests source code

1 module requests;
2 
3 public import requests.http;
4 public import requests.ftp;
5 public import requests.streams;
6 public import requests.base;
7 public import requests.uri;
8 public import requests.request;
9 public import requests.pool;
10 
11 import std.datetime;
12 import std.conv;
13 import std.experimental.logger;
14 import requests.utils;
15 
16 ///
17 package unittest {
18     import std.algorithm;
19     import std.range;
20     import std.array;
21     import std.json;
22     import std.stdio;
23     import std.string;
24     import std.exception;
25 
26     string httpbinUrl = httpTestServer();
27 
28     version(vibeD) {
29     }
30     else {
31         import httpbin;
32         auto server = httpbinApp();
33         server.start();
34         scope(exit) {
35             server.stop();
36         }
37     }
38 
39     globalLogLevel(LogLevel.info);
40 
41     infof("testing Request");
42     Request rq;
43     Response rs;
44     //
45     rs = rq.get(httpbinUrl);
46     assert(rs.code==200);
47     assert(rs.responseBody.length > 0);
48     rs = rq.get(httpbinUrl ~ "get", ["c":" d", "a":"b"]);
49     assert(rs.code == 200);
50     auto json = parseJSON(cast(string)rs.responseBody.data).object["args"].object;
51     assert(json["c"].str == " d");
52     assert(json["a"].str == "b");
53     
54     rq = Request();
55     rq.keepAlive = true;
56     // handmade json
57     info("Check POST json");
58     rs = rq.post(httpbinUrl ~ "post?b=x", `{"a":"b ", "c":[1,2,3]}`, "application/json");
59     assert(rs.code==200);
60     json = parseJSON(cast(string)rs.responseBody.data).object["args"].object;
61     assert(json["b"].str == "x");
62     json = parseJSON(cast(string)rs.responseBody.data).object["json"].object;
63     assert(json["a"].str == "b ");
64     assert(json["c"].array.map!(a=>a.integer).array == [1,2,3]);
65     {
66         import std.file;
67         import std.path;
68         auto tmpd = tempDir();
69         auto tmpfname = tmpd ~ dirSeparator ~ "request_test.txt";
70         auto f = File(tmpfname, "wb");
71         f.rawWrite("abcdefgh\n12345678\n");
72         f.close();
73         // files
74         info("Check POST files");
75         PostFile[] files = [
76             {fileName: tmpfname, fieldName:"abc", contentType:"application/octet-stream"}, 
77             {fileName: tmpfname}
78         ];
79         rs = rq.post(httpbinUrl ~ "post", files);
80         assert(rs.code==200);
81         info("Check POST chunked from file.byChunk");
82         f = File(tmpfname, "rb");
83         rs = rq.post(httpbinUrl ~ "post", f.byChunk(3), "application/octet-stream");
84         if (httpbinUrl != "http://httpbin.org/" ) {
85             assert(rs.code==200);
86             auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]);
87             assert(data=="abcdefgh\n12345678\n");
88         }
89         f.close();
90     }
91     // ranges
92     {
93         info("Check POST chunked from lineSplitter");
94         auto s = lineSplitter("one,\ntwo,\nthree.");
95         rs = rq.exec!"POST"(httpbinUrl ~ "post", s, "application/octet-stream");
96         if (httpbinUrl != "http://httpbin.org/" ) {
97             assert(rs.code==200);
98             auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]);
99             assert(data=="one,two,three.");
100         }
101     }
102     {
103         info("Check POST chunked from array");
104         auto s = ["one,", "two,", "three."];
105         rs = rq.post(httpbinUrl ~ "post", s, "application/octet-stream");
106         if (httpbinUrl != "http://httpbin.org/" ) {
107             assert(rs.code==200);
108             auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]);
109             assert(data=="one,two,three.");
110         }
111     }
112     {
113         info("Check POST chunked using std.range.chunks()");
114         auto s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
115         rs = rq.post(httpbinUrl ~ "post", s.representation.chunks(10), "application/octet-stream");
116         if (httpbinUrl != "http://httpbin.org/") {
117             assert(rs.code==200);
118             auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]);
119             assert(data==s);
120         }
121     }
122     // associative array
123     rs = rq.post(httpbinUrl ~ "post", ["a":"b ", "c":"d"]);
124     assert(rs.code==200);
125     auto form = parseJSON(cast(string)rs.responseBody.data).object["form"].object;
126     assert(form["a"].str == "b ");
127     assert(form["c"].str == "d");
128     info("Check HEAD");
129     rs = rq.exec!"HEAD"(httpbinUrl);
130     assert(rs.code==200);
131     info("Check DELETE");
132     rs = rq.exec!"DELETE"(httpbinUrl ~ "delete");
133     assert(rs.code==200);
134     info("Check PUT");
135     rs = rq.exec!"PUT"(httpbinUrl ~ "put",  `{"a":"b", "c":[1,2,3]}`, "application/json");
136     assert(rs.code==200);
137     info("Check PATCH");
138     rs = rq.exec!"PATCH"(httpbinUrl ~ "patch", "привiт, свiт!", "application/octet-stream");
139     assert(rs.code==200);
140     
141     info("Check compressed content");
142     rq = Request();
143     rq.keepAlive = true;
144     rq.addHeaders(["X-Header": "test"]);
145     rs = rq.get(httpbinUrl ~ "gzip");
146     assert(rs.code==200);
147     info("gzip - ok");
148     rs = rq.get(httpbinUrl ~ "deflate");
149     assert(rs.code==200);
150     info("deflate - ok");
151     
152     info("Check redirects");
153     rq = Request();
154     rq.keepAlive = true;
155     rs = rq.get(httpbinUrl ~ "relative-redirect/2");
156     assert((cast(HTTPResponse)rs).history.length == 2);
157     assert((cast(HTTPResponse)rs).code==200);
158 
159     info("Check cookie");
160     rq = Request();
161     rs = rq.get(httpbinUrl ~ "cookies/set?A=abcd&b=cdef");
162     assert(rs.code == 200);
163     json = parseJSON(cast(string)rs.responseBody.data).object["cookies"].object;
164     assert(json["A"].str == "abcd");
165     assert(json["b"].str == "cdef");
166     auto cookie = rq.cookie();
167     foreach(c; rq.cookie) {
168         final switch(c.attr) {
169             case "A":
170                 assert(c.value == "abcd");
171                 break;
172             case "b":
173                 assert(c.value == "cdef");
174                 break;
175         }
176     }
177     rs = rq.get(httpbinUrl ~ "absolute-redirect/2");
178     assert((cast(HTTPResponse)rs).history.length == 2);
179     assert((cast(HTTPResponse)rs).code==200);
180     //    rq = Request();
181     rq.maxRedirects = 2;
182     rq.keepAlive = false;
183     assertThrown!MaxRedirectsException(rq.get(httpbinUrl ~ "absolute-redirect/3"));
184 
185     info("Check chunked content");
186     rq = Request();
187     rq.keepAlive = true;
188     rq.bufferSize = 16*1024;
189     rs = rq.get(httpbinUrl ~ "range/1024");
190     assert(rs.code==200);
191     assert(rs.responseBody.length==1024);
192     
193     info("Check basic auth");
194     rq = Request();
195     rq.authenticator = new BasicAuthentication("user", "passwd");
196     rs = rq.get(httpbinUrl ~ "basic-auth/user/passwd");
197     assert(rs.code==200);
198     
199     info("Check limits");
200     rq = Request();
201     rq.maxContentLength = 1;
202     assertThrown!RequestException(rq.get(httpbinUrl));
203     rq = Request();
204     rq.maxHeadersLength = 1;
205     assertThrown!RequestException(rq.get(httpbinUrl));
206 
207     info("Test getContent");
208     auto r = getContent(httpbinUrl ~ "stream/20");
209     assert(r.splitter('\n').filter!("a.length>0").count == 20);
210     r = getContent(httpbinUrl ~ "get", ["a":"b", "c":"d"]);
211     string name = "user", sex = "male";
212     int    age = 42;
213     r = getContent(httpbinUrl ~ "get", "name", name, "age", age, "sex", sex);
214 
215     info("Test receiveAsRange with GET");
216     rq = Request();
217     rq.useStreaming = true;
218     rq.bufferSize = 16;
219     rs = rq.get(httpbinUrl ~ "stream/20");
220     auto stream = rs.receiveAsRange();
221     ubyte[] streamedContent;
222     while( !stream.empty() ) {
223         streamedContent ~= stream.front;
224         stream.popFront();
225     }
226     rq = Request();
227     rs = rq.get(httpbinUrl ~ "stream/20");
228     assert(streamedContent.length == rs.responseBody.data.length);
229     info("Test postContent");
230     r = postContent(httpbinUrl ~ "post", `{"a":"b", "c":1}`, "application/json");
231     assert(parseJSON(cast(string)r).object["json"].object["c"].integer == 1);
232 
233     /// Posting to forms (for small data)
234     ///
235     /// posting query parameters using "application/x-www-form-urlencoded"
236     info("Test postContent using query params");
237     postContent(httpbinUrl ~ "post", queryParams("first", "a", "second", 2));
238     
239     /// posting using multipart/form-data (large data and files). See docs fot HTTPRequest
240     info("Test postContent form");
241     MultipartForm mpform;
242     mpform.add(formData(/* field name */ "greeting", /* content */ cast(ubyte[])"hello"));
243     postContent(httpbinUrl ~ "post", mpform);
244     
245     /// you can do this using Request struct to access response details
246     info("Test postContent form via Request()");
247     rq = Request();
248     mpform = MultipartForm().add(formData(/* field name */ "greeting", /* content */ cast(ubyte[])"hello"));
249     rs = rq.post(httpbinUrl ~ "post", mpform);
250     assert(rs.code == 200);
251     
252     info("Test receiveAsRange with POST");
253     streamedContent.length = 0;
254     rq = Request();
255     rq.useStreaming = true;
256     rq.bufferSize = 16;
257     string s = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
258     rs = rq.post(httpbinUrl ~ "post", s.representation.chunks(10), "application/octet-stream");
259     stream = rs.receiveAsRange();
260     while( !stream.empty() ) {
261         streamedContent ~= stream.front;
262         stream.popFront();
263     }
264     rq = Request();
265     rs = rq.post(httpbinUrl ~ "post", s.representation.chunks(10), "application/octet-stream");
266     assert(streamedContent == rs.responseBody.data);
267     info("Test get in parallel");
268     {
269         import std.stdio;
270         import std.parallelism;
271         import std.algorithm;
272         import std.string;
273         import core.atomic;
274         
275         immutable auto urls = [
276             "stream/10",
277             "stream/20",
278             "stream/30",
279             "stream/40",
280             "stream/50",
281             "stream/60",
282             "stream/70",
283         ].map!(a => httpbinUrl ~ a).array.idup;
284         
285         defaultPoolThreads(4);
286         
287         shared short lines;
288         
289         foreach(url; parallel(urls)) {
290             atomicOp!"+="(lines, getContent(url).splitter("\n").count);
291         }
292         assert(lines == 287);
293         
294     }
295 }
296 
297 auto queryParams(A...)(A args) pure @safe nothrow {
298     QueryParam[] res;
299     static if ( args.length >= 2 ) {
300         res = QueryParam(args[0].to!string, args[1].to!string) ~ queryParams(args[2..$]);
301     }
302     return res;
303 }
304 /**
305  * Call GET, and return response content.
306  * This is the simplest case, when all you need is the response body and have no parameters.
307  * Returns:
308  * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
309  */
310 public auto ref getContent(A...)(string url) {
311     auto rq = Request();
312     auto rs = rq.get(url);
313     return rs.responseBody;
314 }
315 /**
316  * Call GET, and return response content.
317  * args = string[string] fo query parameters.
318  * Returns:
319  * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
320  */
321 public auto ref getContent(A...)(string url, string[string] args) {
322     auto rq = Request();
323     auto rs = rq.get(url, args);
324     return rs.responseBody;
325 }
326 /**
327  * Call GET, and return response content.
328  * args = QueryParam[] of parameters.
329  * Returns:
330  * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
331  */
332 public auto ref getContent(A...)(string url, QueryParam[] args) {
333     auto rq = Request();
334     auto rs = rq.get(url, args);
335     return rs.responseBody;
336 }
337 /**
338  * Call GET, and return response content.
339  * args = variadic args to supply parameter names and values.
340  * Returns:
341  * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
342  */
343 public auto ref getContent(A...)(string url, A args) if (args.length > 1 && args.length % 2 == 0 ) {
344     return Request().
345             get(url, queryParams(args)).
346             responseBody;
347 }
348 
349 ///
350 /// Call post and return response content.
351 ///
352 public auto postContent(A...)(string url, A args) {
353     auto rq = Request();
354     auto rs = rq.post(url, args);
355     return rs.responseBody;
356 }
357 
358 ///
359 package unittest {
360     import std.json;
361     import std.string;
362     import std.stdio;
363     import std.range;
364 
365     globalLogLevel(LogLevel.info);
366 
367     /// ftp upload from range
368     info("Test getContent(ftp)");
369     auto r = getContent("ftp://speedtest.tele2.net/1KB.zip");
370     assert(r.length == 1024);
371     
372     info("Test postContent ftp");
373     r = postContent("ftp://speedtest.tele2.net/upload/TEST.TXT", "test, ignore please\n".representation);
374     assert(r.length == 0);
375 
376     info("Test receiveAsRange with GET(ftp)");
377     ubyte[] streamedContent;
378     auto rq = Request();
379     rq.useStreaming = true;
380     streamedContent.length = 0;
381     auto rs = rq.get("ftp://speedtest.tele2.net/1KB.zip");
382     auto stream = rs.receiveAsRange;
383     while( !stream.empty() ) {
384         streamedContent ~= stream.front;
385         stream.popFront();
386     }
387     assert(streamedContent.length == 1024);
388     //
389     info("ftp post ", "ftp://speedtest.tele2.net/upload/TEST.TXT");
390     rs = rq.post("ftp://speedtest.tele2.net/upload/TEST.TXT", "test, ignore please\n".representation);
391     assert(rs.code == 226);
392     info("ftp get  ", "ftp://speedtest.tele2.net/nonexistent", ", in same session.");
393     rs = rq.get("ftp://speedtest.tele2.net/nonexistent");
394     assert(rs.code != 226);
395     rq.useStreaming = false;
396     info("ftp get  ", "ftp://speedtest.tele2.net/1KB.zip", ", in same session.");
397     rs = rq.get("ftp://speedtest.tele2.net/1KB.zip");
398     assert(rs.code == 226);
399     assert(rs.responseBody.length == 1024);
400     info("ftp post ", "ftp://speedtest.tele2.net/upload/TEST.TXT");
401     rs = rq.post("ftp://speedtest.tele2.net/upload/TEST.TXT", "another test, ignore please\n".representation);
402     assert(rs.code == 226);
403     info("ftp get  ", "ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT");
404     rs = rq.get("ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT");
405     assert(rs.code == 226);
406     rq.authenticator = new BasicAuthentication("anonymous", "request@");
407     rs = rq.get("ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT");
408     assert(rs.code == 226);
409     info("testing ftp - done.");
410 }
411