requests source code

1 module requests;
2 
3 public import requests.http;
4 public import requests.ftp;
5 public import requests.streams;
6 public import requests.base;
7 public import requests.uri;
8 public import requests.request;
9 public import requests.pool;
10 
11 import std.datetime;
12 import std.conv;
13 import std.experimental.logger;
14 import requests.utils;
15 
16 ///
17 package unittest {
18     import std.algorithm;
19     import std.range;
20     import std.array;
21     import std.json;
22     import std.stdio;
23     import std.string;
24     import std.exception;
25 
26     string httpbinUrl = httpTestServer();
27 
28     version(vibeD) {
29     }
30     else {
31         import httpbin;
32         auto server = httpbinApp();
33         server.start();
34         scope(exit) {
35             server.stop();
36         }
37     }
38 
39     globalLogLevel(LogLevel.info);
40 
41     infof("testing Request");
42     Request rq;
43     Response rs;
44     //
45     rs = rq.get(httpbinUrl);
46     assert(rs.code==200);
47     assert(rs.responseBody.length > 0);
48     rs = rq.get(httpbinUrl ~ "get", ["c":" d", "a":"b"]);
49     assert(rs.code == 200);
50     auto json = parseJSON(cast(string)rs.responseBody.data).object["args"].object;
51     assert(json["c"].str == " d");
52     assert(json["a"].str == "b");
53 
54     
55     rq = Request();
56     rq.keepAlive = true;
57     {
58         info("Check handling incomplete status line");
59         rs = rq.get(httpbinUrl ~ "incomplete");
60         if (httpbinUrl != "http://httpbin.org/") {
61             assert(rs.code==600);
62         }
63     }
64     // handmade json
65     info("Check POST json");
66     rs = rq.post(httpbinUrl ~ "post?b=x", `{"a":"b ", "c":[1,2,3]}`, "application/json");
67     assert(rs.code==200);
68     json = parseJSON(cast(string)rs.responseBody.data).object["args"].object;
69     assert(json["b"].str == "x");
70     json = parseJSON(cast(string)rs.responseBody.data).object["json"].object;
71     assert(json["a"].str == "b ");
72     assert(json["c"].array.map!(a=>a.integer).array == [1,2,3]);
73     {
74         import std.file;
75         import std.path;
76         auto tmpd = tempDir();
77         auto tmpfname = tmpd ~ dirSeparator ~ "request_test.txt";
78         auto f = File(tmpfname, "wb");
79         f.rawWrite("abcdefgh\n12345678\n");
80         f.close();
81         // files
82         info("Check POST files");
83         PostFile[] files = [
84             {fileName: tmpfname, fieldName:"abc", contentType:"application/octet-stream"}, 
85             {fileName: tmpfname}
86         ];
87         rs = rq.post(httpbinUrl ~ "post", files);
88         assert(rs.code==200);
89         info("Check POST chunked from file.byChunk");
90         f = File(tmpfname, "rb");
91         rs = rq.post(httpbinUrl ~ "post", f.byChunk(3), "application/octet-stream");
92         if (httpbinUrl != "http://httpbin.org/" ) {
93             assert(rs.code==200);
94             auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]);
95             assert(data=="abcdefgh\n12345678\n");
96         }
97         f.close();
98     }
99     // ranges
100     {
101         info("Check POST chunked from lineSplitter");
102         auto s = lineSplitter("one,\ntwo,\nthree.");
103         rs = rq.exec!"POST"(httpbinUrl ~ "post", s, "application/octet-stream");
104         if (httpbinUrl != "http://httpbin.org/" ) {
105             assert(rs.code==200);
106             auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]);
107             assert(data=="one,two,three.");
108         }
109     }
110     {
111         info("Check POST chunked from array");
112         auto s = ["one,", "two,", "three."];
113         rs = rq.post(httpbinUrl ~ "post", s, "application/octet-stream");
114         if (httpbinUrl != "http://httpbin.org/" ) {
115             assert(rs.code==200);
116             auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]);
117             assert(data=="one,two,three.");
118         }
119     }
120     {
121         info("Check POST chunked using std.range.chunks()");
122         auto s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
123         rs = rq.post(httpbinUrl ~ "post", s.representation.chunks(10), "application/octet-stream");
124         if (httpbinUrl != "http://httpbin.org/") {
125             assert(rs.code==200);
126             auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]);
127             assert(data==s);
128         }
129     }
130     // associative array
131     rs = rq.post(httpbinUrl ~ "post", ["a":"b ", "c":"d"]);
132     assert(rs.code==200);
133     auto form = parseJSON(cast(string)rs.responseBody.data).object["form"].object;
134     assert(form["a"].str == "b ");
135     assert(form["c"].str == "d");
136     info("Check HEAD");
137     rs = rq.exec!"HEAD"(httpbinUrl);
138     assert(rs.code==200);
139     info("Check DELETE");
140     rs = rq.exec!"DELETE"(httpbinUrl ~ "delete");
141     assert(rs.code==200);
142     info("Check PUT");
143     rs = rq.exec!"PUT"(httpbinUrl ~ "put",  `{"a":"b", "c":[1,2,3]}`, "application/json");
144     assert(rs.code==200);
145     info("Check PATCH");
146     rs = rq.exec!"PATCH"(httpbinUrl ~ "patch", "привiт, свiт!", "application/octet-stream");
147     assert(rs.code==200);
148     
149     info("Check compressed content");
150     rq = Request();
151     rq.keepAlive = true;
152     rq.addHeaders(["X-Header": "test"]);
153     rs = rq.get(httpbinUrl ~ "gzip");
154     assert(rs.code==200);
155     info("gzip - ok");
156     rs = rq.get(httpbinUrl ~ "deflate");
157     assert(rs.code==200);
158     info("deflate - ok");
159     
160     info("Check redirects");
161     rq = Request();
162     rq.keepAlive = true;
163     rs = rq.get(httpbinUrl ~ "relative-redirect/2");
164     assert((cast(HTTPResponse)rs).history.length == 2);
165     assert((cast(HTTPResponse)rs).code==200);
166 
167     info("Check cookie");
168     rq = Request();
169     rs = rq.get(httpbinUrl ~ "cookies/set?A=abcd&b=cdef");
170     assert(rs.code == 200);
171     json = parseJSON(cast(string)rs.responseBody.data).object["cookies"].object;
172     assert(json["A"].str == "abcd");
173     assert(json["b"].str == "cdef");
174     auto cookie = rq.cookie();
175     foreach(c; rq.cookie) {
176         final switch(c.attr) {
177             case "A":
178                 assert(c.value == "abcd");
179                 break;
180             case "b":
181                 assert(c.value == "cdef");
182                 break;
183         }
184     }
185     rs = rq.get(httpbinUrl ~ "absolute-redirect/2");
186     assert((cast(HTTPResponse)rs).history.length == 2);
187     assert((cast(HTTPResponse)rs).code==200);
188     //    rq = Request();
189     rq.maxRedirects = 2;
190     rq.keepAlive = false;
191     assertThrown!MaxRedirectsException(rq.get(httpbinUrl ~ "absolute-redirect/3"));
192 
193     info("Check chunked content");
194     rq = Request();
195     rq.keepAlive = true;
196     rq.bufferSize = 16*1024;
197     rs = rq.get(httpbinUrl ~ "range/1024");
198     assert(rs.code==200);
199     assert(rs.responseBody.length==1024);
200     
201     info("Check basic auth");
202     rq = Request();
203     rq.authenticator = new BasicAuthentication("user", "passwd");
204     rs = rq.get(httpbinUrl ~ "basic-auth/user/passwd");
205     assert(rs.code==200);
206     
207     info("Check limits");
208     rq = Request();
209     rq.maxContentLength = 1;
210     assertThrown!RequestException(rq.get(httpbinUrl));
211     rq = Request();
212     rq.maxHeadersLength = 1;
213     assertThrown!RequestException(rq.get(httpbinUrl));
214 
215     info("Test getContent");
216     auto r = getContent(httpbinUrl ~ "stream/20");
217     assert(r.splitter('\n').filter!("a.length>0").count == 20);
218     r = getContent(httpbinUrl ~ "get", ["a":"b", "c":"d"]);
219     string name = "user", sex = "male";
220     int    age = 42;
221     r = getContent(httpbinUrl ~ "get", "name", name, "age", age, "sex", sex);
222 
223     info("Test receiveAsRange with GET");
224     rq = Request();
225     rq.useStreaming = true;
226     rq.bufferSize = 16;
227     rs = rq.get(httpbinUrl ~ "stream/20");
228     auto stream = rs.receiveAsRange();
229     ubyte[] streamedContent;
230     while( !stream.empty() ) {
231         streamedContent ~= stream.front;
232         stream.popFront();
233     }
234     rq = Request();
235     rs = rq.get(httpbinUrl ~ "stream/20");
236     assert(streamedContent.length == rs.responseBody.data.length);
237     info("Test postContent");
238     r = postContent(httpbinUrl ~ "post", `{"a":"b", "c":1}`, "application/json");
239     assert(parseJSON(cast(string)r).object["json"].object["c"].integer == 1);
240 
241     /// Posting to forms (for small data)
242     ///
243     /// posting query parameters using "application/x-www-form-urlencoded"
244     info("Test postContent using query params");
245     postContent(httpbinUrl ~ "post", queryParams("first", "a", "second", 2));
246     
247     /// posting using multipart/form-data (large data and files). See docs fot HTTPRequest
248     info("Test postContent form");
249     MultipartForm mpform;
250     mpform.add(formData(/* field name */ "greeting", /* content */ cast(ubyte[])"hello"));
251     postContent(httpbinUrl ~ "post", mpform);
252     
253     /// you can do this using Request struct to access response details
254     info("Test postContent form via Request()");
255     rq = Request();
256     mpform = MultipartForm().add(formData(/* field name */ "greeting", /* content */ cast(ubyte[])"hello"));
257     rs = rq.post(httpbinUrl ~ "post", mpform);
258     assert(rs.code == 200);
259     
260     info("Test receiveAsRange with POST");
261     streamedContent.length = 0;
262     rq = Request();
263     rq.useStreaming = true;
264     rq.bufferSize = 16;
265     string s = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
266     rs = rq.post(httpbinUrl ~ "post", s.representation.chunks(10), "application/octet-stream");
267     stream = rs.receiveAsRange();
268     while( !stream.empty() ) {
269         streamedContent ~= stream.front;
270         stream.popFront();
271     }
272     rq = Request();
273     rs = rq.post(httpbinUrl ~ "post", s.representation.chunks(10), "application/octet-stream");
274     assert(streamedContent == rs.responseBody.data);
275 
276 
277     info("Test POST'ing from Rank(2) range with user-provided Content-Length");
278     rq = Request();
279     rq.addHeaders(["content-length": to!string(s.length)]);
280     rs = rq.post(httpbinUrl ~ "post", s.representation.chunks(10), "application/octet-stream");
281     auto flat_content = parseJSON(cast(string)rs.responseBody().data).object["data"];
282     if ( flat_content.type == JSON_TYPE.STRING ) {
283         // httpbin.org returns string in "data"
284         assert(s == flat_content.str);
285     } else {
286         // internal httpbin server return array ob bytes
287         assert(s.representation == flat_content.array.map!(i => i.integer).array);
288     }
289 
290     info("Test get in parallel");
291     {
292         import std.stdio;
293         import std.parallelism;
294         import std.algorithm;
295         import std.string;
296         import core.atomic;
297         
298         immutable auto urls = [
299             "stream/10",
300             "stream/20",
301             "stream/30",
302             "stream/40",
303             "stream/50",
304             "stream/60",
305             "stream/70",
306         ].map!(a => httpbinUrl ~ a).array.idup;
307         
308         defaultPoolThreads(4);
309         
310         shared short lines;
311         
312         foreach(url; parallel(urls)) {
313             atomicOp!"+="(lines, getContent(url).splitter("\n").count);
314         }
315         assert(lines == 287);
316         
317     }
318 }
319 
320 auto queryParams(A...)(A args) pure @safe nothrow {
321     QueryParam[] res;
322     static if ( args.length >= 2 ) {
323         res = QueryParam(args[0].to!string, args[1].to!string) ~ queryParams(args[2..$]);
324     }
325     return res;
326 }
327 /**
328  * Call GET, and return response content.
329  * This is the simplest case, when all you need is the response body and have no parameters.
330  * Returns:
331  * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
332  */
333 public auto ref getContent(A...)(string url) {
334     auto rq = Request();
335     auto rs = rq.get(url);
336     return rs.responseBody;
337 }
338 /**
339  * Call GET, and return response content.
340  * args = string[string] fo query parameters.
341  * Returns:
342  * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
343  */
344 public auto ref getContent(A...)(string url, string[string] args) {
345     auto rq = Request();
346     auto rs = rq.get(url, args);
347     return rs.responseBody;
348 }
349 /**
350  * Call GET, and return response content.
351  * args = QueryParam[] of parameters.
352  * Returns:
353  * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
354  */
355 public auto ref getContent(A...)(string url, QueryParam[] args) {
356     auto rq = Request();
357     auto rs = rq.get(url, args);
358     return rs.responseBody;
359 }
360 /**
361  * Call GET, and return response content.
362  * args = variadic args to supply parameter names and values.
363  * Returns:
364  * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
365  */
366 public auto ref getContent(A...)(string url, A args) if (args.length > 1 && args.length % 2 == 0 ) {
367     return Request().
368             get(url, queryParams(args)).
369             responseBody;
370 }
371 
372 ///
373 /// Call post and return response content.
374 ///
375 public auto postContent(A...)(string url, A args) {
376     auto rq = Request();
377     auto rs = rq.post(url, args);
378     return rs.responseBody;
379 }
380 
381 ///
382 package unittest {
383     import std.json;
384     import std.string;
385     import std.stdio;
386     import std.range;
387 
388     globalLogLevel(LogLevel.info);
389 
390     /// ftp upload from range
391     info("Test getContent(ftp)");
392     auto r = getContent("ftp://speedtest.tele2.net/1KB.zip");
393     assert(r.length == 1024);
394     
395     info("Test postContent ftp");
396     r = postContent("ftp://speedtest.tele2.net/upload/TEST.TXT", "test, ignore please\n".representation);
397     assert(r.length == 0);
398 
399     info("Test receiveAsRange with GET(ftp)");
400     ubyte[] streamedContent;
401     auto rq = Request();
402     rq.useStreaming = true;
403     streamedContent.length = 0;
404     auto rs = rq.get("ftp://speedtest.tele2.net/1KB.zip");
405     auto stream = rs.receiveAsRange;
406     while( !stream.empty() ) {
407         streamedContent ~= stream.front;
408         stream.popFront();
409     }
410     assert(streamedContent.length == 1024);
411     //
412     info("ftp post ", "ftp://speedtest.tele2.net/upload/TEST.TXT");
413     rs = rq.post("ftp://speedtest.tele2.net/upload/TEST.TXT", "test, ignore please\n".representation);
414     assert(rs.code == 226);
415     info("ftp get  ", "ftp://speedtest.tele2.net/nonexistent", ", in same session.");
416     rs = rq.get("ftp://speedtest.tele2.net/nonexistent");
417     assert(rs.code != 226);
418     rq.useStreaming = false;
419     info("ftp get  ", "ftp://speedtest.tele2.net/1KB.zip", ", in same session.");
420     rs = rq.get("ftp://speedtest.tele2.net/1KB.zip");
421     assert(rs.code == 226);
422     assert(rs.responseBody.length == 1024);
423     info("ftp post ", "ftp://speedtest.tele2.net/upload/TEST.TXT");
424     rs = rq.post("ftp://speedtest.tele2.net/upload/TEST.TXT", "another test, ignore please\n".representation);
425     assert(rs.code == 226);
426     info("ftp get  ", "ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT");
427     rs = rq.get("ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT");
428     assert(rs.code == 226);
429     rq.authenticator = new BasicAuthentication("anonymous", "request@");
430     rs = rq.get("ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT");
431     assert(rs.code == 226);
432     info("testing ftp - done.");
433 }
434