requests source code

1 module requests;
2 
3 public import requests.http;
4 public import requests.ftp;
5 public import requests.streams;
6 public import requests.base;
7 public import requests.uri;
8 public import requests.request;
9 public import requests.pool;
10 
11 import std.datetime;
12 import std.conv;
13 import std.experimental.logger;
14 import requests.utils;
15 
16 ///
17 package unittest {
18     import std.algorithm;
19     import std.range;
20     import std.array;
21     import std.json;
22     import std.stdio;
23     import std.string;
24     import std.exception;
25 
26     string httpbinUrl = httpTestServer();
27 
28     version(vibeD) {
29     }
30     else {
31         import httpbin;
32         auto server = httpbinApp();
33         server.start();
34         scope(exit) {
35             server.stop();
36         }
37     }
38 
39     globalLogLevel(LogLevel.info);
40 
41     infof("testing Request");
42     Request rq;
43     Response rs;
44     //
45     rs = rq.get(httpbinUrl);
46     assert(rs.code==200);
47     assert(rs.responseBody.length > 0);
48     rs = rq.get(httpbinUrl ~ "get", ["c":" d", "a":"b"]);
49     assert(rs.code == 200);
50     auto json = parseJSON(cast(string)rs.responseBody.data).object["args"].object;
51     assert(json["c"].str == " d");
52     assert(json["a"].str == "b");
53 
54     
55     rq = Request();
56     rq.keepAlive = true;
57     {
58         info("Check handling incomplete status line");
59         rs = rq.get(httpbinUrl ~ "incomplete");
60         if (httpbinUrl != "http://httpbin.org/") {
61             assert(rs.code==600);
62         }
63     }
64     // handmade json
65     info("Check POST json");
66     rs = rq.post(httpbinUrl ~ "post?b=x", `{"a":"b ", "c":[1,2,3]}`, "application/json");
67     assert(rs.code==200);
68     json = parseJSON(cast(string)rs.responseBody.data).object["args"].object;
69     assert(json["b"].str == "x");
70     json = parseJSON(cast(string)rs.responseBody.data).object["json"].object;
71     assert(json["a"].str == "b ");
72     assert(json["c"].array.map!(a=>a.integer).array == [1,2,3]);
73     {
74         import std.file;
75         import std.path;
76         auto tmpd = tempDir();
77         auto tmpfname = tmpd ~ dirSeparator ~ "request_test.txt";
78         auto f = File(tmpfname, "wb");
79         f.rawWrite("abcdefgh\n12345678\n");
80         f.close();
81         // files
82         info("Check POST files");
83         PostFile[] files = [
84             {fileName: tmpfname, fieldName:"abc", contentType:"application/octet-stream"}, 
85             {fileName: tmpfname}
86         ];
87         rs = rq.post(httpbinUrl ~ "post", files);
88         assert(rs.code==200);
89         info("Check POST chunked from file.byChunk");
90         f = File(tmpfname, "rb");
91         rs = rq.post(httpbinUrl ~ "post", f.byChunk(3), "application/octet-stream");
92         if (httpbinUrl != "http://httpbin.org/" ) {
93             assert(rs.code==200);
94             auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]);
95             assert(data=="abcdefgh\n12345678\n");
96         }
97         f.close();
98     }
99     // ranges
100     {
101         info("Check POST chunked from lineSplitter");
102         auto s = lineSplitter("one,\ntwo,\nthree.");
103         rs = rq.exec!"POST"(httpbinUrl ~ "post", s, "application/octet-stream");
104         if (httpbinUrl != "http://httpbin.org/" ) {
105             assert(rs.code==200);
106             auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]);
107             assert(data=="one,two,three.");
108         }
109     }
110     {
111         info("Check POST chunked from array");
112         auto s = ["one,", "two,", "three."];
113         rs = rq.post(httpbinUrl ~ "post", s, "application/octet-stream");
114         if (httpbinUrl != "http://httpbin.org/" ) {
115             assert(rs.code==200);
116             auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]);
117             assert(data=="one,two,three.");
118         }
119     }
120     {
121         info("Check POST chunked using std.range.chunks()");
122         auto s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
123         rs = rq.post(httpbinUrl ~ "post", s.representation.chunks(10), "application/octet-stream");
124         if (httpbinUrl != "http://httpbin.org/") {
125             assert(rs.code==200);
126             auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]);
127             assert(data==s);
128         }
129     }
130     // associative array
131     rs = rq.post(httpbinUrl ~ "post", ["a":"b ", "c":"d"]);
132     assert(rs.code==200);
133     auto form = parseJSON(cast(string)rs.responseBody.data).object["form"].object;
134     assert(form["a"].str == "b ");
135     assert(form["c"].str == "d");
136     info("Check HEAD");
137     rs = rq.exec!"HEAD"(httpbinUrl);
138     assert(rs.code==200);
139     info("Check DELETE");
140     rs = rq.exec!"DELETE"(httpbinUrl ~ "delete");
141     assert(rs.code==200);
142     info("Check PUT");
143     rs = rq.exec!"PUT"(httpbinUrl ~ "put",  `{"a":"b", "c":[1,2,3]}`, "application/json");
144     assert(rs.code==200);
145     info("Check PATCH");
146     rs = rq.exec!"PATCH"(httpbinUrl ~ "patch", "привiт, свiт!", "application/octet-stream");
147     assert(rs.code==200);
148     
149     info("Check compressed content");
150     rq = Request();
151     rq.keepAlive = true;
152     rq.addHeaders(["X-Header": "test"]);
153     rs = rq.get(httpbinUrl ~ "gzip");
154     assert(rs.code==200);
155     info("gzip - ok");
156     rs = rq.get(httpbinUrl ~ "deflate");
157     assert(rs.code==200);
158     info("deflate - ok");
159     
160     info("Check redirects");
161     rq = Request();
162     rq.keepAlive = true;
163     rs = rq.get(httpbinUrl ~ "relative-redirect/2");
164     assert((cast(HTTPResponse)rs).history.length == 2);
165     assert((cast(HTTPResponse)rs).code==200);
166 
167     info("Check cookie");
168     rq = Request();
169     rs = rq.get(httpbinUrl ~ "cookies/set?A=abcd&b=cdef");
170     assert(rs.code == 200);
171     json = parseJSON(cast(string)rs.responseBody.data).object["cookies"].object;
172     assert(json["A"].str == "abcd");
173     assert(json["b"].str == "cdef");
174     auto cookie = rq.cookie();
175     foreach(c; rq.cookie) {
176         final switch(c.attr) {
177             case "A":
178                 assert(c.value == "abcd");
179                 break;
180             case "b":
181                 assert(c.value == "cdef");
182                 break;
183         }
184     }
185     rs = rq.get(httpbinUrl ~ "absolute-redirect/2");
186     assert((cast(HTTPResponse)rs).history.length == 2);
187     assert((cast(HTTPResponse)rs).code==200);
188     //    rq = Request();
189     rq.maxRedirects = 2;
190     rq.keepAlive = false;
191     assertThrown!MaxRedirectsException(rq.get(httpbinUrl ~ "absolute-redirect/3"));
192 
193     info("Check chunked content");
194     rq = Request();
195     rq.keepAlive = true;
196     rq.bufferSize = 16*1024;
197     rs = rq.get(httpbinUrl ~ "range/1024");
198     assert(rs.code==200);
199     assert(rs.responseBody.length==1024);
200     
201     info("Check basic auth");
202     rq = Request();
203     rq.authenticator = new BasicAuthentication("user", "passwd");
204     rs = rq.get(httpbinUrl ~ "basic-auth/user/passwd");
205     assert(rs.code==200);
206     
207     info("Check limits");
208     rq = Request();
209     rq.maxContentLength = 1;
210     assertThrown!RequestException(rq.get(httpbinUrl));
211     rq = Request();
212     rq.maxHeadersLength = 1;
213     assertThrown!RequestException(rq.get(httpbinUrl));
214 
215     info("Test getContent");
216     auto r = getContent(httpbinUrl ~ "stream/20");
217     assert(r.splitter('\n').filter!("a.length>0").count == 20);
218     r = getContent(httpbinUrl ~ "get", ["a":"b", "c":"d"]);
219     string name = "user", sex = "male";
220     int    age = 42;
221     r = getContent(httpbinUrl ~ "get", "name", name, "age", age, "sex", sex);
222 
223     info("Test receiveAsRange with GET");
224     rq = Request();
225     rq.useStreaming = true;
226     rq.bufferSize = 16;
227     rs = rq.get(httpbinUrl ~ "stream/20");
228     auto stream = rs.receiveAsRange();
229     ubyte[] streamedContent;
230     while( !stream.empty() ) {
231         streamedContent ~= stream.front;
232         stream.popFront();
233     }
234     rq = Request();
235     rs = rq.get(httpbinUrl ~ "stream/20");
236     assert(streamedContent.length == rs.responseBody.data.length,
237             "streamedContent.length(%d) == rs.responseBody.data.length(%d)".format(streamedContent.length, rs.responseBody.data.length));
238     info("Test postContent");
239     r = postContent(httpbinUrl ~ "post", `{"a":"b", "c":1}`, "application/json");
240     assert(parseJSON(cast(string)r).object["json"].object["c"].integer == 1);
241 
242     /// Posting to forms (for small data)
243     ///
244     /// posting query parameters using "application/x-www-form-urlencoded"
245     info("Test postContent using query params");
246     postContent(httpbinUrl ~ "post", queryParams("first", "a", "second", 2));
247     
248     /// posting using multipart/form-data (large data and files). See docs fot HTTPRequest
249     info("Test postContent form");
250     MultipartForm mpform;
251     mpform.add(formData(/* field name */ "greeting", /* content */ cast(ubyte[])"hello"));
252     postContent(httpbinUrl ~ "post", mpform);
253     
254     /// you can do this using Request struct to access response details
255     info("Test postContent form via Request()");
256     rq = Request();
257     mpform = MultipartForm().add(formData(/* field name */ "greeting", /* content */ cast(ubyte[])"hello"));
258     rs = rq.post(httpbinUrl ~ "post", mpform);
259     assert(rs.code == 200);
260     
261     info("Test receiveAsRange with POST");
262     streamedContent.length = 0;
263     rq = Request();
264     rq.useStreaming = true;
265     rq.bufferSize = 16;
266     string s = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
267     rs = rq.post(httpbinUrl ~ "post", s.representation, "application/octet-stream");
268     stream = rs.receiveAsRange();
269     while( !stream.empty() ) {
270         streamedContent ~= stream.front;
271         stream.popFront();
272     }
273     rq = Request();
274     rs = rq.post(httpbinUrl ~ "post", s.representation, "application/octet-stream");
275     assert(streamedContent == rs.responseBody.data);
276 
277 
278     info("Test POST'ing from Rank(2) range with user-provided Content-Length");
279     rq = Request();
280     rq.addHeaders(["content-length": to!string(s.length)]);
281     rs = rq.post(httpbinUrl ~ "post", s.representation.chunks(10), "application/octet-stream");
282     auto flat_content = parseJSON(cast(string)rs.responseBody().data).object["data"];
283     if ( flat_content.type == JSON_TYPE.STRING ) {
284         // httpbin.org returns string in "data"
285         assert(s == flat_content.str);
286     } else {
287         // internal httpbin server return array ob bytes
288         assert(s.representation == flat_content.array.map!(i => i.integer).array);
289     }
290 
291     info("Test get in parallel");
292     {
293         import std.stdio;
294         import std.parallelism;
295         import std.algorithm;
296         import std.string;
297         import core.atomic;
298         
299         immutable auto urls = [
300             "stream/10",
301             "stream/20",
302             "stream/30",
303             "stream/40",
304             "stream/50",
305             "stream/60",
306             "stream/70",
307         ].map!(a => httpbinUrl ~ a).array.idup;
308         
309         defaultPoolThreads(4);
310         
311         shared short lines;
312         
313         foreach(url; parallel(urls)) {
314             atomicOp!"+="(lines, getContent(url).splitter("\n").count);
315         }
316         assert(lines == 287);
317         
318     }
319 }
320 
321 auto queryParams(A...)(A args) pure @safe nothrow {
322     QueryParam[] res;
323     static if ( args.length >= 2 ) {
324         res = QueryParam(args[0].to!string, args[1].to!string) ~ queryParams(args[2..$]);
325     }
326     return res;
327 }
328 /**
329  * Call GET, and return response content.
330  * This is the simplest case, when all you need is the response body and have no parameters.
331  * Returns:
332  * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
333  */
334 public auto ref getContent(A...)(string url) {
335     auto rq = Request();
336     auto rs = rq.get(url);
337     return rs.responseBody;
338 }
339 /**
340  * Call GET, and return response content.
341  * args = string[string] fo query parameters.
342  * Returns:
343  * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
344  */
345 public auto ref getContent(A...)(string url, string[string] args) {
346     auto rq = Request();
347     auto rs = rq.get(url, args);
348     return rs.responseBody;
349 }
350 /**
351  * Call GET, and return response content.
352  * args = QueryParam[] of parameters.
353  * Returns:
354  * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
355  */
356 public auto ref getContent(A...)(string url, QueryParam[] args) {
357     auto rq = Request();
358     auto rs = rq.get(url, args);
359     return rs.responseBody;
360 }
361 /**
362  * Call GET, and return response content.
363  * args = variadic args to supply parameter names and values.
364  * Returns:
365  * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
366  */
367 public auto ref getContent(A...)(string url, A args) if (args.length > 1 && args.length % 2 == 0 ) {
368     return Request().
369             get(url, queryParams(args)).
370             responseBody;
371 }
372 
373 ///
374 /// Call post and return response content.
375 ///
376 public auto postContent(A...)(string url, A args) {
377     auto rq = Request();
378     auto rs = rq.post(url, args);
379     return rs.responseBody;
380 }
381 
382 ///
383 package unittest {
384     import std.json;
385     import std.string;
386     import std.stdio;
387     import std.range;
388     import std.process;
389 
390     globalLogLevel(LogLevel.info);
391     // while we have no internal ftp server we can run tests in non-reloable networking environment
392     bool unreliable_network = environment.get("UNRELIABLENETWORK", "false") == "true";
393 
394     /// ftp upload from range
395     info("Test getContent(ftp)");
396     auto r = getContent("ftp://speedtest.tele2.net/1KB.zip");
397     assert(unreliable_network || r.length == 1024);
398     
399     info("Test postContent ftp");
400     r = postContent("ftp://speedtest.tele2.net/upload/TEST.TXT", "test, ignore please\n".representation);
401     assert(unreliable_network || r.length == 0);
402 
403     info("Test receiveAsRange with GET(ftp)");
404     ubyte[] streamedContent;
405     auto rq = Request();
406     rq.useStreaming = true;
407     streamedContent.length = 0;
408     auto rs = rq.get("ftp://speedtest.tele2.net/1KB.zip");
409     auto stream = rs.receiveAsRange;
410     while( !stream.empty() ) {
411         streamedContent ~= stream.front;
412         stream.popFront();
413     }
414     assert(unreliable_network || streamedContent.length == 1024);
415     //
416     info("ftp post ", "ftp://speedtest.tele2.net/upload/TEST.TXT");
417     rs = rq.post("ftp://speedtest.tele2.net/upload/TEST.TXT", "test, ignore please\n".representation);
418     assert(unreliable_network || rs.code == 226);
419     info("ftp get  ", "ftp://speedtest.tele2.net/nonexistent", ", in same session.");
420     rs = rq.get("ftp://speedtest.tele2.net/nonexistent");
421     assert(unreliable_network || rs.code != 226);
422     rq.useStreaming = false;
423     info("ftp get  ", "ftp://speedtest.tele2.net/1KB.zip", ", in same session.");
424     rs = rq.get("ftp://speedtest.tele2.net/1KB.zip");
425     assert(unreliable_network || rs.code == 226);
426     assert(unreliable_network || rs.responseBody.length == 1024);
427     info("ftp post ", "ftp://speedtest.tele2.net/upload/TEST.TXT");
428     rs = rq.post("ftp://speedtest.tele2.net/upload/TEST.TXT", "another test, ignore please\n".representation);
429     assert(unreliable_network || rs.code == 226);
430     info("ftp get  ", "ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT");
431     try {
432         rs = rq.get("ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT");
433     } catch (ConnectError e)
434     {
435     }
436     assert(unreliable_network || rs.code == 226);
437     rq.authenticator = new BasicAuthentication("anonymous", "request@");
438     try {
439         rs = rq.get("ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT");
440     } catch (ConnectError e)
441     {
442     }
443     assert(unreliable_network || rs.code == 226);
444     info("testing ftp - done.");
445 }
446