requests source code

1 module requests;
2 
3 public import requests.http;
4 public import requests.ftp;
5 public import requests.streams;
6 public import requests.base;
7 public import requests.uri;
8 public import requests.request;
9 public import requests.pool;
10 
11 import std.datetime;
12 import std.conv;
13 import std.experimental.logger;
14 import requests.utils;
15 
16 ///
17 package unittest {
18     import std.algorithm;
19     import std.range;
20     import std.array;
21     import std.json;
22     import std.stdio;
23     import std.string;
24     import std.exception;
25 
26     string httpbinUrl = httpTestServer();
27     globalLogLevel(LogLevel.info);
28 
29     version(vibeD) {
30     }
31     else {
32         import httpbin;
33         auto server = httpbinApp();
34         server.start();
35         scope(exit) {
36             server.stop();
37         }
38     }
39 
40 
41     infof("testing Request");
42     Request rq;
43     Response rs;
44     //
45     rs = rq.get(httpbinUrl);
46     assert(rs.code==200);
47     assert(rs.responseBody.length > 0);
48     rs = rq.get(httpbinUrl ~ "get", ["c":" d", "a":"b"]);
49     assert(rs.code == 200);
50     auto json = parseJSON(cast(string)rs.responseBody.data).object["args"].object;
51     assert(json["c"].str == " d");
52     assert(json["a"].str == "b");
53 
54     
55     rq = Request();
56     rq.keepAlive = false; // disable keepalive on idempotents requests
57     {
58         info("Check handling incomplete status line");
59         rs = rq.get(httpbinUrl ~ "incomplete");
60         if (httpbinUrl != "http://httpbin.org/") {
61             // 600 when test direct server respond, or 502 if test through squid
62             assert(rs.code==600 || rs.code == 502);
63         }
64     }
65     // handmade json
66     info("Check POST json");
67     rs = rq.post(httpbinUrl ~ "post?b=x", `{"a":"b ", "c":[1,2,3]}`, "application/json");
68     assert(rs.code==200);
69     json = parseJSON(cast(string)rs.responseBody.data).object["args"].object;
70     assert(json["b"].str == "x");
71     json = parseJSON(cast(string)rs.responseBody.data).object["json"].object;
72     assert(json["a"].str == "b ");
73     assert(json["c"].array.map!(a=>a.integer).array == [1,2,3]);
74     {
75         import std.file;
76         import std.path;
77         auto tmpd = tempDir();
78         auto tmpfname = tmpd ~ dirSeparator ~ "request_test.txt";
79         auto f = File(tmpfname, "wb");
80         f.rawWrite("abcdefgh\n12345678\n");
81         f.close();
82         // files
83         info("Check POST files");
84         PostFile[] files = [
85             {fileName: tmpfname, fieldName:"abc", contentType:"application/octet-stream"}, 
86             {fileName: tmpfname}
87         ];
88         rs = rq.post(httpbinUrl ~ "post", files);
89         assert(rs.code==200);
90         info("Check POST chunked from file.byChunk");
91         f = File(tmpfname, "rb");
92         rs = rq.post(httpbinUrl ~ "post", f.byChunk(3), "application/octet-stream");
93         if (httpbinUrl != "http://httpbin.org/" ) {
94             assert(rs.code==200);
95             auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]);
96             assert(data=="abcdefgh\n12345678\n");
97         }
98         f.close();
99     }
100     // ranges
101     {
102         info("Check POST chunked from lineSplitter");
103         auto s = lineSplitter("one,\ntwo,\nthree.");
104         rs = rq.exec!"POST"(httpbinUrl ~ "post", s, "application/octet-stream");
105         if (httpbinUrl != "http://httpbin.org/" ) {
106             assert(rs.code==200);
107             auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]);
108             assert(data=="one,two,three.");
109         }
110     }
111     {
112         info("Check POST chunked from array");
113         auto s = ["one,", "two,", "three."];
114         rs = rq.post(httpbinUrl ~ "post", s, "application/octet-stream");
115         if (httpbinUrl != "http://httpbin.org/" ) {
116             assert(rs.code==200);
117             auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]);
118             assert(data=="one,two,three.");
119         }
120     }
121     {
122         info("Check POST chunked using std.range.chunks()");
123         auto s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
124         rs = rq.post(httpbinUrl ~ "post", s.representation.chunks(10), "application/octet-stream");
125         if (httpbinUrl != "http://httpbin.org/") {
126             assert(rs.code==200);
127             auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]);
128             assert(data==s);
129         }
130     }
131     // associative array
132     rs = rq.post(httpbinUrl ~ "post", ["a":"b ", "c":"d"]);
133     assert(rs.code==200);
134     auto form = parseJSON(cast(string)rs.responseBody.data).object["form"].object;
135     assert(form["a"].str == "b ");
136     assert(form["c"].str == "d");
137     info("Check HEAD");
138     rs = rq.exec!"HEAD"(httpbinUrl);
139     assert(rs.code==200);
140     info("Check DELETE");
141     rs = rq.exec!"DELETE"(httpbinUrl ~ "delete");
142     assert(rs.code==200);
143     info("Check PUT");
144     rs = rq.exec!"PUT"(httpbinUrl ~ "put",  `{"a":"b", "c":[1,2,3]}`, "application/json");
145     assert(rs.code==200);
146     info("Check PATCH");
147     rs = rq.exec!"PATCH"(httpbinUrl ~ "patch", "привiт, свiт!", "application/octet-stream");
148     assert(rs.code==200);
149 
150     info("Check compressed content");
151     rq = Request();
152     rq.keepAlive = true;
153     rq.addHeaders(["X-Header": "test"]);
154     rs = rq.get(httpbinUrl ~ "gzip");
155     assert(rs.code==200);
156     info("gzip - ok");
157     rs = rq.get(httpbinUrl ~ "deflate");
158     assert(rs.code==200);
159     info("deflate - ok");
160     
161     info("Check redirects");
162     rq = Request();
163     rq.keepAlive = true;
164     rs = rq.get(httpbinUrl ~ "relative-redirect/2");
165     assert((cast(HTTPResponse)rs).history.length == 2);
166     assert((cast(HTTPResponse)rs).code==200);
167 
168     info("Check cookie");
169     rq = Request();
170     rs = rq.get(httpbinUrl ~ "cookies/set?A=abcd&b=cdef");
171     assert(rs.code == 200);
172     json = parseJSON(cast(string)rs.responseBody.data).object["cookies"].object;
173     assert(json["A"].str == "abcd");
174     assert(json["b"].str == "cdef");
175     foreach(c; rq.cookie) {
176         final switch(c.attr) {
177             case "A":
178                 assert(c.value == "abcd");
179                 break;
180             case "b":
181                 assert(c.value == "cdef");
182                 break;
183         }
184     }
185 
186     info("Check redirects");
187     rs = rq.get(httpbinUrl ~ "absolute-redirect/2");
188     assert((cast(HTTPResponse)rs).history.length == 2);
189     assert((cast(HTTPResponse)rs).code==200);
190     //    rq = Request();
191     info("Check maxredirects");
192     rq.maxRedirects = 2;
193     rq.keepAlive = false;
194     assertThrown!MaxRedirectsException(rq.get(httpbinUrl ~ "absolute-redirect/3"));
195 
196 
197     info("Check chunked content");
198     rq = Request();
199     rq.keepAlive = true;
200     rq.bufferSize = 16*1024;
201     rs = rq.get(httpbinUrl ~ "range/1024");
202     assert(rs.code==200);
203     assert(rs.responseBody.length==1024);
204     
205     info("Check basic auth");
206     rq = Request();
207     rq.authenticator = new BasicAuthentication("user", "passwd");
208     rs = rq.get(httpbinUrl ~ "basic-auth/user/passwd");
209     assert(rs.code==200);
210     
211     info("Check limits");
212     rq = Request();
213     rq.maxContentLength = 1;
214     assertThrown!RequestException(rq.get(httpbinUrl));
215     rq = Request();
216     rq.maxHeadersLength = 1;
217     assertThrown!RequestException(rq.get(httpbinUrl));
218 
219     info("Test getContent");
220     auto r = getContent(httpbinUrl ~ "stream/20");
221     assert(r.splitter('\n').filter!("a.length>0").count == 20);
222     r = getContent(httpbinUrl ~ "get", ["a":"b", "c":"d"]);
223     string name = "user", sex = "male";
224     immutable age = 42;
225     r = getContent(httpbinUrl ~ "get", "name", name, "age", age, "sex", sex);
226 
227     info("Test receiveAsRange with GET");
228     rq = Request();
229     rq.useStreaming = true;
230     rq.bufferSize = 16;
231     rs = rq.get(httpbinUrl ~ "stream/20");
232     auto stream = rs.receiveAsRange();
233     ubyte[] streamedContent;
234     while( !stream.empty() ) {
235         streamedContent ~= stream.front;
236         stream.popFront();
237     }
238     rq = Request();
239     rs = rq.get(httpbinUrl ~ "stream/20");
240     assert(streamedContent.length == rs.responseBody.data.length,
241             "streamedContent.length(%d) == rs.responseBody.data.length(%d)".
242             format(streamedContent.length, rs.responseBody.data.length));
243     info("Test postContent");
244     r = postContent(httpbinUrl ~ "post", `{"a":"b", "c":1}`, "application/json");
245     assert(parseJSON(cast(string)r).object["json"].object["c"].integer == 1);
246 
247     /// Posting to forms (for small data)
248     ///
249     /// posting query parameters using "application/x-www-form-urlencoded"
250     info("Test postContent using query params");
251     postContent(httpbinUrl ~ "post", queryParams("first", "a", "second", 2));
252     
253     /// posting using multipart/form-data (large data and files). See docs fot HTTPRequest
254     info("Test postContent form");
255     MultipartForm mpform;
256     mpform.add(formData(/* field name */ "greeting", /* content */ cast(ubyte[])"hello"));
257     postContent(httpbinUrl ~ "post", mpform);
258     
259     /// you can do this using Request struct to access response details
260     info("Test postContent form via Request()");
261     rq = Request();
262     mpform = MultipartForm().add(formData(/* field name */ "greeting", /* content */ cast(ubyte[])"hello"));
263     rs = rq.post(httpbinUrl ~ "post", mpform);
264     assert(rs.code == 200);
265     
266     info("Test receiveAsRange with POST");
267     streamedContent.length = 0;
268     rq = Request();
269     rq.useStreaming = true;
270     rq.bufferSize = 16;
271     string s = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
272     rs = rq.post(httpbinUrl ~ "post", s.representation, "application/octet-stream");
273     stream = rs.receiveAsRange();
274     while( !stream.empty() ) {
275         streamedContent ~= stream.front;
276         stream.popFront();
277     }
278     rq = Request();
279     rs = rq.post(httpbinUrl ~ "post", s.representation, "application/octet-stream");
280     assert(streamedContent == rs.responseBody.data);
281 
282 
283     info("Test POST'ing from Rank(2) range with user-provided Content-Length");
284     rq = Request();
285     rq.addHeaders(["content-length": to!string(s.length)]);
286     rs = rq.post(httpbinUrl ~ "post", s.representation.chunks(10), "application/octet-stream");
287     auto flat_content = parseJSON(cast(string)rs.responseBody().data).object["data"];
288     if ( flat_content.type == JSON_TYPE.STRING ) {
289         // httpbin.org returns string in "data"
290         assert(s == flat_content.str);
291     } else {
292         // internal httpbin server return array ob bytes
293         assert(s.representation == flat_content.array.map!(i => i.integer).array);
294     }
295 
296     info("Test get in parallel");
297     {
298         import std.stdio;
299         import std.parallelism;
300         import std.algorithm;
301         import std.string;
302         import core.atomic;
303         
304         immutable auto urls = [
305             "stream/10",
306             "stream/20",
307             "stream/30",
308             "stream/40",
309             "stream/50",
310             "stream/60",
311             "stream/70",
312         ].map!(a => httpbinUrl ~ a).array.idup;
313         
314         defaultPoolThreads(4);
315         
316         shared short lines;
317         
318         foreach(url; parallel(urls)) {
319             atomicOp!"+="(lines, getContent(url).splitter("\n").count);
320         }
321         assert(lines == 287);
322         
323     }
324 }
325 ///
326 /// Create array of query params from args
327 ///
328 auto queryParams(A...)(A args) pure @safe nothrow {
329     QueryParam[] res;
330     static if ( args.length >= 2 ) {
331         res = [QueryParam(args[0].to!string, args[1].to!string)] ~ queryParams(args[2..$]);
332     }
333     return res;
334 }
335 /**
336  * Call GET, and return response content.
337  * This is the simplest case, when all you need is the response body and have no parameters.
338  * Returns:
339  * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
340  */
341 public auto ref getContent(A...)(string url) {
342     auto rq = Request();
343     auto rs = rq.get(url);
344     return rs.responseBody;
345 }
346 /**
347  * Call GET, and return response content.
348  * args = string[string] fo query parameters.
349  * Returns:
350  * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
351  */
352 public auto ref getContent(A...)(string url, string[string] args) {
353     auto rq = Request();
354     auto rs = rq.get(url, args);
355     return rs.responseBody;
356 }
357 /**
358  * Call GET, and return response content.
359  * args = QueryParam[] of parameters.
360  * Returns:
361  * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
362  */
363 public auto ref getContent(A...)(string url, QueryParam[] args) {
364     auto rq = Request();
365     auto rs = rq.get(url, args);
366     return rs.responseBody;
367 }
368 /**
369  * Call GET, and return response content.
370  * args = variadic args to supply parameter names and values.
371  * Returns:
372  * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
373  */
374 public auto ref getContent(A...)(string url, A args) if (args.length > 1 && args.length % 2 == 0 ) {
375     return Request().
376             get(url, queryParams(args)).
377             responseBody;
378 }
379 
380 ///
381 /// Call post and return response content.
382 ///
383 public auto postContent(A...)(string url, A args) {
384     auto rq = Request();
385     auto rs = rq.post(url, args);
386     return rs.responseBody;
387 }
388 
389 ///
390 package unittest {
391     import std.json;
392     import std.string;
393     import std.stdio;
394     import std.range;
395     import std.process;
396 
397     globalLogLevel(LogLevel.info);
398     // while we have no internal ftp server we can run tests in non-reloable networking environment
399     immutable unreliable_network = environment.get("UNRELIABLENETWORK", "false") == "true";
400 
401     /// ftp upload from range
402     info("Test getContent(ftp)");
403     auto r = getContent("ftp://speedtest.tele2.net/1KB.zip");
404     assert(unreliable_network || r.length == 1024);
405     
406     info("Test postContent ftp");
407     r = postContent("ftp://speedtest.tele2.net/upload/TEST.TXT", "test, ignore please\n".representation);
408     assert(unreliable_network || r.length == 0);
409 
410     info("Test receiveAsRange with GET(ftp)");
411     ubyte[] streamedContent;
412     auto rq = Request();
413     rq.useStreaming = true;
414     streamedContent.length = 0;
415     auto rs = rq.get("ftp://speedtest.tele2.net/1KB.zip");
416     auto stream = rs.receiveAsRange;
417     while( !stream.empty() ) {
418         streamedContent ~= stream.front;
419         stream.popFront();
420     }
421     assert(unreliable_network || streamedContent.length == 1024);
422     //
423     info("ftp post ", "ftp://speedtest.tele2.net/upload/TEST.TXT");
424     rs = rq.post("ftp://speedtest.tele2.net/upload/TEST.TXT", "test, ignore please\n".representation);
425     assert(unreliable_network || rs.code == 226);
426     info("ftp get  ", "ftp://speedtest.tele2.net/nonexistent", ", in same session.");
427     rs = rq.get("ftp://speedtest.tele2.net/nonexistent");
428     assert(unreliable_network || rs.code != 226);
429     rq.useStreaming = false;
430     info("ftp get  ", "ftp://speedtest.tele2.net/1KB.zip", ", in same session.");
431     rs = rq.get("ftp://speedtest.tele2.net/1KB.zip");
432     assert(unreliable_network || rs.code == 226);
433     assert(unreliable_network || rs.responseBody.length == 1024);
434     info("ftp post ", "ftp://speedtest.tele2.net/upload/TEST.TXT");
435     rs = rq.post("ftp://speedtest.tele2.net/upload/TEST.TXT", "another test, ignore please\n".representation);
436     assert(unreliable_network || rs.code == 226);
437     info("ftp get  ", "ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT");
438     try {
439         rs = rq.get("ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT");
440     } catch (ConnectError e)
441     {
442     }
443     assert(unreliable_network || rs.code == 226);
444     rq.authenticator = new BasicAuthentication("anonymous", "request@");
445     try {
446         rs = rq.get("ftp://ftp.iij.ad.jp/pub/FreeBSD/README.TXT");
447     } catch (ConnectError e)
448     {
449     }
450     assert(unreliable_network || rs.code == 226);
451     info("testing ftp - done.");
452 }
453