1 /**********************************************************************************
2
3 HTTP client library, inspired by python-requests with goals:
4
5 $(UL
6 $(LI small memory footprint)
7 $(LI performance)
8 $(LI simple, high level API)
9 $(LI native D implementation)
10 )
11 */
12 module requests;
13
14 public import requests.http;
15 public import requests.ftp;
16 public import requests.streams;
17 public import requests.base;
18 public import requests.uri;
19 public import requests.request;
20 public import requests.pool;
21 public import requests.utils;
22 public import requests.rangeadapter;
23
24 import std.datetime;
25 import std.conv;
26 import std.experimental.logger;
27 import requests.utils;
28
29 ///
30 package unittest {
31 import std.algorithm;
32 import std.range;
33 import std.array;
34 import std.json;
35 import std.stdio;
36 import std.string;
37 import std.exception;
38
39 string httpbinUrl = httpTestServer();
40 globalLogLevel(LogLevel.info);
41
42 version(vibeD) {
43 }
44 else {
45 import httpbin;
46 auto server = httpbinApp();
47 server.start();
48 scope(exit) {
49 server.stop();
50 }
51 }
52
53
54 infof("testing Request");
55 Request rq;
56 Response rs;
57 // moved from http.d
58
59 URI uri = URI(httpbinUrl);
60 rs = rq.get(httpbinUrl);
61 assert(rs.code==200);
62 assert(rs.responseBody.length > 0);
63 assert(rq.format("%m|%h|%p|%P|%q|%U") ==
64 "GET|%s|%d|%s||%s"
65 .format(uri.host, uri.port, uri.path, httpbinUrl));
66
67 //
68 rs = rq.get(httpbinUrl);
69 assert(rs.code==200);
70 assert(rs.responseBody.length > 0);
71 rs = rq.get(httpbinUrl ~ "get", ["c":" d", "a":"b"]);
72 assert(rs.code == 200);
73 auto json = parseJSON(cast(string)rs.responseBody.data).object["args"].object;
74 assert(json["c"].str == " d");
75 assert(json["a"].str == "b");
76
77
78 rq = Request();
79 rq.keepAlive = false; // disable keepalive on idempotents requests
80 {
81 info("Check handling incomplete status line");
82 rs = rq.get(httpbinUrl ~ "incomplete");
83 if (httpbinUrl != "http://httpbin.org/") {
84 // 600 when test direct server respond, or 502 if test through squid
85 assert(rs.code==600 || rs.code == 502);
86 }
87 }
88 // handmade json
89 info("Check POST json");
90 rs = rq.post(httpbinUrl ~ "post?b=x", `{"a":"b ", "c":[1,2,3]}`, "application/json");
91 assert(rs.code==200);
92 json = parseJSON(cast(string)rs.responseBody.data).object["args"].object;
93 assert(json["b"].str == "x");
94 json = parseJSON(cast(string)rs.responseBody.data).object["json"].object;
95 assert(json["a"].str == "b ");
96 assert(json["c"].array.map!(a=>a.integer).array == [1,2,3]);
97 {
98 import std.file;
99 import std.path;
100 auto tmpd = tempDir();
101 auto tmpfname = tmpd ~ dirSeparator ~ "request_test.txt";
102 auto f = File(tmpfname, "wb");
103 f.rawWrite("abcdefgh\n12345678\n");
104 f.close();
105 // files
106 info("Check POST files");
107 PostFile[] files = [
108 {fileName: tmpfname, fieldName:"abc", contentType:"application/octet-stream"},
109 {fileName: tmpfname}
110 ];
111 rs = rq.post(httpbinUrl ~ "post", files);
112 assert(rs.code==200);
113 info("Check POST chunked from file.byChunk");
114 f = File(tmpfname, "rb");
115 rs = rq.post(httpbinUrl ~ "post", f.byChunk(3), "application/octet-stream");
116 if (httpbinUrl != "http://httpbin.org/" ) {
117 assert(rs.code==200);
118 auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]);
119 assert(data=="abcdefgh\n12345678\n");
120 }
121 f.close();
122 }
123 // ranges
124 {
125 info("Check POST chunked from lineSplitter");
126 auto s = lineSplitter("one,\ntwo,\nthree.");
127 rs = rq.exec!"POST"(httpbinUrl ~ "post", s, "application/octet-stream");
128 if (httpbinUrl != "http://httpbin.org/" ) {
129 assert(rs.code==200);
130 auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]);
131 assert(data=="one,two,three.");
132 }
133 }
134 {
135 info("Check POST chunked from array");
136 auto s = ["one,", "two,", "three."];
137 rs = rq.post(httpbinUrl ~ "post", s, "application/octet-stream");
138 if (httpbinUrl != "http://httpbin.org/" ) {
139 assert(rs.code==200);
140 auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]);
141 assert(data=="one,two,three.");
142 }
143 }
144 {
145 info("Check POST chunked using std.range.chunks()");
146 auto s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
147 rs = rq.post(httpbinUrl ~ "post", s.representation.chunks(10), "application/octet-stream");
148 if (httpbinUrl != "http://httpbin.org/") {
149 assert(rs.code==200);
150 auto data = fromJsonArrayToStr(parseJSON(cast(string)rs.responseBody).object["data"]);
151 assert(data==s);
152 }
153 }
154 info("Check POST from QueryParams");
155 {
156 rs = rq.post(httpbinUrl ~ "post", queryParams("name[]", "first", "name[]", 2));
157 assert(rs.code==200);
158 auto data = parseJSON(cast(string)rs.responseBody).object["form"].object;
159 string[] a;
160 try {
161 a = to!(string[])(data["name[]"].str);
162 }
163 catch (JSONException e) {
164 a = data["name[]"].array.map!"a.str".array;
165 }
166 assert(equal(["first", "2"], a));
167 }
168 info("Check POST json");
169 {
170 rs = rq.post(httpbinUrl ~ "post?b=x", `{"a":"a b", "c":[1,2,3]}`, "application/json");
171 assert(rs.code==200);
172 auto j = parseJSON(cast(string)rs.responseBody).object["args"].object;
173 assert(j["b"].str == "x");
174 j = parseJSON(cast(string)rs.responseBody).object["json"].object;
175 assert(j["a"].str == "a b");
176 assert(j["c"].array.map!(a=>a.integer).array == [1,2,3]);
177 }
178 // associative array
179 info("Check POST from AA");
180 rs = rq.post(httpbinUrl ~ "post", ["a":"b ", "c":"d"]);
181 assert(rs.code==200);
182 auto form = parseJSON(cast(string)rs.responseBody.data).object["form"].object;
183 assert(form["a"].str == "b ");
184 assert(form["c"].str == "d");
185 info("Check HEAD");
186 rs = rq.exec!"HEAD"(httpbinUrl);
187 assert(rs.code==200);
188 rs = rq.execute("HEAD", httpbinUrl);
189 assert(rs.code==200);
190 info("Check DELETE");
191 rs = rq.exec!"DELETE"(httpbinUrl ~ "delete");
192 assert(rs.code==200);
193 rs = rq.execute("DELETE", httpbinUrl ~ "delete");
194 assert(rs.code==200);
195 info("Check PUT");
196 rs = rq.exec!"PUT"(httpbinUrl ~ "put", `{"a":"b", "c":[1,2,3]}`, "application/json");
197 assert(rs.code==200);
198 rs = rq.execute("PUT", httpbinUrl ~ "put", `{"a":"b", "c":[1,2,3]}`, "application/json");
199 assert(rs.code==200);
200 info("Check PATCH");
201 rs = rq.exec!"PATCH"(httpbinUrl ~ "patch", "привiт, свiт!", "application/octet-stream");
202 assert(rs.code==200);
203 rs = rq.execute("PATCH", httpbinUrl ~ "patch", "привiт, свiт!", "application/octet-stream");
204 assert(rs.code==200);
205
206 info("Check compressed content");
207 rq = Request();
208 rq.keepAlive = true;
209 rq.addHeaders(["X-Header": "test"]);
210 rs = rq.get(httpbinUrl ~ "gzip");
211 assert(rs.code==200);
212 info("gzip - ok");
213 rs = rq.get(httpbinUrl ~ "deflate");
214 assert(rs.code==200);
215 info("deflate - ok");
216
217 info("Check cookie");
218 rq = Request();
219 rs = rq.get(httpbinUrl ~ "cookies/set?A=abcd&b=cdef");
220 assert(rs.code == 200);
221 json = parseJSON(cast(string)rs.responseBody.data).object["cookies"].object;
222 assert(json["A"].str == "abcd");
223 assert(json["b"].str == "cdef");
224 foreach(c; rq.cookie._array) {
225 final switch(c.attr) {
226 case "A":
227 assert(c.value == "abcd");
228 break;
229 case "b":
230 assert(c.value == "cdef");
231 break;
232 }
233 }
234
235 info("Check redirects");
236 rq = Request();
237 rq.keepAlive = true;
238 rs = rq.get(httpbinUrl ~ "relative-redirect/2");
239 assert((cast(HTTPResponse)rs).history.length == 2);
240 assert((cast(HTTPResponse)rs).code==200);
241
242 rs = rq.get(httpbinUrl ~ "absolute-redirect/2");
243 assert((cast(HTTPResponse)rs).history.length == 2);
244 assert((cast(HTTPResponse)rs).code==200);
245 // rq = Request();
246 info("Check maxredirects");
247 rq.maxRedirects = 2;
248 rq.keepAlive = false;
249 assertThrown!MaxRedirectsException(rq.get(httpbinUrl ~ "absolute-redirect/3"));
250
251 rq.maxRedirects = 0;
252 rs = rq.get(httpbinUrl ~ "absolute-redirect/1");
253 assert(rs.code==302);
254 rq.maxRedirects = 10;
255
256 info("Check chunked content");
257 rq = Request();
258 rq.keepAlive = true;
259 rq.bufferSize = 16*1024;
260 rs = rq.get(httpbinUrl ~ "range/1024");
261 assert(rs.code==200);
262 assert(rs.responseBody.length==1024);
263
264 info("Check basic auth");
265 rq = Request();
266 rq.authenticator = new BasicAuthentication("user", "passwd");
267 rs = rq.get(httpbinUrl ~ "basic-auth/user/passwd");
268 assert(rs.code==200);
269
270 info("Check limits");
271 rq = Request();
272 rq.maxContentLength = 1;
273 assertThrown!RequestException(rq.get(httpbinUrl));
274 rq = Request();
275 rq.maxHeadersLength = 1;
276 assertThrown!RequestException(rq.get(httpbinUrl));
277
278 info("Test getContent");
279 auto r = getContent(httpbinUrl ~ "stream/20");
280 assert(r.splitter('\n').filter!("a.length>0").count == 20);
281 r = getContent(httpbinUrl ~ "get", ["a":"b", "c":"d"]);
282 string name = "user", sex = "male";
283 immutable age = 42;
284 r = getContent(httpbinUrl ~ "get", "name", name, "age", age, "sex", sex);
285
286 info("Test receiveAsRange with GET");
287 rq = Request();
288 rq.useStreaming = true;
289 rq.bufferSize = 16;
290 rs = rq.get(httpbinUrl ~ "stream/20");
291 auto stream = rs.receiveAsRange();
292 ubyte[] streamedContent;
293 while( !stream.empty() ) {
294 streamedContent ~= stream.front;
295 stream.popFront();
296 }
297 rq = Request();
298 rs = rq.get(httpbinUrl ~ "stream/20");
299 assert(streamedContent.length == rs.responseBody.data.length,
300 "streamedContent.length(%d) == rs.responseBody.data.length(%d)".
301 format(streamedContent.length, rs.responseBody.data.length));
302 info("Test postContent");
303 r = postContent(httpbinUrl ~ "post", `{"a":"b", "c":1}`, "application/json");
304 assert(parseJSON(cast(string)r).object["json"].object["c"].integer == 1);
305
306 /// Posting to forms (for small data)
307 ///
308 /// posting query parameters using "application/x-www-form-urlencoded"
309 info("Test postContent using query params");
310 r = postContent(httpbinUrl ~ "post", queryParams("first", "a", "second", 2));
311 assert(parseJSON(cast(string)r).object["form"].object["first"].str == "a");
312
313 /// posting using multipart/form-data (large data and files). See docs fot HTTPRequest
314 info("Test postContent form");
315 MultipartForm mpform;
316 mpform.add(formData(/* field name */ "greeting", /* content */ cast(ubyte[])"hello"));
317 r = postContent(httpbinUrl ~ "post", mpform);
318 assert(parseJSON(cast(string)r).object["form"].object["greeting"].str == "hello");
319
320 /// you can do this using Request struct to access response details
321 info("Test postContent form via Request()");
322 rq = Request();
323 mpform = MultipartForm().add(formData(/* field name */ "greeting", /* content */ cast(ubyte[])"hello"));
324 rs = rq.post(httpbinUrl ~ "post", mpform);
325 assert(rs.code == 200);
326 assert(parseJSON(cast(string)(rs.responseBody().data)).object["form"].object["greeting"].str == "hello");
327
328 info("Test receiveAsRange with POST");
329 streamedContent.length = 0;
330 rq = Request();
331 rq.useStreaming = true;
332 rq.bufferSize = 16;
333 string s = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
334 rs = rq.post(httpbinUrl ~ "post", s.representation, "application/octet-stream");
335 stream = rs.receiveAsRange();
336 while( !stream.empty() ) {
337 streamedContent ~= stream.front;
338 stream.popFront();
339 }
340 rq = Request();
341 rs = rq.post(httpbinUrl ~ "post", s.representation, "application/octet-stream");
342 assert(streamedContent == rs.responseBody.data);
343
344 streamedContent.length = 0;
345 rq = Request();
346 rq.useStreaming = true;
347 rq.bufferSize = 16;
348 mpform = MultipartForm().add(formData(/* field name */ "greeting", /* content */ cast(ubyte[])"hello"));
349 rs = rq.post(httpbinUrl ~ "post", mpform);
350 stream = rs.receiveAsRange();
351 while( !stream.empty() ) {
352 streamedContent ~= stream.front;
353 stream.popFront();
354 }
355 info("Check POST files using multiPartForm");
356 {
357 /// This is example on usage files with MultipartForm data.
358 /// For this example we have to create files which will be sent.
359 import std.file;
360 import std.path;
361 /// preapare files
362 auto tmpd = tempDir();
363 auto tmpfname1 = tmpd ~ dirSeparator ~ "request_test1.txt";
364 auto f = File(tmpfname1, "wb");
365 f.rawWrite("file1 content\n");
366 f.close();
367 auto tmpfname2 = tmpd ~ dirSeparator ~ "request_test2.txt";
368 f = File(tmpfname2, "wb");
369 f.rawWrite("file2 content\n");
370 f.close();
371 ///
372 /// Ok, files ready.
373 /// Now we will prepare Form data
374 ///
375 File f1 = File(tmpfname1, "rb");
376 File f2 = File(tmpfname2, "rb");
377 scope(exit) {
378 f1.close();
379 f2.close();
380 }
381 ///
382 /// for each part we have to set field name, source (ubyte array or opened file) and optional filename and content-type
383 ///
384 MultipartForm mForm = MultipartForm().
385 add(formData("Field1", cast(ubyte[])"form field from memory")).
386 add(formData("Field2", cast(ubyte[])"file field from memory", ["filename":"data2"])).
387 add(formData("File1", f1, ["filename":"file1", "Content-Type": "application/octet-stream"])).
388 add(formData("File2", f2, ["filename":"file2", "Content-Type": "application/octet-stream"]));
389 /// everything ready, send request
390 rs = rq.post(httpbinUrl ~ "post", mForm);
391 }
392
393 rq = Request();
394 mpform = MultipartForm().add(formData(/* field name */ "greeting", /* content */ cast(ubyte[])"hello"));
395 rs = rq.post(httpbinUrl ~ "post", mpform);
396 assert(parseJSON(cast(string)(rs.responseBody().data)).object["form"].object["greeting"].str ==
397 parseJSON(cast(string)streamedContent).object["form"].object["greeting"].str);
398
399
400 info("Test POST'ing from Rank(2) range with user-provided Content-Length");
401 rq.addHeaders(["content-length": to!string(s.length)]);
402 rs = rq.post(httpbinUrl ~ "post", s.representation.chunks(10), "application/octet-stream");
403 auto flat_content = parseJSON(cast(string)rs.responseBody().data).object["data"];
404 if ( flat_content.type == JSONType..string ) {
405 // httpbin.org returns string in "data"
406 assert(s == flat_content.str);
407 } else {
408 // internal httpbin server return array ob bytes
409 assert(s.representation == flat_content.array.map!(i => i.integer).array);
410 }
411
412 /// Putting to forms (for small data)
413 ///
414 /// putting query parameters using "application/x-www-form-urlencoded"
415 info("Test putContent using query params");
416 r = putContent(httpbinUrl ~ "put", queryParams("first", "a", "second", 2));
417 assert(parseJSON(cast(string)r).object["form"].object["first"].str == "a");
418
419 /// putting using multipart/form-data (large data and files). See docs fot HTTPRequest
420 info("Test putContent form");
421 mpform = MultipartForm();
422 mpform.add(formData(/* field name */ "greeting", /* content */ cast(ubyte[])"hello"));
423 r = putContent(httpbinUrl ~ "put", mpform);
424 assert(parseJSON(cast(string)r).object["form"].object["greeting"].str == "hello");
425
426 /// Patching to forms (for small data)
427 ///
428 /// patching query parameters using "application/x-www-form-urlencoded"
429 info("Test patchContent using query params");
430 r = patchContent(httpbinUrl ~ "patch", queryParams("first", "a", "second", 2));
431 assert(parseJSON(cast(string)r).object["form"].object["first"].str == "a");
432
433 /// patching using multipart/form-data (large data and files). See docs fot HTTPRequest
434 info("Test patchContent form");
435 mpform = MultipartForm();
436 mpform.add(formData(/* field name */ "greeting", /* content */ cast(ubyte[])"hello"));
437 r = patchContent(httpbinUrl ~ "patch", mpform);
438 assert(parseJSON(cast(string)r).object["form"].object["greeting"].str == "hello");
439
440 info("Check exception handling, error messages and timeouts are OK");
441 rq.clearHeaders();
442 rq.timeout = 1.seconds;
443 assertThrown!TimeoutException(rq.get(httpbinUrl ~ "delay/3"));
444
445 info("Test get in parallel");
446 {
447 import std.stdio;
448 import std.parallelism;
449 import std.algorithm;
450 import std.string;
451 import core.atomic;
452
453 immutable auto urls = [
454 "stream/10",
455 "stream/20",
456 "stream/30",
457 "stream/40",
458 "stream/50",
459 "stream/60",
460 "stream/70",
461 ].map!(a => httpbinUrl ~ a).array.idup;
462
463 defaultPoolThreads(4);
464
465 shared short lines;
466
467 foreach(url; parallel(urls)) {
468 atomicOp!"+="(lines, getContent(url).splitter("\n").count);
469 }
470 assert(lines == 287);
471
472 }
473 }
474 /**
475 * Call GET, and return response content.
476 *
477 * This is the simplest case, when all you need is the response body and have no parameters.
478 * Returns:
479 * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
480 */
481 public auto ref getContent(string url) {
482 auto rq = Request();
483 auto rs = rq.get(url);
484 return rs.responseBody;
485 }
486 /**
487 Call GET and lazily convert server response into InputRange of ubyte[], splitted by '\n'.
488 */
489 public auto getContentByLine(string url)
490 {
491 auto rq = Request();
492 rq.useStreaming = true;
493 auto rs = rq.get(url);
494 _LineReader lr = _LineReader(rq, rs);
495 return lr;
496 }
497 /**
498 * Call GET, with parameters, and return response content.
499 *
500 * args = string[string] fo query parameters.
501 * Returns:
502 * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
503 */
504 public auto ref getContent(string url, string[string] args) {
505 auto rq = Request();
506 auto rs = rq.get(url, args);
507 return rs.responseBody;
508 }
509 /**
510 Call GET and lazily convert server response into InputRange of ubyte[], splitted by '\n'.
511 */
512 public auto getContentByLine(string url, string[string] args)
513 {
514 auto rq = Request();
515 rq.useStreaming = true;
516 auto rs = rq.get(url, args);
517 _LineReader lr = _LineReader(rq, rs);
518 return lr;
519 }
520 /**
521 * Call GET, with parameters, and return response content.
522 *
523 * args = QueryParam[] of parameters.
524 * Returns:
525 * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
526 */
527 public auto ref getContent(string url, QueryParam[] args) {
528 auto rq = Request();
529 auto rs = rq.get(url, args);
530 return rs.responseBody;
531 }
532 /**
533 Call GET and lazily convert server response into InputRange of ubyte[], splitted by '\n'.
534 */
535 public auto getContentByLine(string url, QueryParam[] args)
536 {
537 auto rq = Request();
538 rq.useStreaming = true;
539 auto rs = rq.get(url, args);
540 _LineReader lr = _LineReader(rq, rs);
541 return lr;
542 }
543 /**
544 * Call GET, and return response content.
545 * args = variadic args to supply parameter names and values.
546 * Returns:
547 * Buffer!ubyte which you can use as ForwardRange or DirectAccessRange, or extract data with .data() method.
548 */
549 public auto ref getContent(A...)(string url, A args) if (args.length > 1 && args.length % 2 == 0 ) {
550 return Request().
551 get(url, queryParams(args)).
552 responseBody;
553 }
554 /**
555 Call GET and lazily convert server response into InputRange of ubyte[], splitted by '\n'.
556 */
557 public auto getContentByLine(A...)(string url, A args) if (args.length > 1 && args.length % 2 == 0 )
558 {
559 auto rq = Request();
560 rq.useStreaming = true;
561 auto rs = rq.get(url, queryParams(args));
562 _LineReader lr = _LineReader(rq, rs);
563 return lr;
564 }
565 ///
566 /// Call post and return response content.
567 ///
568 public auto postContent(A...)(string url, A args) {
569 auto rq = Request();
570 auto rs = rq.post(url, args);
571 return rs.responseBody;
572 }
573 /**
574 Call POST and lazily convert server response into InputRange of ubyte[], splitted by '\n'.
575 */
576 public auto postContentByLine(A...)(string url, A args)
577 {
578 auto rq = Request();
579 rq.useStreaming = true;
580 auto rs = rq.post(url, args);
581 _LineReader lr = _LineReader(rq, rs);
582 return lr;
583 }
584
585 ///
586 /// Call put and return response content.
587 ///
588 public auto putContent(A...)(string url, A args) {
589 auto rq = Request();
590 auto rs = rq.put(url, args);
591 return rs.responseBody;
592 }
593 /**
594 Call PUT and lazily convert server response into InputRange of ubyte[], splitted by '\n'.
595 */
596 public auto putContentByLine(A...)(string url, A args)
597 {
598 auto rq = Request();
599 rq.useStreaming = true;
600 auto rs = rq.put(url, args);
601 _LineReader lr = _LineReader(rq, rs);
602 return lr;
603 }
604
605 ///
606 /// Call patch and return response content.
607 ///
608 public auto patchContent(A...)(string url, A args) {
609 auto rq = Request();
610 auto rs = rq.patch(url, args);
611 return rs.responseBody;
612 }
613
614 unittest
615 {
616 import std.algorithm;
617 info("Test ByLine interfaces");
618 auto r = getContentByLine("https://httpbin.org/stream/50");
619 assert(r.count == 50);
620 r = getContentByLine("https://httpbin.org/stream/0");
621 assert(r.count == 0);
622 r = getContentByLine("https://httpbin.org/anything", ["1":"1"]);
623 assert(r.map!"cast(string)a".filter!(a => a.canFind("data")).count == 1);
624 r = getContentByLine("https://httpbin.org/anything", queryParams("1","1"));
625 assert(r.map!"cast(string)a".filter!(a => a.canFind("data")).count == 1);
626 r = postContentByLine("https://httpbin.org/anything", "123");
627 assert(r.map!"cast(string)a".filter!(a => a.canFind("data")).count == 1);
628 r = putContentByLine("https://httpbin.org/anything", "123");
629 assert(r.map!"cast(string)a".filter!(a => a.canFind("data")).count == 1);
630 }