1 module requests.base;
2 
3 import requests.streams;
4 import requests.utils;
5 import requests.uri;
6 
7 import std.format;
8 import std.datetime;
9 import core.time;
10 import std.stdio;
11 import std.algorithm;
12 import std.string;
13 import std.exception;
14 import std.bitmanip;
15 import std.conv;
16 
17 /++ 
18     Interface to provide user info and http headers requred for server auth.
19 +/
20 public interface Auth {
21     string[string] authHeaders(string domain); /// Create headers for authentication
22     string         userName();                 /// Returns user name
23     string         password();                 /// Returns user password
24 }
25 /**
26  * Basic authentication.
27  * Adds $(B Authorization: Basic) header to request
28  * Example:
29  * ---
30  * import requests;
31  * void main() {
32  * rq = Request();
33  * rq.authenticator = new BasicAuthentication("user", "passwd");
34  * rs = rq.get("http://httpbin.org/basic-auth/user/passwd");
35  * }
36  * ---
37 */
38 public class BasicAuthentication: Auth {
39     private {
40         string   _username, _password;
41         string[] _domains;
42     }
43     /// Constructor.
44     /// Params:
45     /// username = username
46     /// password = password
47     /// domains = not used now
48     ///
49     this(string username, string password, string[] domains = []) {
50         _username = username;
51         _password = password;
52         _domains = domains;
53     }
54     /// create Basic Auth header
55     override string[string] authHeaders(string domain) {
56         import std.base64;
57         string[string] auth;
58         auth["Authorization"] = "Basic " ~ to!string(Base64.encode(cast(ubyte[])"%s:%s".format(_username, _password)));
59         return auth;
60     }
61     /// returns username
62     override string userName() {
63         return _username;
64     }
65     /// return user password
66     override string password() {
67         return _password;
68     }
69 }
70 
71 /**
72  * Struct to send multiple files in POST request.
73  */
74 public struct PostFile {
75     /// Path to the file to send.
76     string fileName;
77     /// Name of the field (if empty - send file base name)
78     string fieldName;
79     /// contentType of the file if not empty
80     string contentType;
81 }
82 ///
83 /// This is File-like interface for sending data to multipart forms
84 ///
85 public interface FiniteReadable {
86     /// size of the content
87     abstract ulong  getSize();
88     /// file-like read()
89     abstract ubyte[] read();
90 }
91 ///
92 /// Helper to create form elements from File.
93 /// Params:
94 /// name = name of the field in form
95 /// f = opened std.stio.File to send to server
96 /// parameters = optional parameters (most important are "filename" and "Content-Type")
97 ///
98 public auto formData(string name, File f, string[string] parameters = null) {
99     return MultipartForm.FormData(name, new FormDataFile(f), parameters);
100 }
101 ///
102 /// Helper to create form elements from ubyte[].
103 /// Params:
104 /// name = name of the field in form
105 /// b = data to send to server
106 /// parameters = optional parameters (can be "filename" and "Content-Type")
107 ///
108 public auto formData(string name, ubyte[] b, string[string] parameters = null) {
109     return MultipartForm.FormData(name, new FormDataBytes(b), parameters);
110 }
111 public auto formData(string name, string b, string[string] parameters = null) {
112     return MultipartForm.FormData(name, new FormDataBytes(b.dup.representation), parameters);
113 }
114 
115 private immutable uint defaultBufferSize = 12*1024;
116 /// Class to provide FiniteReadable from user-provided ubyte[]
117 public class FormDataBytes : FiniteReadable {
118     private {
119         ulong   _size;
120         ubyte[] _data;
121         size_t  _offset;
122         bool    _exhausted;
123     }
124     /// constructor from ubyte[]
125     this(ubyte[] data) {
126         _data = data;
127         _size = data.length;
128     }
129     final override ulong getSize() {
130         return _size;
131     }
132     final override ubyte[] read() {
133         enforce( !_exhausted, "You can't read froum exhausted source" );
134         size_t toRead = min(defaultBufferSize, _size - _offset);
135         auto result = _data[_offset.._offset+toRead];
136         _offset += toRead;
137         if ( toRead == 0 ) {
138             _exhausted = true;
139         }
140         return result;
141     }
142 }
143 /// Class to provide FiniteReadable from File
144 public class FormDataFile : FiniteReadable {
145     import  std.file;
146     private {
147         File    _fileHandle;
148         ulong   _fileSize;
149         size_t  _processed;
150         bool    _exhausted;
151     }
152     /// constructor from File object
153     this(File file) {
154         import std.file;
155         _fileHandle = file;
156         _fileSize = std.file.getSize(file.name);
157     }
158     final override ulong getSize() pure nothrow @safe {
159         return _fileSize;
160     }
161     final override ubyte[] read() {
162         enforce( !_exhausted, "You can't read froum exhausted source" );
163         auto b = new ubyte[defaultBufferSize];
164         auto r = _fileHandle.rawRead(b);
165         auto toRead = min(r.length, _fileSize - _processed);
166         if ( toRead == 0 ) {
167             _exhausted = true;
168         }
169         _processed += toRead;
170         return r[0..toRead];
171     }
172 }
173 ///
174 /// This struct used to bulld POST's to forms.
175 /// Each part have name and data. data is something that can be read-ed and have size.
176 /// For example this can be string-like object (wrapped for reading) or opened File.
177 ///
178 public struct MultipartForm {
179     package struct FormData {
180         FiniteReadable  input;
181         string          name;
182         string[string]  parameters;
183         this(string name, FiniteReadable i, string[string] parameters = null) {
184             this.input = i;
185             this.name = name;
186             this.parameters = parameters;
187         }
188     }
189 
190     package FormData[] _sources;
191     auto add(FormData d) {
192         _sources ~= d;
193         return this;
194     }
195     auto add(string name, FiniteReadable i, string[string]parameters = null) {
196         _sources ~= FormData(name, i, parameters);
197         return this;
198     }
199     bool empty() const
200     {
201         return _sources.length == 0;
202     }
203 }
204 ///
205 
206 /// General type exception from Request
207 public class RequestException: Exception {
208     this(string msg, string file = __FILE__, size_t line = __LINE__, Throwable next = null) @safe pure nothrow {
209         super(msg, file, line, next);
210     }
211 }
212 
213 /**
214     ReceiveAsRange is InputRange used to supply client with data from response.
215     Each popFront fetch next data portion.
216 */
217 public struct ReceiveAsRange {
218     bool empty() {
219         return data.length == 0;
220     }
221     ubyte[] front() {
222         return data;
223     }
224     void popFront() {
225         if ( read ) {
226             // get new portion
227             data = read();
228         } else {
229             // we can't read any new data
230             data.length = 0;
231         }
232     }
233     package {
234         bool            activated;
235         ubyte[]         data;
236         /// HTTP or FTP module set up delegate read() for reading next portion of data.
237         ubyte[]         delegate() read;
238     }
239 }
240 
241 /**
242     Response
243 */
244 public class Response {
245     package {
246         /// Server status code
247         ushort           _code;
248         /// Response body
249         Buffer!ubyte     _responseBody;
250         /// Response headers
251         string[string]   _responseHeaders;
252         /// Initial URI
253         URI              _uri;
254         /// Final URI. Can differ from uri() if request go through redirections.
255         URI              _finalURI;
256         /// stream range stored here
257         ReceiveAsRange   _receiveAsRange;
258         SysTime          _startedAt,
259                          _connectedAt,
260                          _requestSentAt,
261                          _finishedAt;
262         /// Length of received content
263         long             _contentReceived;
264         /// Server-supplied content length (can be -1 when unknown)
265         long             _contentLength = -1;
266         mixin(Setter!ushort("code"));
267         mixin(Setter!URI("uri"));
268         mixin(Setter!URI("finalURI"));
269     }
270     mixin(Getter("code"));
271     mixin(Getter("contentReceived"));
272     mixin(Getter("contentLength"));
273     mixin(Getter("uri"));
274     mixin(Getter("finalURI"));
275 
276     @property auto getStats() const pure @safe {
277         import std.typecons: Tuple;
278         alias statTuple = Tuple!(Duration, "connectTime",
279             Duration, "sendTime",
280             Duration, "recvTime");
281         statTuple stat;
282         stat.connectTime = _connectedAt - _startedAt;
283         stat.sendTime = _requestSentAt - _connectedAt;
284         stat.recvTime = _finishedAt - _requestSentAt;
285         return stat;
286     }
287     @property auto ref responseBody() @safe nothrow {
288         return _responseBody;
289     }
290     @property auto ref responseHeaders() pure @safe nothrow {
291         return _responseHeaders;
292     }
293     @property auto ref receiveAsRange() pure @safe nothrow {
294         return _receiveAsRange;
295     }
296     /// string representation of response
297     override string toString() const {
298         return "Response(%d, %s)".format(_code, _finalURI.uri());
299     }
300     /// format response to string (hpPqsBTUS).
301     /**
302 
303         %h - remote hostname
304 
305         %p - remote port
306 
307         %P - remote path
308 
309         %q - query parameters
310 
311         %s - string representation
312 
313         %B - received bytes
314 
315         %T - resquest total time
316 
317         %U - request uri()
318 
319         %S - status code
320     */
321     string format(string fmt) const {
322         import std.array;
323         auto a = appender!string();
324         auto f = FormatSpec!char(fmt);
325         while (f.writeUpToNextSpec(a)) {
326             switch (f.spec) {
327                 case 'h':
328                     // Remote hostname.
329                     a.put(_uri.host);
330                     break;
331                 case 'p':
332                     // Remote port.
333                     a.put("%d".format(_uri.port));
334                     break;
335                 case 'P':
336                     // Path.
337                     a.put(_uri.path);
338                     break;
339                 case 'q':
340                     // query parameters supplied with url.
341                     a.put(_uri.query);
342                     break;
343                 case 's':
344                     a.put("Response(%d, %s)".format(_code, _finalURI.uri()));
345                     break;
346                 case 'B': // received bytes
347                     a.put("%d".format(_responseBody.length));
348                     break;
349                 case 'T': // request total time, ms
350                     a.put("%d".format((_finishedAt - _startedAt).total!"msecs"));
351                     break;
352                 case 'U':
353                     a.put(_uri.uri());
354                     break;
355                 case 'S':
356                     a.put("%d".format(_code));
357                     break;
358                 default:
359                     throw new FormatException("Unknown Response format specifier: %" ~ f.spec);
360             }
361         }
362         return a.data();
363     }
364 }
365 
366 struct _UH {
367     // flags for each important header, added by user using addHeaders
368     mixin(bitfields!(
369     bool, "Host", 1,
370     bool, "UserAgent", 1,
371     bool, "ContentLength", 1,
372     bool, "Connection", 1,
373     bool, "AcceptEncoding", 1,
374     bool, "ContentType", 1,
375     bool, "Cookie", 1,
376     uint, "", 1
377     ));
378 }