1 module requests.base; 2 3 import requests.streams; 4 import requests.utils; 5 import requests.uri; 6 7 import std.format; 8 import std.datetime; 9 import core.time; 10 import std.stdio; 11 import std.algorithm; 12 import std..string; 13 import std.exception; 14 import std.bitmanip; 15 import std.conv; 16 17 /++ 18 Interface to provide user info and http headers requred for server auth. 19 +/ 20 public interface Auth { 21 string[string] authHeaders(string domain); /// Create headers for authentication 22 string userName(); /// Returns user name 23 string password(); /// Returns user password 24 } 25 /** 26 * Basic authentication. 27 * Adds $(B Authorization: Basic) header to request 28 * Example: 29 * --- 30 * import requests; 31 * void main() { 32 * rq = Request(); 33 * rq.authenticator = new BasicAuthentication("user", "passwd"); 34 * rs = rq.get("http://httpbin.org/basic-auth/user/passwd"); 35 * } 36 * --- 37 */ 38 public class BasicAuthentication: Auth { 39 private { 40 string _username, _password; 41 string[] _domains; 42 } 43 /// Constructor. 44 /// Params: 45 /// username = username 46 /// password = password 47 /// domains = not used now 48 /// 49 this(string username, string password, string[] domains = []) { 50 _username = username; 51 _password = password; 52 _domains = domains; 53 } 54 /// create Basic Auth header 55 override string[string] authHeaders(string domain) { 56 import std.base64; 57 string[string] auth; 58 auth["Authorization"] = "Basic " ~ to!string(Base64.encode(cast(ubyte[])"%s:%s".format(_username, _password))); 59 return auth; 60 } 61 /// returns username 62 override string userName() { 63 return _username; 64 } 65 /// return user password 66 override string password() { 67 return _password; 68 } 69 } 70 71 /** 72 * Struct to send multiple files in POST request. 73 */ 74 public struct PostFile { 75 /// Path to the file to send. 76 string fileName; 77 /// Name of the field (if empty - send file base name) 78 string fieldName; 79 /// contentType of the file if not empty 80 string contentType; 81 } 82 /// 83 /// This is File-like interface for sending data to multipart forms 84 /// 85 public interface FiniteReadable { 86 /// size of the content 87 abstract ulong getSize(); 88 /// file-like read() 89 abstract ubyte[] read(); 90 } 91 /// 92 /// Helper to create form elements from File. 93 /// Params: 94 /// name = name of the field in form 95 /// f = opened std.stio.File to send to server 96 /// parameters = optional parameters (most important are "filename" and "Content-Type") 97 /// 98 public auto formData(string name, File f, string[string] parameters = null) { 99 return MultipartForm.FormData(name, new FormDataFile(f), parameters); 100 } 101 /// 102 /// Helper to create form elements from ubyte[]. 103 /// Params: 104 /// name = name of the field in form 105 /// b = data to send to server 106 /// parameters = optional parameters (can be "filename" and "Content-Type") 107 /// 108 public auto formData(string name, ubyte[] b, string[string] parameters = null) { 109 return MultipartForm.FormData(name, new FormDataBytes(b), parameters); 110 } 111 public auto formData(string name, string b, string[string] parameters = null) { 112 return MultipartForm.FormData(name, new FormDataBytes(b.dup.representation), parameters); 113 } 114 115 private immutable uint defaultBufferSize = 12*1024; 116 /// Class to provide FiniteReadable from user-provided ubyte[] 117 public class FormDataBytes : FiniteReadable { 118 private { 119 ulong _size; 120 ubyte[] _data; 121 size_t _offset; 122 bool _exhausted; 123 } 124 /// constructor from ubyte[] 125 this(ubyte[] data) { 126 _data = data; 127 _size = data.length; 128 } 129 final override ulong getSize() { 130 return _size; 131 } 132 final override ubyte[] read() { 133 enforce( !_exhausted, "You can't read froum exhausted source" ); 134 size_t toRead = min(defaultBufferSize, _size - _offset); 135 auto result = _data[_offset.._offset+toRead]; 136 _offset += toRead; 137 if ( toRead == 0 ) { 138 _exhausted = true; 139 } 140 return result; 141 } 142 } 143 /// Class to provide FiniteReadable from File 144 public class FormDataFile : FiniteReadable { 145 import std.file; 146 private { 147 File _fileHandle; 148 ulong _fileSize; 149 size_t _processed; 150 bool _exhausted; 151 } 152 /// constructor from File object 153 this(File file) { 154 import std.file; 155 _fileHandle = file; 156 _fileSize = std.file.getSize(file.name); 157 } 158 final override ulong getSize() pure nothrow @safe { 159 return _fileSize; 160 } 161 final override ubyte[] read() { 162 enforce( !_exhausted, "You can't read froum exhausted source" ); 163 auto b = new ubyte[defaultBufferSize]; 164 auto r = _fileHandle.rawRead(b); 165 auto toRead = min(r.length, _fileSize - _processed); 166 if ( toRead == 0 ) { 167 _exhausted = true; 168 } 169 _processed += toRead; 170 return r[0..toRead]; 171 } 172 } 173 /// 174 /// This struct used to bulld POST's to forms. 175 /// Each part have name and data. data is something that can be read-ed and have size. 176 /// For example this can be string-like object (wrapped for reading) or opened File. 177 /// 178 public struct MultipartForm { 179 package struct FormData { 180 FiniteReadable input; 181 string name; 182 string[string] parameters; 183 this(string name, FiniteReadable i, string[string] parameters = null) { 184 this.input = i; 185 this.name = name; 186 this.parameters = parameters; 187 } 188 } 189 190 package FormData[] _sources; 191 auto add(FormData d) { 192 _sources ~= d; 193 return this; 194 } 195 auto add(string name, FiniteReadable i, string[string]parameters = null) { 196 _sources ~= FormData(name, i, parameters); 197 return this; 198 } 199 bool empty() const 200 { 201 return _sources.length == 0; 202 } 203 } 204 /// 205 206 /// General type exception from Request 207 public class RequestException: Exception { 208 this(string msg, string file = __FILE__, size_t line = __LINE__, Throwable next = null) @safe pure nothrow { 209 super(msg, file, line, next); 210 } 211 } 212 213 /** 214 ReceiveAsRange is InputRange used to supply client with data from response. 215 Each popFront fetch next data portion. 216 */ 217 public struct ReceiveAsRange { 218 bool empty() { 219 return data.length == 0; 220 } 221 ubyte[] front() { 222 return data; 223 } 224 void popFront() { 225 if ( read ) { 226 // get new portion 227 data = read(); 228 } else { 229 // we can't read any new data 230 data.length = 0; 231 } 232 } 233 package { 234 bool activated; 235 ubyte[] data; 236 /// HTTP or FTP module set up delegate read() for reading next portion of data. 237 ubyte[] delegate() read; 238 } 239 } 240 241 /** 242 Response 243 */ 244 public class Response { 245 package { 246 /// Server status code 247 ushort _code; 248 /// Response body 249 Buffer!ubyte _responseBody; 250 /// Response headers 251 string[string] _responseHeaders; 252 /// Initial URI 253 URI _uri; 254 /// Final URI. Can differ from uri() if request go through redirections. 255 URI _finalURI; 256 /// stream range stored here 257 ReceiveAsRange _receiveAsRange; 258 SysTime _startedAt, 259 _connectedAt, 260 _requestSentAt, 261 _finishedAt; 262 /// Length of received content 263 long _contentReceived; 264 /// Server-supplied content length (can be -1 when unknown) 265 long _contentLength = -1; 266 mixin(Setter!ushort("code")); 267 mixin(Setter!URI("uri")); 268 mixin(Setter!URI("finalURI")); 269 } 270 mixin(Getter("code")); 271 mixin(Getter("contentReceived")); 272 mixin(Getter("contentLength")); 273 mixin(Getter("uri")); 274 mixin(Getter("finalURI")); 275 276 @property auto getStats() const pure @safe { 277 import std.typecons: Tuple; 278 alias statTuple = Tuple!(Duration, "connectTime", 279 Duration, "sendTime", 280 Duration, "recvTime"); 281 statTuple stat; 282 stat.connectTime = _connectedAt - _startedAt; 283 stat.sendTime = _requestSentAt - _connectedAt; 284 stat.recvTime = _finishedAt - _requestSentAt; 285 return stat; 286 } 287 @property auto ref responseBody() @safe nothrow { 288 return _responseBody; 289 } 290 @property auto ref responseHeaders() pure @safe nothrow { 291 return _responseHeaders; 292 } 293 @property auto ref receiveAsRange() pure @safe nothrow { 294 return _receiveAsRange; 295 } 296 /// string representation of response 297 override string toString() const { 298 return "Response(%d, %s)".format(_code, _finalURI.uri()); 299 } 300 /// format response to string (hpPqsBTUS). 301 /** 302 303 %h - remote hostname 304 305 %p - remote port 306 307 %P - remote path 308 309 %q - query parameters 310 311 %s - string representation 312 313 %B - received bytes 314 315 %T - resquest total time 316 317 %U - request uri() 318 319 %S - status code 320 */ 321 string format(string fmt) const { 322 import std.array; 323 auto a = appender!string(); 324 auto f = FormatSpec!char(fmt); 325 while (f.writeUpToNextSpec(a)) { 326 switch (f.spec) { 327 case 'h': 328 // Remote hostname. 329 a.put(_uri.host); 330 break; 331 case 'p': 332 // Remote port. 333 a.put("%d".format(_uri.port)); 334 break; 335 case 'P': 336 // Path. 337 a.put(_uri.path); 338 break; 339 case 'q': 340 // query parameters supplied with url. 341 a.put(_uri.query); 342 break; 343 case 's': 344 a.put("Response(%d, %s)".format(_code, _finalURI.uri())); 345 break; 346 case 'B': // received bytes 347 a.put("%d".format(_responseBody.length)); 348 break; 349 case 'T': // request total time, ms 350 a.put("%d".format((_finishedAt - _startedAt).total!"msecs")); 351 break; 352 case 'U': 353 a.put(_uri.uri()); 354 break; 355 case 'S': 356 a.put("%d".format(_code)); 357 break; 358 default: 359 throw new FormatException("Unknown Response format specifier: %" ~ f.spec); 360 } 361 } 362 return a.data(); 363 } 364 } 365 366 struct _UH { 367 // flags for each important header, added by user using addHeaders 368 mixin(bitfields!( 369 bool, "Host", 1, 370 bool, "UserAgent", 1, 371 bool, "ContentLength", 1, 372 bool, "Connection", 1, 373 bool, "AcceptEncoding", 1, 374 bool, "ContentType", 1, 375 bool, "Cookie", 1, 376 uint, "", 1 377 )); 378 }