1 /++ 2 `std.curl` wrapper 3 +/ 4 module sily.curl; 5 6 import std.conv: to; 7 import std.concurrency; 8 import std.net.curl; 9 import std.typecons; 10 import std.datetime: Duration, seconds; 11 12 import sily.async; 13 14 15 /// Performs HTTP request 16 HTTPRequest fetch(string url, FetchConfig conf = FetchConfig()) { 17 HTTPRequest req = new HTTPRequest(); 18 auto http = HTTP(url); 19 20 foreach (key; conf.headers.keys) { 21 http.addRequestHeader(key, conf.headers[key]); 22 } 23 24 http.postData = conf.data; 25 26 http.verbose = conf.verbose; 27 28 http.method = cast(HTTP.Method) conf.method; 29 30 string result = ""; 31 32 http.onReceive((ubyte[] data) { 33 // import std.stdio; 34 // writeln(http.statusLine.code); 35 // writeln(http.statusLine.reason); 36 37 if (http.statusLine.code >= 300) { 38 (cast(HTTPRequest) req).reject(new HTTPStatusException(http.statusLine.code, http.statusLine.reason)); 39 return data.length; 40 } 41 42 result ~= (cast(immutable(char)*)data)[0..data.length]; 43 44 return data.length; 45 }); 46 47 http.onProgress = delegate int(size_t dl, size_t dln, size_t ul, size_t uln) { 48 if (http.statusLine.code >= 300) return 0; 49 // import std.stdio; 50 // writeln(dl, " ", dln); 51 if (dl != 0 && dln != 0 && dl == dln) { 52 req.resolve(result); 53 } 54 if (ul != 0 && uln != 0 && ul == uln) { 55 req.resolve(result); 56 } 57 return 0; 58 }; 59 60 http.dataTimeout = conf.dataTimeout; 61 http.operationTimeout = conf.operationTimeout; 62 http.connectTimeout = conf.connectTimeout; 63 http.dnsTimeout = conf.dnsTimeout; 64 65 http.authenticationMethod = conf.authMethod; 66 67 http.maxRedirects = conf.redirects; 68 69 if (conf.proxy._isInit) { 70 http.proxy = conf.proxy.host; 71 http.proxyPort = conf.proxy.port; 72 http.proxyType = conf.proxy.type; 73 } 74 75 if (conf.netInterface.length) { 76 http.netInterface = conf.netInterface; 77 } 78 79 if (conf.auth.username.length) { 80 http.setAuthentication(conf.auth.username, conf.auth.password, conf.auth.domain); 81 } 82 83 if (conf.proxyAuth.username.length) { 84 http.setProxyAuthentication(conf.proxyAuth.username, conf.proxyAuth.password); 85 } 86 87 if (conf.port != 0) http.localPort = conf.port; 88 if (conf.portRange != 0) http.localPortRange = conf.portRange; 89 90 http.tcpNoDelay = conf.noDelay; 91 92 if (conf.userAgent.length) http.setUserAgent = conf.userAgent; 93 if (conf.noUserAgent) http.setUserAgent = ""; 94 95 string cookies; 96 foreach (key; conf.cookie) { 97 cookies ~= key ~ "=" ~ conf.cookie[key] ~ ";"; 98 } 99 100 if (cookies.length) { 101 http.setCookie = cookies[0..$-1]; 102 } 103 104 if (conf.cookieJar.length) http.setCookieJar = conf.cookieJar; 105 106 if (conf.contentLength != 0) http.contentLength = conf.contentLength; 107 108 http.perform(No.throwOnError); 109 // TODO: async 110 111 // void performHttp(shared HTTP p_http) { 112 // (cast(HTTP) p_http).perform(No.throwOnError); 113 // } 114 115 // void fn() { 116 // performHttp(cast(shared) http); 117 // } 118 119 // spawn(cast(shared) &performHttp, cast(shared) http); 120 // Thread t = new Thread(&fn); 121 // t.start(); 122 // t.join(false); 123 // new Thread({import std.stdio; writeln("Message from thread"); (cast(HTTPRequest) req).resolve(`{"val"="v"}`);}).start(); 124 125 return req; 126 } 127 128 /// Simplified version of fetch GET 129 HTTPRequest get(string url, string[string] headers = null) { 130 return fetch(url, FetchConfig(GET, headers)); 131 } 132 133 /// Simplified version of fetch POST 134 HTTPRequest post(string url, string data, string[string] headers = null) { 135 return fetch(url, FetchConfig(POST, headers, data)); 136 } 137 138 /// Simplified version of fetch PUT 139 HTTPRequest put(string url, string data, string[string] headers = null) { 140 return fetch(url, FetchConfig(PUT, headers, data)); 141 } 142 143 /// Simplified version of fetch DELETE 144 HTTPRequest del(string url, string[string] headers = null) { 145 return fetch(url, FetchConfig(DELETE, headers)); 146 } 147 148 /// Simplified version of fetch PATCH 149 HTTPRequest patch(string url, string data, string[string] headers = null) { 150 return fetch(url, FetchConfig(PATCH, headers, data)); 151 } 152 153 /// Full fetch configuration 154 struct FetchConfig { 155 /// Fetch method 156 FetchMethod method = GET; 157 /// Headers to send 158 string[string] headers; 159 /// Request body 160 string data; 161 /// Ditto 162 alias body_ = data; 163 /// Sets curl authorisation method 164 AuthMethod authMethod = AuthMethod.basic; 165 /// Sets timeout for activity on connection 166 Duration dataTimeout = seconds(3600); 167 /// Sets maximum time an operation is allowed to take 168 Duration operationTimeout = Duration.max; 169 /// Sets timeout for connecting 170 Duration connectTimeout = Duration.max; 171 /// Sets timeout for connecting 172 Duration dnsTimeout = Duration.max; 173 /// Set max allowed redirections 174 uint redirects = uint.max; 175 /// Curl proxy 176 Proxy proxy; 177 /// Network interface to use in ofrm of the IP 178 string netInterface; 179 /// Outgoing port to use 180 ushort port = 0; 181 /// Port range (`port` to `port + portRange`) 182 ushort portRange = 0; 183 /// Sets tcp no-delay socket option on/off 184 bool noDelay = true; 185 /// Authentification 186 Auth auth; 187 /// Orixy authentification 188 Auth proxyAuth; 189 /// User agent request header 190 string userAgent; 191 /// Passes empty string into user agent if true 192 bool noUserAgent = false; 193 /// Sets active cookie strings 194 string[string] cookie; 195 /// Sets file path for cookies to be stored 196 string cookieJar; 197 /++ 198 The content length in bytes when using request that has content e.g. 199 POST/PUT and not using chunked transfer. 200 +/ 201 ulong contentLength = 0; 202 /// Curl verbosity 203 bool verbose = false; 204 } 205 206 /++ 207 Curl proxy. Must be init with this(host, port, type), otherwise not valid 208 +/ 209 struct Proxy { 210 /// Proxy 211 string host = ""; 212 /// Proxy port 213 ushort port = 0; 214 /// Proxy type 215 CurlProxy type = CurlProxy.http; 216 217 private bool _isInit = false; 218 219 this(string _host, ushort _port, CurlProxy _type) { 220 host = _host; 221 port = _port; 222 type = _type; 223 _isInit = true; 224 } 225 } 226 227 /++ 228 Curl auth. 229 +/ 230 struct Auth { 231 /// Username 232 string username; 233 /// Password 234 string password; 235 /// Domain (can be none) 236 string domain = ""; 237 } 238 239 alias AuthMethod = HTTP.AuthMethod; 240 alias CurlProxy = HTTP.CurlProxy; 241 242 alias FetchMethod = int; 243 244 enum: FetchMethod { 245 HEAD = HTTP.Method.head, 246 GET = HTTP.Method.get, 247 POST = HTTP.Method.post, 248 PUT = HTTP.Method.put, 249 DELETE = HTTP.Method.del, 250 OPTIONS = HTTP.Method.options, 251 TRACE = HTTP.Method.trace, 252 CONNECT = HTTP.Method.connect, 253 PATCH = HTTP.Method.patch 254 } 255