private System.Net.Http.HttpClient GetClient(TimeSpan timeout) { var client = new System.Net.Http.HttpClient(_handler); client.DefaultRequestHeaders.Add("User-Agent", _userAgentBuilder.GetUserAgent(true)); client.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate"); client.Timeout = timeout; return(client); }
public void DownloadFile(string url, string fileName) { try { var fileInfo = new FileInfo(fileName); if (fileInfo.Directory != null && !fileInfo.Directory.Exists) { fileInfo.Directory.Create(); } _logger.Debug("Downloading [{0}] to [{1}]", url, fileName); var stopWatch = Stopwatch.StartNew(); var webClient = new GZipWebClient(); webClient.Headers.Add(HttpRequestHeader.UserAgent, _userAgentBuilder.GetUserAgent()); webClient.DownloadFile(url, fileName); stopWatch.Stop(); _logger.Debug("Downloading Completed. took {0:0}s", stopWatch.Elapsed.Seconds); } catch (WebException e) { _logger.Warn("Failed to get response from: {0} {1}", url, e.Message); throw; } catch (Exception e) { _logger.Warn(e, "Failed to get response from: " + url); throw; } }
private VersionMessage CreateVersionMessage() { var version = new VersionMessage { Version = KnownVersion.CurrentVersion, Timestamp = _dateTimeProvider.GetTimeOffset(), Nonce = _randomNumberGenerator.GetUint64(), UserAgent = _userAgentBuilder.GetUserAgent(), }; return(version); }
private VersionMessage CreateVersionMessage() { var version = new VersionMessage { Version = KnownVersion.CurrentVersion, /// TODO: it's part of the node settings and depends on the configured features/shards, shouldn't be hard coded /// if/when pruned will be implemented, remember to remove Network service flag /// ref: https://github.com/bitcoin/bitcoin/blob/99813a9745fe10a58bedd7a4cb721faf14f907a4/src/init.cpp#L1671-L1680 Services = (ulong)_localServiceProvider.GetServices(), Timestamp = _dateTimeProvider.GetTimeOffset(), ReceiverAddress = new Types.NetworkAddressNoTime { EndPoint = PeerContext.RemoteEndPoint }, SenderAddress = new Types.NetworkAddressNoTime { EndPoint = PeerContext.PublicEndPoint ?? PeerContext.LocalEndPoint }, Nonce = _randomNumberGenerator.GetUint64(), UserAgent = _userAgentBuilder.GetUserAgent(), StartHeight = _headersTree.GetTip().Height, Relay = true //this.IsRelay, TODO: it's part of the node settings }; return(version); }
public HttpResponse GetResponse(HttpRequest request, CookieContainer cookies) { var webRequest = (HttpWebRequest)WebRequest.Create((Uri)request.Url); if (PlatformInfo.IsMono) { // On Mono GZipStream/DeflateStream leaks memory if an exception is thrown, use an intermediate buffer in that case. webRequest.AutomaticDecompression = DecompressionMethods.None; webRequest.Headers.Add("Accept-Encoding", "gzip"); } else { // Deflate is not a standard and could break depending on implementation. // we should just stick with the more compatible Gzip //http://stackoverflow.com/questions/8490718/how-to-decompress-stream-deflated-with-java-util-zip-deflater-in-net webRequest.AutomaticDecompression = DecompressionMethods.GZip; } webRequest.Method = request.Method.ToString(); webRequest.UserAgent = _userAgentBuilder.GetUserAgent(request.UseSimplifiedUserAgent); webRequest.KeepAlive = request.ConnectionKeepAlive; webRequest.AllowAutoRedirect = false; webRequest.CookieContainer = cookies; if (request.RequestTimeout != TimeSpan.Zero) { webRequest.Timeout = (int)Math.Ceiling(request.RequestTimeout.TotalMilliseconds); } AddProxy(webRequest, request); if (request.Headers != null) { AddRequestHeaders(webRequest, request.Headers); } HttpWebResponse httpWebResponse; try { if (request.ContentData != null) { webRequest.ContentLength = request.ContentData.Length; using (var writeStream = webRequest.GetRequestStream()) { writeStream.Write(request.ContentData, 0, request.ContentData.Length); } } httpWebResponse = (HttpWebResponse)webRequest.GetResponse(); } catch (WebException e) { if (e.Status == WebExceptionStatus.SecureChannelFailure && OsInfo.IsWindows) { SecurityProtocolPolicy.DisableTls12(); } httpWebResponse = (HttpWebResponse)e.Response; if (httpWebResponse == null) { // Workaround for mono not closing connections properly in certain situations. AbortWebRequest(webRequest); // The default messages for WebException on mono are pretty horrible. if (e.Status == WebExceptionStatus.NameResolutionFailure) { throw new WebException($"DNS Name Resolution Failure: '{webRequest.RequestUri.Host}'", e.Status); } else if (e.ToString().Contains("TLS Support not")) { throw new TlsFailureException(webRequest, e); } else if (e.ToString().Contains("The authentication or decryption has failed.")) { throw new TlsFailureException(webRequest, e); } else if (OsInfo.IsNotWindows) { throw new WebException($"{e.Message}: '{webRequest.RequestUri}'", e, e.Status, e.Response); } else { throw; }; } } byte[] data = null; using (var responseStream = httpWebResponse.GetResponseStream()) { if (responseStream != null && responseStream != Stream.Null) { try { data = responseStream.ToBytes(); if (PlatformInfo.IsMono && httpWebResponse.ContentEncoding == "gzip") { using (var compressedStream = new MemoryStream(data)) using (var gzip = new GZipStream(compressedStream, CompressionMode.Decompress)) using (var decompressedStream = new MemoryStream()) { gzip.CopyTo(decompressedStream); data = decompressedStream.ToArray(); } httpWebResponse.Headers.Remove("Content-Encoding"); } } catch (Exception ex) { throw new WebException("Failed to read complete http response", ex, WebExceptionStatus.ReceiveFailure, httpWebResponse); } } } return(new HttpResponse(request, new HttpHeader(httpWebResponse.Headers), data, httpWebResponse.StatusCode)); }
public async Task <HttpResponse> GetResponseAsync(HttpRequest request, CookieContainer cookies) { var webRequest = (HttpWebRequest)WebRequest.Create((Uri)request.Url); // Deflate is not a standard and could break depending on implementation. // we should just stick with the more compatible Gzip //http://stackoverflow.com/questions/8490718/how-to-decompress-stream-deflated-with-java-util-zip-deflater-in-net webRequest.AutomaticDecompression = DecompressionMethods.GZip; webRequest.Method = request.Method.ToString(); webRequest.UserAgent = _userAgentBuilder.GetUserAgent(request.UseSimplifiedUserAgent); webRequest.KeepAlive = request.ConnectionKeepAlive; webRequest.AllowAutoRedirect = false; webRequest.CookieContainer = cookies; if (request.RequestTimeout != TimeSpan.Zero) { webRequest.Timeout = (int)Math.Ceiling(request.RequestTimeout.TotalMilliseconds); } webRequest.Proxy = request.Proxy ?? GetProxy(request.Url); if (request.Headers != null) { AddRequestHeaders(webRequest, request.Headers); } HttpWebResponse httpWebResponse; var sw = new Stopwatch(); sw.Start(); try { if (request.ContentData != null) { webRequest.ContentLength = request.ContentData.Length; using (var writeStream = webRequest.GetRequestStream()) { writeStream.Write(request.ContentData, 0, request.ContentData.Length); } } httpWebResponse = (HttpWebResponse)await webRequest.GetResponseAsync(); } catch (WebException e) { httpWebResponse = (HttpWebResponse)e.Response; if (httpWebResponse == null) { // The default messages for WebException on mono are pretty horrible. if (e.Status == WebExceptionStatus.NameResolutionFailure) { throw new WebException($"DNS Name Resolution Failure: '{webRequest.RequestUri.Host}'", e.Status); } else if (e.ToString().Contains("TLS Support not")) { throw new TlsFailureException(webRequest, e); } else if (e.ToString().Contains("The authentication or decryption has failed.")) { throw new TlsFailureException(webRequest, e); } else if (OsInfo.IsNotWindows) { throw new WebException($"{e.Message}: '{webRequest.RequestUri}'", e, e.Status, e.Response); } else { throw; } } } byte[] data = null; using (var responseStream = httpWebResponse.GetResponseStream()) { if (responseStream != null && responseStream != Stream.Null) { try { data = await responseStream.ToBytes(); } catch (Exception ex) { throw new WebException("Failed to read complete http response", ex, WebExceptionStatus.ReceiveFailure, httpWebResponse); } } } sw.Stop(); return(new HttpResponse(request, new HttpHeader(httpWebResponse.Headers), httpWebResponse.Cookies, data, sw.ElapsedMilliseconds, httpWebResponse.StatusCode)); }
public HttpResponse GetResponse(HttpRequest request, CookieContainer cookies) { var webRequest = (HttpWebRequest)WebRequest.Create((Uri)request.Url); if (PlatformInfo.IsMono && request.ResponseStream == null) { // On Mono GZipStream/DeflateStream leaks memory if an exception is thrown, use an intermediate buffer in that case. webRequest.AutomaticDecompression = DecompressionMethods.None; webRequest.Headers.Add("Accept-Encoding", "gzip"); } else { // Deflate is not a standard and could break depending on implementation. // we should just stick with the more compatible Gzip //http://stackoverflow.com/questions/8490718/how-to-decompress-stream-deflated-with-java-util-zip-deflater-in-net webRequest.AutomaticDecompression = DecompressionMethods.GZip; } webRequest.Method = request.Method.ToString(); webRequest.UserAgent = _userAgentBuilder.GetUserAgent(request.UseSimplifiedUserAgent); webRequest.KeepAlive = request.ConnectionKeepAlive; webRequest.AllowAutoRedirect = false; webRequest.CookieContainer = cookies; if (request.Credentials != null) { if (request.Credentials is BasicNetworkCredential nc) { // Manually set header to avoid initial challenge response var authInfo = nc.UserName + ":" + nc.Password; authInfo = Convert.ToBase64String(Encoding.GetEncoding("ISO-8859-1").GetBytes(authInfo)); webRequest.Headers.Add("Authorization", "Basic " + authInfo); } else { webRequest.PreAuthenticate = true; webRequest.Credentials = request.Credentials; } } if (request.RequestTimeout != TimeSpan.Zero) { webRequest.Timeout = (int)Math.Ceiling(request.RequestTimeout.TotalMilliseconds); } AddProxy(webRequest, request); if (request.Headers != null) { AddRequestHeaders(webRequest, request.Headers); } HttpWebResponse httpWebResponse; try { if (request.ContentData != null) { webRequest.ContentLength = request.ContentData.Length; using (var writeStream = webRequest.GetRequestStream()) { writeStream.Write(request.ContentData, 0, request.ContentData.Length); } } httpWebResponse = (HttpWebResponse)webRequest.GetResponse(); } catch (WebException e) { httpWebResponse = (HttpWebResponse)e.Response; if (httpWebResponse == null) { // Workaround for mono not closing connections properly in certain situations. AbortWebRequest(webRequest); // The default messages for WebException on mono are pretty horrible. if (e.Status == WebExceptionStatus.NameResolutionFailure) { throw new WebException($"DNS Name Resolution Failure: '{webRequest.RequestUri.Host}'", e.Status); } else if (e.ToString().Contains("TLS Support not")) { throw new TlsFailureException(webRequest, e); } else if (e.ToString().Contains("The authentication or decryption has failed.")) { throw new TlsFailureException(webRequest, e); } else if (OsInfo.IsNotWindows) { throw new WebException($"{e.Message}: '{webRequest.RequestUri}'", e, e.Status, e.Response); } else { throw; } } } byte[] data = null; using (var responseStream = httpWebResponse.GetResponseStream()) { if (responseStream != null && responseStream != Stream.Null) { try { if (request.ResponseStream != null) { // A target ResponseStream was specified, write to that instead. responseStream.CopyTo(request.ResponseStream); } else { data = responseStream.ToBytes(); if (PlatformInfo.IsMono && httpWebResponse.ContentEncoding == "gzip") { data = data.Decompress(); httpWebResponse.Headers.Remove("Content-Encoding"); } } } catch (Exception ex) { throw new WebException("Failed to read complete http response", ex, WebExceptionStatus.ReceiveFailure, httpWebResponse); } } } return(new HttpResponse(request, new HttpHeader(httpWebResponse.Headers), data, httpWebResponse.StatusCode)); }
public HttpResponse GetResponse(HttpRequest request, CookieContainer cookies) { if (!CheckAvailability()) { throw new ApplicationException("Curl failed to initialize."); } lock (CurlGlobalHandle.Instance) { Stream responseStream = new MemoryStream(); Stream headerStream = new MemoryStream(); using (var curlEasy = new CurlEasy()) { curlEasy.AutoReferer = false; curlEasy.WriteFunction = (b, s, n, o) => { responseStream.Write(b, 0, s * n); return(s * n); }; curlEasy.HeaderFunction = (b, s, n, o) => { headerStream.Write(b, 0, s * n); return(s * n); }; AddProxy(curlEasy, request); curlEasy.Url = request.Url.FullUri; switch (request.Method) { case HttpMethod.GET: curlEasy.HttpGet = true; break; case HttpMethod.POST: curlEasy.Post = true; break; case HttpMethod.PUT: curlEasy.Put = true; break; default: throw new NotSupportedException($"HttpCurl method {request.Method} not supported"); } curlEasy.UserAgent = _userAgentBuilder.GetUserAgent(request.UseSimplifiedUserAgent); curlEasy.FollowLocation = false; if (request.RequestTimeout != TimeSpan.Zero) { curlEasy.Timeout = (int)Math.Ceiling(request.RequestTimeout.TotalSeconds); } if (OsInfo.IsWindows) { curlEasy.CaInfo = _caBundleFilePath; } if (cookies != null) { curlEasy.Cookie = cookies.GetCookieHeader((Uri)request.Url); } if (request.ContentData != null) { curlEasy.PostFieldSize = request.ContentData.Length; curlEasy.SetOpt(CurlOption.CopyPostFields, new string(Array.ConvertAll(request.ContentData, v => (char)v))); } // Yes, we have to keep a ref to the object to prevent corrupting the unmanaged state using (var httpRequestHeaders = SerializeHeaders(request)) { curlEasy.HttpHeader = httpRequestHeaders; var result = curlEasy.Perform(); if (result != CurlCode.Ok) { switch (result) { case CurlCode.SslCaCert: case (CurlCode)77: throw new WebException(string.Format("Curl Error {0} for Url {1}, issues with your operating system SSL Root Certificate Bundle (ca-bundle).", result, curlEasy.Url)); default: throw new WebException(string.Format("Curl Error {0} for Url {1}", result, curlEasy.Url)); } } } var webHeaderCollection = ProcessHeaderStream(request, cookies, headerStream); var responseData = ProcessResponseStream(request, responseStream, webHeaderCollection); var httpHeader = new HttpHeader(webHeaderCollection); return(new HttpResponse(request, httpHeader, responseData, (HttpStatusCode)curlEasy.ResponseCode)); } } }
public HttpResponse GetResponse(HttpRequest request, CookieContainer cookies) { var requestMessage = new HttpRequestMessage(request.Method, (Uri)request.Url); requestMessage.Headers.UserAgent.ParseAdd(_userAgentBuilder.GetUserAgent(request.UseSimplifiedUserAgent)); requestMessage.Headers.ConnectionClose = !request.ConnectionKeepAlive; var cookieHeader = cookies.GetCookieHeader((Uri)request.Url); if (cookieHeader.IsNotNullOrWhiteSpace()) { requestMessage.Headers.Add("Cookie", cookieHeader); } if (request.Credentials != null) { if (request.Credentials is BasicNetworkCredential bc) { // Manually set header to avoid initial challenge response var authInfo = bc.UserName + ":" + bc.Password; authInfo = Convert.ToBase64String(Encoding.GetEncoding("ISO-8859-1").GetBytes(authInfo)); requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Basic", authInfo); } else if (request.Credentials is NetworkCredential nc) { var creds = GetCredentialCache(); foreach (var authtype in new[] { "Basic", "Digest" }) { creds.Remove((Uri)request.Url, authtype); creds.Add((Uri)request.Url, authtype, nc); } } } using var cts = new CancellationTokenSource(); if (request.RequestTimeout != TimeSpan.Zero) { cts.CancelAfter(request.RequestTimeout); } else { // The default for System.Net.Http.HttpClient cts.CancelAfter(TimeSpan.FromSeconds(100)); } if (request.ContentData != null) { requestMessage.Content = new ByteArrayContent(request.ContentData); } if (request.Headers != null) { AddRequestHeaders(requestMessage, request.Headers); } var httpClient = GetClient(request.Url); HttpResponseMessage responseMessage; try { responseMessage = httpClient.Send(requestMessage, cts.Token); } catch (HttpRequestException e) { _logger.Error(e, "HttpClient error"); throw; } byte[] data = null; using (var responseStream = responseMessage.Content.ReadAsStream()) { if (responseStream != null && responseStream != Stream.Null) { try { if (request.ResponseStream != null && responseMessage.StatusCode == HttpStatusCode.OK) { // A target ResponseStream was specified, write to that instead. // But only on the OK status code, since we don't want to write failures and redirects. responseStream.CopyTo(request.ResponseStream); } else { data = responseStream.ToBytes(); } } catch (Exception ex) { throw new WebException("Failed to read complete http response", ex, WebExceptionStatus.ReceiveFailure, null); } } } var headers = responseMessage.Headers.ToNameValueCollection(); headers.Add(responseMessage.Content.Headers.ToNameValueCollection()); return(new HttpResponse(request, new HttpHeader(headers), data, responseMessage.StatusCode)); }
public HttpResponse GetResponse(HttpRequest request, CookieContainer cookies) { var requestMessage = new HttpRequestMessage(request.Method, (Uri)request.Url); requestMessage.Headers.UserAgent.ParseAdd(_userAgentBuilder.GetUserAgent(request.UseSimplifiedUserAgent)); requestMessage.Headers.ConnectionClose = !request.ConnectionKeepAlive; var cookieHeader = cookies.GetCookieHeader((Uri)request.Url); if (cookieHeader.IsNotNullOrWhiteSpace()) { requestMessage.Headers.Add("Cookie", cookieHeader); } using var cts = new CancellationTokenSource(); if (request.RequestTimeout != TimeSpan.Zero) { cts.CancelAfter(request.RequestTimeout); } else { // The default for System.Net.Http.HttpClient cts.CancelAfter(TimeSpan.FromSeconds(100)); } if (request.Credentials != null) { if (request.Credentials is BasicNetworkCredential bc) { // Manually set header to avoid initial challenge response var authInfo = bc.UserName + ":" + bc.Password; authInfo = Convert.ToBase64String(Encoding.GetEncoding("ISO-8859-1").GetBytes(authInfo)); requestMessage.Headers.Add("Authorization", "Basic " + authInfo); } else if (request.Credentials is NetworkCredential nc) { var creds = GetCredentialCache(); foreach (var authtype in new[] { "Basic", "Digest" }) { creds.Remove((Uri)request.Url, authtype); creds.Add((Uri)request.Url, authtype, nc); } } } if (request.ContentData != null) { requestMessage.Content = new ByteArrayContent(request.ContentData); } if (request.Headers != null) { AddRequestHeaders(requestMessage, request.Headers); } var httpClient = GetClient(request.Url); using var responseMessage = httpClient.Send(requestMessage, HttpCompletionOption.ResponseHeadersRead, cts.Token); { byte[] data = null; try { if (request.ResponseStream != null && responseMessage.StatusCode == HttpStatusCode.OK) { responseMessage.Content.CopyTo(request.ResponseStream, null, cts.Token); } else { data = responseMessage.Content.ReadAsByteArrayAsync(cts.Token).GetAwaiter().GetResult(); } } catch (Exception ex) { throw new WebException("Failed to read complete http response", ex, WebExceptionStatus.ReceiveFailure, null); } var headers = responseMessage.Headers.ToNameValueCollection(); headers.Add(responseMessage.Content.Headers.ToNameValueCollection()); return(new HttpResponse(request, new HttpHeader(headers), data, responseMessage.StatusCode)); } }
public HttpResponse GetResponse(HttpRequest request, CookieContainer cookies) { var requestMessage = new HttpRequestMessage(request.Method, (Uri)request.Url); requestMessage.Headers.UserAgent.ParseAdd(_userAgentBuilder.GetUserAgent(request.UseSimplifiedUserAgent)); requestMessage.Headers.ConnectionClose = !request.ConnectionKeepAlive; var cookieHeader = cookies.GetCookieHeader((Uri)request.Url); if (cookieHeader.IsNotNullOrWhiteSpace()) { requestMessage.Headers.Add("Cookie", cookieHeader); } using var cts = new CancellationTokenSource(); if (request.RequestTimeout != TimeSpan.Zero) { cts.CancelAfter(request.RequestTimeout); } else { // The default for System.Net.Http.HttpClient cts.CancelAfter(TimeSpan.FromSeconds(100)); } if (request.ContentData != null) { requestMessage.Content = new ByteArrayContent(request.ContentData); } if (request.Headers != null) { AddRequestHeaders(requestMessage, request.Headers); } var httpClient = GetClient(request.Url); using var responseMessage = httpClient.Send(requestMessage, HttpCompletionOption.ResponseHeadersRead, cts.Token); { byte[] data = null; try { if (request.ResponseStream != null && responseMessage.StatusCode == HttpStatusCode.OK) { responseMessage.Content.CopyTo(request.ResponseStream, null, cts.Token); } else { data = responseMessage.Content.ReadAsByteArrayAsync(cts.Token).GetAwaiter().GetResult(); } } catch (Exception ex) { throw new WebException("Failed to read complete http response", ex, WebExceptionStatus.ReceiveFailure, null); } var headers = responseMessage.Headers.ToNameValueCollection(); headers.Add(responseMessage.Content.Headers.ToNameValueCollection()); return(new HttpResponse(request, new HttpHeader(headers), data, responseMessage.StatusCode)); } }
public HttpResponse GetResponse(HttpRequest request, CookieContainer cookies) { var webRequest = (HttpWebRequest)WebRequest.Create((Uri)request.Url); // Deflate is not a standard and could break depending on implementation. // we should just stick with the more compatible Gzip //http://stackoverflow.com/questions/8490718/how-to-decompress-stream-deflated-with-java-util-zip-deflater-in-net webRequest.AutomaticDecompression = DecompressionMethods.GZip; webRequest.Method = request.Method.ToString(); webRequest.UserAgent = _userAgentBuilder.GetUserAgent(request.UseSimplifiedUserAgent); webRequest.KeepAlive = request.ConnectionKeepAlive; webRequest.AllowAutoRedirect = request.AllowAutoRedirect; webRequest.CookieContainer = cookies; if (request.RequestTimeout != TimeSpan.Zero) { webRequest.Timeout = (int)Math.Ceiling(request.RequestTimeout.TotalMilliseconds); } AddProxy(webRequest, request); if (request.Headers != null) { AddRequestHeaders(webRequest, request.Headers); } if (request.ContentData != null) { webRequest.ContentLength = request.ContentData.Length; using (var writeStream = webRequest.GetRequestStream()) { writeStream.Write(request.ContentData, 0, request.ContentData.Length); } } HttpWebResponse httpWebResponse; try { httpWebResponse = (HttpWebResponse)webRequest.GetResponse(); } catch (WebException e) { if (e.Status == WebExceptionStatus.SecureChannelFailure && OsInfo.IsWindows) { SecurityProtocolPolicy.DisableTls12(); } httpWebResponse = (HttpWebResponse)e.Response; if (httpWebResponse == null) { throw; } } byte[] data = null; using (var responseStream = httpWebResponse.GetResponseStream()) { if (responseStream != null) { data = responseStream.ToBytes(); } } return(new HttpResponse(request, new HttpHeader(httpWebResponse.Headers), data, httpWebResponse.StatusCode)); }