private void DownloadToStream(string url, Stream stream, bool urlIsGzipFile) { byte[] buffer = new byte[1024]; try { WebRequest webRequest = WebRequest.Create(url); using (WebResponse webResponse = WebRequestHelper.GetResponse(webRequest)) { using (Stream remoteStream = WebRequestHelper.GetResponseStream(webResponse)) { var readStream = !urlIsGzipFile ? remoteStream : new GZipStream(remoteStream, CompressionMode.Decompress); try { int bytesRead = 0; do { bytesRead = readStream.Read(buffer, 0, buffer.Length); if (bytesRead > 0) { stream.Write(buffer, 0, bytesRead); } } while(bytesRead > 0); stream.Flush(); } finally { if (urlIsGzipFile && readStream != null) { readStream.Dispose(); } } } } } catch (Exception ex) { // The standard exception is not very helpful - in particular it doesn't report the URL that we're downloading throw new InvalidOperationException($"Could not download {url}: {ex.Message}", ex); } }
static void Connect() { //string url = "https://www.google.co.uk"; string url = "https://api.nuget.org/v3/index.json"; try { var response = WebRequestHelper.GetResponse(() => { var request = (HttpWebRequest)WebRequest.Create(url); return(request); }); if (response.StatusCode == System.Net.HttpStatusCode.OK) { Console.WriteLine("OK"); } else { Console.Write(response.StatusCode); } } catch (WebException ex) { var webResponse = ex.Response as HttpWebResponse; if (webResponse != null) { if (webResponse.StatusCode == HttpStatusCode.Unauthorized) { Console.WriteLine("Unauthorized"); } } Console.WriteLine(ex); } }
private WebRequestResult <AirportDataThumbnailsJson> RequestThumbnails(string icao, string registration, int maxThumbnails) { var requestUrl = String.Format("http://www.airport-data.com/api/ac_thumb.json?m={0}&r={1}&n={2}", HttpUtility.UrlEncode(icao), HttpUtility.UrlEncode(registration ?? ""), maxThumbnails); var request = HttpWebRequest.Create(requestUrl); request.Timeout = ThumbnailTimeout; var result = new WebRequestResult <AirportDataThumbnailsJson>(); try { using (var response = (HttpWebResponse)WebRequestHelper.GetResponse(request)) { result.HttpStatusCode = response.StatusCode; if (result.HttpStatusCode == HttpStatusCode.OK) { using (var responseStream = WebRequestHelper.GetResponseStream(response)) { var deserialiser = new DataContractJsonSerializer(typeof(AirportDataThumbnailsJson)); result.Result = (AirportDataThumbnailsJson)deserialiser.ReadObject(responseStream); } } } } catch (WebException ex) { var webResponse = ex.Response as HttpWebResponse; if (webResponse != null) { result.HttpStatusCode = webResponse.StatusCode; } else { throw; } } return(result); }
public virtual string GetAuthorityAccessToken(string code) { if (string.IsNullOrEmpty(code)) { return(null); } var url = string.Format( "https://graph.qq.com/oauth2.0/token?client_id={0}&client_secret={1}&code={2}&grant_type=authorization_code&redirect_uri={3}", appId, appSecret, code, redirecturl); HttpWebRequest request = WebRequest.Create(url) as HttpWebRequest; var json = WebRequestHelper.GetResponse(request, "utf-8"); if (string.IsNullOrEmpty(json)) { return(null); } if (!json.Contains("access_token")) { return(null); } var dis = json.Split('&').Where(it => it.Contains("access_token")).FirstOrDefault(); var accessToken = dis.Split('=')[1]; return(accessToken); }
/// <summary> /// Executes search request, forms and returns product information models. /// </summary> /// <returns>Returns array of models containing information about product.</returns> public override ProductInfo[] ExecuteSearchRequest() { var firstResultNodes = new List<HtmlNode>(); var result = new List<ProductInfo>(); _isCategorizied = CheckIfCategorizied(ref firstResultNodes); if (!_isCategorizied) { return base.ExecuteSearchRequest(); } var doc = new HtmlDocument(); GetProducts(doc.DocumentNode, ref firstResultNodes); var products = ProccedResultPage(firstResultNodes); if (products.Length == 0) return new ProductInfo[0]; result.AddRange(products); int i = 2; while (result.Count < Limit) { var requestUrl = GetNonFirstRequestUrl(i); var response = WebRequestHelper.GetResponse(requestUrl); var newProducts = ProccedResultPage(WebRequestHelper.GetResponseHtml(response, Encoding)).ToArray(); if (newProducts.Length == 0) { break; } result.AddRange(newProducts); i++; } return result.Take(Limit).ToArray(); }
/// <summary> /// Executes search request, forms and returns product information models. /// </summary> /// <returns>Returns array of models containing information about product.</returns> public override ProductInfo[] ExecuteSearchRequest() { var products = new List <ProductInfo>(); var url = FirstRequestUrl; var firstResponse = WebRequestHelper.GetResponse(url, RequestHeaders); var firstResponseHtmlDocument = new HtmlDocument(); firstResponseHtmlDocument.LoadHtml(WebRequestHelper.GetResponseHtml(firstResponse, Encoding.UTF8)); products.AddRange(ProccedResultPage(firstResponseHtmlDocument).ToArray()); try { var responseHtmlDocument = firstResponseHtmlDocument; while (products.Count < Limit) { var requestUrl = GetNextPageRequestUrl(responseHtmlDocument); if (requestUrl == null) { break; } var response = WebRequestHelper.GetResponse(requestUrl, RequestHeaders); responseHtmlDocument = new HtmlDocument(); responseHtmlDocument.LoadHtml(WebRequestHelper.GetResponseHtml(response, Encoding)); ProductInfo[] newProducts = ProccedResultPage(responseHtmlDocument).ToArray(); if (newProducts.Length == 0) { break; } products.AddRange(newProducts); } } catch (Exception) { } return(products.Take(Limit).ToArray()); }
protected override WebResponse GetWebResponse(WebRequest request) { #pragma warning disable CS0618 // Type or member is obsolete. Have to use WebRequest with the DiscoveryClientProtocol return(WebRequestHelper.GetResponse(() => CreateWebRequest(request.RequestUri))); #pragma warning restore CS0618 // Type or member is obsolete. }
/// <summary> /// Gets search request response. /// </summary> /// <param name="jSessionId">Cookie containing jSessionid of search session on source website.</param> /// <param name="pageNum">Page number.</param> /// <returns>String containing html response.</returns> private string GetSearchRequestResponse(Cookie jSessionId, int pageNum) { var url = pageNum == 1 ? "http://www.annonsborsen.se/search/searchFast.jspx?" : String.Format("http://www.annonsborsen.se/search/searchFast.jspx?offset={0}", 20 * (pageNum - 1)); var resp = WebRequestHelper.GetResponse(url, null, new[] { jSessionId }); return(WebRequestHelper.GetResponseHtml(resp, Encoding)); }
public void TestResponse() { var url = "https://www.google.com"; var webRequestHelper = new WebRequestHelper(); var request = webRequestHelper.Create(url); var response = webRequestHelper.GetResponse(request); response.Should().BeOfType <HttpWebResponse>(); }
/// <summary> /// See interface docs. /// </summary> /// <param name="url"></param> /// <returns></returns> public string DownloadFileContent(string url) { string result = null; var webRequest = WebRequest.Create(url); using (var webResponse = WebRequestHelper.GetResponse(webRequest)) { using (var streamReader = new StreamReader(WebRequestHelper.GetResponseStream(webResponse))) { result = streamReader.ReadToEnd(); } } return(result); }
public void CheckManufacturerResponse() { string manufacturerUrl = AppSettingsUtility.GetString(AppSettingsKeys.ManufacturerUrl); var modifiedManufacturerUrl = manufacturerUrl.Replace("{index}", "0"); var webRequestHelper = new WebRequestHelper(modifiedManufacturerUrl); string response = webRequestHelper.GetResponse(); Assert.AreNotEqual <string>(response, string.Empty); var manufacturers = Program.GetManufaturersFromHtml(response); //Manufacturer count should be 50 for first page Assert.AreEqual <int>(manufacturers.Count, 50); }
public static ServiceDescription ReadServiceDescription(string uri) { var desc = new ServiceDescription(); try { WebResponse response = WebRequestHelper.GetResponse(() => (HttpWebRequest)WebRequest.Create(uri)); desc = ServiceDescription.Read(response.GetResponseStream()); response.Close(); desc.RetrievalUrl = uri; } catch (Exception) {} return(desc); }
/// <summary> /// 获取用户的基本信息 /// </summary> /// <param name="token"></param> /// <param name="openId"></param> /// <returns></returns> public virtual Dictionary <string, string> GetUserInfo(string token, string openId) { if (string.IsNullOrEmpty(token)) { return(null); } var url = $"https://graph.qq.com/user/get_user_info?access_token={token}&openid={openId}&oauth_consumer_key={appId}"; HttpWebRequest request = WebRequest.Create(url) as HttpWebRequest; var json = WebRequestHelper.GetResponse(request, "utf-8"); var dis = json.DeserializeJson <Dictionary <string, string> >(); if (dis.ContainsKey("ret") && dis["ret"] != "0") { return(null); } return(dis); }
public override HttpWebResponse GetSearchResultPage(string requestUrl) { int scope = 2; if (ProductText.ToLower().Contains("uddevalla")) { scope = 0; } else { if (ProductText.ToLower().Contains("göteborgs")) { scope = 1; } } var cookie = new Cookie("scope", scope.ToString(CultureInfo.InvariantCulture), "/", "www.uddevallatorget.se"); return(WebRequestHelper.GetResponse(requestUrl, null, new[] { cookie })); }
/// <summary> /// Defines the entry point of the application. /// </summary> static void Main() { string manufacturerUrl = AppSettingsUtility.GetString(AppSettingsKeys.ManufacturerUrl); for (int count = 0; count < 53; count++) { var modifiedManufacturerUrl = manufacturerUrl.Replace("{index}", count.ToString()); var webRequestHelper = new WebRequestHelper(modifiedManufacturerUrl); string response = webRequestHelper.GetResponse(); var manufacturers = GetManufaturersFromHtml(response); // Add to database if (manufacturers.Count > 0) { manufaturerRepository.Add(manufacturers); } Console.Clear(); Console.Write("Manufacturer Updated: " + count * 50); } }
/// <summary> /// 获取OpenId和UnionId /// </summary> /// <param name="token"></param> /// <returns></returns> public virtual Dictionary <string, string> GetAuthorityOpendIdAndUnionId(string token) { if (string.IsNullOrEmpty(token)) { return(null); } var url = $"https://graph.qq.com/oauth2.0/me?access_token={token}&unionid=1"; HttpWebRequest request = WebRequest.Create(url) as HttpWebRequest; request.Method = "GET"; request.ContentType = "application/x-www-form-urlencoded"; var json = WebRequestHelper.GetResponse(request, "utf-8"); if (string.IsNullOrEmpty(json) || json.Contains("error") || !json.Contains("callback")) { return(null); } Regex reg = new Regex(@"\(([^)]*)\)"); Match m = reg.Match(json); var dis = m.Result("$1").DeserializeJson <Dictionary <string, string> >(); return(dis); }
public static async Task LaunchBrowser(string appUrl, string launchUrl, Task processTask) { launchUrl = launchUrl ?? ""; Uri launchUri; //Check if lanuchUrl is valid absolute url and use it if it is... if (!Uri.TryCreate(launchUrl, UriKind.Absolute, out launchUri)) { //Otherwise check if appUrl is valid absolute and lanuchUrl is relative then concat them... Uri appUri; if (!Uri.TryCreate(appUrl, UriKind.Absolute, out appUri)) { LoggingService.LogWarning("Failed to launch browser because invalid launch and app urls."); return; } if (!Uri.TryCreate(launchUrl, UriKind.Relative, out launchUri)) { LoggingService.LogWarning("Failed to launch browser because invalid launch url."); return; } launchUri = new Uri(appUri, launchUri); } // Try to connect every 50ms while process is running // Give up after 2 minutes TimeSpan maximumWaitTime = TimeSpan.FromMinutes(2); var waitStartTime = DateTime.UtcNow; while (!processTask.IsCompleted) { var currentTime = DateTime.UtcNow; if (currentTime - waitStartTime > maximumWaitTime) { LoggingService.LogWarning("Failed to launch browser because no response was ever received from the launch url."); return; } await Task.Delay(TimeSpan.FromMilliseconds(50)); HttpWebRequest httpRequest = null; try { httpRequest = WebRequest.CreateHttp(launchUri); httpRequest.Timeout = (int)TimeSpan.FromSeconds(30).TotalMilliseconds; } catch (NotSupportedException) { LoggingService.LogWarning("Failed to launch browser because launch url is not an http or https request."); return; } catch (SecurityException) { LoggingService.LogWarning("Failed to launch browser because caller does not have permission to connect to the requested URI or a URI that the request is redirected to."); return; } try { using (var response = WebRequestHelper.GetResponse(() => httpRequest)) { if (response != null) { break; } } } catch { } } if (processTask.IsCompleted) { LoggingService.LogDebug("Failed to launch browser because process exited before server started listening."); return; } // Process is still alive hence we succesfully connected inside loop to web server, launch browser DesktopService.ShowUrl(launchUri.AbsoluteUri); }