private void DownloadToStream(string url, Stream stream, bool urlIsGzipFile)
{
byte[] buffer = new byte[1024];
try {
WebRequest webRequest = WebRequest.Create(url);
using (WebResponse webResponse = WebRequestHelper.GetResponse(webRequest)) {
using (Stream remoteStream = WebRequestHelper.GetResponseStream(webResponse)) {
var readStream = !urlIsGzipFile ? remoteStream : new GZipStream(remoteStream, CompressionMode.Decompress);
try {
int bytesRead = 0;
do
{
bytesRead = readStream.Read(buffer, 0, buffer.Length);
if (bytesRead > 0)
{
stream.Write(buffer, 0, bytesRead);
}
} while(bytesRead > 0);
stream.Flush();
} finally {
if (urlIsGzipFile && readStream != null)
{
readStream.Dispose();
}
}
}
}
} catch (Exception ex) {
// The standard exception is not very helpful - in particular it doesn't report the URL that we're downloading
throw new InvalidOperationException($"Could not download {url}: {ex.Message}", ex);
}
}
static void Connect()
{
//string url = "https://www.google.co.uk";
string url = "https://api.nuget.org/v3/index.json";
try {
var response = WebRequestHelper.GetResponse(() => {
var request = (HttpWebRequest)WebRequest.Create(url);
return(request);
});
if (response.StatusCode == System.Net.HttpStatusCode.OK)
{
Console.WriteLine("OK");
}
else
{
Console.Write(response.StatusCode);
}
} catch (WebException ex) {
var webResponse = ex.Response as HttpWebResponse;
if (webResponse != null)
{
if (webResponse.StatusCode == HttpStatusCode.Unauthorized)
{
Console.WriteLine("Unauthorized");
}
}
Console.WriteLine(ex);
}
}
private WebRequestResult <AirportDataThumbnailsJson> RequestThumbnails(string icao, string registration, int maxThumbnails)
{
var requestUrl = String.Format("http://www.airport-data.com/api/ac_thumb.json?m={0}&r={1}&n={2}", HttpUtility.UrlEncode(icao), HttpUtility.UrlEncode(registration ?? ""), maxThumbnails);
var request = HttpWebRequest.Create(requestUrl);
request.Timeout = ThumbnailTimeout;
var result = new WebRequestResult <AirportDataThumbnailsJson>();
try {
using (var response = (HttpWebResponse)WebRequestHelper.GetResponse(request)) {
result.HttpStatusCode = response.StatusCode;
if (result.HttpStatusCode == HttpStatusCode.OK)
{
using (var responseStream = WebRequestHelper.GetResponseStream(response)) {
var deserialiser = new DataContractJsonSerializer(typeof(AirportDataThumbnailsJson));
result.Result = (AirportDataThumbnailsJson)deserialiser.ReadObject(responseStream);
}
}
}
} catch (WebException ex) {
var webResponse = ex.Response as HttpWebResponse;
if (webResponse != null)
{
result.HttpStatusCode = webResponse.StatusCode;
}
else
{
throw;
}
}
return(result);
}
public virtual string GetAuthorityAccessToken(string code)
{
if (string.IsNullOrEmpty(code))
{
return(null);
}
var url =
string.Format(
"https://graph.qq.com/oauth2.0/token?client_id={0}&client_secret={1}&code={2}&grant_type=authorization_code&redirect_uri={3}",
appId, appSecret, code, redirecturl);
HttpWebRequest request = WebRequest.Create(url) as HttpWebRequest;
var json = WebRequestHelper.GetResponse(request, "utf-8");
if (string.IsNullOrEmpty(json))
{
return(null);
}
if (!json.Contains("access_token"))
{
return(null);
}
var dis = json.Split('&').Where(it => it.Contains("access_token")).FirstOrDefault();
var accessToken = dis.Split('=')[1];
return(accessToken);
}
/// <summary>
/// Executes search request, forms and returns product information models.
/// </summary>
/// <returns>Returns array of models containing information about product.</returns>
public override ProductInfo[] ExecuteSearchRequest()
{
var firstResultNodes = new List<HtmlNode>();
var result = new List<ProductInfo>();
_isCategorizied = CheckIfCategorizied(ref firstResultNodes);
if (!_isCategorizied)
{
return base.ExecuteSearchRequest();
}
var doc = new HtmlDocument();
GetProducts(doc.DocumentNode, ref firstResultNodes);
var products = ProccedResultPage(firstResultNodes);
if (products.Length == 0)
return new ProductInfo[0];
result.AddRange(products);
int i = 2;
while (result.Count < Limit)
{
var requestUrl = GetNonFirstRequestUrl(i);
var response = WebRequestHelper.GetResponse(requestUrl);
var newProducts = ProccedResultPage(WebRequestHelper.GetResponseHtml(response, Encoding)).ToArray();
if (newProducts.Length == 0)
{
break;
}
result.AddRange(newProducts);
i++;
}
return result.Take(Limit).ToArray();
}
/// <summary>
/// Executes search request, forms and returns product information models.
/// </summary>
/// <returns>Returns array of models containing information about product.</returns>
public override ProductInfo[] ExecuteSearchRequest()
{
var products = new List <ProductInfo>();
var url = FirstRequestUrl;
var firstResponse = WebRequestHelper.GetResponse(url, RequestHeaders);
var firstResponseHtmlDocument = new HtmlDocument();
firstResponseHtmlDocument.LoadHtml(WebRequestHelper.GetResponseHtml(firstResponse, Encoding.UTF8));
products.AddRange(ProccedResultPage(firstResponseHtmlDocument).ToArray());
try
{
var responseHtmlDocument = firstResponseHtmlDocument;
while (products.Count < Limit)
{
var requestUrl = GetNextPageRequestUrl(responseHtmlDocument);
if (requestUrl == null)
{
break;
}
var response = WebRequestHelper.GetResponse(requestUrl, RequestHeaders);
responseHtmlDocument = new HtmlDocument();
responseHtmlDocument.LoadHtml(WebRequestHelper.GetResponseHtml(response, Encoding));
ProductInfo[] newProducts = ProccedResultPage(responseHtmlDocument).ToArray();
if (newProducts.Length == 0)
{
break;
}
products.AddRange(newProducts);
}
}
catch (Exception)
{
}
return(products.Take(Limit).ToArray());
}
protected override WebResponse GetWebResponse(WebRequest request)
{
#pragma warning disable CS0618 // Type or member is obsolete. Have to use WebRequest with the DiscoveryClientProtocol
return(WebRequestHelper.GetResponse(() => CreateWebRequest(request.RequestUri)));
#pragma warning restore CS0618 // Type or member is obsolete.
}
/// <summary>
/// Gets search request response.
/// </summary>
/// <param name="jSessionId">Cookie containing jSessionid of search session on source website.</param>
/// <param name="pageNum">Page number.</param>
/// <returns>String containing html response.</returns>
private string GetSearchRequestResponse(Cookie jSessionId, int pageNum)
{
var url = pageNum == 1
? "http://www.annonsborsen.se/search/searchFast.jspx?"
: String.Format("http://www.annonsborsen.se/search/searchFast.jspx?offset={0}", 20 * (pageNum - 1));
var resp = WebRequestHelper.GetResponse(url, null, new[] { jSessionId });
return(WebRequestHelper.GetResponseHtml(resp, Encoding));
}
public void TestResponse()
{
var url = "https://www.google.com";
var webRequestHelper = new WebRequestHelper();
var request = webRequestHelper.Create(url);
var response = webRequestHelper.GetResponse(request);
response.Should().BeOfType <HttpWebResponse>();
}
/// <summary>
/// See interface docs.
/// </summary>
/// <param name="url"></param>
/// <returns></returns>
public string DownloadFileContent(string url)
{
string result = null;
var webRequest = WebRequest.Create(url);
using (var webResponse = WebRequestHelper.GetResponse(webRequest)) {
using (var streamReader = new StreamReader(WebRequestHelper.GetResponseStream(webResponse))) {
result = streamReader.ReadToEnd();
}
}
return(result);
}
public void CheckManufacturerResponse()
{
string manufacturerUrl = AppSettingsUtility.GetString(AppSettingsKeys.ManufacturerUrl);
var modifiedManufacturerUrl = manufacturerUrl.Replace("{index}", "0");
var webRequestHelper = new WebRequestHelper(modifiedManufacturerUrl);
string response = webRequestHelper.GetResponse();
Assert.AreNotEqual <string>(response, string.Empty);
var manufacturers = Program.GetManufaturersFromHtml(response);
//Manufacturer count should be 50 for first page
Assert.AreEqual <int>(manufacturers.Count, 50);
}
public static ServiceDescription ReadServiceDescription(string uri)
{
var desc = new ServiceDescription();
try
{
WebResponse response = WebRequestHelper.GetResponse(() => (HttpWebRequest)WebRequest.Create(uri));
desc = ServiceDescription.Read(response.GetResponseStream());
response.Close();
desc.RetrievalUrl = uri;
}
catch (Exception) {}
return(desc);
}
/// <summary>
/// 获取用户的基本信息
/// </summary>
/// <param name="token"></param>
/// <param name="openId"></param>
/// <returns></returns>
public virtual Dictionary <string, string> GetUserInfo(string token, string openId)
{
if (string.IsNullOrEmpty(token))
{
return(null);
}
var url = $"https://graph.qq.com/user/get_user_info?access_token={token}&openid={openId}&oauth_consumer_key={appId}";
HttpWebRequest request = WebRequest.Create(url) as HttpWebRequest;
var json = WebRequestHelper.GetResponse(request, "utf-8");
var dis = json.DeserializeJson <Dictionary <string, string> >();
if (dis.ContainsKey("ret") && dis["ret"] != "0")
{
return(null);
}
return(dis);
}
public override HttpWebResponse GetSearchResultPage(string requestUrl)
{
int scope = 2;
if (ProductText.ToLower().Contains("uddevalla"))
{
scope = 0;
}
else
{
if (ProductText.ToLower().Contains("göteborgs"))
{
scope = 1;
}
}
var cookie = new Cookie("scope", scope.ToString(CultureInfo.InvariantCulture), "/", "www.uddevallatorget.se");
return(WebRequestHelper.GetResponse(requestUrl, null, new[] { cookie }));
}
/// <summary>
/// Defines the entry point of the application.
/// </summary>
static void Main()
{
string manufacturerUrl = AppSettingsUtility.GetString(AppSettingsKeys.ManufacturerUrl);
for (int count = 0; count < 53; count++)
{
var modifiedManufacturerUrl = manufacturerUrl.Replace("{index}", count.ToString());
var webRequestHelper = new WebRequestHelper(modifiedManufacturerUrl);
string response = webRequestHelper.GetResponse();
var manufacturers = GetManufaturersFromHtml(response);
// Add to database
if (manufacturers.Count > 0)
{
manufaturerRepository.Add(manufacturers);
}
Console.Clear();
Console.Write("Manufacturer Updated: " + count * 50);
}
}
/// <summary>
/// 获取OpenId和UnionId
/// </summary>
/// <param name="token"></param>
/// <returns></returns>
public virtual Dictionary <string, string> GetAuthorityOpendIdAndUnionId(string token)
{
if (string.IsNullOrEmpty(token))
{
return(null);
}
var url = $"https://graph.qq.com/oauth2.0/me?access_token={token}&unionid=1";
HttpWebRequest request = WebRequest.Create(url) as HttpWebRequest;
request.Method = "GET";
request.ContentType = "application/x-www-form-urlencoded";
var json = WebRequestHelper.GetResponse(request, "utf-8");
if (string.IsNullOrEmpty(json) || json.Contains("error") || !json.Contains("callback"))
{
return(null);
}
Regex reg = new Regex(@"\(([^)]*)\)");
Match m = reg.Match(json);
var dis = m.Result("$1").DeserializeJson <Dictionary <string, string> >();
return(dis);
}
public static async Task LaunchBrowser(string appUrl, string launchUrl, Task processTask)
{
launchUrl = launchUrl ?? "";
Uri launchUri;
//Check if lanuchUrl is valid absolute url and use it if it is...
if (!Uri.TryCreate(launchUrl, UriKind.Absolute, out launchUri))
{
//Otherwise check if appUrl is valid absolute and lanuchUrl is relative then concat them...
Uri appUri;
if (!Uri.TryCreate(appUrl, UriKind.Absolute, out appUri))
{
LoggingService.LogWarning("Failed to launch browser because invalid launch and app urls.");
return;
}
if (!Uri.TryCreate(launchUrl, UriKind.Relative, out launchUri))
{
LoggingService.LogWarning("Failed to launch browser because invalid launch url.");
return;
}
launchUri = new Uri(appUri, launchUri);
}
// Try to connect every 50ms while process is running
// Give up after 2 minutes
TimeSpan maximumWaitTime = TimeSpan.FromMinutes(2);
var waitStartTime = DateTime.UtcNow;
while (!processTask.IsCompleted)
{
var currentTime = DateTime.UtcNow;
if (currentTime - waitStartTime > maximumWaitTime)
{
LoggingService.LogWarning("Failed to launch browser because no response was ever received from the launch url.");
return;
}
await Task.Delay(TimeSpan.FromMilliseconds(50));
HttpWebRequest httpRequest = null;
try {
httpRequest = WebRequest.CreateHttp(launchUri);
httpRequest.Timeout = (int)TimeSpan.FromSeconds(30).TotalMilliseconds;
} catch (NotSupportedException) {
LoggingService.LogWarning("Failed to launch browser because launch url is not an http or https request.");
return;
} catch (SecurityException) {
LoggingService.LogWarning("Failed to launch browser because caller does not have permission to connect to the requested URI or a URI that the request is redirected to.");
return;
}
try {
using (var response = WebRequestHelper.GetResponse(() => httpRequest)) {
if (response != null)
{
break;
}
}
} catch {
}
}
if (processTask.IsCompleted)
{
LoggingService.LogDebug("Failed to launch browser because process exited before server started listening.");
return;
}
// Process is still alive hence we succesfully connected inside loop to web server, launch browser
DesktopService.ShowUrl(launchUri.AbsoluteUri);
}
