private string GET(string Url, string Data, out HttpStatusCode statusCode)
{
WebRequest req = WebRequest.Create(Url + "?" + Data);
try
{
_doRequest += 1;
WebResponse resp = req.GetResponse();
HttpWebResponse httpResponse = (HttpWebResponse)resp;
statusCode = httpResponse.StatusCode;
Stream stream = resp.GetResponseStream();
string limit = resp.Headers["X-RateLimit-Remaining"];
var resetTime = resp.Headers["X-RateLimit-Reset"];
_limitRequst = Int32.Parse(limit);
_resetTimeLimits = UPCDB.UnixTimeStampToDateTime(Double.Parse(resetTime));
StreamReader sr = new StreamReader(stream);
string Out = sr.ReadToEnd();
sr.Close();
System.Threading.Thread.Sleep(1000 * _timeDelay);
return(Out);
}
catch (WebException e)
{
HttpWebResponse httpResponse = (HttpWebResponse)e.Response;
statusCode = httpResponse.StatusCode;
string limit = httpResponse.Headers["X-RateLimit-Remaining"];
var resetTime = httpResponse.Headers["X-RateLimit-Reset"];
_limitRequst = Int32.Parse(limit);
_resetTimeLimits = UPCDB.UnixTimeStampToDateTime(Double.Parse(resetTime));
if (statusCode == HttpStatusCode.NotFound)
{
//если артикул не найден, его нужно добавить в файл markets но без размеров, чтобы повторно его не запрашивать на следующий день парсинга
}
System.Threading.Thread.Sleep(1000 * _timeDelay);
return(null);
}
}
public string GetWithCrawlera(string Url, string Data, out HttpStatusCode statusCode)
{
ServicePointManager.ServerCertificateValidationCallback = (sender, cert, chain, sslPolicyErrors) => true;
var myProxy = new WebProxy("http://proxy.crawlera.com:8010");
myProxy.Credentials = new NetworkCredential("36f14b90c38c4005a81ccbed16a31f58", "");
//string url = "https://twitter.com/";
//string url = "https://api.upcitemdb.com/prod/trial/search?s=nike%20859524-005&match_mode=1&type=product";
string url = Url + "?" + Data;
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
var encodedApiKey = Helper.Base64Encode("36f14b90c38c4005a81ccbed16a31f58:");
request.Headers.Add("Proxy-Authorization", "Basic " + encodedApiKey);
//request.Proxy = proxy;
//request.PreAuthenticate = true;
request.Proxy = myProxy;
request.PreAuthenticate = true;
request.Method = "GET";
request.Accept = "application/json";
try
{
_doRequest += 1;
WebResponse response = request.GetResponse();
HttpWebResponse httpResponse = (HttpWebResponse)response;
statusCode = httpResponse.StatusCode;
//Console.WriteLine("Response Status: "
// + ((HttpWebResponse)response).StatusDescription);
//Console.WriteLine("\nResponse Headers:\n"
// + ((HttpWebResponse)response).Headers);
Stream dataStream = response.GetResponseStream();
string limit = response.Headers["X-RateLimit-Remaining"];
var resetTime = response.Headers["X-RateLimit-Reset"];
_limitRequst = Int32.Parse(limit);
_resetTimeLimits = UPCDB.UnixTimeStampToDateTime(Double.Parse(resetTime));
var reader = new StreamReader(dataStream);
string responseFromServer = reader.ReadToEnd();
//Console.WriteLine("Response Body:\n" + responseFromServer);
reader.Close();
response.Close();
JsonRootObject jsonObj = JsonConvert.DeserializeObject <JsonRootObject>(responseFromServer);
return(responseFromServer);
}
catch (WebException e)
{
HttpWebResponse httpResponse = (HttpWebResponse)e.Response;
statusCode = httpResponse.StatusCode;
if (statusCode == HttpStatusCode.NotFound)
{
//если артикул не найден, его нужно добавить в файл markets но без размеров, чтобы повторно его не запрашивать на следующий день парсинга
}
System.Threading.Thread.Sleep(1000 * _timeDelay);
return(null);
}
}