public void obhodFullCatalog()
{
int i = 0;
int j = 0;
foreach (var sneaker in fullCatalog.sneakers)
{
var jsonSneaker = myUPCDB.sneakers.Find(x => x.sku == sneaker.sku);
var stopListSku = MyStopList.skuList.Find(x => x == sneaker.sku);
if (jsonSneaker == null && stopListSku == null)
{
bool isLimits = false;
JsonRootObject json = GetAllSizes(sneaker.brand, sneaker.sku, out isLimits);
//if (_limitRequst == 0) isLimits = true;
if (isLimits)
{
Program.Logger.Info("Добавлено артикулов: " + i);
Program.Logger.Info("Добавлено размеров: " + j);
Program.Logger.Info("Выполнено запросов: " + _doRequest);
Program.Logger.Info("Остаток запросов: " + _limitRequst);
Program.Logger.Info("Время перезагрузки лимитов: " + _resetTimeLimits);
break;
}
if (json != null)
{
jsonSneaker = new SneakerJson();
jsonSneaker.sku = sneaker.sku;
jsonSneaker.title = sneaker.title;
jsonSneaker.brand = sneaker.brand;
jsonSneaker.sizes = json.items;
jsonSneaker.category = sneaker.category;
jsonSneaker.sex = sneaker.sex;
i += 1;
j += jsonSneaker.sizes.Count;
myUPCDB.sneakers.Add(jsonSneaker);
//todo убрать, сейчас по 1 кроссовку добавляется
//break;
Program.Logger.Info("Добавили артикул: " + sneaker.sku + ". Размеров: " + jsonSneaker.sizes.Count + ". Осталось запросов: " + _limitRequst);
}
else
{
bool test = true;
}
}
}
//Program.logger.Info("Обход фуллкаталога завершен!");
}
public string GetWithCrawlera(string Url, string Data, out HttpStatusCode statusCode)
{
ServicePointManager.ServerCertificateValidationCallback = (sender, cert, chain, sslPolicyErrors) => true;
var myProxy = new WebProxy("http://proxy.crawlera.com:8010");
myProxy.Credentials = new NetworkCredential("36f14b90c38c4005a81ccbed16a31f58", "");
//string url = "https://twitter.com/";
//string url = "https://api.upcitemdb.com/prod/trial/search?s=nike%20859524-005&match_mode=1&type=product";
string url = Url + "?" + Data;
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
var encodedApiKey = Helper.Base64Encode("36f14b90c38c4005a81ccbed16a31f58:");
request.Headers.Add("Proxy-Authorization", "Basic " + encodedApiKey);
//request.Proxy = proxy;
//request.PreAuthenticate = true;
request.Proxy = myProxy;
request.PreAuthenticate = true;
request.Method = "GET";
request.Accept = "application/json";
try
{
_doRequest += 1;
WebResponse response = request.GetResponse();
HttpWebResponse httpResponse = (HttpWebResponse)response;
statusCode = httpResponse.StatusCode;
//Console.WriteLine("Response Status: "
// + ((HttpWebResponse)response).StatusDescription);
//Console.WriteLine("\nResponse Headers:\n"
// + ((HttpWebResponse)response).Headers);
Stream dataStream = response.GetResponseStream();
string limit = response.Headers["X-RateLimit-Remaining"];
var resetTime = response.Headers["X-RateLimit-Reset"];
_limitRequst = Int32.Parse(limit);
_resetTimeLimits = UPCDB.UnixTimeStampToDateTime(Double.Parse(resetTime));
var reader = new StreamReader(dataStream);
string responseFromServer = reader.ReadToEnd();
//Console.WriteLine("Response Body:\n" + responseFromServer);
reader.Close();
response.Close();
JsonRootObject jsonObj = JsonConvert.DeserializeObject <JsonRootObject>(responseFromServer);
return(responseFromServer);
}
catch (WebException e)
{
HttpWebResponse httpResponse = (HttpWebResponse)e.Response;
statusCode = httpResponse.StatusCode;
if (statusCode == HttpStatusCode.NotFound)
{
//если артикул не найден, его нужно добавить в файл markets но без размеров, чтобы повторно его не запрашивать на следующий день парсинга
}
System.Threading.Thread.Sleep(1000 * _timeDelay);
return(null);
}
}
public JsonRootObject GetAllSizes2(string brand, string sku, out bool isLimits)
{
//http://www.upcitemdb.com/api/explorer#!/search/get_trial_search
//string brand = "nike";
//string sku = "859524-005";
isLimits = false;
string sku2 = sku.Replace("-", "%20");
int offset = 0;
string data = "s=" + brand + "%20" + sku2 + "&offset=" + offset + "&match_mode=1&type=product";
HttpStatusCode statusCode;
//string textJson = GET("https://api.upcitemdb.com/prod/trial/search", data, out statusCode);
string textJson = GetWithCrawlera("https://api.upcitemdb.com/prod/trial/search", data, out statusCode);
if (!string.IsNullOrEmpty(textJson))
{
JsonRootObject jsonObj = JsonConvert.DeserializeObject <JsonRootObject>(textJson);
if (jsonObj.code == "OK")
{
if (jsonObj.total <= 5)
{
return(jsonObj);
}
else
{
var currentOffset = jsonObj.offset;
do
{
data = "s=" + brand + "%20" + sku2 + "&offset=" + currentOffset + "&match_mode=1&type=product";
//textJson = GET("https://api.upcitemdb.com/prod/trial/search", data, out statusCode);
textJson = GetWithCrawlera("https://api.upcitemdb.com/prod/trial/search", data, out statusCode);
if (!string.IsNullOrEmpty(textJson))
{
JsonRootObject currentJsonObj = JsonConvert.DeserializeObject <JsonRootObject>(textJson);
foreach (var item in currentJsonObj.items)
{
jsonObj.items.Add(item);
}
currentOffset = currentJsonObj.offset;
}
else
{
return(null);
}
}while (currentOffset != 0);
return(jsonObj);
//todo дописать чтобы норм работало если больше 5 резульатов
//return null;
}
}
else
{
return(null);
}
}
else
{
//если ошибка в запросе и вернул пустую строку
if (statusCode == HttpStatusCode.NotFound)
{
JsonRootObject json = new JsonRootObject();
json.items = new List <JsonItem>();
return(json);
}
if ((int)statusCode == 429)
{
MyStopList.skuList.Add(sku);
isLimits = true;
return(null);
}
return(null);
}
}
