Exemplo n.º 1
0
        public void obhodFullCatalog()
        {
            int i = 0;
            int j = 0;

            foreach (var sneaker in fullCatalog.sneakers)
            {
                var jsonSneaker = myUPCDB.sneakers.Find(x => x.sku == sneaker.sku);
                var stopListSku = MyStopList.skuList.Find(x => x == sneaker.sku);
                if (jsonSneaker == null && stopListSku == null)
                {
                    bool           isLimits = false;
                    JsonRootObject json     = GetAllSizes(sneaker.brand, sneaker.sku, out isLimits);
                    //if (_limitRequst == 0) isLimits = true;
                    if (isLimits)
                    {
                        Program.Logger.Info("Добавлено артикулов: " + i);
                        Program.Logger.Info("Добавлено размеров: " + j);
                        Program.Logger.Info("Выполнено запросов: " + _doRequest);
                        Program.Logger.Info("Остаток запросов: " + _limitRequst);
                        Program.Logger.Info("Время перезагрузки лимитов: " + _resetTimeLimits);
                        break;
                    }

                    if (json != null)
                    {
                        jsonSneaker          = new SneakerJson();
                        jsonSneaker.sku      = sneaker.sku;
                        jsonSneaker.title    = sneaker.title;
                        jsonSneaker.brand    = sneaker.brand;
                        jsonSneaker.sizes    = json.items;
                        jsonSneaker.category = sneaker.category;
                        jsonSneaker.sex      = sneaker.sex;

                        i += 1;
                        j += jsonSneaker.sizes.Count;
                        myUPCDB.sneakers.Add(jsonSneaker);

                        //todo убрать, сейчас по 1 кроссовку добавляется
                        //break;
                        Program.Logger.Info("Добавили артикул: " + sneaker.sku + ". Размеров: " + jsonSneaker.sizes.Count + ". Осталось запросов: " + _limitRequst);
                    }
                    else
                    {
                        bool test = true;
                    }
                }
            }
            //Program.logger.Info("Обход фуллкаталога завершен!");
        }
Exemplo n.º 2
0
        public string GetWithCrawlera(string Url, string Data, out HttpStatusCode statusCode)
        {
            ServicePointManager.ServerCertificateValidationCallback = (sender, cert, chain, sslPolicyErrors) => true;

            var myProxy = new WebProxy("http://proxy.crawlera.com:8010");

            myProxy.Credentials = new NetworkCredential("36f14b90c38c4005a81ccbed16a31f58", "");

            //string url = "https://twitter.com/";
            //string url = "https://api.upcitemdb.com/prod/trial/search?s=nike%20859524-005&match_mode=1&type=product";
            string         url     = Url + "?" + Data;
            HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);

            var encodedApiKey = Helper.Base64Encode("36f14b90c38c4005a81ccbed16a31f58:");

            request.Headers.Add("Proxy-Authorization", "Basic " + encodedApiKey);
            //request.Proxy = proxy;
            //request.PreAuthenticate = true;

            request.Proxy           = myProxy;
            request.PreAuthenticate = true;

            request.Method = "GET";
            request.Accept = "application/json";

            try
            {
                _doRequest += 1;
                WebResponse response = request.GetResponse();

                HttpWebResponse httpResponse = (HttpWebResponse)response;
                statusCode = httpResponse.StatusCode;

                //Console.WriteLine("Response Status: "
                //  + ((HttpWebResponse)response).StatusDescription);
                //Console.WriteLine("\nResponse Headers:\n"
                //  + ((HttpWebResponse)response).Headers);

                Stream dataStream = response.GetResponseStream();

                string limit     = response.Headers["X-RateLimit-Remaining"];
                var    resetTime = response.Headers["X-RateLimit-Reset"];
                _limitRequst     = Int32.Parse(limit);
                _resetTimeLimits = UPCDB.UnixTimeStampToDateTime(Double.Parse(resetTime));

                var    reader             = new StreamReader(dataStream);
                string responseFromServer = reader.ReadToEnd();
                //Console.WriteLine("Response Body:\n" + responseFromServer);
                reader.Close();

                response.Close();

                JsonRootObject jsonObj = JsonConvert.DeserializeObject <JsonRootObject>(responseFromServer);
                return(responseFromServer);
            }
            catch (WebException e)
            {
                HttpWebResponse httpResponse = (HttpWebResponse)e.Response;
                statusCode = httpResponse.StatusCode;
                if (statusCode == HttpStatusCode.NotFound)
                {
                    //если артикул не найден, его нужно добавить в файл markets но без размеров, чтобы повторно его не запрашивать на следующий день парсинга
                }
                System.Threading.Thread.Sleep(1000 * _timeDelay);
                return(null);
            }
        }
Exemplo n.º 3
0
        public JsonRootObject GetAllSizes2(string brand, string sku, out bool isLimits)
        {
            //http://www.upcitemdb.com/api/explorer#!/search/get_trial_search

            //string brand = "nike";
            //string sku = "859524-005";
            isLimits = false;
            string         sku2   = sku.Replace("-", "%20");
            int            offset = 0;
            string         data   = "s=" + brand + "%20" + sku2 + "&offset=" + offset + "&match_mode=1&type=product";
            HttpStatusCode statusCode;
            //string textJson = GET("https://api.upcitemdb.com/prod/trial/search", data, out statusCode);
            string textJson = GetWithCrawlera("https://api.upcitemdb.com/prod/trial/search", data, out statusCode);

            if (!string.IsNullOrEmpty(textJson))
            {
                JsonRootObject jsonObj = JsonConvert.DeserializeObject <JsonRootObject>(textJson);
                if (jsonObj.code == "OK")
                {
                    if (jsonObj.total <= 5)
                    {
                        return(jsonObj);
                    }
                    else
                    {
                        var currentOffset = jsonObj.offset;
                        do
                        {
                            data = "s=" + brand + "%20" + sku2 + "&offset=" + currentOffset + "&match_mode=1&type=product";
                            //textJson = GET("https://api.upcitemdb.com/prod/trial/search", data, out statusCode);
                            textJson = GetWithCrawlera("https://api.upcitemdb.com/prod/trial/search", data, out statusCode);
                            if (!string.IsNullOrEmpty(textJson))
                            {
                                JsonRootObject currentJsonObj = JsonConvert.DeserializeObject <JsonRootObject>(textJson);

                                foreach (var item in currentJsonObj.items)
                                {
                                    jsonObj.items.Add(item);
                                }

                                currentOffset = currentJsonObj.offset;
                            }
                            else
                            {
                                return(null);
                            }
                        }while (currentOffset != 0);

                        return(jsonObj);
                        //todo дописать чтобы норм работало если больше 5 резульатов
                        //return null;
                    }
                }
                else
                {
                    return(null);
                }
            }
            else
            {
                //если ошибка в запросе и вернул пустую строку
                if (statusCode == HttpStatusCode.NotFound)
                {
                    JsonRootObject json = new JsonRootObject();
                    json.items = new List <JsonItem>();
                    return(json);
                }
                if ((int)statusCode == 429)
                {
                    MyStopList.skuList.Add(sku);
                    isLimits = true;
                    return(null);
                }
                return(null);
            }
        }