예제 #1
0
        private static void CrawlZeroPay()
        {
            StartTime = (long)(DateTime.UtcNow.Subtract(new DateTime(1970, 1, 1))).TotalSeconds;

            int pointerIndex = 1;

            WebClient crawlerClient = new WebClient {
                Encoding = Encoding.UTF8
            };

            while (true)
            {
                if (!_moduleRunFlag)
                {
                    LogManager.NewLog(LogType.WindowManager, LogLevel.Info, "CrawlZeroPay", "CrawlZeroPay Thread Stop : Safe");
                    break;
                }

                if (CrawlingData == null)
                {
                    LogManager.NewLog(LogType.WindowManager, LogLevel.Info, "CrawlZeroPay", "CrawlZeroPay Thread Stop : Collection Null");
                    break;
                }

                if (CrawlingData.Count > 0)
                {
                    for (int i = 0; i < CrawlingData.Count; i++)
                    {
                        CrawlData data = CrawlingData[i];
                        if (data.Status == CrawlDataResult.Finish)
                        {
                            continue;
                        }
                        if (data.Status == CrawlDataResult.Error || data.Status == CrawlDataResult.Queue)
                        {
                            continue;
                        }

                        data.Status = CrawlDataResult.Queue;
                        if (DatabaseManager.GetInstance().InsertShop(data))
                        {
                            CrawlingData[i].Status = CrawlDataResult.Finish;
                        }
                        else
                        {
                            CrawlingData[i].Status = CrawlDataResult.Error;
                            return;
                        }
                    }
                    for (int i = 0; i < CrawlingData.Count; i++)
                    {
                        CrawlData data = CrawlingData[i];
                        if (data.Status == CrawlDataResult.Finish)
                        {
                            CrawlingData.RemoveAt(i);
                            i--;
                        }
                    }
                }

                if (CrawlingData.Count >= 100)
                {
                    // TODO REFRESH HOLDING DATA STATUS
                    LogManager.NewLog(LogType.WindowManager, LogLevel.Info, "CrawlZeroPay", "CrawlZeroPay Parse Limit");
                    continue;
                }

                string requestURL = $"https://www.zeropay.or.kr/intro/frncSrchList_json.do?firstIndex={pointerIndex}&lastIndex={pointerIndex+9}&searchCondition=&tryCode={FilterSiDo:00}";
                if (FilterSiGunGu != 0)
                {
                    requestURL += $"&skkCode={FilterSiGunGu:00}";
                }
                if (FilterType != null && FilterType.Equals(string.Empty))
                {
                    requestURL += $"&bztypName={FilterType}";
                }
                if (FilterName != null && FilterName.Equals(string.Empty))
                {
                    requestURL += $"&pobsAfstrName={FilterName}";
                }
                var webResult = crawlerClient.DownloadString(requestURL);

                try
                {
                    JObject zeroJson = JObject.Parse(webResult);
                    JToken  zeroResult;
                    if (!zeroJson.TryGetValue("result", out zeroResult) || zeroResult.ToString().Equals("FAIL"))
                    {
                        LogManager.NewLog(LogType.WindowManager, LogLevel.Error, "CrawlZeroPay", "CrawlZeroPay Parse Fail");
                        break;
                    }

                    JToken zeroTotal;
                    if (!zeroJson.TryGetValue("totalCnt", out zeroTotal) || pointerIndex >= zeroTotal.Value <int>())
                    {
                        LogManager.NewLog(LogType.WindowManager, LogLevel.Info, "CrawlZeroPay", "CrawlZeroPay Parse Checked",
                                          $"Size : {zeroTotal}, SiDo : {SiDoDictionary.FirstOrDefault(x => x.Value == CrawlManager.FilterSiDo).Key}, " +
                                          $"SiGunGu : {SiGunGuDictionary[FilterSiDo].FirstOrDefault(x => x.Value == CrawlManager.FilterSiGunGu).Key}");
                        break;
                    }

                    JToken zeroList;
                    if (!zeroJson.TryGetValue("list", out zeroList) || (zeroList as JArray).Count == 0)
                    {
                        LogManager.NewLog(LogType.WindowManager, LogLevel.Info, "CrawlZeroPay", "CrawlZeroPay Parse Empty",
                                          $"Size : {zeroTotal}, SiDo : {SiDoDictionary.FirstOrDefault(x => x.Value == CrawlManager.FilterSiDo).Key}, " +
                                          $"SiGunGu : {SiGunGuDictionary[FilterSiDo].FirstOrDefault(x => x.Value == CrawlManager.FilterSiGunGu).Key}");
                        break;
                    }

                    int zeroListCount = zeroList.Count();
                    for (int i = 0; i < zeroListCount; i++)
                    {
                        JToken    zeroObject = zeroList[i];
                        CrawlData zeroData   = new CrawlData();
                        zeroData.Name = zeroObject["pobsAfstrName"].Value <string>();
                        zeroData.Type = zeroObject["bztypName"].Value <string>();
                        string zeroAddress = zeroObject["pobsBaseAddr"].Value <string>();
                        zeroAddress += " " + zeroObject["pobsDtlAddr"].Value <string>();
                        zeroAddress  = zeroAddress.Replace("  ", " ");
                        try
                        {
                            string[] zeroAddressArray = zeroAddress.Split(' ');
                            zeroData.AddressSiDo    = zeroAddressArray[0];
                            zeroData.AddressSiGunGu = zeroAddressArray[1];
                            zeroData.Address        = zeroAddress.Substring(zeroAddressArray[0].Length + zeroAddressArray[1].Length + 2);
                        }
                        catch (Exception exception)
                        {
                            zeroData.AddressSiDo    = string.Empty;
                            zeroData.AddressSiGunGu = string.Empty;
                            zeroData.Address        = zeroAddress;
                        }
                        zeroData.CodeSido    = GetCodeBySido(zeroData.AddressSiDo);
                        zeroData.CodeSigungu = GetCodeBySigungu(zeroData.CodeSido, zeroData.AddressSiGunGu);
                        zeroData.Status      = CrawlDataResult.Ready;
                        CrawlingData.Add(zeroData);
                    }
                }
                catch (Exception exception)
                {
                    LogManager.NewLog(LogType.WindowManager, LogLevel.Error, "CrawlZeroPay", "CrawlZeroPay Parse Error", exception.StackTrace);
                    break;
                }

                pointerIndex += 10;

                Application.Current.Dispatcher?.BeginInvoke(DispatcherPriority.Normal, new Action(WindowManager.UpdateCrawlData));
            }
        }
        public bool InsertShop(CrawlData crawlData)
        {
            Shop shop = new Shop(crawlData);

            return(InsertShop(shop));
        }