private static void CrawlZeroPay()
{
StartTime = (long)(DateTime.UtcNow.Subtract(new DateTime(1970, 1, 1))).TotalSeconds;
int pointerIndex = 1;
WebClient crawlerClient = new WebClient {
Encoding = Encoding.UTF8
};
while (true)
{
if (!_moduleRunFlag)
{
LogManager.NewLog(LogType.WindowManager, LogLevel.Info, "CrawlZeroPay", "CrawlZeroPay Thread Stop : Safe");
break;
}
if (CrawlingData == null)
{
LogManager.NewLog(LogType.WindowManager, LogLevel.Info, "CrawlZeroPay", "CrawlZeroPay Thread Stop : Collection Null");
break;
}
if (CrawlingData.Count > 0)
{
for (int i = 0; i < CrawlingData.Count; i++)
{
CrawlData data = CrawlingData[i];
if (data.Status == CrawlDataResult.Finish)
{
continue;
}
if (data.Status == CrawlDataResult.Error || data.Status == CrawlDataResult.Queue)
{
continue;
}
data.Status = CrawlDataResult.Queue;
if (DatabaseManager.GetInstance().InsertShop(data))
{
CrawlingData[i].Status = CrawlDataResult.Finish;
}
else
{
CrawlingData[i].Status = CrawlDataResult.Error;
return;
}
}
for (int i = 0; i < CrawlingData.Count; i++)
{
CrawlData data = CrawlingData[i];
if (data.Status == CrawlDataResult.Finish)
{
CrawlingData.RemoveAt(i);
i--;
}
}
}
if (CrawlingData.Count >= 100)
{
// TODO REFRESH HOLDING DATA STATUS
LogManager.NewLog(LogType.WindowManager, LogLevel.Info, "CrawlZeroPay", "CrawlZeroPay Parse Limit");
continue;
}
string requestURL = $"https://www.zeropay.or.kr/intro/frncSrchList_json.do?firstIndex={pointerIndex}&lastIndex={pointerIndex+9}&searchCondition=&tryCode={FilterSiDo:00}";
if (FilterSiGunGu != 0)
{
requestURL += $"&skkCode={FilterSiGunGu:00}";
}
if (FilterType != null && FilterType.Equals(string.Empty))
{
requestURL += $"&bztypName={FilterType}";
}
if (FilterName != null && FilterName.Equals(string.Empty))
{
requestURL += $"&pobsAfstrName={FilterName}";
}
var webResult = crawlerClient.DownloadString(requestURL);
try
{
JObject zeroJson = JObject.Parse(webResult);
JToken zeroResult;
if (!zeroJson.TryGetValue("result", out zeroResult) || zeroResult.ToString().Equals("FAIL"))
{
LogManager.NewLog(LogType.WindowManager, LogLevel.Error, "CrawlZeroPay", "CrawlZeroPay Parse Fail");
break;
}
JToken zeroTotal;
if (!zeroJson.TryGetValue("totalCnt", out zeroTotal) || pointerIndex >= zeroTotal.Value <int>())
{
LogManager.NewLog(LogType.WindowManager, LogLevel.Info, "CrawlZeroPay", "CrawlZeroPay Parse Checked",
$"Size : {zeroTotal}, SiDo : {SiDoDictionary.FirstOrDefault(x => x.Value == CrawlManager.FilterSiDo).Key}, " +
$"SiGunGu : {SiGunGuDictionary[FilterSiDo].FirstOrDefault(x => x.Value == CrawlManager.FilterSiGunGu).Key}");
break;
}
JToken zeroList;
if (!zeroJson.TryGetValue("list", out zeroList) || (zeroList as JArray).Count == 0)
{
LogManager.NewLog(LogType.WindowManager, LogLevel.Info, "CrawlZeroPay", "CrawlZeroPay Parse Empty",
$"Size : {zeroTotal}, SiDo : {SiDoDictionary.FirstOrDefault(x => x.Value == CrawlManager.FilterSiDo).Key}, " +
$"SiGunGu : {SiGunGuDictionary[FilterSiDo].FirstOrDefault(x => x.Value == CrawlManager.FilterSiGunGu).Key}");
break;
}
int zeroListCount = zeroList.Count();
for (int i = 0; i < zeroListCount; i++)
{
JToken zeroObject = zeroList[i];
CrawlData zeroData = new CrawlData();
zeroData.Name = zeroObject["pobsAfstrName"].Value <string>();
zeroData.Type = zeroObject["bztypName"].Value <string>();
string zeroAddress = zeroObject["pobsBaseAddr"].Value <string>();
zeroAddress += " " + zeroObject["pobsDtlAddr"].Value <string>();
zeroAddress = zeroAddress.Replace(" ", " ");
try
{
string[] zeroAddressArray = zeroAddress.Split(' ');
zeroData.AddressSiDo = zeroAddressArray[0];
zeroData.AddressSiGunGu = zeroAddressArray[1];
zeroData.Address = zeroAddress.Substring(zeroAddressArray[0].Length + zeroAddressArray[1].Length + 2);
}
catch (Exception exception)
{
zeroData.AddressSiDo = string.Empty;
zeroData.AddressSiGunGu = string.Empty;
zeroData.Address = zeroAddress;
}
zeroData.CodeSido = GetCodeBySido(zeroData.AddressSiDo);
zeroData.CodeSigungu = GetCodeBySigungu(zeroData.CodeSido, zeroData.AddressSiGunGu);
zeroData.Status = CrawlDataResult.Ready;
CrawlingData.Add(zeroData);
}
}
catch (Exception exception)
{
LogManager.NewLog(LogType.WindowManager, LogLevel.Error, "CrawlZeroPay", "CrawlZeroPay Parse Error", exception.StackTrace);
break;
}
pointerIndex += 10;
Application.Current.Dispatcher?.BeginInvoke(DispatcherPriority.Normal, new Action(WindowManager.UpdateCrawlData));
}
}
public bool InsertShop(CrawlData crawlData)
{
Shop shop = new Shop(crawlData);
return(InsertShop(shop));
}