private void ReadResponse(FileInfo file) { string site = file.Name.Replace(file.Extension, ""); string taskDomain = AppConfig.Instance.GetTaskDomain(site); if (string.IsNullOrEmpty(taskDomain)) { Info("unknow site: " + site); return; } string siteBase, searchInterFace; siteBase = AppConfig.Instance.GetWebSiteConfig(site, out searchInterFace); ConfigModify configModify = new ConfigModify(siteBase, searchInterFace); configModify.Modify(); string[] lines = File.ReadAllLines(file.FullName); QuidsiSearchSpider spider = new QuidsiSearchSpider(); foreach (string line in lines) { try { string url = line.Trim(); if (url == string.Empty) continue; if (AppConfig.Instance.IsPLPUrl(url) == false) { Info("not a plp url: " + url); continue; } string realUrl = url.Replace(UrlUtility.GetDomain(url), taskDomain); SpiderTask spiderTask = new SpiderTask(); spiderTask.Url = realUrl; spider.GetMercadoResponseWithMercadoTestPage(spiderTask); if (spiderTask.SpiderTaskItems == null || spiderTask.SpiderTaskItems.Count == 0) { Info("capture task string empty, url= " + realUrl); continue; } SaveResponse(file, realUrl, spiderTask); Info("finish url: " + realUrl); } catch (Exception ex) { Info("error: " + ex.Message + "; url:" + line); } } UnDunplicateMappingFile(file); configModify.Restore(); }
private void ReadResponse(FileInfo file) { string site = file.Name.Replace(file.Extension, ""); string taskDomain = AppConfig.Instance.GetTaskDomain(site); if (string.IsNullOrEmpty(taskDomain)) { Info("unknow site: " + site); return; } string siteBase, searchInterFace; siteBase = AppConfig.Instance.GetWebSiteConfig(site, out searchInterFace); ConfigModify configModify = new ConfigModify(siteBase, searchInterFace); configModify.Modify(); string[] lines = File.ReadAllLines(file.FullName); QuidsiSearchSpider spider = new QuidsiSearchSpider(); foreach (string line in lines) { try { string url = line.Trim(); if (url == string.Empty) { continue; } if (AppConfig.Instance.IsPLPUrl(url) == false) { Info("not a plp url: " + url); continue; } string realUrl = url.Replace(UrlUtility.GetDomain(url), taskDomain); SpiderTask spiderTask = new SpiderTask(); spiderTask.Url = realUrl; spider.GetMercadoResponseWithMercadoTestPage(spiderTask); if (spiderTask.SpiderTaskItems == null || spiderTask.SpiderTaskItems.Count == 0) { Info("capture task string empty, url= " + realUrl); continue; } SaveResponse(file, realUrl, spiderTask); Info("finish url: " + realUrl); } catch (Exception ex) { Info("error: " + ex.Message + "; url:" + line); } } UnDunplicateMappingFile(file); configModify.Restore(); }