private void SaveResponse(FileInfo file, string url, SpiderTask spiderTask) { var dir = CreateDirectory(file); foreach (SpiderTaskItem spiderTaskItem in spiderTask.SpiderTaskItems) { if (string.IsNullOrEmpty(spiderTaskItem.ResponseXml)) { Info(string.Format("capture mercado response empty, url={0} , Task String = {1}", url, spiderTaskItem.TaskString)); continue; } string taskHash = MD5Hasher.Md5Hash(spiderTaskItem.TaskString); if (AppConfig.Instance.LogTaskString) { logger.Log("==url==" + url); logger.Log("==task has==" + taskHash); logger.Log(spiderTaskItem.TaskString); logger.Log("========================================="); logger.FlushToFile(); } //save response string responseFile = Path.Combine(dir.FullName, taskHash + ".xml"); File.WriteAllText(responseFile, spiderTaskItem.ResponseXml); //save mapping string url_taskHas = taskHash + "\t" + url + Environment.NewLine; File.AppendAllText(Path.Combine(dir.FullName, hashCodeUrlMapping), url_taskHas); } }
public void GetMercadoResponseWithMercadoTestPage(SpiderTask spiderTask) { List <string> taskStrings = new List <string>(); string pyramid, webType; string mercadoTestUrl = GetMercadoTestUrl(spiderTask.Url, out pyramid, out taskStrings, out webType); string postResponse = string.Empty; if (webType == "qs") { foreach (string taskString in taskStrings) { string postData = BuildMVCPostData(taskString); mercadoTestUrl = AddMercadoResultMethod(mercadoTestUrl); postResponse = NetRequest.PostRequest(mercadoTestUrl, postData); string mercadoXml = GetMercadoXmlFromPageSource(postResponse); SpiderTaskItem spiderTaskItem = new SpiderTaskItem(); spiderTaskItem.TaskString = taskString; spiderTaskItem.ResponseXml = mercadoXml; spiderTask.SpiderTaskItems.Add(spiderTaskItem); } } else { foreach (string taskString in taskStrings) { string testPageSource = NetRequest.GetRequest(mercadoTestUrl); string eventId = GetEventIdFromPageSource(testPageSource); string postData = BuildAspxPostData(eventId, taskString); postResponse = NetRequest.PostRequest(mercadoTestUrl, postData); string mercadoXml = GetMercadoXmlFromPageSource(postResponse); SpiderTaskItem spiderTaskItem = new SpiderTaskItem(); spiderTaskItem.TaskString = taskString; spiderTaskItem.ResponseXml = mercadoXml; spiderTask.SpiderTaskItems.Add(spiderTaskItem); } } }
public void GetMercadoResponseWithMercadoTestPage(SpiderTask spiderTask) { List<string> taskStrings = new List<string>(); string pyramid, webType; string mercadoTestUrl = GetMercadoTestUrl(spiderTask.Url, out pyramid, out taskStrings, out webType); string postResponse = string.Empty; if (webType == "qs") { foreach (string taskString in taskStrings) { string postData = BuildMVCPostData(taskString); mercadoTestUrl = AddMercadoResultMethod(mercadoTestUrl); postResponse = NetRequest.PostRequest(mercadoTestUrl, postData); string mercadoXml = GetMercadoXmlFromPageSource(postResponse); SpiderTaskItem spiderTaskItem = new SpiderTaskItem(); spiderTaskItem.TaskString = taskString; spiderTaskItem.ResponseXml = mercadoXml; spiderTask.SpiderTaskItems.Add(spiderTaskItem); } } else { foreach (string taskString in taskStrings) { string testPageSource = NetRequest.GetRequest(mercadoTestUrl); string eventId = GetEventIdFromPageSource(testPageSource); string postData = BuildAspxPostData(eventId, taskString); postResponse = NetRequest.PostRequest(mercadoTestUrl, postData); string mercadoXml = GetMercadoXmlFromPageSource(postResponse); SpiderTaskItem spiderTaskItem = new SpiderTaskItem(); spiderTaskItem.TaskString = taskString; spiderTaskItem.ResponseXml = mercadoXml; spiderTask.SpiderTaskItems.Add(spiderTaskItem); } } }
private void ReadResponse(FileInfo file) { string site = file.Name.Replace(file.Extension, ""); string taskDomain = AppConfig.Instance.GetTaskDomain(site); if (string.IsNullOrEmpty(taskDomain)) { Info("unknow site: " + site); return; } string siteBase, searchInterFace; siteBase = AppConfig.Instance.GetWebSiteConfig(site, out searchInterFace); ConfigModify configModify = new ConfigModify(siteBase, searchInterFace); configModify.Modify(); string[] lines = File.ReadAllLines(file.FullName); QuidsiSearchSpider spider = new QuidsiSearchSpider(); foreach (string line in lines) { try { string url = line.Trim(); if (url == string.Empty) continue; if (AppConfig.Instance.IsPLPUrl(url) == false) { Info("not a plp url: " + url); continue; } string realUrl = url.Replace(UrlUtility.GetDomain(url), taskDomain); SpiderTask spiderTask = new SpiderTask(); spiderTask.Url = realUrl; spider.GetMercadoResponseWithMercadoTestPage(spiderTask); if (spiderTask.SpiderTaskItems == null || spiderTask.SpiderTaskItems.Count == 0) { Info("capture task string empty, url= " + realUrl); continue; } SaveResponse(file, realUrl, spiderTask); Info("finish url: " + realUrl); } catch (Exception ex) { Info("error: " + ex.Message + "; url:" + line); } } UnDunplicateMappingFile(file); configModify.Restore(); }
private void ReadResponse(FileInfo file) { string site = file.Name.Replace(file.Extension, ""); string taskDomain = AppConfig.Instance.GetTaskDomain(site); if (string.IsNullOrEmpty(taskDomain)) { Info("unknow site: " + site); return; } string siteBase, searchInterFace; siteBase = AppConfig.Instance.GetWebSiteConfig(site, out searchInterFace); ConfigModify configModify = new ConfigModify(siteBase, searchInterFace); configModify.Modify(); string[] lines = File.ReadAllLines(file.FullName); QuidsiSearchSpider spider = new QuidsiSearchSpider(); foreach (string line in lines) { try { string url = line.Trim(); if (url == string.Empty) { continue; } if (AppConfig.Instance.IsPLPUrl(url) == false) { Info("not a plp url: " + url); continue; } string realUrl = url.Replace(UrlUtility.GetDomain(url), taskDomain); SpiderTask spiderTask = new SpiderTask(); spiderTask.Url = realUrl; spider.GetMercadoResponseWithMercadoTestPage(spiderTask); if (spiderTask.SpiderTaskItems == null || spiderTask.SpiderTaskItems.Count == 0) { Info("capture task string empty, url= " + realUrl); continue; } SaveResponse(file, realUrl, spiderTask); Info("finish url: " + realUrl); } catch (Exception ex) { Info("error: " + ex.Message + "; url:" + line); } } UnDunplicateMappingFile(file); configModify.Restore(); }