private void SaveResponse(FileInfo file, string url, SpiderTask spiderTask)
{
var dir = CreateDirectory(file);
foreach (SpiderTaskItem spiderTaskItem in spiderTask.SpiderTaskItems)
{
if (string.IsNullOrEmpty(spiderTaskItem.ResponseXml))
{
Info(string.Format("capture mercado response empty, url={0} , Task String = {1}", url, spiderTaskItem.TaskString));
continue;
}
string taskHash = MD5Hasher.Md5Hash(spiderTaskItem.TaskString);
if (AppConfig.Instance.LogTaskString)
{
logger.Log("==url==" + url);
logger.Log("==task has==" + taskHash);
logger.Log(spiderTaskItem.TaskString);
logger.Log("=========================================");
logger.FlushToFile();
}
//save response
string responseFile = Path.Combine(dir.FullName, taskHash + ".xml");
File.WriteAllText(responseFile, spiderTaskItem.ResponseXml);
//save mapping
string url_taskHas = taskHash + "\t" + url + Environment.NewLine;
File.AppendAllText(Path.Combine(dir.FullName, hashCodeUrlMapping), url_taskHas);
}
}
public void GetMercadoResponseWithMercadoTestPage(SpiderTask spiderTask)
{
List <string> taskStrings = new List <string>();
string pyramid, webType;
string mercadoTestUrl = GetMercadoTestUrl(spiderTask.Url, out pyramid, out taskStrings, out webType);
string postResponse = string.Empty;
if (webType == "qs")
{
foreach (string taskString in taskStrings)
{
string postData = BuildMVCPostData(taskString);
mercadoTestUrl = AddMercadoResultMethod(mercadoTestUrl);
postResponse = NetRequest.PostRequest(mercadoTestUrl, postData);
string mercadoXml = GetMercadoXmlFromPageSource(postResponse);
SpiderTaskItem spiderTaskItem = new SpiderTaskItem();
spiderTaskItem.TaskString = taskString;
spiderTaskItem.ResponseXml = mercadoXml;
spiderTask.SpiderTaskItems.Add(spiderTaskItem);
}
}
else
{
foreach (string taskString in taskStrings)
{
string testPageSource = NetRequest.GetRequest(mercadoTestUrl);
string eventId = GetEventIdFromPageSource(testPageSource);
string postData = BuildAspxPostData(eventId, taskString);
postResponse = NetRequest.PostRequest(mercadoTestUrl, postData);
string mercadoXml = GetMercadoXmlFromPageSource(postResponse);
SpiderTaskItem spiderTaskItem = new SpiderTaskItem();
spiderTaskItem.TaskString = taskString;
spiderTaskItem.ResponseXml = mercadoXml;
spiderTask.SpiderTaskItems.Add(spiderTaskItem);
}
}
}
public void GetMercadoResponseWithMercadoTestPage(SpiderTask spiderTask)
{
List<string> taskStrings = new List<string>();
string pyramid, webType;
string mercadoTestUrl = GetMercadoTestUrl(spiderTask.Url, out pyramid, out taskStrings, out webType);
string postResponse = string.Empty;
if (webType == "qs")
{
foreach (string taskString in taskStrings)
{
string postData = BuildMVCPostData(taskString);
mercadoTestUrl = AddMercadoResultMethod(mercadoTestUrl);
postResponse = NetRequest.PostRequest(mercadoTestUrl, postData);
string mercadoXml = GetMercadoXmlFromPageSource(postResponse);
SpiderTaskItem spiderTaskItem = new SpiderTaskItem();
spiderTaskItem.TaskString = taskString;
spiderTaskItem.ResponseXml = mercadoXml;
spiderTask.SpiderTaskItems.Add(spiderTaskItem);
}
}
else
{
foreach (string taskString in taskStrings)
{
string testPageSource = NetRequest.GetRequest(mercadoTestUrl);
string eventId = GetEventIdFromPageSource(testPageSource);
string postData = BuildAspxPostData(eventId, taskString);
postResponse = NetRequest.PostRequest(mercadoTestUrl, postData);
string mercadoXml = GetMercadoXmlFromPageSource(postResponse);
SpiderTaskItem spiderTaskItem = new SpiderTaskItem();
spiderTaskItem.TaskString = taskString;
spiderTaskItem.ResponseXml = mercadoXml;
spiderTask.SpiderTaskItems.Add(spiderTaskItem);
}
}
}
private void ReadResponse(FileInfo file)
{
string site = file.Name.Replace(file.Extension, "");
string taskDomain = AppConfig.Instance.GetTaskDomain(site);
if (string.IsNullOrEmpty(taskDomain))
{
Info("unknow site: " + site);
return;
}
string siteBase, searchInterFace;
siteBase = AppConfig.Instance.GetWebSiteConfig(site, out searchInterFace);
ConfigModify configModify = new ConfigModify(siteBase, searchInterFace);
configModify.Modify();
string[] lines = File.ReadAllLines(file.FullName);
QuidsiSearchSpider spider = new QuidsiSearchSpider();
foreach (string line in lines)
{
try
{
string url = line.Trim();
if (url == string.Empty) continue;
if (AppConfig.Instance.IsPLPUrl(url) == false)
{
Info("not a plp url: " + url);
continue;
}
string realUrl = url.Replace(UrlUtility.GetDomain(url), taskDomain);
SpiderTask spiderTask = new SpiderTask();
spiderTask.Url = realUrl;
spider.GetMercadoResponseWithMercadoTestPage(spiderTask);
if (spiderTask.SpiderTaskItems == null || spiderTask.SpiderTaskItems.Count == 0)
{
Info("capture task string empty, url= " + realUrl);
continue;
}
SaveResponse(file, realUrl, spiderTask);
Info("finish url: " + realUrl);
}
catch (Exception ex)
{
Info("error: " + ex.Message + "; url:" + line);
}
}
UnDunplicateMappingFile(file);
configModify.Restore();
}
private void SaveResponse(FileInfo file, string url, SpiderTask spiderTask)
{
var dir = CreateDirectory(file);
foreach (SpiderTaskItem spiderTaskItem in spiderTask.SpiderTaskItems)
{
if (string.IsNullOrEmpty(spiderTaskItem.ResponseXml))
{
Info(string.Format("capture mercado response empty, url={0} , Task String = {1}", url, spiderTaskItem.TaskString));
continue;
}
string taskHash = MD5Hasher.Md5Hash(spiderTaskItem.TaskString);
if (AppConfig.Instance.LogTaskString)
{
logger.Log("==url==" + url);
logger.Log("==task has==" + taskHash);
logger.Log(spiderTaskItem.TaskString);
logger.Log("=========================================");
logger.FlushToFile();
}
//save response
string responseFile = Path.Combine(dir.FullName, taskHash + ".xml");
File.WriteAllText(responseFile, spiderTaskItem.ResponseXml);
//save mapping
string url_taskHas = taskHash + "\t" + url + Environment.NewLine;
File.AppendAllText(Path.Combine(dir.FullName, hashCodeUrlMapping), url_taskHas);
}
}
private void ReadResponse(FileInfo file)
{
string site = file.Name.Replace(file.Extension, "");
string taskDomain = AppConfig.Instance.GetTaskDomain(site);
if (string.IsNullOrEmpty(taskDomain))
{
Info("unknow site: " + site);
return;
}
string siteBase, searchInterFace;
siteBase = AppConfig.Instance.GetWebSiteConfig(site, out searchInterFace);
ConfigModify configModify = new ConfigModify(siteBase, searchInterFace);
configModify.Modify();
string[] lines = File.ReadAllLines(file.FullName);
QuidsiSearchSpider spider = new QuidsiSearchSpider();
foreach (string line in lines)
{
try
{
string url = line.Trim();
if (url == string.Empty)
{
continue;
}
if (AppConfig.Instance.IsPLPUrl(url) == false)
{
Info("not a plp url: " + url);
continue;
}
string realUrl = url.Replace(UrlUtility.GetDomain(url), taskDomain);
SpiderTask spiderTask = new SpiderTask();
spiderTask.Url = realUrl;
spider.GetMercadoResponseWithMercadoTestPage(spiderTask);
if (spiderTask.SpiderTaskItems == null || spiderTask.SpiderTaskItems.Count == 0)
{
Info("capture task string empty, url= " + realUrl);
continue;
}
SaveResponse(file, realUrl, spiderTask);
Info("finish url: " + realUrl);
}
catch (Exception ex)
{
Info("error: " + ex.Message + "; url:" + line);
}
}
UnDunplicateMappingFile(file);
configModify.Restore();
}
