protected override Task <bool> ExecuteAsync(PipelineContext context)
{
if (context.Site != null)
{
var requestSite = context.Site;
requestSite.Referer = "https://wannianrili.bmcx.com/";
requestSite.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8";
requestSite.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36";
// 重试3次。
Page responsePage = null;
var index = 4;
do
{
if (index < 4)
{
Logger.Warn($"重试：{requestSite.Url}");
if (responsePage != null)
{
Logger.Trace(responsePage.HtmlSource);
}
Thread.Sleep(60 * 1000);
}
responsePage = Options.Downloader.GetPage(requestSite);
index--;
} while (index > 0 && (responsePage.HttpStatusCode != 200 || responsePage.DocumentNode == null));
if (responsePage.HttpStatusCode == 200 && responsePage.DocumentNode != null)
{
//获取基础信息
var allDayList = responsePage.DocumentNode.SelectNodes("//div[@class=\"wnrl_k_you\"]");
//获取详细信息
var allDayDetailList = responsePage.DocumentNode.SelectNodes("//div[@class=\"wnrl_k_xia_nr\"]");
if (allDayList != null && allDayList.Count > 0)
{
var month = new HuangliMonth();
AnalysisHelper.AnalysisMonth(month, responsePage);
for (int i = 0; i < allDayList.Count; i++)
{
var day = AnalysisHelper.GetDay(allDayList[i], responsePage, i);
month.HuangliDays.Add(day);
}
if (allDayDetailList != null && allDayDetailList.Count > 0)
{
for (int i = 0; i < allDayDetailList.Count; i++)
{
var daydetail = AnalysisHelper.GetDayDetail(allDayDetailList[i], responsePage, i);
month.HuangliDaysDetails.Add(daydetail);
}
}
foreach (var item in month.HuangliDays)
{
var detailInfo = month.HuangliDaysDetails.Where(x => x.Id == item.Id).FirstOrDefault();
item.JieQi = detailInfo.JieQi;
month.resultList.Add(item);
}
_huangliScheduler.Push(month);
Logger.Info($"{month.Date:yyyy-MM} 爬取页面完成");
return(Task.FromResult(true));
}
else
{
Logger.Fatal($"链接爬取失败：{requestSite.Url}");
}
}
else
{
Logger.Fatal($"链接爬取失败：{requestSite.Url}");
}
}
return(Task.FromResult(false));
}
public static void AnalysisMonth(HuangliMonth month, Page page)
{
var dateNode = page.DocumentNode.SelectSingleNode("//span[@class=\"wnrl_xuanze_top_wenzi\"]");
month.Date = DateTime.Parse(dateNode.InnerText);
}
