/// <summary>
/// 爬取数据
/// </summary>
public static void CrawlHotMovie()
{
Task.Factory.StartNew(() =>
{
try
{
LogHelper.Info("CrawlHotMovie Start...");
var htmlDoc = HTTPHelper.GetHTMLByURL("http://www.dy2018.com/");
var dom = htmlParser.Parse(htmlDoc);
var lstDivInfo = dom.QuerySelectorAll("div.co_content222");
if (lstDivInfo != null)
{
//前三个DIV为新电影
foreach (var divInfo in lstDivInfo.Take(3))
{
divInfo.QuerySelectorAll("a").Where(a => a.GetAttribute("href").Contains("/i/")).ToList().ForEach(
a =>
{
var onlineURL = "http://www.dy2018.com" + a.GetAttribute("href");
if (!hotMovieList.IsContainsMoive(onlineURL))
{
MovieInfo movieInfo = Dy2018MoviceInfoHelper.GetMovieInfoFromOnlineURL(onlineURL);
if (movieInfo != null && movieInfo.XunLeiDownLoadURLList != null && movieInfo.XunLeiDownLoadURLList.Count != 0)
{
hotMovieList.AddToMovieDic(movieInfo);
}
}
});
}
}
LogHelper.Info("CrawlHotMovie Finish...");
}
catch (Exception ex)
{
LogHelper.Error("CrawlHotMovie Exception", ex);
}
});
}
/// <summary>
/// 从在线网页提取数据
/// </summary>
/// <param name="i"></param>
private static void FillMovieFromOnline(string indexURL)
{
var htmlDoc = HTTPHelper.GetHTMLByURL(indexURL);
var dom = htmlParser.Parse(htmlDoc);
var lstDivInfo = dom.QuerySelectorAll("div.co_content8");
if (lstDivInfo != null)
{
lstDivInfo.FirstOrDefault().QuerySelectorAll("a").Where(a => a.GetAttribute("href").Contains("/i/")).ToList()
.ForEach(a =>
{
var onlineURL = "http://www.dy2018.com" + a.GetAttribute("href");
if (!latestMovieList.IsContainsMoive(onlineURL))
{
MovieInfo movieInfo = Dy2018MoviceInfoHelper.GetMovieInfoFromOnlineURL(onlineURL);
if (movieInfo != null && movieInfo.XunLeiDownLoadURLList != null && movieInfo.XunLeiDownLoadURLList.Count != 0)
{
latestMovieList.AddToMovieDic(movieInfo);
}
}
});
}
}