public SpiderTests()
{
var credentials = GetCredentials();
var repository = new SpiderRepository(credentials);
_spiderAppService = new SpiderAppService(repository);
}
public override void Process(ResultItems resultItems)
{
var repository = new SpiderRepository();
foreach (WandaFilm entry in resultItems.Results["VideoResult"])
{
repository.SaveFilmData(new Models.Movie()
{
CreatedBy = "SpiderMachine",
CreatedDate = DateTime.Now,
Description = entry.Description,
Name = entry.MovieName,
ShowDate = entry.MovieTime,
DetailsUrl = entry.Url,
FilmId = entry.FilmId,
CanWatch = entry.CanWatch
});
}
// Other actions like save data to DB. 可以自由实现插入数据库或保存到文件
}
public static void WandaDetailsSpiderRun()
{
var rep = new SpiderRepository();
var site = new Site {
EncodingName = "UTF-8", RemoveOutboundLinks = true
};
foreach (var item in rep.GetFilmData())
{
site.AddStartUrl(item.DetailsUrl);
}
Spider spider = Spider.Create(site,
new QueueDuplicateRemovedScheduler(),
new WandaDetailPageProcessor())
.AddPipeline(new WandaDetailPipeline())
.SetDownloader(new HttpClientDownloader())
.SetThreadNum(1);
spider.EmptySleepTime = 3000;
spider.Run();
}
