public SpiderTests() { var credentials = GetCredentials(); var repository = new SpiderRepository(credentials); _spiderAppService = new SpiderAppService(repository); }
public override void Process(ResultItems resultItems) { var repository = new SpiderRepository(); foreach (WandaFilm entry in resultItems.Results["VideoResult"]) { repository.SaveFilmData(new Models.Movie() { CreatedBy = "SpiderMachine", CreatedDate = DateTime.Now, Description = entry.Description, Name = entry.MovieName, ShowDate = entry.MovieTime, DetailsUrl = entry.Url, FilmId = entry.FilmId, CanWatch = entry.CanWatch }); } // Other actions like save data to DB. 可以自由实现插入数据库或保存到文件 }
public static void WandaDetailsSpiderRun() { var rep = new SpiderRepository(); var site = new Site { EncodingName = "UTF-8", RemoveOutboundLinks = true }; foreach (var item in rep.GetFilmData()) { site.AddStartUrl(item.DetailsUrl); } Spider spider = Spider.Create(site, new QueueDuplicateRemovedScheduler(), new WandaDetailPageProcessor()) .AddPipeline(new WandaDetailPipeline()) .SetDownloader(new HttpClientDownloader()) .SetThreadNum(1); spider.EmptySleepTime = 3000; spider.Run(); }