public static Task Run() { //var spider = Create<VnexpressSpider>(); var builder = new SpiderBuilder(); builder.AddSerilog(); builder.ConfigureAppConfiguration(); builder.UseStandalone(); var settings = new ProjectDefinition() { ProjectName = "Vnexpress Spider", Site = "Vnexpress/Kinh Doanh", ItemUrlsSelector = "", Urls = "https://vnexpress.net/kinh-doanh", FileFormat = "*.html", FileStorage = @"P:\Neil.Test\Spider Storage\Vnexpress", PageLimit = 4, }; builder.Services.AddSingleton <ProjectDefinition>(settings); builder.AddSpider <VnexpressSpider>(); // builder.Services.AddSingleton<IDynamicMessageQueue, InMemoryMessageQueue>((s)=> null); //builder.Services.AddSingleton<IDynamicMessageQueue,InMemoryMessageQueue>(); builder.UseDynamicMessageQueue(); var factory = builder.Build(); var spider = factory.Create <VnexpressSpider>(); return(spider.RunAsync()); }
public static Task Run() { //var spider = Create<VnexpressSpider>(); var builder = new SpiderBuilder(); builder.AddSerilog(); builder.Services.AddSingleton <IProxyValidator, FakeProxyValidator>(); //builder.Services.AddSingleton<IProxyValidator, DefaultProxyValidator>(); builder.ConfigureAppConfiguration(null, args: new string[] { "/ProxySupplyUrl=http://localhost:52445/api/proxies" }, true); builder.UseStandalone(); var settings = new ProjectDefinition() { ProjectName = "Vnexpress Spider", Site = "Vnexpress/Kinh Doanh", ItemUrlsSelector = "//article/h1[@class='title_news']/a[1];//article[@class='list_news']/h4[@class='title_news']/a[1]", Urls = "https://vnexpress.net/kinh-doanh", FileStorage = @"P:\Neil.Test\Spider Storage\Vnexpress", FileFormat = "*.json", PageLimit = 4, Deepth = 2, NextPageSelector = "//p[@id='pagination']/a[@class='next']", NumberOfConcurrentRequests = 5, Mapping = new ItemMapping { ItemCssSelector = "//section[@id='left_calculator']", Mapping = new FieldMapping[] { new FieldMapping { Field = "Title", CssSelector = "//h1[@class='title_news_detail mb10']" }, new FieldMapping { Field = "Description", CssSelector = "//p[@class='description']" }, } } }; builder.Services.AddSingleton <ProjectDefinition>(settings); builder.AddSpider <HttpClientSpider>(); // builder.Services.AddSingleton<IDynamicMessageQueue, InMemoryMessageQueue>((s)=> null); //builder.Services.AddSingleton<IDynamicMessageQueue,InMemoryMessageQueue>(); builder.UseDynamicMessageQueue(); var factory = builder.Build(); var spider = factory.Create <HttpClientSpider>(); return(spider.RunAsync()); }