static async Task Main(string[] args) { var configure = new LoggerConfiguration() #if DEBUG .MinimumLevel.Verbose() #else .MinimumLevel.Information() #endif .MinimumLevel.Override("Microsoft", LogEventLevel.Warning) .Enrich.FromLogContext() .WriteTo.Console().WriteTo .RollingFile("dotnet-spider.log"); Log.Logger = configure.CreateLogger(); await BaseUsage.Run(); // await DistributedSpider.Run(); Console.Read(); }
public static void Main(string[] args) { RegexTestEntitySpider spider2 = new RegexTestEntitySpider(); spider2.Run(); EntitySpider spider = new EntitySpider(new Core.Site()); spider.AddStartUrl("http://www.baidu.com"); // Custmize processor and pipeline 完全自定义页面解析和数据管道 BaseUsage.CustmizeProcessorAndPipeline(); Console.WriteLine("Press any key to continue..."); Console.Read(); // Crawler pages without traverse 采集指定页面不做遍历 BaseUsage.CrawlerPagesWithoutTraverse(); Console.WriteLine("Press any key to continue..."); Console.Read(); // Crawler pages traversal 遍历整站 BaseUsage.CrawlerPagesTraversal(); Console.WriteLine("Press any key to continue..."); Console.Read(); DDengEntitySpider dDengEntitySpider = new DDengEntitySpider(); dDengEntitySpider.Run(); Console.WriteLine("Press any key to continue..."); Console.Read(); Cnblogs.Run(); Console.WriteLine("Press any key to continue..."); Console.Read(); CasSpider casSpider = new CasSpider(); casSpider.Run(); Console.WriteLine("Press any key to continue..."); Console.Read(); BaiduSearchSpider baiduSearchSpider = new BaiduSearchSpider(); baiduSearchSpider.Run(); Console.WriteLine("Press any key to continue..."); Console.Read(); JdShopDetailSpider jdShopDetailSpider = new JdShopDetailSpider(); jdShopDetailSpider.Run(); Console.WriteLine("Press any key to continue..."); Console.Read(); JdSkuSampleSpider jdSkuSampleSpider = new JdSkuSampleSpider(); jdSkuSampleSpider.Run(); Console.WriteLine("Press any key to continue..."); Console.Read(); Situoli.Run(); }
private static void MyTest() { BaseUsage.CustmizeProcessorAndPipeline(); }
public static void Main(string[] args) { BaseUsage.Run(); }
public static void Main(string[] args) { //ShanxizhaoshengSpider spider = new ShanxizhaoshengSpider(); //spider.Run(); // Crawler pages traversal 遍历整站 BaseUsage.CrawlerPagesTraversal(); Console.WriteLine("Press any key to continue..."); Console.Read(); // Custmize processor and pipeline 完全自定义页面解析和数据管道 BaseUsage.CustmizeProcessorAndPipeline(); Console.WriteLine("Press any key to continue..."); Console.Read(); // Crawler pages without traverse 采集指定页面不做遍历 BaseUsage.CrawlerPagesWithoutTraverse(); Console.WriteLine("Press any key to continue..."); Console.Read(); // Crawler pages traversal 遍历整站 BaseUsage.CrawlerPagesTraversal(); Console.WriteLine("Press any key to continue..."); Console.Read(); DDengEntitySpider dDengEntitySpider = new DDengEntitySpider(); dDengEntitySpider.Run(); Console.WriteLine("Press any key to continue..."); Console.Read(); Cnblogs.Run(); Console.WriteLine("Press any key to continue..."); Console.Read(); CasSpider casSpider = new CasSpider(); casSpider.Run(); Console.WriteLine("Press any key to continue..."); Console.Read(); BaiduSearchSpider baiduSearchSpider = new BaiduSearchSpider(); baiduSearchSpider.Run(); Console.WriteLine("Press any key to continue..."); Console.Read(); JdShopDetailSpider jdShopDetailSpider = new JdShopDetailSpider(); jdShopDetailSpider.Run(); Console.WriteLine("Press any key to continue..."); Console.Read(); JdSkuSampleSpider jdSkuSampleSpider = new JdSkuSampleSpider(); jdSkuSampleSpider.Run(); Console.WriteLine("Press any key to continue..."); Console.Read(); Situoli.Run(); }