static async Task Main(string[] args)
{
var configure = new LoggerConfiguration()
#if DEBUG
.MinimumLevel.Verbose()
#else
.MinimumLevel.Information()
#endif
.MinimumLevel.Override("Microsoft", LogEventLevel.Warning)
.Enrich.FromLogContext()
.WriteTo.Console().WriteTo
.RollingFile("dotnet-spider.log");
Log.Logger = configure.CreateLogger();
await BaseUsage.Run();
// await DistributedSpider.Run();
Console.Read();
}
public static void Main(string[] args)
{
RegexTestEntitySpider spider2 = new RegexTestEntitySpider();
spider2.Run();
EntitySpider spider = new EntitySpider(new Core.Site());
spider.AddStartUrl("http://www.baidu.com");
// Custmize processor and pipeline 完全自定义页面解析和数据管道
BaseUsage.CustmizeProcessorAndPipeline();
Console.WriteLine("Press any key to continue...");
Console.Read();
// Crawler pages without traverse 采集指定页面不做遍历
BaseUsage.CrawlerPagesWithoutTraverse();
Console.WriteLine("Press any key to continue...");
Console.Read();
// Crawler pages traversal 遍历整站
BaseUsage.CrawlerPagesTraversal();
Console.WriteLine("Press any key to continue...");
Console.Read();
DDengEntitySpider dDengEntitySpider = new DDengEntitySpider();
dDengEntitySpider.Run();
Console.WriteLine("Press any key to continue...");
Console.Read();
Cnblogs.Run();
Console.WriteLine("Press any key to continue...");
Console.Read();
CasSpider casSpider = new CasSpider();
casSpider.Run();
Console.WriteLine("Press any key to continue...");
Console.Read();
BaiduSearchSpider baiduSearchSpider = new BaiduSearchSpider();
baiduSearchSpider.Run();
Console.WriteLine("Press any key to continue...");
Console.Read();
JdShopDetailSpider jdShopDetailSpider = new JdShopDetailSpider();
jdShopDetailSpider.Run();
Console.WriteLine("Press any key to continue...");
Console.Read();
JdSkuSampleSpider jdSkuSampleSpider = new JdSkuSampleSpider();
jdSkuSampleSpider.Run();
Console.WriteLine("Press any key to continue...");
Console.Read();
Situoli.Run();
}
private static void MyTest()
{
BaseUsage.CustmizeProcessorAndPipeline();
}
public static void Main(string[] args)
{
BaseUsage.Run();
}
public static void Main(string[] args)
{
//ShanxizhaoshengSpider spider = new ShanxizhaoshengSpider();
//spider.Run();
// Crawler pages traversal 遍历整站
BaseUsage.CrawlerPagesTraversal();
Console.WriteLine("Press any key to continue...");
Console.Read();
// Custmize processor and pipeline 完全自定义页面解析和数据管道
BaseUsage.CustmizeProcessorAndPipeline();
Console.WriteLine("Press any key to continue...");
Console.Read();
// Crawler pages without traverse 采集指定页面不做遍历
BaseUsage.CrawlerPagesWithoutTraverse();
Console.WriteLine("Press any key to continue...");
Console.Read();
// Crawler pages traversal 遍历整站
BaseUsage.CrawlerPagesTraversal();
Console.WriteLine("Press any key to continue...");
Console.Read();
DDengEntitySpider dDengEntitySpider = new DDengEntitySpider();
dDengEntitySpider.Run();
Console.WriteLine("Press any key to continue...");
Console.Read();
Cnblogs.Run();
Console.WriteLine("Press any key to continue...");
Console.Read();
CasSpider casSpider = new CasSpider();
casSpider.Run();
Console.WriteLine("Press any key to continue...");
Console.Read();
BaiduSearchSpider baiduSearchSpider = new BaiduSearchSpider();
baiduSearchSpider.Run();
Console.WriteLine("Press any key to continue...");
Console.Read();
JdShopDetailSpider jdShopDetailSpider = new JdShopDetailSpider();
jdShopDetailSpider.Run();
Console.WriteLine("Press any key to continue...");
Console.Read();
JdSkuSampleSpider jdSkuSampleSpider = new JdSkuSampleSpider();
jdSkuSampleSpider.Run();
Console.WriteLine("Press any key to continue...");
Console.Read();
Situoli.Run();
}