Exemplos de DotnetSpider.Sample BaseUsage em C# (CSharp)

Exemplo n.º 1

Arquivo: Program.cs Projeto: c7ming/DotnetSpider

        static async Task Main(string[] args)
        {
            var configure = new LoggerConfiguration()
#if DEBUG
                            .MinimumLevel.Verbose()
#else
                            .MinimumLevel.Information()
#endif
                            .MinimumLevel.Override("Microsoft", LogEventLevel.Warning)
                            .Enrich.FromLogContext()
                            .WriteTo.Console().WriteTo
                            .RollingFile("dotnet-spider.log");

            Log.Logger = configure.CreateLogger();

            await BaseUsage.Run();

            // await DistributedSpider.Run();
            Console.Read();
        }

Exemplo n.º 2

        public static void Main(string[] args)
        {
            RegexTestEntitySpider spider2 = new RegexTestEntitySpider();

            spider2.Run();

            EntitySpider spider = new EntitySpider(new Core.Site());

            spider.AddStartUrl("http://www.baidu.com");


            // Custmize processor and pipeline 完全自定义页面解析和数据管道
            BaseUsage.CustmizeProcessorAndPipeline();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            // Crawler pages without traverse 采集指定页面不做遍历
            BaseUsage.CrawlerPagesWithoutTraverse();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            // Crawler pages traversal 遍历整站
            BaseUsage.CrawlerPagesTraversal();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            DDengEntitySpider dDengEntitySpider = new DDengEntitySpider();

            dDengEntitySpider.Run();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            Cnblogs.Run();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            CasSpider casSpider = new CasSpider();

            casSpider.Run();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            BaiduSearchSpider baiduSearchSpider = new BaiduSearchSpider();

            baiduSearchSpider.Run();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            JdShopDetailSpider jdShopDetailSpider = new JdShopDetailSpider();

            jdShopDetailSpider.Run();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            JdSkuSampleSpider jdSkuSampleSpider = new JdSkuSampleSpider();

            jdSkuSampleSpider.Run();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            Situoli.Run();
        }

Exemplo n.º 3

 private static void MyTest()
 {
     BaseUsage.CustmizeProcessorAndPipeline();
 }

Exemplo n.º 4

Arquivo: Program.cs Projeto: raycdut/DotnetSpider

 public static void Main(string[] args)
 {
     BaseUsage.Run();
 }

Exemplo n.º 5

Arquivo: Program.cs Projeto: zhaoyingju/DotnetSpider

        public static void Main(string[] args)
        {
            //ShanxizhaoshengSpider spider = new ShanxizhaoshengSpider();
            //spider.Run();

            // Crawler pages traversal 遍历整站
            BaseUsage.CrawlerPagesTraversal();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            // Custmize processor and pipeline 完全自定义页面解析和数据管道
            BaseUsage.CustmizeProcessorAndPipeline();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            // Crawler pages without traverse 采集指定页面不做遍历
            BaseUsage.CrawlerPagesWithoutTraverse();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            // Crawler pages traversal 遍历整站
            BaseUsage.CrawlerPagesTraversal();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            DDengEntitySpider dDengEntitySpider = new DDengEntitySpider();

            dDengEntitySpider.Run();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            Cnblogs.Run();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            CasSpider casSpider = new CasSpider();

            casSpider.Run();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            BaiduSearchSpider baiduSearchSpider = new BaiduSearchSpider();

            baiduSearchSpider.Run();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            JdShopDetailSpider jdShopDetailSpider = new JdShopDetailSpider();

            jdShopDetailSpider.Run();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            JdSkuSampleSpider jdSkuSampleSpider = new JdSkuSampleSpider();

            jdSkuSampleSpider.Run();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            Situoli.Run();
        }