示例#1
0
        public static void Main(string[] args)
        {
#if NETCOREAPP2_0
            Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
#else
            ThreadPool.SetMinThreads(200, 200);
            OcrDemo.Process();
#endif

            MyTest();

            Startup.Run(new string[] { "-s:BaiduSearch", "-tid:BaiduSearch", "-i:guid", "-a:" });

            Startup.Run(new string[] { "-s:CustomSpider1", "-tid:CustomSpider1", "-i:CustomSpider1" });

            Startup.Run(new string[] { "-s:DefaultMySqlPipeline", "-tid:DefaultMySqlPipeline", "-i:guid", "-a:" });

            //ConfigurableSpider.Run();

            // Custmize processor and pipeline 完全自定义页面解析和数据管道
            BaseUsage.CustmizeProcessorAndPipeline();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            // Crawler pages without traverse 采集指定页面不做遍历
            BaseUsage.CrawlerPagesWithoutTraverse();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            // Crawler pages traversal 遍历整站
            BaseUsage.CrawlerPagesTraversal();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            DDengEntitySpider dDengEntitySpider = new DDengEntitySpider();
            dDengEntitySpider.Run();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            Cnblogs.Run();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            CasSpider casSpider = new CasSpider();
            casSpider.Run();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            BaiduSearchSpider baiduSearchSpider = new BaiduSearchSpider();
            baiduSearchSpider.Run();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            JdSkuSampleSpider jdSkuSampleSpider = new JdSkuSampleSpider();
            jdSkuSampleSpider.Run();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            Situoli.Run();
        }
示例#2
0
        public static void Main(string[] args)
        {
            Spider1 s = new Spider1();

            s.Run();
            //ConfigurableSpider.Run();

            //// Custmize processor and pipeline 完全自定义页面解析和数据管道
            //BaseUsage.CustmizeProcessorAndPipeline();
            //Console.WriteLine("Press any key to continue...");
            //Console.Read();

            //// Crawler pages without traverse 采集指定页面不做遍历
            //BaseUsage.CrawlerPagesWithoutTraverse();
            //Console.WriteLine("Press any key to continue...");
            //Console.Read();

            //// Crawler pages traversal 遍历整站
            //BaseUsage.CrawlerPagesTraversal();
            //Console.WriteLine("Press any key to continue...");
            //Console.Read();

            DDengEntitySpider dDengEntitySpider = new DDengEntitySpider();

            dDengEntitySpider.Run();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            Cnblogs.Run();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            CasSpider casSpider = new CasSpider();

            casSpider.Run();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            JluSpider jluSpider = new JluSpider();

            jluSpider.Run();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            BaiduSearchSpider baiduSearchSpider = new BaiduSearchSpider();

            baiduSearchSpider.Run();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            JdSkuSampleSpider jdSkuSampleSpider = new JdSkuSampleSpider();

            jdSkuSampleSpider.Run();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            Situoli.Run();
        }
示例#3
0
        public static void Main(string[] args)
        {
            RegexTestEntitySpider spider2 = new RegexTestEntitySpider();

            spider2.Run();

            EntitySpider spider = new EntitySpider(new Core.Site());

            spider.AddStartUrl("http://www.baidu.com");


            // Custmize processor and pipeline 完全自定义页面解析和数据管道
            BaseUsage.CustmizeProcessorAndPipeline();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            // Crawler pages without traverse 采集指定页面不做遍历
            BaseUsage.CrawlerPagesWithoutTraverse();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            // Crawler pages traversal 遍历整站
            BaseUsage.CrawlerPagesTraversal();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            DDengEntitySpider dDengEntitySpider = new DDengEntitySpider();

            dDengEntitySpider.Run();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            Cnblogs.Run();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            CasSpider casSpider = new CasSpider();

            casSpider.Run();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            BaiduSearchSpider baiduSearchSpider = new BaiduSearchSpider();

            baiduSearchSpider.Run();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            JdShopDetailSpider jdShopDetailSpider = new JdShopDetailSpider();

            jdShopDetailSpider.Run();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            JdSkuSampleSpider jdSkuSampleSpider = new JdSkuSampleSpider();

            jdSkuSampleSpider.Run();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            Situoli.Run();
        }
示例#4
0
        public static void Main(string[] args)
        {
            //ShanxizhaoshengSpider spider = new ShanxizhaoshengSpider();
            //spider.Run();

            // Crawler pages traversal 遍历整站
            BaseUsage.CrawlerPagesTraversal();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            // Custmize processor and pipeline 完全自定义页面解析和数据管道
            BaseUsage.CustmizeProcessorAndPipeline();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            // Crawler pages without traverse 采集指定页面不做遍历
            BaseUsage.CrawlerPagesWithoutTraverse();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            // Crawler pages traversal 遍历整站
            BaseUsage.CrawlerPagesTraversal();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            DDengEntitySpider dDengEntitySpider = new DDengEntitySpider();

            dDengEntitySpider.Run();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            Cnblogs.Run();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            CasSpider casSpider = new CasSpider();

            casSpider.Run();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            BaiduSearchSpider baiduSearchSpider = new BaiduSearchSpider();

            baiduSearchSpider.Run();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            JdShopDetailSpider jdShopDetailSpider = new JdShopDetailSpider();

            jdShopDetailSpider.Run();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            JdSkuSampleSpider jdSkuSampleSpider = new JdSkuSampleSpider();

            jdSkuSampleSpider.Run();
            Console.WriteLine("Press any key to continue...");
            Console.Read();

            Situoli.Run();
        }