예제 #1
0
        public FrmCrawlerRaoVatConsumer(int ConfigID, ETypeCrawlRaoVat typeCrawler)
        {
            InitializeComponent();

            this.ConfigID    = ConfigID;
            this.typeCrawler = typeCrawler;
        }
예제 #2
0
 /// <summary>
 /// 0-Full.
 /// 1-RealTime.
 /// </summary>
 /// <param name="iType"></param>
 public SimpleCrawlerRaoVat(int ConfigID, ETypeCrawlRaoVat TypeCrawler)
 {
     this.sqlDb            = new SqlDb(QT.Entities.Server.ConnectionString);
     this.iTypeCrawler     = TypeCrawler;
     this.setAddedQueue    = new SetCrawlerRaoVat(this.sqlDb, ConfigID, Convert.ToInt32(this.iTypeCrawler));
     this.queueWaitRun     = new QueueCrawlerRaoVat(this.sqlDb, ConfigID, Convert.ToInt32(this.iTypeCrawler));
     this.sqlRaoVatAdapter = new RaoVatSQLAdapter(this.sqlDb);
     this.configXPath      = sqlRaoVatAdapter.GetConfigByID(ConfigID);
     this.Domain           = this.configXPath.domain;
     this.mongoDbAdapter   = new MongoDbRaoVat();
     this.dicMapClassificationAndCategories = this.sqlRaoVatAdapter.GetDicMapClassificationAndCategories(this.configXPath.website_id);
     this.dicCityAndRegex = this.sqlRaoVatAdapter.GetDicCityAndRegex();
 }
예제 #3
0
        public void doCrawl(int configID, ETypeCrawlRaoVat typeCrawler)
        {
            SqlDb            sqlDb         = new SqlDb(QT.Entities.Server.ConnectionStringCrawler);
            RaoVatSQLAdapter adapter       = new Entities.RaoVat.RaoVatSQLAdapter(sqlDb);
            ConfigXPaths     configuration = adapter.GetConfigByID(this.ConfigID);

            while (!this.bPause)
            {
                try
                {
                    SimpleCrawlerRaoVat crawler = new SimpleCrawlerRaoVat(configID, typeCrawler);

                    crawler.eventWhenStart += new Crawler.AbstractionCrawler.EventReportRun(delegate(object sender, string mss)
                    {
                        WriteLog("eventWhenStart", mss);
                    });

                    crawler.eventWhenGetJob += new Crawler.AbstractionCrawler.EventReportRun(delegate(object sender, string mss)
                    {
                        WriteLog("eventWhenGetJob", mss);

                        this.Invoke(new Action(() =>
                        {
                            this.spinnumberVisitedLink.Value = Convert.ToInt32(spinnumberVisitedLink.Value) + 1;
                        }));
                    });

                    crawler.eventWhenPushJob += new Crawler.AbstractionCrawler.EventReportRun(delegate(object sender, string mss)
                    {
                        WriteLog("eventWhenPushJob", mss);
                    });

                    crawler.eventWhenEnd += new Crawler.AbstractionCrawler.EventReportRun(delegate(object sender, string mss)
                    {
                        WriteLog("eventWhenEnd", mss);
                    });

                    crawler.eventWhenSuccessProduct += new Crawler.AbstractionCrawler.EventReportRun(delegate(object sender, string mss)
                    {
                        WriteLog("eventWhenSuccessProduct", mss);
                        this.Invoke(new Action(() =>
                        {
                            this.spinNumberProduct.Value = Convert.ToInt32(spinNumberProduct.Value) + 1;
                        }));
                    });

                    crawler.StartCrawler();
                    Thread.Sleep(20000);
                }
                catch (ThreadAbortException ex1)
                {
                    break;
                }
                catch (Exception ex)
                {
                    log.ErrorFormat(ex.Message);
                    WriteLog("Exception", string.Format("Exception:{0}", ex.Message));
                    Thread.Sleep(20000);
                }
            }
        }