Пример #1
0
 public FrmMainSale()
 {
     InitializeComponent();
     sqlDb = new QT.Entities.Data.SqlDb(QT.Entities.Server.ConnectionStringCrawler);
     configXPathAdapter = new RaoVatSQLAdapter(sqlDb);
     foreach (var item in this.configXPathAdapter.GetListConfig())
     {
         this.dicConfigXPath.Add(item.ID, item);
     }
 }
Пример #2
0
 /// <summary>
 /// 0-Full.
 /// 1-RealTime.
 /// </summary>
 /// <param name="iType"></param>
 public SimpleCrawlerRaoVat(int ConfigID, ETypeCrawlRaoVat TypeCrawler)
 {
     this.sqlDb            = new SqlDb(QT.Entities.Server.ConnectionString);
     this.iTypeCrawler     = TypeCrawler;
     this.setAddedQueue    = new SetCrawlerRaoVat(this.sqlDb, ConfigID, Convert.ToInt32(this.iTypeCrawler));
     this.queueWaitRun     = new QueueCrawlerRaoVat(this.sqlDb, ConfigID, Convert.ToInt32(this.iTypeCrawler));
     this.sqlRaoVatAdapter = new RaoVatSQLAdapter(this.sqlDb);
     this.configXPath      = sqlRaoVatAdapter.GetConfigByID(ConfigID);
     this.Domain           = this.configXPath.domain;
     this.mongoDbAdapter   = new MongoDbRaoVat();
     this.dicMapClassificationAndCategories = this.sqlRaoVatAdapter.GetDicMapClassificationAndCategories(this.configXPath.website_id);
     this.dicCityAndRegex = this.sqlRaoVatAdapter.GetDicCityAndRegex();
 }
Пример #3
0
        private void InitData()
        {
            try
            {
                RaoVatSQLAdapter raovat = new RaoVatSQLAdapter(new SqlDb(QT.Entities.Server.ConnectionStringCrawler));
                this.CboWebSiteID.DataSource    = raovat.GetTblWebSite();
                this.CboWebSiteID.ValueMember   = "id";
                this.CboWebSiteID.DisplayMember = "domain";

                string Webdomain      = "raovat";
                string URLSolrConnect = "http://118.70.205.94:9104/solr/raovat";
                slr = SolrRaoVatDriver.GetDriver(null);

                this.LoadEventCheckRegex(new RichTextBox[] {
                    this.NoProductUrlRegexTextBox,
                    this.noVisitUrlsRegexTextBox,
                    this.ProductUrlsRegexTextBox,
                    this.visitUrlsRegexTextBox
                });

                this.LoadEventCheckXPaths(new RichTextBox[] {
                    this.PhoneSalerXPathsTextBox,
                    this.AddressXPathsTextBox,
                    this.AvaiableXPathsTextBox,
                    this.QualityXPathsTextBox,
                    this.ImageUrlsXPathsTextBox,
                    this.LastChangeXPathsTextBox,
                    this.PostDateXPathsTextBox,
                    this.PriceXPathsTextBox,
                    this.ProvinceXPathsTextBox,
                    this.UserNameXPathsTextBox,
                    this.TitleXPathsTextBox,
                    this.ContentXPathTextbox,
                    this.txtTagXPaths,
                    this.webCategoryXPathsTextBox,
                    this.txtLastEditXPaths
                });

                this.factoryMQ     = new ConnectionFactory();
                factoryMQ.UserName = QT.Entities.Server.RabbitMQ_User;
                factoryMQ.Password = QT.Entities.Server.RabbitMQ_Pass;
                factoryMQ.Protocol = Protocols.DefaultProtocol;
                factoryMQ.HostName = QT.Entities.Server.RabbitMQ_Host;
                factoryMQ.Port     = QT.Entities.Server.RabbitMQ_Port;
            }
            catch (Exception ex)
            {
                MessageBox.Show(ex.Message);
            }
        }
Пример #4
0
        public void RunSync(int numberItem)
        {
            MySqlAdapterRaoVat       mySqlAdapterRaoVat = null;
            Dictionary <int, string> dicCity            = null;
            RaoVatSQLAdapter         configXPathAdapter = new RaoVatSQLAdapter(
                new SqlDb(QT.Entities.Server.ConnectionStringCrawler));

            MongoDbRaoVat mongoDb = new MongoDbRaoVat();

            while (!this.bPause)
            {
                try
                {
                    if (TypeProcess == TypeRunSync.ReloadAllProuduct)
                    {
                        bool bOK = ReloadAllProduct(numberItem, mongoDb).Result;
                        WriteLog(string.Format("Commit {0} data", numberItem));
                        Thread.Sleep(1000);
                    }
                    else if (TypeProcess == TypeRunSync.AnalyscFieldProduct)
                    {
                        mySqlAdapterRaoVat = new MySqlAdapterRaoVat();
                        dicCity            = mySqlAdapterRaoVat.GetDicCity();
                        bool bOK = AnalysicField(numberItem, mongoDb, dicCity).Result;
                        WriteLog(string.Format("Commit {0} data", numberItem));
                        Thread.Sleep(1000);
                    }
                    else if (TypeProcess == TypeRunSync.FixSlug)
                    {
                        bool OK = AnalysicKeyword(10000, mongoDb, null).Result;
                        WriteLog(string.Format("Commit {0} data", numberItem));
                        Thread.Sleep(1000);
                    }
                    else if (TypeProcess == TypeRunSync.ReloadAllKeyWord)
                    {
                        bool iProductAnalysic = AnalysicKeyword(10000, mongoDb, null).Result;
                    }
                }
                catch (OperationAbortedException ex1)
                {
                    break;
                }
                catch (Exception ex)
                {
                    WriteLog(ex.Message);
                    Thread.Sleep(10000);
                }
            }
        }
Пример #5
0
        public FrmConfigXPath()
        {
            try
            {
                InitializeComponent();


                sqlDb            = new SqlDb(QT.Entities.Server.ConnectionStringCrawler);
                raovatSqlAdapter = new RaoVatSQLAdapter(sqlDb);
                productAdapter   = new ProductSaleNewDataAdapter(sqlDb);

                this.hanlerContentOfHtml = new QT.Entities.RaoVat.HandlerContentOfHtml();
                InitData();
            }
            catch (Exception ex)
            {
            }
        }
Пример #6
0
        private void RunData(string textProcess)
        {
            this.Invoke(new Action(() =>
            {
                this.btnStart.Visible = false;
            }));

            SqlDb sqlDb = new SqlDb(QT.Entities.Server.ConnectionStringCrawler);
            MySqlAdapterRaoVat mySqlAdapterRaoVat = new MySqlAdapterRaoVat();
            RaoVatSQLAdapter   sqlAdapterRaoVat   = new RaoVatSQLAdapter(sqlDb);

            if (textProcess == "cities")
            {
                var dicCity = mySqlAdapterRaoVat.GetDicCity();
                for (int i = 0; i < dicCity.Count; i++)
                {
                    var itemCity = dicCity.ElementAt(i);
                    if (sqlAdapterRaoVat.CheckExistCity(itemCity.Key))
                    {
                        sqlAdapterRaoVat.UpdateCity(itemCity.Key, itemCity.Value);
                    }
                    else
                    {
                        sqlAdapterRaoVat.InsertCity(itemCity.Key, itemCity.Value);
                    }
                    this.Invoke(new Action(() =>
                    {
                        this.progressBar1.Value = Convert.ToInt32(((double)i / (double)dicCity.Count) * 100);
                    }));
                }
            }
            else if (textProcess == "categories")
            {
                DataTable tblcategories = mySqlAdapterRaoVat.GetTableCategories();
                for (int i = 0; i < tblcategories.Rows.Count; i++)
                {
                    var itemCity = tblcategories.Rows[i];
                    if (sqlAdapterRaoVat.CheckExistCategories(Convert.ToInt32(itemCity["id"])))
                    {
                        sqlAdapterRaoVat.UpdateCategories(new QT.Entities.RaoVat.RaoVatCategory()
                        {
                            id        = Convert.ToInt32(itemCity["id"]),
                            level     = Convert.ToInt32(itemCity["level"]),
                            name      = Common.Obj2String(itemCity["name"]),
                            parent_id = Common.Obj2Int(itemCity["parent_id"]),
                            slug      = Common.Obj2String(itemCity["slug"]),
                            path      = Common.Obj2String(itemCity["path"])
                        });
                    }
                    else
                    {
                        sqlAdapterRaoVat.InsertCategory(new QT.Entities.RaoVat.RaoVatCategory()
                        {
                            id        = Convert.ToInt32(itemCity["id"]),
                            level     = Convert.ToInt32(itemCity["level"]),
                            name      = Common.Obj2String(itemCity["name"]),
                            parent_id = Common.Obj2Int(itemCity["parent_id"]),
                            slug      = Common.Obj2String(itemCity["slug"]),
                            path      = Common.Obj2String(itemCity["path"])
                        });
                    }
                    this.Invoke(new Action(() =>
                    {
                        this.progressBar1.Value = Convert.ToInt32(((double)i / (double)tblcategories.Rows.Count) * 100);
                    }));
                }
            }

            this.Invoke(new Action(() =>
            {
                this.btnStart.Visible = true;
            }));
        }