Пример #1
0
        public bool UpdateProduct(ProductSaleNew product)
        {
            try
            {
                string sql = string.Format(@"UPDATE ProductSaleNew 
            {0}=@{0}, {1}=@{1}, {2}=@{2}, {3}=@{3}, {4}=@{4}, {5}=@{5}, {6}=@{6}, {7}=@{7}, {8}=@{8}, {9}=@{9}, {10}=@{10}, {11}=@{11}, {12}=@{12}, {13}=@{13}, {14}=@{14}, {15}=@{15}, {16}=@{16}, {17}=@{17}, {18}=@{18}, {19}=@{19}, {20}=@{20}, {21}=@{21}, {22}=@{22}, {23}=@{23} WHERE @{0} = @{0}"
                                           , "ID", "Content", "Title", "Price",
                                           "PostDate", "LastChange", "Province", "PhoneSaler", "Quality", "Address", "Avaiable",
                                           "FieldS01", "FieldS02", "FieldS03", "FieldS04", "FieldS05", "FieldS06", "FieldS07", "FieldS08", "FieldS09",
                                           "FieldS10", "FieldI01", "FieldI02", "FieldI03");
                SqlParameter[] arPara = new SqlParameter[]
                {
                    this.sqlDb.CreateParamteter("ID", product.id, System.Data.SqlDbType.BigInt),
                    this.sqlDb.CreateParamteter("Title", product.name, System.Data.SqlDbType.NVarChar),
                    this.sqlDb.CreateParamteter("Price", product.price, System.Data.SqlDbType.Decimal),
                    this.sqlDb.CreateParamteter("PostDate", product.post_date, System.Data.SqlDbType.DateTime),
                    this.sqlDb.CreateParamteter("LastChange", product.source_updated_at, System.Data.SqlDbType.DateTime),
                    this.sqlDb.CreateParamteter("Province", product.province, System.Data.SqlDbType.NVarChar),
                    this.sqlDb.CreateParamteter("PhoneSaler", product.phone_saler, System.Data.SqlDbType.NVarChar),
                    this.sqlDb.CreateParamteter("Quality", product.quality, System.Data.SqlDbType.NVarChar),
                    this.sqlDb.CreateParamteter("Address", product.address, System.Data.SqlDbType.NVarChar),
                    this.sqlDb.CreateParamteter("Content", product.content, System.Data.SqlDbType.NText),
                    this.sqlDb.CreateParamteter("Avaiable", product.Avaiable, System.Data.SqlDbType.Int)
                };

                this.sqlDb.RunQuery(sql, System.Data.CommandType.Text, arPara);
                return(true);
            }
            catch (Exception ex)
            {
                log.ErrorFormat("Exception:{0}", ex.Message);
                return(false);
            }
        }
Пример #2
0
        private void RunLoadData(string querySelectData)
        {
            try
            {
                List <ProductSaleNew> lstExtract = new List <ProductSaleNew>();
                var listExtraction = mongoDb.GetListExtractKeyWord(querySelectData, 100, 1).Result;

                foreach (var item in listExtraction)
                {
                    ProductSaleNew productSaleNew = new ProductSaleNew();
                    productSaleNew.id                = item["id"].AsInt64;
                    productSaleNew._id               = item["_id"].AsObjectId.ToString();
                    productSaleNew.name              = item["name"].AsString;
                    productSaleNew.updated_at        = item["updated_at"].ToUniversalTime().AddHours(7);
                    productSaleNew.source_updated_at = item["source_updated_at"].ToUniversalTime().AddHours(7);
                    lstExtract.Add(productSaleNew);
                }

                this.Invoke(new Action(() =>
                {
                    try
                    {
                        this.gridControl2.DataSource = lstExtract;
                    }
                    catch (Exception ex)
                    {
                        log.ErrorFormat(ex.Message);
                    }
                }));
            }
            catch (Exception ex)
            {
                log.ErrorFormat(ex.Message);
            }
        }
Пример #3
0
        public void SaveKeyWord(ProductSaleNew productSaleNew)
        {
            string strQuery  = string.Format(@"IF NOT EXISTS(SELECT KeyHash FROM KeyWords WHERE KeyHash=@KeyHash) BEGIN INSERT INTO KeyWords ({0}, {1}, {2}) VALUES
                                            (@{0}, @{1}, @{2}) END", "KeyHash", "KeyName", "ProductID");
            string strQuery1 = string.Format(@"IF NOT EXISTS(SELECT KeyHash, ProductID FROM KeyWordProduct WHERE KeyHash=@KeyHash AND ProductID=@ProductID) BEGIN
                                              INSERT INTO KeyWordProduct(KeyHash, ProductID) VALUES (@KeyHash, @ProductID)
                                             END");

            try
            {
                List <string> list = productSaleNew.tags;
                foreach (string str in list)
                {
                    long hashKeyWord = Math.Abs(GABIZ.Base.Tools.getCRC32(str));
                    this.sqlDb.RunQuery(
                        strQuery, CommandType.Text, new SqlParameter[] {
                        this.sqlDb.CreateParamteter("KeyHash", hashKeyWord, SqlDbType.BigInt),
                        this.sqlDb.CreateParamteter("KeyName", str, SqlDbType.NVarChar),
                        this.sqlDb.CreateParamteter("ProductID", productSaleNew.id, SqlDbType.BigInt)
                    });
                    this.sqlDb.RunQuery(strQuery1, CommandType.Text, new SqlParameter[]
                    {
                        this.sqlDb.CreateParamteter("KeyHash", hashKeyWord, SqlDbType.BigInt),
                        this.sqlDb.CreateParamteter("ProductID", productSaleNew.id, SqlDbType.BigInt),
                    });
                }
            }
            catch (Exception ex)
            {
                log.ErrorFormat("Exeception:SaveKeyWord:0{}", ex.Message);
            }
        }
Пример #4
0
        private void btnTestProduct_Click(object sender, EventArgs e)
        {
            string urlTest = urlTestTextBox.Text;

            if (!string.IsNullOrWhiteSpace(urlTest))
            {
                string url  = urlTestTextBox.Text;
                string html = GABIZ.Base.HtmlUrl.HTMLTransmitter.getHTML(url, 45, 2);
                GABIZ.Base.HtmlAgilityPack.HtmlDocument doc = new GABIZ.Base.HtmlAgilityPack.HtmlDocument();

                WebExceptionStatus status = WebExceptionStatus.Success;
                html = GABIZ.Base.HtmlUrl.HTMLTransmitter.getHTML(url, 45, 2, out status);
                doc.LoadHtml(html);

                ConfigXPaths config = this.raovatSqlAdapter.GetConfigByID((int)this.configXPathIDSpinEdit.Value);
                if (config == null)
                {
                    config = new ConfigXPaths()
                    {
                        ID = -1
                    }
                }
                ;
                if (this.LoadFormToConfig(ref config))
                {
                    var product = new ProductSaleNew();
                    int iError  = this.hanlerContentOfHtml.AnalyticsProductSaleNew(config.domain, urlTest, config, product,
                                                                                   this.raovatSqlAdapter.GetDicMapClassificationAndCategories(config.website_id),
                                                                                   this.raovatSqlAdapter.GetDicCityAndRegex());

                    FrmDataShow frmDataShow = new FrmDataShow(product.ToString());
                    frmDataShow.btnSave.Click += new EventHandler(delegate(object obj, EventArgs eventArg)
                    {
                        if (MessageBox.Show("Save to Cassandra?", "Warning", MessageBoxButtons.YesNo, MessageBoxIcon.Warning) == System.Windows.Forms.DialogResult.Yes)
                        {
                            bool bExits = this.mongoDbAdapter.CheckExistsProductSalenew(product.id);
                            if (bExits)
                            {
                                mongoDbAdapter.UpdateProduct(product);
                                mongoDbAdapter.SaveHtml(product.id, html, bExits);
                            }
                            else
                            {
                                mongoDbAdapter.InsertProduct(product);
                                mongoDbAdapter.SaveHtml(product.id, html, bExits);
                            }
                        }
                    });
                    frmDataShow.ShowDialog();
                }
            }
            else
            {
                MessageBox.Show("Not url test");
            }
        }
Пример #5
0
 private void btnUnCheckAll_Click(object sender, EventArgs e)
 {
     int[] arSelected = this.gridView1.GetSelectedRows();
     foreach (var itemRow in arSelected)
     {
         ProductSaleNew ptsn = this.gridView1.GetRow(itemRow) as ProductSaleNew;
         ptsn.is_selected = false;
     }
     this.gridView1.RefreshData();
 }
Пример #6
0
        private void AnalysicData(int[] arNumberWord, bool UserRegex)
        {
            Dictionary <long, KeywordSaleNew> lstResultKeyWord = new Dictionary <long, KeywordSaleNew>();
            Dictionary <long, KeywordSaleNew> lstBlackLink     = new Dictionary <long, KeywordSaleNew>();

            for (int iRow = 0; iRow < this.gridView2.DataRowCount; iRow++)
            {
                ProductSaleNew productSaleNew = this.gridView2.GetRow(iRow) as ProductSaleNew;
                string         titleData      = productSaleNew.name.ToLower();
                List <string>  lstTitle       = Common.ExtractBySentence(titleData);
                foreach (var title in lstTitle)
                {
                    List <string> lstWordNGram = new List <string>();

                    foreach (var item in arNumberWord)
                    {
                        lstWordNGram.AddRange(GABIZ.Base.Tools.NGramDocument(item, title));
                    }

                    for (int i = lstWordNGram.Count - 1; i >= 0; i--)
                    {
                        if (Common.ValidKeyword(lstWordNGram[i]) &&
                            (UserRegex == false || Common.ValidKeyWordGrammar(lstWordNGram[i])))
                        {
                            lstWordNGram.RemoveAt(i);
                        }
                    }

                    foreach (var itemCanKeyword in lstWordNGram)
                    {
                        long idKeyword = Math.Abs(Common.GetID_Keywords64(itemCanKeyword));
                        if (!CheckInBlackLink(idKeyword))
                        {
                            if (!lstResultKeyWord.ContainsKey(idKeyword))
                            {
                                lstResultKeyWord.Add(idKeyword, new KeywordSaleNew()
                                {
                                    crc          = idKeyword,
                                    category_ids = productSaleNew.category_ids,
                                    name         = itemCanKeyword,
                                    is_selected  = false
                                });
                            }
                        }
                    }
                }
            }

            this.Invoke(new Action(() =>
            {
                this.gridControlGoodKey.DataSource = lstResultKeyWord.Values.ToList();
                this.gridControlKey.DataSource     = lstBlackLink.Values.ToList();
            }));
        }
Пример #7
0
        public bool InsertFirstTimeProduct(ProductSaleNew product)
        {
            return(true);
//            long ID = product.id;
//            string Title = product.name;
//            decimal Price = product.price;
//            DateTime PostDate = product.post_date;
//            DateTime LastChange = DateTime.Now;
//            string Province = product.province;
//            string PhoneSaler = product.phone_saler;
//            string Quality = product.quality;
//            string Address = product.address;
//            string Content = product.content;
//            int Avaiable = product.Avaiable;

//            try
//            {
//                string sql = string.Format(@"INSERT INTO ProductSaleNew (
//            {0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}, {8}, {9}, {10}, {11}, {12}, {13}, {14}, {15}, {16}, {17}, {18}, {19}, {20}, {21}, {22}, {23}, {24}, {25}, {26}
//            ) VALUES (
//@{0}, @{1}, @{2}, @{3}, @{4}, @{5}, @{6}, @{7}, @{8}, @{9}, @{10}, @{11}, @{12}, @{13}, @{14}, @{15}, @{16}, @{17}, @{18}, @{19}, @{20}, @{21}, @{22}, @{23}, @{24}, @{25}, @{26}
//               )"
//                    , "ID", "Content", "Title", "Price",
//                    "PostDate", "LastChange", "Province", "PhoneSaler", "Quality", "Address", "Avaiable",
//                    "FieldS01", "FieldS02", "FieldS03", "FieldS04", "FieldS05", "FieldS06", "FieldS07", "FieldS08", "FieldS09",
//                    "FieldS10", "FieldI01", "FieldI02", "FieldI03", "Category", "CategoryID", "ImageUrls");
//                SqlParameter[] arPara = new SqlParameter[]
//               {
//                  this.sqlDb.CreateParamteter("ID",ID,System.Data.SqlDbType.BigInt),
//                  this.sqlDb.CreateParamteter("Title",Title,System.Data.SqlDbType.NVarChar),
//                  this.sqlDb.CreateParamteter("Price",Price,System.Data.SqlDbType.Decimal),
//                  this.sqlDb.CreateParamteter("PostDate",PostDate,System.Data.SqlDbType.DateTime),
//                  this.sqlDb.CreateParamteter("LastChange",LastChange,System.Data.SqlDbType.DateTime),
//                  this.sqlDb.CreateParamteter("Province",Province,System.Data.SqlDbType.NVarChar),
//                  this.sqlDb.CreateParamteter("PhoneSaler",PhoneSaler,System.Data.SqlDbType.NVarChar),
//                  this.sqlDb.CreateParamteter("Quality",Quality,System.Data.SqlDbType.NVarChar),
//                  this.sqlDb.CreateParamteter("Address",Address,System.Data.SqlDbType.NVarChar),
//                  this.sqlDb.CreateParamteter("Content",Content,System.Data.SqlDbType.NText),
//                  this.sqlDb.CreateParamteter("Avaiable",Avaiable,System.Data.SqlDbType.Int),
//                  this.sqlDb.CreateParamteter("Category",Convert.ToString(product.Category),System.Data.SqlDbType.NVarChar),
//                  this.sqlDb.CreateParamteter("CategoryID",Convert.ToInt32(product.category_id),System.Data.SqlDbType.Int),
//                  this.sqlDb.CreateParamteter("ImageUrls",Common.ConvertToString( product.images,";"),System.Data.SqlDbType.NVarChar)
//               };

//                this.sqlDb.RunQuery(sql, System.Data.CommandType.Text, arPara);
//                return true;
//            }
//            catch (Exception ex)
//            {
//                log.ErrorFormat("Exception:{0}", ex.Message);
//                return false;
//            }
        }
Пример #8
0
        public ProductSaleNew GetProductById(long ProductID)
        {
            RowSet            rowSet  = this._session.Execute(string.Format("SELECT * FROM sale.product WHERE id = {0} ", ProductID));
            IEnumerable <Row> rows    = rowSet.GetRows();
            ProductSaleNew    product = new ProductSaleNew();

            foreach (Row row in rows)
            {
                ParseProduct(ref product, row);
                return(product);
            }
            return(product);
        }
Пример #9
0
        private void ProcessData_ReloadAllField(object objSender, BsonDocument productNeedProcess)
        {
            ProductSaleNew productSaleNew = new ProductSaleNew();
            ConfigXPaths   configXPath    = this.dicConfigXPath[productNeedProcess["source_id"].AsInt32];
            int            iFail          = handerContent.AnalyticsProductSaleNew(configXPath.domain,
                                                                                  productNeedProcess["url"].AsString, configXPath, productSaleNew,
                                                                                  null, null);

            if (productSaleNew.IsDetailSucess)
            {
                //Cập nhật hàng hóa.
                mon.UpdateProduct(productSaleNew);
                mon.colProduct.UpdateOneAsync(
                    Builders <BsonDocument> .Filter.Eq("_id", productNeedProcess["_id"].AsObjectId)
                    , Builders <BsonDocument> .Update
                    .Set("processed", true));
            }
            else
            {
                int iNumberFail    = 0;
                int iStatusCurrent = productNeedProcess["status"].AsInt32;
                if (productNeedProcess.Contains("fail_count"))
                {
                    iNumberFail = productNeedProcess["fail_count"].AsInt32;
                }
                iNumberFail++;
                if (iNumberFail >= 1 && iStatusCurrent == 1)
                {
                    iStatusCurrent = 2;
                }
                mon.colProduct.UpdateOneAsync(
                    Builders <BsonDocument> .Filter.Eq("_id", productNeedProcess["_id"].AsObjectId)
                    , Builders <BsonDocument> .Update
                    .Set("status", iStatusCurrent)
                    .Set("fail_count", iNumberFail)
                    .Set("is_solr_updated", false)
                    .CurrentDate("updated_at"));
            }
        }
Пример #10
0
        private void ShowProduct(ProductSaleNew productSaleNew)
        {
            try
            {
                this.Invoke(new Action(() =>
                {
                    string s = string.Format("\n\r-----------------------\n\r   ID:{0} LastUpdate:{4} \n\r Name:{1}  \n\r  Price:{2}  \n\r Classification:{3}\n\r   Categories:{5}",
                                             productSaleNew.id,
                                             (productSaleNew.name.Length < 100) ? productSaleNew.name : productSaleNew.name.Substring(0, 100),
                                             productSaleNew.price.ToString(),
                                             (productSaleNew.web_category.Length < 100) ? productSaleNew.web_category : productSaleNew.web_category,
                                             productSaleNew.source_updated_at.ToString("yyyy-MM-dd HH:mm:ss"),
                                             Common.ConvertToString(productSaleNew.category_ids, "_")
                                             );
                    this.richTextBox1.AppendText(s);

                    this.spinProduct.Value = this.ProductCount++;
                }));
            }
            catch (Exception ex02)
            {
            }
        }
Пример #11
0
        private void btnHide_Click(object sender, EventArgs e)
        {
            Thread threadHideData = new Thread(() =>
            {
                MongoDbRaoVat mongoDb = new MongoDbRaoVat();
                try
                {
                    List <string> lstDelItem = new List <string>();


                    int iRowCount = this.gridView1.DataRowCount;
                    for (int iRow = 0; iRow < iRowCount; iRow++)
                    {
                        ProductSaleNew productSalenew = this.gridView1.GetRow(iRow) as ProductSaleNew;
                        if (productSalenew.is_selected)
                        {
                            lstDelItem.Add(productSalenew._id);
                        }
                    }
                    //Xóa dữ liệu trên solr.
                    this.solrRaoVat.DelProductById(lstDelItem);
                    //Ẩn dữ liệu từ mongo.
                    foreach (var item in lstDelItem)
                    {
                        mongoDb.UpdateStatusOfProduct(item, 0);
                    }
                }
                catch (Exception ex)
                {
                    log.ErrorFormat(ex.Message);
                }
            });

            threadHideData.Start();
            threadHideData.Join();
            MessageBox.Show("Hoan Tat");
        }
Пример #12
0
        private ProductSaleNew ParesProductFromRow(Row row)
        {
            ProductSaleNew product = new ProductSaleNew();

            product.id                = Common.Obj2Int64(row["ID"]);
            product.address           = Common.Obj2String(row["address"]);
            product.Avaiable          = Common.Obj2Int(row["avaiable"]);
            product.website_id        = Common.Obj2Int(row["config_crawl_id"]);
            product.content           = Common.Obj2String(row["content"]);
            product.is_standard       = Common.Obj2Bool(row["is_standard"]);
            product.source_updated_at = Common.ObjectToDataTime(row["last_change"]);
            product.phone_saler       = Common.Obj2String(row["phone_saler"]);
            product.post_date         = Common.ObjectToDataTime(row["post_date"]);
            product.price             = Convert.ToInt64(Common.Obj2Decimal(row["price"]));
            product.province          = Common.Obj2String(row["province"]);
            product.quality           = Common.Obj2String(row["quality"]);
            product.tags              = Common.GetListXPathFromString(Common.Obj2String(row["tags"]));
            product.name              = Common.Obj2String(row["title"]);
            product.url               = Common.Obj2String(row["url"]);
            product.web_category      = Common.Obj2String(row["web_category"]);
            product.address           = Common.Obj2String(row["address"]);

            return(product);
        }
Пример #13
0
        private void DoCrawler()
        {
            Dictionary <long, int[]>   dicMapClassAndCategori = this.raovatSqlAdapter.GetDicMapClassificationAndCategories(this.websiteRaoVat.id);
            Dictionary <int, string[]> dicMapCity             = this.raovatSqlAdapter.GetDicCityAndRegex();

            while (true)
            {
                try
                {
                    int igone = 0;

                    //Khởi tạo.
                    Queue <JobCrawlerSale>    queueUrl   = new Queue <JobCrawlerSale>();
                    Dictionary <long, string> dicVisited = new Dictionary <long, string>();
                    foreach (var item in this.RunnerCrawler.root_link)
                    {
                        queueUrl.Enqueue(new JobCrawlerSale()
                        {
                            deep = 0,
                            url  = item
                        });
                    }
                    this.ShowQueue(queueUrl.Count);

                    while (!this.Pause && queueUrl != null && queueUrl.Count > 0)
                    {
                        JobCrawlerSale job = queueUrl.Dequeue();

                        ShowUrlCurrent(job.url);
                        ShowQueue(queueUrl.Count);

                        if (configXPath.TimeDelay > 0)
                        {
                            Thread.Sleep(configXPath.TimeDelay);
                        }
                        string html = GABIZ.Base.HtmlUrl.HTMLTransmitter.getHTML(job.url, 45, 2);
                        if (!string.IsNullOrEmpty(html))
                        {
                            GABIZ.Base.HtmlAgilityPack.HtmlDocument doc = new GABIZ.Base.HtmlAgilityPack.HtmlDocument();
                            doc.LoadHtml(html);

                            //Extraction.
                            var a_nodes = doc.DocumentNode.SelectNodes("//a[@href]");
                            if (a_nodes != null)
                            {
                                foreach (var a_node in a_nodes)
                                {
                                    string url1       = QT.Entities.Common.GetAbsoluteUrl(a_node.Attributes["href"].Value, this.websiteRaoVat.base_link);
                                    string compacLink = QT.Entities.Common.CompactUrl(url1);
                                    long   s_crc      = Math.Abs(GABIZ.Base.Tools.getCRC64(compacLink));
                                    if (!dicVisited.ContainsKey(s_crc))
                                    {
                                        dicVisited.Add(s_crc, "");
                                        ShowVisited(dicVisited.Count);

                                        bool bRegexProduct = QT.Entities.Common.CheckRegex(compacLink, configXPath.ProductUrlsRegex, configXPath.NoProductUrlRegex, false);
                                        bool bRegexExtract = QT.Entities.Common.CheckRegex(compacLink, configXPath.VisitUrlsRegex, configXPath.NoVisitUrlRegex, false);
                                        if (bRegexExtract)
                                        {
                                            if (job.deep + 1 < this.RunnerCrawler.max_deep)
                                            {
                                                queueUrl.Enqueue(new JobCrawlerSale()
                                                {
                                                    url  = url1,
                                                    deep = job.deep + 1
                                                });
                                                ShowQueue(queueUrl.Count);
                                            }
                                        }
                                        else
                                        {
                                            if (bRegexProduct)
                                            {
                                                queueUrl.Enqueue(new JobCrawlerSale()
                                                {
                                                    url  = url1,
                                                    deep = job.deep + 1
                                                });
                                                ShowQueue(queueUrl.Count);
                                            }
                                        }
                                    }
                                }
                            }

                            //AnalysicData.
                            if (QT.Entities.Common.CheckRegex(
                                    QT.Entities.Common.CompactUrl(job.url), configXPath.ProductUrlsRegex, configXPath.NoProductUrlRegex, false))
                            {
                                QT.Entities.RaoVat.HandlerContentOfHtml handlerContentHtml = new Entities.RaoVat.HandlerContentOfHtml();
                                ProductSaleNew productSaleNew = new ProductSaleNew();
                                handlerContentHtml.AnalyticsProductSaleNew(websiteRaoVat.domain, job.url, doc, configXPath
                                                                           , productSaleNew, dicMapClassAndCategori, dicMapCity);

                                if (productSaleNew.IsDetailSucess)
                                {
                                    //SaveClassification
                                    try
                                    {
                                        this.raovatSqlAdapter.SaveClassification(productSaleNew.website_id, productSaleNew.web_category);
                                    }
                                    catch (Exception ex01)
                                    {
                                    }

                                    if (!this.mongoDbAdapter.CheckExistsProductSalenew(productSaleNew.id))
                                    {
                                        this.mongoDbAdapter.InsertProduct(productSaleNew);
                                    }
                                    else
                                    {
                                        this.mongoDbAdapter.UpdateProduct(productSaleNew);
                                    }
                                    ShowProduct(productSaleNew);
                                }
                                else
                                {
                                    ShowIgone(igone++);
                                }
                            }
                        }
                    }

                    this.Invoke(new Action(() =>
                    {
                        richTextBox1.AppendText("\n\rWait to next run!");
                    }));

                    Thread.Sleep(10000);
                }
                catch (ThreadAbortException threadAbortException)
                {
                    return;
                }
            }
        }
Пример #14
0
        public override void ProcessProductData(Job job, GABIZ.Base.HtmlAgilityPack.HtmlDocument document)
        {
            bool bSkip = false;

            //long a = QT.Entities.Common.PhanTichGiaTuNoiDung(document.ToString(), 0, 10000000);
            //if (iTypeCrawler==ETypeCrawlRaoVat.CrawlerKeyWord)
            //{
            //    bSkip = false;
            //}
            //else if (configXPath.is_find_new && configXPath.is_reload)
            //{
            //    bSkip = false;
            //}
            //else if (!configXPath.is_find_new && !CheckProductExists(Common.CrcProductID(job.url)))
            //{
            //    bSkip = true;
            //}
            //else if (!configXPath.is_reload && CheckProductExists(Common.CrcProductID(job.url)))
            //{
            //    bSkip = true;
            //}


            if (!bSkip)
            {
                ProductSaleNew product = new ProductSaleNew();
                int            iError  = (new HandlerContentOfHtml()).AnalyticsProductSaleNew(configXPath.domain, job.url, configXPath, product
                                                                                              , this.dicMapClassificationAndCategories, dicCityAndRegex);

                if (iTypeCrawler == ETypeCrawlRaoVat.CrawlerKeyWord && product.tags != null && product.tags.Count > 0)
                {
                    foreach (var keyword in product.tags)
                    {
                        long crcKeyword   = Math.Abs(GABIZ.Base.Tools.getCRC64(keyword));
                        int  statusInsert = sqlRaoVatAdapter.UpdateKeyword(crcKeyword, keyword);
                        if (statusInsert == 1 && eventWhenSuccessProduct != null)
                        {
                            this.eventWhenSuccessProduct(this, string.Format("Insert keyword {0}", keyword));
                        }
                    }
                }
                else if (product.IsDetailSucess)
                {
                    if (product.classification_id > 0 && !dicClassification.ContainsKey(product.classification_id))
                    {
                        dicClassification.Add(product.classification_id, new RaoVatClassification()
                        {
                            id           = product.classification_id,
                            category_ids = new int[] { },
                            name         = product.web_category,
                            website_id   = configXPath.website_id
                        });

                        bool bInsert = sqlRaoVatAdapter.InserClassification(new RaoVatClassification()
                        {
                            id         = product.classification_id,
                            name       = product.web_category,
                            website_id = configXPath.website_id
                        });
                    }

                    //Phân tích category.
                    if (product.category_ids == null || product.category_ids.Count() == 0)
                    {
                        product.category_ids = new List <int> {
                            31
                        };
                        sqlRaoVatAdapter.InsertToNeedAdapter(product.id, product.url, product.web_category);
                    }

                    if (this.eventWhenSuccessProduct != null)
                    {
                        this.eventWhenSuccessProduct(this, product.ToString());
                    }

                    string _id    = this.mongoDbAdapter.GetProductbyID(product.id).Result;
                    bool   bExits = _id != "";
                    if (bExits)
                    {
                        mongoDbAdapter.UpdateProduct(product);
                    }
                    else
                    {
                        mongoDbAdapter.InsertProduct(product);
                    }
                }
            }
        }
Пример #15
0
 private void ParseProduct(ref ProductSaleNew product, Row row)
 {
 }
Пример #16
0
 private void ParseProduct(ref ProductSaleNew product, Row row)
 {
     //product.id = Common.CellToInt64(row, "id", 0);
     //product.name = Common.CellToString(row, "title", "");
     //product.phone_saler = Common.CellToString(row, "phone_saler", "");
     //product.address = Common.CellToString(row, "address", "");
     //product.price = Convert.ToInt64(Common.CellToInt64(row, "price", 0));
     //product.content = Common.CellToString(row, "content", "");
     //product.quality = Common.CellToString(row, "quality", "");
     //product.Avaiable = Common.CellToInt(row, "avaiable", 0);
     //product.post_date = Common.CellToDateTime(row, "post_date", SqlDb.MinDateDb);
     //product.last_edit = DateTime.Now;
     //product.url = Common.CellToString(row, "url", "");
     //product.web_category = Common.CellToString(row, "web_category", "");
     //product.wss_last_update = Common.CellToDateTime(row, "wss_last_update", SqlDb.MinDateDb);
     //product.website_id = Common.CellToInt(row, "config_crawl_id", 0);
     //product.tags = Common.ConvertToList(Common.CellToString(row, "tag", ""));
     //product.images = Common.ConvertToList(Common.CellToString(row, "images", ""));
     //product.car_all = Common.CellToString(row, "car_all", "");
     //product.car_basic = Common.CellToString(row, "car_basic", "");
     //product.car_basic_warranty = Common.CellToString(row, "car_basic_warranty", "");
     //product.car_bluetooth = Common.CellToString(row, "car_bluetooth", "");
     //product.car_bore = Common.CellToString(row, "car_bore", "");
     //product.car_camp_type = Common.CellToString(row, "car_camp_type", "");
     //product.car_camshaft = Common.CellToString(row, "car_camshaft", "");
     //product.car_compression_ratio = Common.CellToString(row, "car_compression_ratio", "");
     //product.car_consumer_rating = Common.CellToString(row, "car_consumer_rating", "");
     //product.car_compression_ratio = Common.CellToString(row, "car_compression_ratio", "");
     //product.car_consumer_rating = Common.CellToString(row, "car_consumer_rating", "");
     //product.car_curb_weight = Common.CellToString(row, "car_curb_weight", "");
     //product.car_cylinders = Common.CellToString(row, "car_cylinders", "");
     //product.car_drive_train = Common.CellToString(row, "car_drive_train", "");
     //product.car_driver_type = Common.CellToString(row, "car_driver_type", "");
     //product.car_engine_size = Common.CellToString(row, "car_engine_size", "");
     //product.car_engine_type = Common.CellToString(row, "car_engine_type", "");
     //product.car_epa_interior_volume = Common.CellToString(row, "car_epa_interior_volume", "");
     //product.car_epa_mileage_est = Common.CellToString(row, "car_epa_mileage_est", "");
     //product.car_epa_mpg = Common.CellToString(row, "car_epa_mpg", "");
     //product.car_free_maintenance = Common.CellToString(row, "car_free_maintenance", "");
     //product.car_fuel_capacity = Common.CellToString(row, "car_fuel_capacity", "");
     //product.car_fuel_type = Common.CellToString(row, "car_fuel_type", "");
     //product.car_gross_vehicle_weight_for_trucks = Common.CellToString(row, "car_gross_vehicle_weight_for_trucks", "");
     //product.car_gross_weight = Common.CellToString(row, "car_gross_weight", "");
     //product.car_head_room = Common.CellToString(row, "car_head_room", "");
     //product.car_heated_seats = Common.CellToString(row, "car_heated_seats", "");
     //product.car_horsepower = Common.CellToString(row, "car_horsepower", "");
     //product.car_height = Common.CellToString(row, "car_height", "");
     //product.car_leg_room = Common.CellToString(row, "car_leg_room", "");
     //product.car_length = Common.CellToString(row, "car_length", "");
     //product.car_maximum_payload = Common.CellToString(row, "car_maximum_payload", "");
     //product.car_navigation = Common.CellToString(row, "car_navigation", "");
     //product.car_number_of_cylinders = Common.CellToString(row, "car_number_of_cylinders", "");
     //product.car_overall_length = Common.CellToString(row, "car_overall_length", "");
     //product.car_payload_capacity_for_trucks = Common.CellToString(row, "car_payload_capacity_for_trucks", "");
     //product.car_price_msrp = Common.CellToString(row, "car_price_msrp", "");
     //product.car_range_in_miles = Common.CellToString(row, "car_range_in_miles", "");
     //product.car_rear_head_room = Common.CellToString(row, "car_rear_head_room", "");
     //product.car_rear_leg_room = Common.CellToString(row, "car_rear_leg_room", "");
     //product.car_rear_shoulder_room = Common.CellToString(row, "car_rear_shoulder_room", "");
     //product.car_seating_capacity = Common.CellToString(row, "car_seating_capacity", "");
     //product.car_shoulder_room = Common.CellToString(row, "car_shoulder_room", "");
     //product.car_stroke = Common.CellToString(row, "car_stroke", "");
     //product.car_tires = Common.CellToString(row, "car_tires", "");
     //product.car_torque = Common.CellToString(row, "car_torque", "");
     //product.car_total_seating = Common.CellToString(row, "car_total_seating", "");
     //product.car_towing_capacity_max = Common.CellToString(row, "car_towing_capacity_max", "");
     //product.car_transmission = Common.CellToString(row, "car_transmission", "");
     //product.car_turning_circle = Common.CellToString(row, "car_turning_circle", "");
     //product.car_valves_per_cylinder = Common.CellToString(row, "car_valves_per_cylinder", "");
     //product.car_wheelbase = Common.CellToString(row, "car_wheelbase", "");
     //product.car_width = Common.CellToString(row, "car_width", "");
     //product.car_cargo_capacity_all_seats_in_place = Common.CellToString(row, "car_cargo_capacity_all_seats_in_place", "");
     //product.car_cargo_capacity_for_cars = Common.CellToString(row, "car_cargo_capacity_for_cars", "");
     //product.car_cargo_maximum_cargo_capacity = Common.CellToString(row, "car_cargo_maximum_cargo_capacity", "");
 }