Beispiel #1
0
        private static void SyncCompanyCrawler()
        {
            int count = 0;
            RedisCacheCompanyCrawler redisCompany            = RedisCacheCompanyCrawler.Instance();
            RedisCompanyWaitCrawler  redisCompanyWaitCrawler = RedisCompanyWaitCrawler.Instance();
            RedisLastUpdateProduct   redisLstProduct         = RedisLastUpdateProduct.Instance();
            CacheProductInfo         cacheProductInfo        = new CacheProductInfo(new SqlDb(Server.ConnectionString));

            ProductAdapter productAdapter = new ProductAdapter(new SqlDb(Server.ConnectionString));
            List <long>    lstCrawler     = productAdapter.GetAllCompanyIdCrawlerReload();

            for (int i = 0; i < lstCrawler.Count; i++)
            {
                redisCompanyWaitCrawler.SetNexReload(lstCrawler[i], -10);

                //long companyID = lstCrawler[i];
                //if (!redisCompanyWaitCrawler.CheckHaveItemReload(companyID))
                //{
                //    count++;
                //    redisCompanyWaitCrawler.SetNexReload(companyID, 1);
                //    Company cmp = new Company(companyID);
                //    redisCompany.SetCompanyInfo(companyID, cmp.Domain, 1, 1);
                //    cacheProductInfo.RefeshCacheProductInfo(companyID);
                //    redisLstProduct.ResetLastUpdateForCompany(companyID, productAdapter.GetListProductIdOfCompany(companyID));
                //}
                log.Info(string.Format("sync company {0} {1}/{2}", count, i, lstCrawler.Count));
            }

            List <long> lstCrawlerFindNew = productAdapter.GetAllCompanyIdCrawlerFindNew();

            for (int i = 0; i < lstCrawler.Count; i++)
            {
                redisCompanyWaitCrawler.SetNexFindNew(lstCrawler[i], -10);
                long companyID = lstCrawler[i];

                //if (!redisCompanyWaitCrawler.CheckHaveItemFindNew(companyID))
                //{
                //    count++;
                //    redisCompanyWaitCrawler.SetNexFindNew(companyID, 1);
                //    Company cmp = new Company(companyID);
                //    redisCompany.SetCompanyInfo(companyID, cmp.Domain, 1, 1);
                //    cacheProductInfo.RefeshCacheProductInfo(companyID);
                //    redisLstProduct.ResetLastUpdateForCompany(companyID, productAdapter.GetListProductIdOfCompany(companyID));

                //}
                log.Info(string.Format("sync company {0} {1}/{2}", count, i, lstCrawler.Count));
            }
            Console.WriteLine("Success sync company crawl!");
        }
Beispiel #2
0
        public void Run(System.Threading.CancellationToken token)
        {
            ProductAdapter productAdapter = new ProductAdapter(new SqlDb("Data Source=42.112.28.93;Initial Catalog=QT_2;Persist Security Info=True;User ID=wss_price;Password=HzlRt4$$axzG-*UlpuL2gYDu;connection timeout=200"));

            log.InfoFormat("Start run at {0}", DateTime.Now.ToString(CultureInfo.InvariantCulture));
            CacheProductHash       cashProductHash        = CacheProductHash.Instance();
            RedisLastUpdateProduct cacheLastUpdateProduct = RedisLastUpdateProduct.Instance();
            int countProduct = 0;

            try
            {
                var lstFn = productAdapter.GetAllCompanyIdCrawlerFindNew();
                var lstRl = productAdapter.GetAllCompanyIdCrawlerReload();
                RedisCompanyWaitCrawler redisCache = RedisCompanyWaitCrawler.Instance();
                redisCache.SyncCompanyFindNew(lstFn);
                redisCache.SyncCompanyReload(lstRl);
            }
            catch (Exception ex)
            {
                log.Error(ex);
            }

            var lst           = new List <QT.Entities.CrawlerProduct.Cache.ProductHash>();
            var lstLastUpdate = new List <long>();
            var lstCompany    = productAdapter.GetAllCompanyIdCrawler();

            foreach (var companyID in lstCompany)
            {
                Company cmp = new Company(companyID);
                productAdapter.DeleteProductUnvalidOfCOmpany(companyID);
                DataTable tbl = productAdapter.GetProductResetColumnDuplicateAndChange(companyID);
                foreach (DataRow rowProduct in tbl.Rows)
                {
                    long   productId        = QT.Entities.Common.Obj2Int64(rowProduct["ID"]);
                    long   originPrice      = QT.Entities.Common.Obj2Int64(rowProduct["OriginPrice"]);
                    string name             = rowProduct["Name"].ToString();
                    long   price            = QT.Entities.Common.Obj2Int64(rowProduct["Price"]);
                    string imageUrl         = Convert.ToString(rowProduct["ImageUrls"]);
                    string detailUrl        = Convert.ToString(rowProduct["DetailUrl"]);
                    int    inStock          = QT.Entities.Common.Obj2Int(rowProduct["InStock"]);
                    bool   valid            = QT.Entities.Common.Obj2Bool(rowProduct["Valid"]);
                    string shortDescription = QT.Entities.Common.CellToString(rowProduct["ShortDescription"], "");
                    long   categoryId       = rowProduct["ClassificationID"] == DBNull.Value ? 0 : QT.Entities.Common.Obj2Int64(rowProduct["ClassificationID"]);
                    long   hashChange       = ProductEntity.GetHashChangeInfo(inStock, valid, price, name, imageUrl, categoryId, shortDescription, originPrice);
                    long   hashDuplicate    = Product.GetHashDuplicate(cmp.Domain, price, name, imageUrl);
                    long   hashImage        = Product.GetHashImageInfo(imageUrl);
                    lst.Add(new QT.Entities.CrawlerProduct.Cache.ProductHash()
                    {
                        HashChange    = hashChange,
                        HashDuplicate = hashDuplicate,
                        HashImage     = hashImage,
                        Id            = productId,
                        Price         = price,
                        url           = detailUrl
                    });
                    lstLastUpdate.Add(productId);
                }
                cashProductHash.SetCacheProductHash(companyID, lst, 100); cacheLastUpdateProduct.RemoveAllLstProduct(companyID);
                cacheLastUpdateProduct.UpdateBathLastUpdateProduct(companyID, lstLastUpdate, DateTime.Now.AddDays(-1));
                productAdapter.UpdateCountProductForCompany(companyID, lstLastUpdate.Count, lstLastUpdate.Count);
                lst.Clear();
                lstLastUpdate.Clear();
                log.Info(string.Format("Complete Company: {0} {1}/{2}", companyID, countProduct++, lstCompany.Count));
            }
            log.Info("Complete all company");
            NextRun = DateTime.Now.AddHours(MAX_HOUR_LOOP);
            log.InfoFormat("End at {0}", DateTime.Now.ToString());
        }