private static void ResetCacheProductInfo()
{
ProductAdapter productAdapter = new ProductAdapter(new SqlDb(Server.ConnectionString));
CacheProductInfo c = new CacheProductInfo(new SqlDb(Server.ConnectionString));
Console.WriteLine("1- Reset all cache by company (domain)");
Console.WriteLine("2- Reset all cache");
Console.WriteLine("3- Reset only 'product_info' all company");
Console.WriteLine("4- Reset cache ProductInfo by domain");
Console.Write("Input choose:");
int i = Convert.ToInt32(Console.ReadLine());
switch (i)
{
case 1:
{
Console.Write("Input domain:");
string strDomain = Console.ReadLine().Trim();
c.RefreshAllCacheAllProduct(productAdapter.GetCompanyIDFromDomain(strDomain));
} break;
case 2:
{
List <long> lstCompany = productAdapter.GetAllCompanyIdCrawler();
for (int j = 0; j < lstCompany.Count; j++)
{
c.RefreshAllCacheAllProduct(lstCompany[j]);
log.Info(string.Format("Update company {0}/{1}", j, lstCompany.Count - 1));
}
} break;
case 3:
{
List <long> lstCompany = productAdapter.GetAllCompanyIdCrawler();
for (int j = 0; j < lstCompany.Count; j++)
{
c.ResetAllCacheProductInfo(lstCompany[j]);
log.Info(string.Format("Update company {0}/{1}", j, lstCompany.Count - 1));
}
} break;
case 4:
{
Console.Write("Input domain:");
string strDomain = Console.ReadLine().Trim();
c.ResetForCompany(productAdapter.GetCompanyIDFromDomain(strDomain));
} break;
}
}
public void Start()
{
ProductAdapter productAdapter = new ProductAdapter(new SqlDb(Server.ConnectionString));
List <long> lstCOmpanyDownloaded = productAdapter.GetCOmpanyDownloadedHTML();
foreach (long companyID in productAdapter.GetAllCompanyIdCrawler())
{
if (!lstCOmpanyDownloaded.Contains(companyID))
{
lst.Enqueue(companyID);
}
}
int MaxThread = 100;
for (int i = 0; i < MaxThread; i++)
{
int jThread = i;
Task.Factory.StartNew(() =>
{
WorkerExportHtml worker = new WorkerExportHtml(jThread);
worker.eventGetCompanyID += new WorkerExportHtml.GetCompanyId(GetCompanyCrawler);
worker.eventProcessHtml += new WorkerExportHtml.DelegateProcessHtml(SaveHtml);
worker.eventWhenFinish += new WorkerExportHtml.DelegateWhenFinish(EventWhenFinsih);
worker.Start();
});
}
}
private static void ResesCacheProduct()
{
ProductAdapter productAdapter = new ProductAdapter(new SqlDb(Server.ConnectionString));
Console.WriteLine("1- Reset all Companies)");
Console.WriteLine("2- Reset for Company");
Console.Write("Input choose:");
int i = Convert.ToInt32(Console.ReadLine());
switch (i)
{
case 1:
{
int countProduct = 0;
var lstCompany = productAdapter.GetAllCompanyIdCrawler();
foreach (var companyID in lstCompany)
{
ResetCacheProduct(companyID);
log.Info(string.Format("Complete Company: {0} {1}/{2}", companyID, countProduct++, lstCompany.Count));
}
log.Info("Complete all company");
} break;
case 2:
{
Console.Write("Input domain:");
string strDomain = Console.ReadLine().Trim();
long companyID = productAdapter.GetCompanyIdByDomain(strDomain);
ResetCacheProduct(companyID);
log.Info(string.Format("Complete Company: {0}", companyID));
} break;
}
}
public void Start()
{
ProductAdapter pa = new ProductAdapter(new SqlDb(ConfigCrawler.ConnectProduct));
QT.Entities.Server.ConnectionString = ConfigCrawler.ConnectProduct;
List <long> companyIds = pa.GetAllCompanyIdCrawler();
for (int i = 0; i < companyIds.Count; i++)
{
long companyId = companyIds[i];
Company company = new Company(companyId);
Configuration configuration = new Configuration(companyId);
ProductParse parse = new ProductParse();
ProductEntity productEntity = new ProductEntity();
HtmlDocument document = new HtmlDocument();
string url = configuration.LinkTest;
string html = GABIZ.Base.HtmlUrl.HTMLTransmitter.getHTML(url, 45, 2);
if (!string.IsNullOrEmpty(html))
{
document.LoadHtml(html);
try
{
parse.Analytics(productEntity, document, configuration.LinkTest, configuration, company.Domain);
if (!productEntity.IsSuccessData(configuration.CheckPrice))
{
pa.GetSqlDb()
.RunQuery("insert into Company_FailConfig (CompanyId) Values (@CompanyId)",
CommandType.Text,
new SqlParameter[]
{
SqlDb.CreateParamteterSQL("@CompanyId", companyId, SqlDbType.BigInt)
});
}
}
catch (Exception ex)
{
pa.GetSqlDb()
.RunQuery("insert into Company_FailConfig (CompanyId, Error) Values (@CompanyId, @Error)",
CommandType.Text,
new SqlParameter[]
{
SqlDb.CreateParamteterSQL("@CompanyId", companyId, SqlDbType.BigInt),
SqlDb.CreateParamteterSQL("@Error", ex.Message + "\n" + ex.StackTrace,
SqlDbType.NVarChar)
});
}
_log.Info(string.Format("Run data {0}/ {1}", i, companyIds.Count));
}
}
}
private static void DeleteAllUnvalidProduct()
{
ProductAdapter productAdapter = new ProductAdapter(new SqlDb(Server.ConnectionString));
List <long> lstCompanyCrawler = productAdapter.GetAllCompanyIdCrawler();
for (int i = 0; i < lstCompanyCrawler.Count; i++)
{
productAdapter.DeleteProductUnvalidOfCOmpany(lstCompanyCrawler[i]);
log.Info(string.Format("Delete unvalid product for company {0}/{1}", i, lstCompanyCrawler.Count - 1));
}
log.Info("Delete unvalid product for company");
}
private static void ManagerCacheLastTime()
{
ProductAdapter productAdapter = new ProductAdapter(new SqlDb(Server.ConnectionString));
RedisLastUpdateProduct redisLastUpdate = RedisLastUpdateProduct.Instance();
Console.WriteLine("1- Reset cache by company (domain)");
Console.WriteLine("2- Cache for AllCompany");
Console.Write("Input choose:");
int i = Convert.ToInt32(Console.ReadLine());
switch (i)
{
case 1:
{
Console.Write("Input domain you need reset cache:");
string strDomain = Console.ReadLine().Trim();
long CompanyID = productAdapter.GetCompanyIDFromDomain(strDomain);
redisLastUpdate.RemoveAllLstProduct(CompanyID);
DataTable tbl = productAdapter.sqlDb.GetTblData("select id from product where company = @CompanyID", CommandType.Text, new System.Data.SqlClient.SqlParameter[] {
SqlDb.CreateParamteterSQL("@CompanyID", CompanyID, SqlDbType.BigInt)
});
List <long> productIDs = new List <long>();
foreach (DataRow rowInfo in tbl.Rows)
{
productIDs.Add(Convert.ToInt64(rowInfo["id"]));
}
redisLastUpdate.UpdateBathLastUpdateProduct(CompanyID, productIDs, new DateTime(1900, 1, 1, 0, 0, 0));
Console.WriteLine("Reseted all product LastUpdate in cache");
} break;
case 2:
{
List <long> companyCrawelrs = productAdapter.GetAllCompanyIdCrawler();
int Count = 0;
foreach (long CompanyID in companyCrawelrs)
{
Count++;
DataTable tbl = productAdapter.sqlDb.GetTblData("select id from product where company = @CompanyID", CommandType.Text, new System.Data.SqlClient.SqlParameter[] {
SqlDb.CreateParamteterSQL("@CompanyID", CompanyID, SqlDbType.BigInt)
});
List <long> productIDs = new List <long>();
foreach (DataRow rowInfo in tbl.Rows)
{
productIDs.Add(Convert.ToInt64(rowInfo["id"]));
}
redisLastUpdate.RemoveAllLstProduct(CompanyID);
redisLastUpdate.UpdateBathLastUpdateProduct(CompanyID, productIDs, new DateTime(1990, 1, 1, 0, 0, 0));
log.Info(string.Format("Reseted all products for company {0}/{1} LastUpdate in cache", Count, companyCrawelrs.Count));
}
} break;
}
}
private static void ResetAllCopmanyInfo()
{
ProductAdapter productAdapter = new ProductAdapter(new SqlDb(Server.ConnectionString));
RedisCacheCompanyCrawler redisCompanyInfo = RedisCacheCompanyCrawler.Instance();
List <long> lstCompanyCrawler = productAdapter.GetAllCompanyIdCrawler();
for (int i = 0; i < lstCompanyCrawler.Count; i++)
{
Company comp = new Company(lstCompanyCrawler[i]);
redisCompanyInfo.SetCompanyInfo(comp.ID, comp.Domain, 12, 12);
log.Info(string.Format("Pushed companyInfo {0}/{1}", i, lstCompanyCrawler.Count - 1));
}
log.Info("Success push companyInfo");
}
private static void DeleteUnvalidProductCrawler()
{
ProductAdapter productAdapter = new ProductAdapter(new SqlDb(Server.ConnectionString));
int COunt = 0;
List <long> lst = productAdapter.GetAllCompanyIdCrawler();
for (int i = 0; i < lst.Count; i++)
{
COunt++;
productAdapter.sqlDb.RunQuery("delete product where company = @company_id and valid = 0 and isnull(IsBlackList,0) = 0",
CommandType.Text, new System.Data.SqlClient.SqlParameter[] {
SqlDb.CreateParamteterSQL("@company_id", lst[i], SqlDbType.BigInt)
}, true, 1000);
log.Info(string.Format("Delete unvalid product for {0}/{1}", i, lst.Count - 1));
}
}
public void Run(System.Threading.CancellationToken token)
{
List <long> companyCrawelrs = productAdapter.GetAllCompanyIdCrawler();
int Count = 0;
foreach (long CompanyID in companyCrawelrs)
{
Count++;
DataTable tbl = productAdapter.sqlDb.GetTblData("select id from product where company = @CompanyID", CommandType.Text, new System.Data.SqlClient.SqlParameter[] {
SqlDb.CreateParamteterSQL("@CompanyID", CompanyID, SqlDbType.BigInt)
});
List <long> productIDs = new List <long>();
foreach (DataRow rowInfo in tbl.Rows)
{
productIDs.Add(Convert.ToInt64(rowInfo["id"]));
}
redisLastUpdate.RemoveAllLstProduct(CompanyID);
redisLastUpdate.UpdateBathLastUpdateProduct(CompanyID, productIDs, DateTime.Now);
log.Info(string.Format("Reseted all products for company {0}/{1} LastUpdate in cache", Count, companyCrawelrs.Count));
}
}
private static void ResetAllCacheProduct()
{
Console.Write("Input sleep:");
int iSleep = Convert.ToInt32(Console.ReadLine());
ProductAdapter productAdapter = new ProductAdapter(new SqlDb(Server.ConnectionString));
RedisCheckDuplicateAdapter redisCheckDuplicate = RedisCheckDuplicateAdapter.Instace();
RedisLastUpdateProduct redisLastUpdate = RedisLastUpdateProduct.Instance();
CacheProductInfo cacheProductInfo = new CacheProductInfo(productAdapter.sqlDb);
int count = 0;
var companyIDs = productAdapter.GetAllCompanyIdCrawler();
foreach (long companyID in companyIDs)
{
Company cmp = new Company(companyID);
redisCheckDuplicate.ResetForCompany(companyID, cmp.Domain, productAdapter.GetSqlDb());
redisLastUpdate.ResetForCompany(companyID, productAdapter.sqlDb);
cacheProductInfo.ResetForCompany(companyID);
log.Info(string.Format("Reset cache company {0}/{1}-{2} {3} products", count++, companyIDs.Count, cmp.Domain, cmp.TotalProduct));
Thread.Sleep(iSleep * 1000);
}
Console.WriteLine("Success reset data!");
}
public void Run(System.Threading.CancellationToken token)
{
ProductAdapter productAdapter = new ProductAdapter(new SqlDb("Data Source=42.112.28.93;Initial Catalog=QT_2;Persist Security Info=True;User ID=wss_price;Password=HzlRt4$$axzG-*UlpuL2gYDu;connection timeout=200"));
log.InfoFormat("Start run at {0}", DateTime.Now.ToString(CultureInfo.InvariantCulture));
CacheProductHash cashProductHash = CacheProductHash.Instance();
RedisLastUpdateProduct cacheLastUpdateProduct = RedisLastUpdateProduct.Instance();
int countProduct = 0;
try
{
var lstFn = productAdapter.GetAllCompanyIdCrawlerFindNew();
var lstRl = productAdapter.GetAllCompanyIdCrawlerReload();
RedisCompanyWaitCrawler redisCache = RedisCompanyWaitCrawler.Instance();
redisCache.SyncCompanyFindNew(lstFn);
redisCache.SyncCompanyReload(lstRl);
}
catch (Exception ex)
{
log.Error(ex);
}
var lst = new List <QT.Entities.CrawlerProduct.Cache.ProductHash>();
var lstLastUpdate = new List <long>();
var lstCompany = productAdapter.GetAllCompanyIdCrawler();
foreach (var companyID in lstCompany)
{
Company cmp = new Company(companyID);
productAdapter.DeleteProductUnvalidOfCOmpany(companyID);
DataTable tbl = productAdapter.GetProductResetColumnDuplicateAndChange(companyID);
foreach (DataRow rowProduct in tbl.Rows)
{
long productId = QT.Entities.Common.Obj2Int64(rowProduct["ID"]);
long originPrice = QT.Entities.Common.Obj2Int64(rowProduct["OriginPrice"]);
string name = rowProduct["Name"].ToString();
long price = QT.Entities.Common.Obj2Int64(rowProduct["Price"]);
string imageUrl = Convert.ToString(rowProduct["ImageUrls"]);
string detailUrl = Convert.ToString(rowProduct["DetailUrl"]);
int inStock = QT.Entities.Common.Obj2Int(rowProduct["InStock"]);
bool valid = QT.Entities.Common.Obj2Bool(rowProduct["Valid"]);
string shortDescription = QT.Entities.Common.CellToString(rowProduct["ShortDescription"], "");
long categoryId = rowProduct["ClassificationID"] == DBNull.Value ? 0 : QT.Entities.Common.Obj2Int64(rowProduct["ClassificationID"]);
long hashChange = ProductEntity.GetHashChangeInfo(inStock, valid, price, name, imageUrl, categoryId, shortDescription, originPrice);
long hashDuplicate = Product.GetHashDuplicate(cmp.Domain, price, name, imageUrl);
long hashImage = Product.GetHashImageInfo(imageUrl);
lst.Add(new QT.Entities.CrawlerProduct.Cache.ProductHash()
{
HashChange = hashChange,
HashDuplicate = hashDuplicate,
HashImage = hashImage,
Id = productId,
Price = price,
url = detailUrl
});
lstLastUpdate.Add(productId);
}
cashProductHash.SetCacheProductHash(companyID, lst, 100); cacheLastUpdateProduct.RemoveAllLstProduct(companyID);
cacheLastUpdateProduct.UpdateBathLastUpdateProduct(companyID, lstLastUpdate, DateTime.Now.AddDays(-1));
productAdapter.UpdateCountProductForCompany(companyID, lstLastUpdate.Count, lstLastUpdate.Count);
lst.Clear();
lstLastUpdate.Clear();
log.Info(string.Format("Complete Company: {0} {1}/{2}", companyID, countProduct++, lstCompany.Count));
}
log.Info("Complete all company");
NextRun = DateTime.Now.AddHours(MAX_HOUR_LOOP);
log.InfoFormat("End at {0}", DateTime.Now.ToString());
}