private static void ParseDecompressedToDB()
{
string folder = @"C:\Temp\spontaneouse\catalogData";
CrawledProducts productsDb = new CrawledProducts();
DBLoader dbLoader = new DBLoader(productsDb.DB, folder);
var statuses = dbLoader.DoLoad();
foreach (var status in statuses)
{
log.InfoFormat("File ={0}, excepton={1}.", status.File, status.ErrorException);
}
}
private static void UpdateExtendedData()
{
CrawledProducts productsDb = new CrawledProducts();
MongoEntityRepositoryBase<ProductBasicData> basicData = new MongoEntityRepositoryBase<ProductBasicData>(productsDb.DB);
List<ProductBasicData> products = basicData.GetAll().ToList();
log.InfoFormat("Found {0} products for fetching", products.Count);
//foreach (var item in allFiles)
//{
//string html = File.ReadAllText(item);
//ProductBasicDataFetcher fetcher = new ProductBasicDataFetcher(html);
ProductFetcher extendDataFetcher = new ProductFetcher();
ProductDataExtendedDataBuilder builder = new ProductDataExtendedDataBuilder();
//act
//products = fetcher.FetchProducts();
int count = 0;
foreach (var product in products)
{
try
{
if (product.ExtendedData != null)
continue;
//product.Category = item;
string crawledHtml = extendDataFetcher.GetProductExtendedData(product);
product.ExtendedData = builder.Build(crawledHtml);
basicData.Update(product);
count++;
int nutritionTableCount = 0;
if (product.ExtendedData != null && product.ExtendedData.NutritionTable != null)
{
nutritionTableCount = product.ExtendedData.NutritionTable.Count;
log.InfoFormat("Product fetched = {0} Product Name = {2} nutritionTableCount = {1}.", count, nutritionTableCount, product.ProductName);
}
}
catch (Exception ex)
{
log.WarnFormat("Problem on updating extended data product={0}, ex={1}.", product, ex);
}
System.Threading.Thread.Sleep(400);
}
}
public ProductCatalogViewModel GetProductByLastCharsOfBarcode(string barcode)
{
using (CrawledProducts productsDb = new CrawledProducts())
{
MongoEntityRepositoryBase<ProductBasicData> basicData =
new MongoEntityRepositoryBase<ProductBasicData>(productsDb.DB);
var allProducts = basicData.GetAll();
List<ProductBasicData> selected = allProducts.Where(c => barcode.EndsWith(c.ProductId)).ToList();
List<ProductCatalogViewModel> productList = selected.ToProductCatalogModel();
if (productList != null && productList.Count > 0)
{
return productList.FirstOrDefault(c => c.ProductId.Length == productList.Max(lenght => lenght.ProductId.Length));
}
else
{
return null;
}
}
}