|
public Build ( string htmlDecodedResponse ) : |
||
| htmlDecodedResponse | string | |
| return | ||
//[TestMethod]
public void AddExtendedDataTest()
{
//arrange
var testdb = new MongoDBForTest().DB;
MongoEntityRepositoryBase<ProductBasicData> basicData = new MongoEntityRepositoryBase<ProductBasicData>(testdb);
List<ProductBasicData> products = basicData.GetAll().ToList();
ProductFetcher extendDataFetcher = new ProductFetcher();
ProductDataExtendedDataBuilder builder = new ProductDataExtendedDataBuilder();
//act
foreach (var product in products)
{
string crawledHtml = extendDataFetcher.GetProductExtendedData(product);
product.ExtendedData = builder.Build(crawledHtml);
}
}
public void ProductDataExtendedDataBuilderTest()
{
//arrange
ProductBasicData productBasicData = new ProductBasicData();
string htmlDecodedResponse = GetProductExampleHTML(productBasicData);
ProductDataExtendedDataBuilder builder = new ProductDataExtendedDataBuilder();
//act
ProductExtendedData extendedData = builder.Build(htmlDecodedResponse);
//assert
Assert.IsNotNull(extendedData);
Assert.IsNotNull(extendedData.NutritionTable);
Assert.AreEqual(8, extendedData.NutritionTable.Count);
Assert.IsFalse(string.IsNullOrEmpty(extendedData.RawHtmlResponse));
Assert.AreEqual("1485019", extendedData.ProductId);
Assert.AreEqual("200 גרם", extendedData.Weight);
Assert.AreEqual("ישראל", extendedData.ProductionCountry);
//Assert.AreEqual("117", extendedData.Energy);
//Assert.AreEqual("11.5", extendedData.Proteins);
//Assert.AreEqual("6.5", extendedData.Carbohydrate);
//Assert.AreEqual("5", extendedData.fa);
}
//[TestMethod]
public void ProductFetcherIntegrationTest()
{
//arrange
string folder = @"C:\Temp\spontaneouse\catalogData";
List<ProductBasicData> products = new List<ProductBasicData>();
var allFiles = Directory.GetFiles(folder, "*.txt");
foreach (var item in allFiles)
{
string html = File.ReadAllText(item);
ProductBasicDataFetcher fetcher = new ProductBasicDataFetcher(html);
ProductFetcher extendDataFetcher = new ProductFetcher();
ProductDataExtendedDataBuilder builder = new ProductDataExtendedDataBuilder();
//act
products = fetcher.FetchProducts();
foreach (var product in products)
{
product.Category = item;
string crawledHtml = extendDataFetcher.GetProductExtendedData(product);
product.ExtendedData = builder.Build(crawledHtml);
}
}
//assert
foreach (var product in products)
{
Assert.IsNotNull(product.ExtendedData);
}
//TODO Add save to DB method.
}
private static void UpdateExtendedData()
{
CrawledProducts productsDb = new CrawledProducts();
MongoEntityRepositoryBase<ProductBasicData> basicData = new MongoEntityRepositoryBase<ProductBasicData>(productsDb.DB);
List<ProductBasicData> products = basicData.GetAll().ToList();
log.InfoFormat("Found {0} products for fetching", products.Count);
//foreach (var item in allFiles)
//{
//string html = File.ReadAllText(item);
//ProductBasicDataFetcher fetcher = new ProductBasicDataFetcher(html);
ProductFetcher extendDataFetcher = new ProductFetcher();
ProductDataExtendedDataBuilder builder = new ProductDataExtendedDataBuilder();
//act
//products = fetcher.FetchProducts();
int count = 0;
foreach (var product in products)
{
try
{
if (product.ExtendedData != null)
continue;
//product.Category = item;
string crawledHtml = extendDataFetcher.GetProductExtendedData(product);
product.ExtendedData = builder.Build(crawledHtml);
basicData.Update(product);
count++;
int nutritionTableCount = 0;
if (product.ExtendedData != null && product.ExtendedData.NutritionTable != null)
{
nutritionTableCount = product.ExtendedData.NutritionTable.Count;
log.InfoFormat("Product fetched = {0} Product Name = {2} nutritionTableCount = {1}.", count, nutritionTableCount, product.ProductName);
}
}
catch (Exception ex)
{
log.WarnFormat("Problem on updating extended data product={0}, ex={1}.", product, ex);
}
System.Threading.Thread.Sleep(400);
}
}