//[TestMethod]
public void AddExtendedDataTest()
{
//arrange
var testdb = new MongoDBForTest().DB;
MongoEntityRepositoryBase<ProductBasicData> basicData = new MongoEntityRepositoryBase<ProductBasicData>(testdb);
List<ProductBasicData> products = basicData.GetAll().ToList();
ProductFetcher extendDataFetcher = new ProductFetcher();
ProductDataExtendedDataBuilder builder = new ProductDataExtendedDataBuilder();
//act
foreach (var product in products)
{
string crawledHtml = extendDataFetcher.GetProductExtendedData(product);
product.ExtendedData = builder.Build(crawledHtml);
}
}
private static void UpdateExtendedData()
{
CrawledProducts productsDb = new CrawledProducts();
MongoEntityRepositoryBase<ProductBasicData> basicData = new MongoEntityRepositoryBase<ProductBasicData>(productsDb.DB);
List<ProductBasicData> products = basicData.GetAll().ToList();
log.InfoFormat("Found {0} products for fetching", products.Count);
//foreach (var item in allFiles)
//{
//string html = File.ReadAllText(item);
//ProductBasicDataFetcher fetcher = new ProductBasicDataFetcher(html);
ProductFetcher extendDataFetcher = new ProductFetcher();
ProductDataExtendedDataBuilder builder = new ProductDataExtendedDataBuilder();
//act
//products = fetcher.FetchProducts();
int count = 0;
foreach (var product in products)
{
try
{
if (product.ExtendedData != null)
continue;
//product.Category = item;
string crawledHtml = extendDataFetcher.GetProductExtendedData(product);
product.ExtendedData = builder.Build(crawledHtml);
basicData.Update(product);
count++;
int nutritionTableCount = 0;
if (product.ExtendedData != null && product.ExtendedData.NutritionTable != null)
{
nutritionTableCount = product.ExtendedData.NutritionTable.Count;
log.InfoFormat("Product fetched = {0} Product Name = {2} nutritionTableCount = {1}.", count, nutritionTableCount, product.ProductName);
}
}
catch (Exception ex)
{
log.WarnFormat("Problem on updating extended data product={0}, ex={1}.", product, ex);
}
System.Threading.Thread.Sleep(400);
}
}
public void ProductFetcher_ExtendedData_Test()
{
//arrange
ProductBasicData product = new ProductBasicData() { ProductId = "1485019" };
ProductFetcher fetcher = new ProductFetcher();
//act
string htmlDecodedResponse = fetcher.GetProductExtendedData(product);
//File.Create(@"c:\Temp\ExtendedData.txt");
using(var writer = File.CreateText(@"c:\Temp\ExtendedData.txt"))
{
writer.Write(htmlDecodedResponse);
}
//assert
Assert.IsNotNull(htmlDecodedResponse);
Assert.IsTrue(htmlDecodedResponse.Contains(product.ProductId));
}
//[TestMethod]
public void ProductFetcherIntegrationTest()
{
//arrange
string folder = @"C:\Temp\spontaneouse\catalogData";
List<ProductBasicData> products = new List<ProductBasicData>();
var allFiles = Directory.GetFiles(folder, "*.txt");
foreach (var item in allFiles)
{
string html = File.ReadAllText(item);
ProductBasicDataFetcher fetcher = new ProductBasicDataFetcher(html);
ProductFetcher extendDataFetcher = new ProductFetcher();
ProductDataExtendedDataBuilder builder = new ProductDataExtendedDataBuilder();
//act
products = fetcher.FetchProducts();
foreach (var product in products)
{
product.Category = item;
string crawledHtml = extendDataFetcher.GetProductExtendedData(product);
product.ExtendedData = builder.Build(crawledHtml);
}
}
//assert
foreach (var product in products)
{
Assert.IsNotNull(product.ExtendedData);
}
//TODO Add save to DB method.
}