public override System.Collections.Generic.IEnumerable<List<Product>> ReadFromDir(string dir)
{
if (!Directory.Exists(dir))
{
Console.WriteLine("Directory not found: " + dir);
yield break;
}
Console.WriteLine("Started reading from: " + dir);
List<Product> products = new List<Product>();
string[] filePaths = Util.ConcatArrays(Directory.GetFiles(dir, "*.xml"), Directory.GetFiles(dir, "*.csv"));
// Initialize XmlValueReader and its keys
XmlValueReader xvr = new XmlValueReader();
xvr.ProductEnd = "product";
xvr.AddKeys("sku", XmlNodeType.Element);
xvr.AddKeys("advertisercategory", XmlNodeType.Element);
xvr.AddKeys("name", XmlNodeType.Element);
xvr.AddKeys("manufacturer", XmlNodeType.Element);
xvr.AddKeys("saleprice", XmlNodeType.Element);
xvr.AddKeys("currency", XmlNodeType.Element);
xvr.AddKeys("buyurl", XmlNodeType.Element);
xvr.AddKeys("imageurl", XmlNodeType.Element);
xvr.AddKeys("description", XmlNodeType.Element);
xvr.AddKeys("lastupdated", XmlNodeType.Element);
xvr.AddKeys("instock", XmlNodeType.Element);
XmlReaderSettings settings = new XmlReaderSettings();
settings.DtdProcessing = DtdProcessing.Ignore;
Product p = new Product();
foreach (string file in filePaths)
{
string fileUrl = Path.GetFileNameWithoutExtension(file).Split(null)[0].Replace('$', '/');
xvr.CreateReader(file, settings);
foreach (DualKeyDictionary<string, XmlNodeType, string> dkd in xvr.ReadProducts())
{
p.SKU = dkd["sku"][XmlNodeType.Element];
p.Title = dkd["name"][XmlNodeType.Element];
p.Brand = dkd["manufacturer"][XmlNodeType.Element];
p.Price = dkd["saleprice"][XmlNodeType.Element];
p.Currency = dkd["currency"][XmlNodeType.Element];
p.Category = dkd["advertisercategory"][XmlNodeType.Element];
p.Url = dkd["buyurl"][XmlNodeType.Element];
p.Image_Loc = dkd["imageurl"][XmlNodeType.Element];
p.Description = dkd["description"][XmlNodeType.Element];
p.LastModified = dkd["lastupdated"][XmlNodeType.Element];
p.Stock = dkd["instock"][XmlNodeType.Element];
p.Affiliate = "CommissionJunction";
p.FileName = file;
p.Webshop = fileUrl;
// Hash the title and the webshop into a unique ID, because CommissionJunctionReader didn't provide any
p.AffiliateProdID = (p.Title + p.Webshop).ToSHA256();
products.Add(p);
p = new Product();
if (products.Count > PackageSize)
{
yield return products;
products.Clear();
}
}
yield return products;
products.Clear();
}
}
public override IEnumerable<List<Product>> ReadFromFile(string file)
{
// Initialize XmlValueReader and its keys
using (XmlValueReader xvr = new XmlValueReader())
{
List<Product> products = new List<Product>();
string fileUrl = Path.GetFileNameWithoutExtension(file).Split(null)[0].Replace('$', '/');
xvr.ProductEnd = "product";
xvr.AddKeys("sku", XmlNodeType.Element);
xvr.AddKeys("advertisercategory", XmlNodeType.Element);
xvr.AddKeys("name", XmlNodeType.Element);
xvr.AddKeys("manufacturer", XmlNodeType.Element);
xvr.AddKeys("saleprice", XmlNodeType.Element);
xvr.AddKeys("currency", XmlNodeType.Element);
xvr.AddKeys("buyurl", XmlNodeType.Element);
xvr.AddKeys("imageurl", XmlNodeType.Element);
xvr.AddKeys("description", XmlNodeType.Element);
xvr.AddKeys("lastupdated", XmlNodeType.Element);
xvr.AddKeys("instock", XmlNodeType.Element);
XmlReaderSettings settings = new XmlReaderSettings();
settings.DtdProcessing = DtdProcessing.Ignore;
settings.CloseInput = true;
Product p = new Product();
xvr.CreateReader(file, settings);
foreach (DualKeyDictionary<string, XmlNodeType, string> dkd in xvr.ReadProducts())
{
p.SKU = dkd["sku"][XmlNodeType.Element];
p.Title = dkd["name"][XmlNodeType.Element];
p.Brand = dkd["manufacturer"][XmlNodeType.Element];
p.Price = dkd["saleprice"][XmlNodeType.Element];
p.Currency = dkd["currency"][XmlNodeType.Element];
p.Category = dkd["advertisercategory"][XmlNodeType.Element];
p.Url = dkd["buyurl"][XmlNodeType.Element];
p.Image_Loc = dkd["imageurl"][XmlNodeType.Element];
p.Description = dkd["description"][XmlNodeType.Element];
p.LastModified = dkd["lastupdated"][XmlNodeType.Element];
p.Stock = dkd["instock"][XmlNodeType.Element];
p.Affiliate = "CommissionJunction";
p.FileName = file;
p.Webshop = fileUrl;
// Hash the title and the webshop into a unique ID, because CommissionJunctionReader didn't provide any
p.AffiliateProdID = (p.Title + p.Webshop).ToSHA256();
products.Add(p);
p = new Product();
if (products.Count >= PackageSize)
{
yield return products;
products.Clear();
}
}
yield return products;
}
}
public override IEnumerable<List<Product>> ReadFromFile(string file)
{
// Initialize XmlValueReader and its keys
using (XmlValueReader xvr = new XmlValueReader())
{
List<Product> products = new List<Product>();
string fileUrl = Path.GetFileNameWithoutExtension(file).Split(null)[0].Replace('$', '/');
xvr.ProductEnd = "product";
xvr.AddKeys("european_article_number", XmlNodeType.Element);
xvr.AddKeys("product_name", XmlNodeType.Element);
xvr.AddKeys("brand", XmlNodeType.Element);
xvr.AddKeys("price", XmlNodeType.Element);
xvr.AddKeys("currency", XmlNodeType.Element);
xvr.AddKeys("deeplink", XmlNodeType.Element);
xvr.AddKeys("image_url", XmlNodeType.Element);
xvr.AddKeys("category", XmlNodeType.Element);
xvr.AddKeys("description", XmlNodeType.Element);
xvr.AddKeys("delivery_cost", XmlNodeType.Element);
xvr.AddKeys("product_id", XmlNodeType.Element);
xvr.AddKeys("delivery_period", XmlNodeType.Element);
xvr.AddKeys("program_id", XmlNodeType.Element);
xvr.AddKeys("validuntil", XmlNodeType.Element);
xvr.AddKeys("last_updated", XmlNodeType.Element);
Product p = new Product();
xvr.CreateReader(file, new XmlReaderSettings { CloseInput = true });
foreach (DualKeyDictionary<string, XmlNodeType, string> dkd in xvr.ReadProducts())
{
// Fill the product with fields
p.EAN = dkd["european_article_number"][XmlNodeType.Element];
p.Title = dkd["product_name"][XmlNodeType.Element];
p.Brand = dkd["brand"][XmlNodeType.Element];
p.Price = dkd["price"][XmlNodeType.Element];
p.Url = dkd["deeplink"][XmlNodeType.Element];
p.Image_Loc = dkd["image_url"][XmlNodeType.Element];
p.Category = dkd["category"][XmlNodeType.Element];
p.Description = dkd["description"][XmlNodeType.Element];
p.DeliveryCost = dkd["delivery_cost"][XmlNodeType.Element];
p.DeliveryTime = dkd["delivery_period"][XmlNodeType.Element];
p.AffiliateProdID = dkd["product_id"][XmlNodeType.Element] + dkd["program_id"][XmlNodeType.Element];
p.Currency = dkd["currency"][XmlNodeType.Element];
p.Affiliate = "Webgains";
p.FileName = file;
p.Webshop = fileUrl;
products.Add(p);
p = new Product();
if (products.Count >= PackageSize)
{
yield return products;
products.Clear();
}
}
yield return products;
}
}
public override System.Collections.Generic.IEnumerable<List<Product>> ReadFromDir(string dir)
{
if (!Directory.Exists(dir))
{
Console.WriteLine("Directory not found: " + dir);
yield break;
}
Console.WriteLine("Started reading from: " + dir);
List<Product> products = new List<Product>();
string[] filePaths = Util.ConcatArrays(Directory.GetFiles(dir, "*.xml"), Directory.GetFiles(dir, "*.csv"));
// Initialize XmlValueReader and its keys
XmlValueReader xvr = new XmlValueReader();
xvr.ProductEnd = "product";
xvr.AddKeys("european_article_number", XmlNodeType.Element);
xvr.AddKeys("product_name", XmlNodeType.Element);
xvr.AddKeys("brand", XmlNodeType.Element);
xvr.AddKeys("price", XmlNodeType.Element);
xvr.AddKeys("currency", XmlNodeType.Element);
xvr.AddKeys("deeplink", XmlNodeType.Element);
xvr.AddKeys("image_url", XmlNodeType.Element);
xvr.AddKeys("category", XmlNodeType.Element);
xvr.AddKeys("description", XmlNodeType.Element);
xvr.AddKeys("delivery_cost", XmlNodeType.Element);
xvr.AddKeys("product_id", XmlNodeType.Element);
xvr.AddKeys("delivery_period", XmlNodeType.Element);
xvr.AddKeys("program_id", XmlNodeType.Element);
xvr.AddKeys("validuntil", XmlNodeType.Element);
xvr.AddKeys("last_updated", XmlNodeType.Element);
Product p = new Product();
foreach (string file in filePaths)
{
string fileUrl = Path.GetFileNameWithoutExtension(file).Split(null)[0].Replace('$', '/');
xvr.CreateReader(file);
foreach (DualKeyDictionary<string, XmlNodeType, string> dkd in xvr.ReadProducts())
{
// Fill the product with fields
p.EAN = dkd["european_article_number"][XmlNodeType.Element];
p.Title = dkd["product_name"][XmlNodeType.Element];
p.Brand = dkd["brand"][XmlNodeType.Element];
p.Price = dkd["price"][XmlNodeType.Element];
p.Url = dkd["deeplink"][XmlNodeType.Element];
p.Image_Loc = dkd["image_url"][XmlNodeType.Element];
p.Category = dkd["category"][XmlNodeType.Element];
p.Description = dkd["description"][XmlNodeType.Element];
p.DeliveryCost = dkd["delivery_cost"][XmlNodeType.Element];
p.DeliveryTime = dkd["delivery_period"][XmlNodeType.Element];
p.AffiliateProdID = dkd["product_id"][XmlNodeType.Element] + dkd["program_id"][XmlNodeType.Element];
p.Currency = dkd["currency"][XmlNodeType.Element];
p.Affiliate = "Webgains";
p.FileName = file;
p.Webshop = fileUrl;
products.Add(p);
p = new Product();
if (products.Count > PackageSize)
{
yield return products;
products.Clear();
}
}
yield return products;
products.Clear();
}
}
public override IEnumerable<List<Product>> ReadFromFile(string file)
{
// Initialize XmlValueReader and its keys
using (XmlValueReader xvr = new XmlValueReader())
{
List<Product> products = new List<Product>();
string fileUrl = Path.GetFileNameWithoutExtension(file).Split(null)[0].Replace('$', '/');
xvr.ProductEnd = "product";
xvr.AddKeys("ean", XmlNodeType.Element);
xvr.AddKeys("sku", XmlNodeType.Element);
xvr.AddKeys("name", XmlNodeType.Element);
xvr.AddKeys("brand", XmlNodeType.Element);
xvr.AddKeys("price", XmlNodeType.Element);
xvr.AddKeys("currency", XmlNodeType.Element);
xvr.AddKeys("productUrl", XmlNodeType.Element);
xvr.AddKeys("imageUrl", XmlNodeType.Element);
xvr.AddKeys("TDCategoryName", XmlNodeType.Element);
xvr.AddKeys("description", XmlNodeType.Element);
xvr.AddKeys("shippingCost", XmlNodeType.Element);
xvr.AddKeys("inStock", XmlNodeType.Element);
xvr.AddKeys("deliveryTime", XmlNodeType.Element);
xvr.AddKeys("TDProductId", XmlNodeType.Element);
Product p = new Product();
xvr.CreateReader(file, new XmlReaderSettings { CloseInput = true });
foreach (DualKeyDictionary<string, XmlNodeType, string> dkd in xvr.ReadProducts())
{
// Fill the product with fields
p.EAN = dkd["ean"][XmlNodeType.Element];
p.SKU = dkd["sku"][XmlNodeType.Element];
p.Title = dkd["name"][XmlNodeType.Element];
p.Brand = dkd["brand"][XmlNodeType.Element];
p.Price = dkd["price"][XmlNodeType.Element];
p.Url = dkd["productUrl"][XmlNodeType.Element];
p.Image_Loc = dkd["imageUrl"][XmlNodeType.Element];
p.Category = dkd["TDCategoryName"][XmlNodeType.Element];
p.Description = dkd["description"][XmlNodeType.Element];
p.DeliveryCost = dkd["shippingCost"][XmlNodeType.Element];
p.DeliveryTime = dkd["deliveryTime"][XmlNodeType.Element];
p.Stock = dkd["inStock"][XmlNodeType.Element];
p.AffiliateProdID = dkd["TDProductId"][XmlNodeType.Element];
p.Currency = dkd["currency"][XmlNodeType.Element];
p.Affiliate = "TradeDoubler";
p.FileName = file;
p.Webshop = fileUrl;
products.Add(p);
p = new Product();
if (products.Count >= PackageSize)
{
yield return products;
products.Clear();
}
}
yield return products;
}
}