protected override void Process()
{
log.Info("Starting import of AlaTest product reviews");
AssortmentServiceSoapClient client = new AssortmentServiceSoapClient();
try
{
var reviews = client.GetProductReviews();
foreach (Connector connector in base.Connectors.Where(c => c.ConnectorType.Has(ConnectorType.WebAssortment)))
{
using (WebsiteDataContext context = new WebsiteDataContext(ConfigurationManager.ConnectionStrings[connector.Connection].ConnectionString))
{
XDocument reviewsDoc = XDocument.Parse(reviews.OuterXml);
foreach (var productReviewElement in reviewsDoc.Element("ProductReviews").Elements("Product"))
{
#region Product reviews
var product = (from p in context.Products where p.ManufacturerID == productReviewElement.Attribute("vendorItemNumber").Value select p).FirstOrDefault();
if (product != null)
{
var ProductReview = (from pr in context.AlatestReviews
where pr.ProductID == product.ProductID
select pr).FirstOrDefault();
if (ProductReview == null)
{
ProductReview = new AlatestReview
{
Product = product
};
context.AlatestReviews.InsertOnSubmit(ProductReview);
}
ProductReview.Review = productReviewElement.Try(c => c.Element("Review").Value, string.Empty);
ProductReview.ReviewSnippet = productReviewElement.Try(c => c.Element("ReviewSnippet").Value, string.Empty);
}
#endregion
}
try
{
using (TransactionScope ts = new TransactionScope(TransactionScopeOption.Suppress, TimeSpan.FromMinutes(3)))
{
context.SubmitChanges();
ts.Complete();
}
}
catch (Exception e)
{
log.Error("Saving product reviews to the database failed for connector" + connector.Name, e);
}
}
}
}
catch (Exception e)
{
log.Fatal("Downloading product reviews failed", e);
}
}
private void ProcessBrandImages(Connector connector)
{
using (WebsiteDataContext context = new WebsiteDataContext(connector.ConnectionString))
{
var brandImages = (from r in images.Root.Elements("Brands").Elements("BrandMedia")
select int.Parse(r.Attribute("BrandID").Value)).Distinct().ToArray();
var brands = (from b in context.Brands
select b).ToList();
var existingProducts = (from c in context.ImageStores
where c.ImageType == ImageType.BrandImage.ToString()
select c).Distinct().ToDictionary(c => c.BrandCode, c => c);
string imageDirectory = GetConfiguration().AppSettings.Settings["ImageDirectory"].Value;
foreach (var r in images.Root.Elements("Brands").Elements("BrandMedia"))
{
//log.DebugFormat("Get BrandImage for {0} url: {1}", r.Attribute("BrandVendorCode").Value, r.Value);
try
{
if (!string.IsNullOrEmpty(r.Value))
{
string url = GetImage(r.Value, ImageType.BrandImage, r.Attribute("BrandID").Value, null, imageDirectory);
if (!string.IsNullOrEmpty(url))
{
ImageStore brandImage = null;
if (existingProducts.ContainsKey(r.Attribute("BrandVendorCode").Value))
{
brandImage = existingProducts[r.Attribute("BrandVendorCode").Value];
brandImage.ImageUrl = url;
brandImage.BrandCode = r.Attribute("BrandVendorCode").Value;
brandImage.BrandID = brands.Where(x => x.BrandCode == r.Attribute("BrandVendorCode").Value).Select(x => x.BrandID).FirstOrDefault();
}
else
{
brandImage = new ImageStore()
{
BrandID = brands.Where(x => x.BrandCode == r.Attribute("BrandVendorCode").Value).Select(x => x.BrandID).FirstOrDefault(),
BrandCode = r.Attribute("BrandVendorCode").Value,
ImageUrl = url,
ImageType = ImageType.BrandImage.ToString()
};
context.ImageStores.InsertOnSubmit(brandImage);
}
}
}
}
catch (Exception ex)
{
log.Error(ex);
}
}
context.SubmitChanges();
}
}
private void ProcessProductGroupImages(Connector connector)
{
using (WebsiteDataContext context = new WebsiteDataContext(connector.ConnectionString))
{
var productGroupImages = (from r in images.Root.Elements("ProductGroups").Elements("ProductGroupImage")
select int.Parse(r.Attribute("ProductGroupID").Value)).ToArray();
var existingProducts = (from c in context.ImageStores
where c.ProductGroupID.HasValue &&
productGroupImages.Contains(c.ProductGroupID.Value) &&
c.ManufacturerID == null &&
c.CustomerProductID == null
select c).ToDictionary(c => c.ProductGroupID.Value, c => c);
string imageDirectory = GetConfiguration().AppSettings.Settings["ImageDirectory"].Value;
foreach (var r in images.Root.Elements("ProductGroups").Elements("ProductGroupImage"))
{
//log.DebugFormat("Get ProductGroup for {0} url: {1}", r.Attribute("ProductGroupID").Value, r.Value);
try
{
if (!string.IsNullOrEmpty(r.Value))
{
string url = GetImage(r.Value, ImageType.ProductGroupImage, r.Attribute("ProductGroupID").Value, null, imageDirectory);
if (!string.IsNullOrEmpty(url))
{
ImageStore productGroupImage = null;
if (existingProducts.ContainsKey(int.Parse(r.Attribute("ProductGroupID").Value)))
{
productGroupImage = existingProducts[int.Parse(r.Attribute("ProductGroupID").Value)];
productGroupImage.ImageUrl = url;
}
else
{
productGroupImage = new ImageStore()
{
ProductGroupID = int.Parse(r.Attribute("ProductGroupID").Value),
ImageUrl = url,
ImageType = ImageType.ProductGroupImage.ToString()
};
context.ImageStores.InsertOnSubmit(productGroupImage);
}
}
}
}
catch (Exception ex)
{
log.Error(ex);
}
}
context.SubmitChanges();
}
}
protected override void Process()
{
ConcentratorSelectorService.SelectorServiceSoapClient serviceClient = new Spider.WebsiteImport.ConcentratorSelectorService.SelectorServiceSoapClient();
using (WebsiteDataContext context = new WebsiteDataContext())
{
try
{
log.InfoFormat("Starting import of related products");
ProcessRelatedProducts(context, serviceClient, 1);
context.SubmitChanges();
log.InfoFormat("Finished import of related products");
}
catch (Exception e)
{
log.Error("Importing of related products for connector " + Name + "failed ", e);
}
}
}
private void ProcessXML()
{
log.Info("Start import of expert product reviews");
AssortmentServiceSoapClient client = new AssortmentServiceSoapClient();
var message = client.GetExpertProductReviews();
XDocument reviews = XDocument.Parse(message);
var productReviews = (from p in reviews.Element("ExpertReviews").Element("Products").Elements("Product")
select new
{
ConcentratorID = int.Parse(p.Attribute("ID").Value),
Reviews = from r in p.Element("Reviews").Elements("Review")
let sourceID = int.Parse(r.Attribute("SourceID").Value)
select new
{
CustomID = r.Attribute("ConcentratorID").Value,
SourceID = sourceID,
isSummary = bool.Parse(r.Attribute("isSummary").Value),
Author = r.Element("Author").Value,
Date = r.Element("Date").Value,
Title = r.Element("Title").Value,
Summary = r.Element("Summary").Value,
Verdict = r.Element("Verdict").Value,
ReviewURL = r.Element("ReviewURL").Value,
RatingImageURL = r.Element("RatingImageURL").Value,
Rating = r.Element("Rating").Try <XElement, int?>(c => int.Parse(c.Value), null),
Source = (from s in reviews.Element("ExpertReviews").Element("Sources").Elements("Source")
where int.Parse(s.Attribute("ID").Value) == sourceID
select new
{
Name = s.Element("Name").Value,
LanguageCode = s.Element("LanguageCode").Value,
CountryCode = s.Element("CountryCode").Value,
SourceURL = s.Element("SourceURL").Value,
SourceLogoURL = s.Element("SourceLogoURL").Value,
SourceRank = s.Element("SourceRank").Try <XElement, int?>(c => int.Parse(c.Value), null)
}).FirstOrDefault()
}
}
);
foreach (Connector connector in base.Connectors.Where(x => ((ConnectorType)x.ConnectorType).Has(ConnectorType.Reviews)))
{
log.InfoFormat("Start expert product review import for connector {0}", connector.Name);
DateTime start = DateTime.Now;
using (WebsiteDataContext context = new WebsiteDataContext(ConfigurationManager.ConnectionStrings[connector.Connection].ConnectionString))
{
var productIDs = (from p in context.Products select p.ConcentratorProductID).ToList();
var sources = (from s in context.ReviewSources select s).ToList();
productIDs.Sort();
foreach (var product in productReviews)
{
if (!productIDs.Contains(product.ConcentratorID))
{
continue;
}
var productItem = (from p in context.Products where p.ConcentratorProductID == product.ConcentratorID select p).FirstOrDefault();
foreach (var review in product.Reviews)
{
#region Source
var src = sources.FirstOrDefault(c => c.CustomSourceID == review.SourceID);
if (src == null)
{
src = new ReviewSource
{
CustomSourceID = review.SourceID
};
context.ReviewSources.InsertOnSubmit(src);
sources.Add(src);
}
src.CountryCode = review.Source.CountryCode;
src.LanguageCode = review.Source.LanguageCode;
src.Name = review.Source.Name;
src.SourceLogoUrl = review.Source.SourceLogoURL;
src.SourceRank = review.Source.SourceRank;
src.SourceUrl = review.Source.SourceURL;
#endregion
var reviewItem =
(from s in context.AlatestExpertReviews where s.CustomID == review.CustomID select s).FirstOrDefault();
if (reviewItem == null)
{
reviewItem = new AlatestExpertReview()
{
Product = productItem,
ReviewSource = src,
CustomID = review.CustomID
};
context.AlatestExpertReviews.InsertOnSubmit(reviewItem);
}
reviewItem.Author = review.Author;
DateTime date;
DateTime.TryParse(review.Date, out date);
reviewItem.Date = review.Date.Try <string, DateTime?>(c => DateTime.Parse(c), null);
reviewItem.isSummary = review.isSummary;
reviewItem.Rating = review.Rating;
reviewItem.RatingImageURL = review.RatingImageURL;
reviewItem.ReviewURL = review.ReviewURL;
reviewItem.Summary = review.Summary;
reviewItem.Title = review.Title;
reviewItem.Verdict = review.Verdict;
}
}
try
{
using (TransactionScope ts = new TransactionScope(TransactionScopeOption.Suppress, TimeSpan.FromMinutes(5)))
{
context.SubmitChanges();
}
}
catch (Exception e)
{
log.AuditError("Import of alatest expert reviews failed for connector: " + connector.Name, e,
"Import Expert Reviews");
}
}
}
}
private void ProcessProductImages(Connector connector)
{
using (WebsiteDataContext context = new WebsiteDataContext(connector.ConnectionString))
{
var productImages = (from r in images.Root.Elements("Products").Elements("ProductImage")
select new
{
ConcentratorProductID = int.Parse(r.Attribute("ProductID").Value),
Sequence = r.Attribute("Sequence").Value
}).ToList().OrderByDescending(x => x.ConcentratorProductID);
var websiteProducts = (from p in context.Products
where p.ConcentratorProductID.HasValue
select p).ToList();
var ImageDB = (from i in context.ImageStores
where i.ImageType == ImageType.ProductImage.ToString()
select i).ToList();
List <ImageStore> existingProducts = new List <ImageStore>();
foreach (var prod in productImages)
{
var websiteProduct = (from p in websiteProducts
where p.ConcentratorProductID.Value == prod.ConcentratorProductID
select p).FirstOrDefault();
if (websiteProduct != null)
{
var product = (from p in ImageDB
where p.CustomerProductID == websiteProduct.ProductID.ToString() &&
p.Sequence == int.Parse(prod.Sequence)
select p).FirstOrDefault();
if (product != null)
{
existingProducts.Add(product);
}
}
}
int counter = productImages.Count();
int showMessage = 0;
string imageDirectory = GetConfiguration().AppSettings.Settings["ImageDirectory"].Value;
foreach (var r in images.Root.Elements("Products").Elements("ProductImage").OrderByDescending(x => x.Attribute("ProductID").Value))
{
var websiteProduct = (from p in websiteProducts
where p.ConcentratorProductID.Value == int.Parse(r.Attribute("ProductID").Value)
select p).FirstOrDefault();
try
{
if (!string.IsNullOrEmpty(r.Value) && websiteProduct != null)
{
string url = GetImage(r.Value, ImageType.ProductImage, r.Attribute("ProductID").Value, int.Parse(r.Attribute("Sequence").Value), imageDirectory);
if (!string.IsNullOrEmpty(url))
{
ImageStore productImage = existingProducts.Where(x => x.Sequence == int.Parse(r.Attribute("Sequence").Value) &&
x.CustomerProductID == websiteProduct.ProductID.ToString()).FirstOrDefault();
if (productImage != null)
{
productImage.ImageUrl = url;
productImage.BrandID = int.Parse(r.Attribute("BrandID").Value);
productImage.ManufacturerID = r.Attribute("ManufacturerID").Value;
productImage.Sequence = int.Parse(r.Attribute("Sequence").Value);
productImage.ConcentratorProductID = int.Parse(r.Attribute("ProductID").Value);
}
else
{
productImage = new ImageStore()
{
ImageUrl = url,
BrandID = int.Parse(r.Attribute("BrandID").Value),
CustomerProductID = websiteProduct.ProductID.ToString(),
ManufacturerID = r.Attribute("ManufacturerID").Value,
ImageType = ImageType.ProductImage.ToString(),
Sequence = int.Parse(r.Attribute("Sequence").Value),
ConcentratorProductID = int.Parse(r.Attribute("ProductID").Value)
};
context.ImageStores.InsertOnSubmit(productImage);
}
}
counter--;
showMessage++;
if (showMessage == 500)
{
log.InfoFormat("{0} images to process", counter);
showMessage = 0;
}
}
}
catch (Exception ex)
{
log.Error(ex);
}
context.SubmitChanges();
}
// context.SubmitChanges();
}
}
private void ProcessProductImages(Connector connector)
{
using (WebsiteDataContext context = new WebsiteDataContext(connector.ConnectionString))
{
var productImages = (from r in images.Root.Elements("Products").Elements("ProductMedia")
select new
{
ConcentratorProductID = int.Parse(r.Attribute("ProductID").Value),
Sequence = r.Attribute("Sequence").Value
}).ToList().OrderByDescending(x => x.ConcentratorProductID);
var websiteProducts = (from p in context.Products
where p.ConcentratorProductID.HasValue
select p).ToList();
var ImageDB = (from i in context.ImageStores
where i.ImageType == ImageType.ProductImage.ToString()
select i).ToList();
var unusedImages = ImageDB;
var brands = (from b in context.Brands
select b).ToList();
List <ImageStore> existingProducts = new List <ImageStore>();
foreach (var prod in productImages)
{
var websiteProduct = (from p in websiteProducts
where p.ConcentratorProductID.Value == prod.ConcentratorProductID
select p).OrderByDescending(x => x.IsVisible).FirstOrDefault();
if (websiteProduct != null)
{
var product = (from p in ImageDB
where p.ConcentratorProductID == prod.ConcentratorProductID &&
p.Sequence == int.Parse(prod.Sequence)
select p).FirstOrDefault();
if (product != null)
{
existingProducts.Add(product);
}
}
}
int counter = productImages.Count();
int showMessage = 0;
bool succes = true;
//string imageDirectory = GetConfiguration().AppSettings.Settings["ImageDirectory"].Value;
foreach (var r in images.Root.Elements("Products").Elements("ProductMedia").OrderByDescending(x => x.Attribute("ProductID").Value))
{
var websiteProduct = (from p in websiteProducts
where p.ConcentratorProductID.Value == int.Parse(r.Attribute("ProductID").Value)
select p).OrderByDescending(x => x.IsVisible).FirstOrDefault();
try
{
if (!string.IsNullOrEmpty(r.Value) && websiteProduct != null)
{
//string url = GetImage(r.Value, ImageType.ProductImage, r.Attribute("ProductID").Value, int.Parse(r.Attribute("Sequence").Value), imageDirectory);
string url = r.Value;
if (!string.IsNullOrEmpty(url))
{
ImageStore productImage = existingProducts.Where(x => x.ConcentratorProductID.Value == int.Parse(r.Attribute("ProductID").Value) && x.Sequence == int.Parse(r.Attribute("Sequence").Value)).FirstOrDefault();
if (productImage != null)
{
if (productImage.ImageUrl != url ||
productImage.BrandID != int.Parse(r.Attribute("BrandID").Value) ||
productImage.ManufacturerID != r.Attribute("ManufacturerID").Value ||
productImage.Sequence != int.Parse(r.Attribute("Sequence").Value) ||
productImage.ConcentratorProductID != int.Parse(r.Attribute("ProductID").Value)
)
{
productImage.ImageUrl = url;
productImage.BrandID = int.Parse(r.Attribute("BrandID").Value);
productImage.ManufacturerID = r.Attribute("ManufacturerID").Value;
productImage.Sequence = int.Parse(r.Attribute("Sequence").Value);
productImage.ConcentratorProductID = int.Parse(r.Attribute("ProductID").Value);
productImage.LastModificationTime = DateTime.Now;
}
unusedImages.Remove(productImage);
}
else
{
productImage = new ImageStore()
{
ImageUrl = url,
BrandID = int.Parse(r.Attribute("BrandID").Value),
CustomerProductID = websiteProduct.ProductID.ToString(),
ManufacturerID = r.Attribute("ManufacturerID").Value,
ImageType = ImageType.ProductImage.ToString(),
Sequence = int.Parse(r.Attribute("Sequence").Value),
ConcentratorProductID = int.Parse(r.Attribute("ProductID").Value),
LastModificationTime = DateTime.Now
};
context.ImageStores.InsertOnSubmit(productImage);
}
}
counter--;
showMessage++;
if (showMessage == 500)
{
log.InfoFormat("{0} images to process", counter);
showMessage = 0;
}
}
context.SubmitChanges();
}
catch (Exception ex)
{
succes = false;
log.Error(ex);
}
}
try
{
if (succes)
{
if (Processor.TableExists(context, "AuctionProducts"))
{
var auctionImages = (from a in context.AuctionProducts
join i in context.ImageStores on a.Product.ConcentratorProductID equals i.ConcentratorProductID
select i).ToList();
unusedImages = unusedImages.Except(auctionImages).ToList();
}
if (Processor.TableExists(context, "RelatedProducts"))
{
var relatedproducts = (from r in context.RelatedProducts
join i in context.ImageStores on r.Product.ConcentratorProductID equals i.ConcentratorProductID
select i).ToList();
unusedImages = unusedImages.Except(relatedproducts).ToList();
}
log.DebugFormat("Try To delete {0} product images", unusedImages.Count);
context.ImageStores.DeleteAllOnSubmit(unusedImages);
context.SubmitChanges();
}
}
catch (Exception ex)
{
log.Error("Error delete unused images", ex);
}
}
}
private void ProcessBrandImages(Connector connector)
{
using (WebsiteDataContext context = new WebsiteDataContext(connector.ConnectionString))
{
var brandImages = (from r in images.Root.Elements("Brands").Elements("BrandMedia")
select int.Parse(r.Attribute("BrandID").Value)).Distinct().ToArray();
var brands = (from b in context.Brands
select b).ToList();
var existingProducts = (from c in context.ImageStores
where c.ImageType == ImageType.BrandImage.ToString()
select c).ToList();
var unusedImages = existingProducts;
string imageDirectory = GetConfiguration().AppSettings.Settings["ImageDirectory"].Value;
foreach (var r in images.Root.Elements("Brands").Elements("BrandMedia"))
{
//log.DebugFormat("Get BrandImage for {0} url: {1}", r.Attribute("BrandVendorCode").Value, r.Value);
try
{
if (!string.IsNullOrEmpty(r.Value))
{
//string url = GetImage(r.Value, ImageType.BrandImage, r.Attribute("BrandID").Value, null, imageDirectory);
string url = r.Value;
if (!string.IsNullOrEmpty(url))
{
ImageStore brandImage = existingProducts.FirstOrDefault(x => x.BrandCode == r.Attribute("BrandID").Value);
if (brandImage != null)
{
if (
brandImage.ImageUrl != url ||
brandImage.BrandCode != r.Attribute("BrandID").Value ||
brandImage.BrandID != brands.Where(x => x.BrandCode == r.Attribute("BrandID").Value).Select(x => x.BrandID).FirstOrDefault()
)
{
brandImage.ImageUrl = url;
brandImage.BrandCode = r.Attribute("BrandID").Value;
brandImage.BrandID = brands.Where(x => x.BrandCode == r.Attribute("BrandID").Value).Select(x => x.BrandID).FirstOrDefault();
brandImage.LastModificationTime = DateTime.Now;
}
unusedImages.Remove(brandImage);
}
else
{
brandImage = new ImageStore()
{
BrandID = brands.Where(x => x.BrandCode == r.Attribute("BrandID").Value).Select(x => x.BrandID).FirstOrDefault(),
BrandCode = r.Attribute("BrandID").Value,
ImageUrl = url,
ImageType = ImageType.BrandImage.ToString(),
LastModificationTime = DateTime.Now
};
context.ImageStores.InsertOnSubmit(brandImage);
}
}
}
}
catch (Exception ex)
{
log.Error(ex);
}
}
context.SubmitChanges();
//try
//{
// log.DebugFormat("Try To delete {0} brand images", unusedImages.Count);
// context.ImageStores.DeleteAllOnSubmit(unusedImages);
// context.SubmitChanges();
//}
//catch (Exception ex)
//{
// log.Error("Error delete unused images", ex);
//}
}
}
protected override void Process()
{
foreach (Connector connector in Connectors.Where(x => x.ZipCodes))
{
log.DebugFormat("Start Process zipcode import for {0}", connector.Name);
DateTime start = DateTime.Now;
try
{
using (ZipCodesSoapClient soap = new ZipCodesSoapClient())
{
using (WebsiteDataContext context = new WebsiteDataContext(ConfigurationManager.ConnectionStrings[connector.Connection].ConnectionString))
{
var zips = (from p in context.Postcodes
select p).ToList();
var ms = soap.GetZipcodes();
Stream s = new MemoryStream(ms);
XmlSerializer x = new XmlSerializer(typeof(Postcode));
List <Postcode> importZips = (List <Postcode>)x.Deserialize(s);
foreach (var newzip in importZips)
{
var existingVar = (from c in zips
where c.ZipCodeID == newzip.ZipCodeID
select c).FirstOrDefault();
if (existingVar == null)
{
log.DebugFormat("Try inserting new zipcode {0}", newzip.PCWIJK);
Postcode newZipCode = new Postcode
{
PCWIJK = newzip.PCWIJK,
PCLETTER = newzip.PCLETTER,
PCREEKSID = newzip.PCREEKSID,
PCREEKSVAN = newzip.PCREEKSVAN,
PCREEKSTOT = newzip.PCREEKSTOT,
PCCITYTPG = newzip.PCCITYTPG,
PCCITYNEN = newzip.PCCITYNEN,
PCSTRTPG = newzip.PCSTRTPG,
PCSTRNEN = newzip.PCSTRNEN,
PCSTROFF = newzip.PCSTROFF,
PCCITYEXT = newzip.PCCITYEXT,
PCGEMEENTEID = newzip.PCGEMEENTEID,
PCGEMEENTENAAM = newzip.PCGEMEENTENAAM,
PCPROVINCIE = newzip.PCPROVINCIE.Value,
PCCEBUCO = newzip.PCCEBUCO,
PCSTREXT = newzip.PCSTREXT
};
context.Postcodes.InsertOnSubmit(newZipCode);
}
else
{
existingVar.PCWIJK = newzip.PCWIJK;
existingVar.PCLETTER = newzip.PCLETTER;
existingVar.PCREEKSID = newzip.PCREEKSID.Value;
existingVar.PCREEKSVAN = newzip.PCREEKSVAN;
existingVar.PCREEKSTOT = newzip.PCREEKSTOT;
existingVar.PCCITYTPG = newzip.PCCITYTPG;
existingVar.PCCITYNEN = newzip.PCCITYNEN;
existingVar.PCSTRTPG = newzip.PCSTRTPG;
existingVar.PCSTRNEN = newzip.PCSTRNEN;
existingVar.PCSTROFF = newzip.PCSTROFF;
existingVar.PCCITYEXT = newzip.PCCITYEXT;
existingVar.PCGEMEENTEID = newzip.PCGEMEENTEID;
existingVar.PCGEMEENTENAAM = newzip.PCGEMEENTENAAM;
existingVar.PCPROVINCIE = newzip.PCPROVINCIE.Value;
existingVar.PCCEBUCO = newzip.PCCEBUCO;
existingVar.PCSTREXT = newzip.PCSTREXT;
}
context.SubmitChanges();
}
}
}
}
catch (Exception ex)
{
log.Fatal(ex);
}
log.DebugFormat("Finish Process zipcode import for {0}", connector.Name);
}
}
protected override void Process()
{
foreach (Connector connector in base.Connectors) //already filtered
{
log.DebugFormat("Start Process auction export for {0}", connector.Name);
DateTime start = DateTime.Now;
try
{
log.Debug("Start import auction items");
AssortmentServiceSoapClient soap = new AssortmentServiceSoapClient();
XDocument products = new XDocument(soap.GetAssortment(connector.ConnectorID, null, true));
Processor processor = new Processor(products, log, connector);
DateTime? lastUpdate = null;
using (WebsiteDataContext ctx = new WebsiteDataContext(connector.ConnectionString))
{
var auctionProductList = (from p in ctx.AuctionProducts
select new
{
AuctionProduct = p,
BrandCode = p.Product.Brand.BrandCode,
Product = p.Product
}).ToList();
if (auctionProductList.Count > 0)
{
lastUpdate = (from lu in auctionProductList
let update = (lu.AuctionProduct.LastModificationTime.HasValue ? lu.AuctionProduct.LastModificationTime.Value : lu.AuctionProduct.CreationTime)
select update).Max();
}
Dictionary <int, Product> existingProducts = new Dictionary <int, Product>();
log.DebugFormat("Xml contains {0} products", products.Root.Elements("Product").Count());
log.Info("Start import brands");
processor.ImportBrands();
log.Info("Finish import brands");
log.Info("Start import productgroups");
processor.ImportProductGroups(false);
log.Info("Finish import productgroups");
var allProductGroups = (from p in ctx.ProductGroups
select p).ToList();
var productGroupMappings = (from p in ctx.ProductGroupMappings
group p by p.ProductGroupID
into grouped
select grouped).ToDictionary(x => x.Key, y => y.ToList());
foreach (var r in products.Root.Elements("Product"))
{
#region Get Concentrator Product ID
int concentratorProductID = Utility.GetConcentratorProductID(connector, r, log);
if (concentratorProductID == 0)
{
continue;
}
AuctionProduct auctionProduct = null;
Product product = null;
if (connector.UseConcentratorProductID)
{
auctionProduct = (from a in auctionProductList
where
a.Product.ConcentratorProductID.HasValue &&
a.Product.ConcentratorProductID.Value == concentratorProductID
select a.AuctionProduct).FirstOrDefault();
product = (from a in ctx.Products
where
a.ConcentratorProductID.HasValue &&
a.ConcentratorProductID.Value == concentratorProductID
select a).FirstOrDefault();
}
else
{
auctionProduct = (from a in auctionProductList
where a.Product.ProductID == concentratorProductID
select a.AuctionProduct).FirstOrDefault();
product = (from a in ctx.Products
where a.ProductID == concentratorProductID
select a).FirstOrDefault();
}
#endregion
try
{
List <ProductGroup> productgroups = new List <ProductGroup>();
foreach (var productGroupNode in r.Element("ProductGroupHierarchy").Elements("ProductGroup"))
{
var productGroups = (from pg in allProductGroups
where pg.BackendProductGroupCode == productGroupNode.Attribute("ID").Value
select pg).ToList();
List <string> parentNodes = new List <string>()
{
productGroupNode.Attribute("ID").Value
};
var parent = productGroupNode.Element("ProductGroup");
while (parent != null)
{
parentNodes.Add(parent.Attribute("ID").Value);
parent = parent.Element("ProductGroup");
}
string path = String.Join("/", parentNodes.ToArray());
ProductGroup group = null;
foreach (var g in productGroups)
{
string groupPath = g.GetProductGroupCodeTree();
if (groupPath == path)
{
group = g;
break;
}
}
if (group != null)
{
productgroups.Add(group);
}
}
if (r.Element("Price") != null)
{
decimal unitprice = decimal.Parse(r.Element("Price").Element("UnitPrice").Value,
CultureInfo.InvariantCulture);
decimal costprice = decimal.Parse(r.Element("Price").Element("CostPrice").Value,
CultureInfo.InvariantCulture);
string shortDescription = r.Element("Content").Attribute("ShortDescription").Value;
string longDescription = r.Element("Content").Attribute("LongDescription").Value;
string backendDescription = string.Empty;
if (connector.ImportCommercialText)
{
backendDescription = longDescription;
//shortDescription = r.Element("Content").Attribute("ShortContentDescription").Value;
//if (!string.IsNullOrEmpty(r.Element("Content").Attribute("LongContentDescription").Value))
//longDescription = r.Element("Content").Attribute("LongContentDescription").Value;
}
int?taxRateID = (from t in ctx.TaxRates
where t.TaxRate1.HasValue
&&
(t.TaxRate1.Value * 100) ==
decimal.Parse(r.Element("Price").Attribute("TaxRate").Value,
CultureInfo.InvariantCulture)
select t.TaxRateID).FirstOrDefault();
if (!taxRateID.HasValue)
{
taxRateID = 1;
}
if (auctionProduct == null)
{
#region Create New (Auction) Product
Brand brand = (from b in ctx.Brands
where b.BrandCode == r.Element("Brands").Element("Brand").Attribute("BrandID").Value.Trim()
select b).FirstOrDefault();
if (brand != null)
{
//DateTime temppdDate = DateTime.MinValue;
//DateTime? pdDate = null;
//if (DateTime.TryParse(r.Element("Stock").Attribute("PromisedDeliveryDate").Value, out temppdDate))
//{
// if (temppdDate == DateTime.MinValue && temppdDate == DateTime.MaxValue)
// pdDate = null;
// else
// pdDate = temppdDate;
//}
if (product == null)
{
if (!connector.UseConcentratorProductID)
{
//create stub product with identity_insert on
string cmd =
String.Format(
@"
INSERT INTO Products (ProductID, ShortDescription,LongDescription,brandid,taxrateid,iscustom,isvisible,extendedcatalog
, canmodifyprice,creationtime,lastmodificationtime
) VALUES( {0}, '{1}','{2}', {3},{4},0,0,1,0,getdate(), getdate())
",
concentratorProductID, shortDescription, longDescription, brand.BrandID, taxRateID.Value);
ctx.ExecuteCommand(cmd);
product = ctx.Products.Single(x => x.ProductID == concentratorProductID);
product.IsVisible = false;
ctx.SubmitChanges();
}
else
{
product = new Product
{
//ProductID = int.Parse(r.Attribute("CustomProductID").Value),
//ProductGroupID = subprodid,
IsVisible = false,
CreationTime = DateTime.Now,
LastModificationTime = DateTime.Now
};
ctx.Products.InsertOnSubmit(product);
}
if (!string.IsNullOrEmpty(backendDescription))
{
product.BackEndDescription = backendDescription;
}
DateTime pdDate;
if (DateTime.TryParse(r.Element("Stock").Attribute("PromisedDeliveryDate").Value, out pdDate))
{
if (pdDate != DateTime.MinValue && pdDate != DateTime.MaxValue)
{
product.PromisedDeliveryDate = pdDate;
}
}
}
if (!product.IsVisible)
{
product.UnitPrice = unitprice;
product.ShortDescription = shortDescription;
product.LongDescription = longDescription;
product.ManufacturerID = r.Attribute("ManufacturerID").Value;
product.BrandID = brand.BrandID;
product.TaxRateID = taxRateID.Value;
product.UnitCost = 0;
product.IsCustom = false;
product.LineType = string.IsNullOrEmpty(r.Attribute("LineType").Value)
? "S"
: r.Attribute("LineType").Value.Trim();
product.ProductStatus = r.Element("Price").Attribute("CommercialStatus").Value;
}
product.CustomProductID = r.Attribute("CustomProductID").Value;
product.ConcentratorProductID = int.Parse(r.Attribute("ProductID").Value);
ctx.SubmitChanges();
}
else
{
log.DebugFormat("Product {0} does not have a price in xml", r.Attribute("ProductID").Value);
}
if (product != null)
{
auctionProduct = new AuctionProduct()
{
AuctionProductID = product.ProductID,
CreationTime = DateTime.Now
};
ctx.AuctionProducts.InsertOnSubmit(auctionProduct);
auctionProductList.Add(new
{
AuctionProduct = auctionProduct,
BrandCode = product.Brand.BrandCode,
Product = product
});
}
else
{
log.WarnFormat("Cannot create new product info for product ID : {0}", concentratorProductID);
continue;
}
#endregion
}
}
else
{
log.DebugFormat("Brand {0} does not exists for {1}",
r.Elements("Brands").Elements("Brand").FirstOrDefault().Attribute("BrandID").Value.Trim(), r.Attribute("ProductID").Value);
continue;
}
if (auctionProduct == null)
{
log.WarnFormat("Cannot create new auction product info for product ID : {0}", concentratorProductID);
continue;
}
#region Parse Stock Data
var bsc = (from b in r.Element("Stock").Element("Retail").Elements("RetailStock")
where b.Attribute("Name").Value == "BSC"
select b.Attribute("InStock").Value).FirstOrDefault();
int bscStock = 0;
int.TryParse(bsc, out bscStock);
var bscOEM = (from b in r.Element("Stock").Element("Retail").Elements("RetailStock")
where b.Attribute("Name").Value == "BSCOEM"
select b.Attribute("InStock").Value).FirstOrDefault();
int bscOEMStock = 0;
int.TryParse(bscOEM, out bscOEMStock);
//DEMO
var bscDemo = (from b in r.Element("Stock").Element("Retail").Elements("RetailStock")
where b.Attribute("Name").Value == "BSCDEMO"
select b.Attribute("InStock").Value).FirstOrDefault();
int bscDemoStock = 0;
int.TryParse(bscDemo, out bscDemoStock);
//DMGBOX
var bscDMGBOX = (from b in r.Element("Stock").Element("Retail").Elements("RetailStock")
where b.Attribute("Name").Value == "BSCDMGBOX"
select b.Attribute("InStock").Value).FirstOrDefault();
int bscDMGBOXStock = 0;
int.TryParse(bscDMGBOX, out bscDMGBOXStock);
//DMGITEM
var bscDMGITEM = (from b in r.Element("Stock").Element("Retail").Elements("RetailStock")
where b.Attribute("Name").Value == "BSCDMGITEM"
select b.Attribute("InStock").Value).FirstOrDefault();
int bscDMGITEMStock = 0;
int.TryParse(bscDMGITEM, out bscDMGITEMStock);
//INCOMPL
var bscINCOMPL = (from b in r.Element("Stock").Element("Retail").Elements("RetailStock")
where b.Attribute("Name").Value == "BSCINCOMPL"
select b.Attribute("InStock").Value).FirstOrDefault();
int bscINCOMPLStock = 0;
int.TryParse(bscINCOMPL, out bscINCOMPLStock);
//RETURN
var bscRETURN = (from b in r.Element("Stock").Element("Retail").Elements("RetailStock")
where b.Attribute("Name").Value == "BSCRETURN"
select b.Attribute("InStock").Value).FirstOrDefault();
int bscRETURNStock = 0;
int.TryParse(bscRETURN, out bscRETURNStock);
//USED
var bscUSED = (from b in r.Element("Stock").Element("Retail").Elements("RetailStock")
where b.Attribute("Name").Value == "BSCUSED"
select b.Attribute("InStock").Value).FirstOrDefault();
int bscUSEDStock = 0;
int.TryParse(bscUSED, out bscUSEDStock);
//COSTPRice
var totalBscStock = bscDemoStock + bscDMGBOXStock + bscDMGITEMStock + bscINCOMPLStock + bscRETURNStock + bscUSEDStock + bscStock;
var bscCostPrices = (from b in r.Element("Stock").Element("Retail").Elements("RetailStock")
where b.Attribute("Name").Value == "BSC"
select b.Attribute("CostPrice").Value).FirstOrDefault();
decimal BSCCostPrice = 0;
if (bscCostPrices != null)
{
BSCCostPrice = decimal.Parse(bscCostPrices, CultureInfo.InvariantCulture);
}
decimal DC10CostPrice = decimal.Parse(r.Element("Price").Element("CostPrice").Value, CultureInfo.InvariantCulture);
#endregion
if (auctionProduct.AuctionBSCStock != totalBscStock ||
auctionProduct.AuctionDC10Stock != int.Parse(r.Element("Stock").Attribute("InStock").Value) ||
auctionProduct.AuctionOEMStock != bscOEMStock ||
auctionProduct.BSCCostPrice != BSCCostPrice ||
auctionProduct.DC10CostPrice != DC10CostPrice ||
auctionProduct.AuctionDEMOStock != bscDemoStock ||
auctionProduct.AuctionDMGBOXStock != bscDMGBOXStock ||
auctionProduct.AuctionDMGITEMStock != bscDMGITEMStock ||
auctionProduct.AuctionINCOMPLStock != bscINCOMPLStock ||
auctionProduct.AuctionRETURNStock != bscRETURNStock ||
auctionProduct.AuctionUSEDStock != bscUSEDStock ||
auctionProduct.AuctionMYVEILStock != bscStock ||
auctionProduct.QuantityToReceive != int.Parse(r.Element("Stock").Attribute("QuantityToReceive").Value) ||
auctionProduct.StockStatus != r.Element("Stock").Attribute("StockStatus").Value)
{
auctionProduct.LastModificationTime = DateTime.Now;
}
auctionProduct.AuctionBSCStock = totalBscStock;
auctionProduct.AuctionDC10Stock = int.Parse(r.Element("Stock").Attribute("InStock").Value);
auctionProduct.AuctionOEMStock = bscOEMStock;
auctionProduct.BSCCostPrice = BSCCostPrice;
auctionProduct.AuctionDEMOStock = bscDemoStock;
auctionProduct.AuctionDMGBOXStock = bscDMGBOXStock;
auctionProduct.AuctionDMGITEMStock = bscDMGITEMStock;
auctionProduct.AuctionINCOMPLStock = bscINCOMPLStock;
auctionProduct.AuctionRETURNStock = bscRETURNStock;
auctionProduct.AuctionUSEDStock = bscUSEDStock;
auctionProduct.AuctionMYVEILStock = bscStock;
auctionProduct.DC10CostPrice = DC10CostPrice;
auctionProduct.QuantityToReceive = int.Parse(r.Element("Stock").Attribute("QuantityToReceive").Value);;
auctionProduct.StockStatus = r.Element("Stock").Attribute("StockStatus").Value;
ctx.SubmitChanges();
processor.ImportProductGroupMapping(ctx, productgroups, productGroupMappings, auctionProduct.AuctionProductID, concentratorProductID);
ctx.SubmitChanges();
if (!product.ConcentratorProductID.HasValue)
{
product.ConcentratorProductID = int.Parse(r.Attribute("ProductID").Value);
ctx.SubmitChanges();
}
}
catch (Exception ex)
{
log.Error("error insert product", ex);
}
}
//log.Info("Start import productgroupmapppings");
//processor.ImportProductGroupMapping(ctx, connector);
//log.Info("Finish import productgroupmapppings");
log.Debug("Start cleanup auctionproducts");
var cItemNumbers = (from r in products.Root.Elements("Product")
select new
{
concentratorProductID = r.Attribute("ProductID").Value,
customProductID = r.Attribute("CustomProductID").Value
}).ToList();
List <string> xmlProducts = new List <string>();
if (connector.UseConcentratorProductID)
{
xmlProducts = (from c in cItemNumbers
where c.concentratorProductID != String.Empty
select c.concentratorProductID).ToList();
}
else
{
foreach (var c in cItemNumbers.Select(x => x.customProductID))
{
int tmp = 0;
if (Int32.TryParse(c, out tmp))
{
xmlProducts.Add(tmp.ToString());
}
}
}
List <AuctionProduct> siteProducts = new List <AuctionProduct>();
if (connector.UseConcentratorProductID)
{
var tempSiteProducts = (from c in ctx.AuctionProducts
where c.Product.ConcentratorProductID.HasValue
select new
{
concentratorProductID = c.Product.ConcentratorProductID.Value.ToString(),
auctionItem = c
}).ToList();
foreach (var p in cItemNumbers)
{
tempSiteProducts.RemoveAll(c => c.concentratorProductID == p.concentratorProductID.ToString());
}
siteProducts = tempSiteProducts.Select(x => x.auctionItem).ToList();
if (siteProducts != null && siteProducts.Count > 0)
{
ctx.AuctionProducts.DeleteAllOnSubmit(siteProducts);
}
ctx.SubmitChanges();
var delProds = (from c in ctx.AuctionProducts
where !c.Product.ConcentratorProductID.HasValue
select c).ToList();
log.DebugFormat("Delete {0} items without concentrator productid", delProds.Count);
ctx.AuctionProducts.DeleteAllOnSubmit(delProds);
ctx.SubmitChanges();
}
else
{
siteProducts = (from c in ctx.AuctionProducts
select c).ToList();
foreach (var p in cItemNumbers)
{
siteProducts.RemoveAll(c => c.AuctionProductID.ToString() == p.customProductID);
}
if (siteProducts != null && siteProducts.Count > 0)
{
ctx.AuctionProducts.DeleteAllOnSubmit(siteProducts);
}
ctx.SubmitChanges();
}
}
log.Debug("Finish cleanup assormtent");
// processor.CleanUpProductGroupMapping();
log.Debug("Start import productattributes");
processor.ProcessAttributes(soap, connector, true, lastUpdate, false);
log.Debug("Finish import productattributes");
log.DebugFormat("Auction Product import Completed On: {0}", DateTime.Now);
}
catch (Exception ex)
{
log.Error("Error import products", ex);
}
log.DebugFormat("Finish Process auction import for {0}", connector.Name);
}
}
protected override void Process()
{
log.Info("Selector import cycle started...");
try
{
foreach (Connector connector in base.Connectors.Where(c => ((ConnectorType)c.ConnectorType).Has(ConnectorType.WebAssortment) && c.Selectors))
{
DateTime start = DateTime.Now;
AssortmentServiceSoapClient contentClient = new AssortmentServiceSoapClient();
SelectorServiceSoapClient serviceClient = new SelectorServiceSoapClient();
using (
WebsiteDataContext context =
new WebsiteDataContext(
ConfigurationManager.ConnectionStrings[connector.Connection].ConnectionString))
{
log.InfoFormat("Start selector import for connector {0}", connector.Name);
var selectorIDs = (from s in context.Selectors select s.SelectorID).ToList();
foreach (int selectorID in selectorIDs)
{
var products = XDocument.Parse(contentClient.GetSelectorProducts(connector.ConnectorID, selectorID));
Processor processor = new Processor(products, log, connector);
log.Debug("Start import brands");
processor.ImportBrands();
log.Debug("Finished import brands");
log.Debug("Start import productgroups");
processor.ImportProductGroups(false);
log.Debug("Finished import productgroups");
log.Debug("Start import Products");
processor.ImportSelectorProducts();
log.Debug("Finished import Products");
log.Debug("Start import Attributes");
processor.processSelectorAttributes(connector);
log.Debug("Finished import Attributes");
log.Debug("Start import images");
ProcessProductImages(connector, products);
log.Debug("Finish import images");
//processor.CleanUpProducts();
//
}
try
{
log.InfoFormat("Starting import of related products");
ImportRelatedProducts(context, serviceClient, contentClient, connector);
log.InfoFormat("Finished import of related products");
}
catch (Exception e)
{
log.Error("Importing of related products for connector " + connector.Name + "failed ", e);
}
try
{
using (TransactionScope scope = new TransactionScope(TransactionScopeOption.Suppress, TimeSpan.FromMinutes(5)))
{
context.SubmitChanges();
scope.Complete();
}
log.AuditSuccess("Finished selector export for connector " + connector.Name, "Selector export");
}
catch (Exception e)
{
log.AuditFatal("There was an error inserting the selectors", e);
}
}
}
}
catch (Exception e)
{
log.Fatal("Selector import cycle failed", e);
}
}
public void ImportRelatedProducts(WebsiteDataContext context, SelectorServiceSoapClient client, Concentrator.Web.ServiceClient.AssortmentService.AssortmentServiceSoapClient contentClient, Connector connector)
{
var selectorIDs = (from s in context.Selectors select s.SelectorID).ToList();
Processor processor = new Processor(new XDocument(), log, connector);
foreach (int selectorID in selectorIDs)
{
var selectorProducts = client.GetAllSelectorProducts(selectorID, connector.ConnectorID);
XDocument comparableProducts = XDocument.Parse(selectorProducts);
int counter = 0;
int total = comparableProducts.Root.Elements("RelatedProduct").Count();
List <Product> products = (from p in context.Products
select p).ToList();
int totalRelatedproducts = total;
foreach (var rProduct in comparableProducts.Root.Elements("RelatedProduct"))
{
if (counter == 50)
{
base.log.InfoFormat("Still need to process {0} of {1}; {2} done;", total, totalRelatedproducts, totalRelatedproducts - total);
counter = 0;
}
total--;
counter++;
int concentratorProductID = int.Parse(rProduct.Attribute("ProductID").Value);
Product product = null;
//try
//{
product = (from p in products
where p.ConcentratorProductID.HasValue && (p.ConcentratorProductID.Value == concentratorProductID)
select p).FirstOrDefault();
if (product == null)
{
continue;
}
foreach (var comparableProduct in rProduct.Elements("ComparableProducts").Elements("ComparableProduct"))
{
int relatedProductID = int.Parse(comparableProduct.Attribute("ProductID").Value);
try
{
Product relatedProduct = (from p in products
where p.ConcentratorProductID.HasValue && (p.ConcentratorProductID.Value == relatedProductID)
select p).FirstOrDefault();
if (relatedProduct == null)
{
continue;
}
//insert related product
RelatedProduct relatedProd = (from rp in context.RelatedProducts
where
rp.ProductID == product.ProductID
&&
rp.RelatedProductID == relatedProduct.ProductID
&&
rp.SelectorID == selectorID
select rp).FirstOrDefault();
if (relatedProd == null)
{
context.RelatedProducts.InsertOnSubmit(new RelatedProduct
{
ProductID = product.ProductID,
RelatedProductID = relatedProduct.ProductID,
SelectorID = selectorID
});
}
context.SubmitChanges();
}
catch (Exception ex)
{
log.ErrorFormat("Error process product {0} error {1}", relatedProductID, ex.StackTrace);
}
}
}
#region Cleanup
var itemNumbers = comparableProducts.Root.Elements("RelatedProduct").ToDictionary(x => x.Attribute("ProductID").Value, x => x);
List <RelatedProduct> unused = (from c in context.RelatedProducts
select c).ToList();
foreach (var p in itemNumbers)
{
foreach (var v in p.Value.Elements("ComparableProducts").Elements("ComparableProduct"))
{
unused.RemoveAll(c => (c.Product.ConcentratorProductID.HasValue && c.Product.ConcentratorProductID.Value.ToString() == p.Key) && (c.Product1.ConcentratorProductID.HasValue && c.Product1.ConcentratorProductID.Value.ToString() == v.Attribute("ProductID").Value));
}
}
#endregion
if (unused != null && unused.Count > 0)
{
context.RelatedProducts.DeleteAllOnSubmit(unused);
context.SubmitChanges();
}
}
}