private void FeedXmlElement(XmlWriter xmlWriter, XmlWriter xmlWriterSecondary, IDataReader reader, StringDictionary dict, string catalog, string identifier, string feedFilePath, ref Tuple <int, int, int> counter) { if (counter == null) { throw new ArgumentNullException("counter"); } var countProcessed = 0; var countError = 0; var countSkipped = 0; var hasTwoGenerators = HasTwoGenerators(); var time = DateTime.Now; PlaRelatedFeedUtils.StartXmlDocument(xmlWriter, _runnerFeed.GetRunFeedType(), time); if (hasTwoGenerators) { PlaRelatedFeedUtils.StartXmlDocument(xmlWriterSecondary, _runnerFeedSecondary.GetRunFeedType(), time); } //<entry> while (reader.Read()) { Log.DebugFormat("{0}::Processing record [{1}]: {2}", identifier, (countProcessed + countError), reader["PID"]); var id = reader[dict["gId"]].ToString(); var title = reader[dict["title"]].ToString(); try { // First check if the product is a sensitive product, and if so, exclude if (dict.ContainsKey("isSensitiveProduct") && int.Parse(reader[dict["isSensitiveProduct"]].ToString()) > 0) { countSkipped++; continue; } var linkSku = reader[dict["linkSku"]].ToString(); var brandName = !dict.ContainsKey("gBrand") ? string.Empty : reader[dict["gBrand"]].ToString(); var cContributors = PlaRelatedFeedUtils.ContributorAttributes(dict, reader, id); var contributor = cContributors ?? null; var defaultCategory = FeedUtils.GetFeedGeneratorIndigoCategory(_feedGeneratorCategoryService, reader, dict, catalog, Log); var productData = FeedUtils.GetProductData(dict, reader, linkSku, catalog, brandName, contributor, defaultCategory); var formatString = dict.ContainsKey("bisacbindingtypeid") ? FeedUtils.GetFormat(reader[dict["bisacbindingtypeid"]].ToString(), id) : string.Empty; var sanitizedTitle = string.IsNullOrWhiteSpace(title) ? string.Empty : FeedUtils.SanitizeString(title); var gAvailability = !dict.ContainsKey("gAvailability") ? FeedUtils.GetGoogleAvailability(1) : FeedUtils.GetGoogleAvailability((int)reader[dict["gAvailability"]]); var availability = !dict.ContainsKey("gAvailability") ? 1 : (int)reader[dict["gAvailability"]]; var recordType = !dict.ContainsKey("recordType") ? string.Empty : reader[dict["recordType"]].ToString(); var hasImage = true; if (!SkipHasImageCheck) { hasImage = (!dict.ContainsKey("hasImage")) || int.Parse(reader[dict["hasImage"]].ToString()) > 0; } string message; var isExcludedDueToData = IndigoBreadcrumbRepositoryUtils.IsExcludedDueToData(_feedId, sanitizedTitle, hasImage, availability, recordType, false, out message); // First determine if the entries are to be excluded or not var isEntryExcluded = isExcludedDueToData || _runnerFeed.IsExcludedFromFeed(productData, false); var isEntrySecondaryExcluded = isExcludedDueToData; if (!isEntrySecondaryExcluded && hasTwoGenerators) { isEntrySecondaryExcluded = _runnerFeedSecondary.IsExcludedFromFeed(productData, false); } // If both entries are excluded, then there is no need to process anything further if ((isEntryExcluded && isEntrySecondaryExcluded) || (isEntryExcluded && !hasTwoGenerators)) { countSkipped++; continue; } var linkCatalog = dict["linkCatalog"]; if (MaxTitleLength > 0) { sanitizedTitle = FeedUtils.GetTruncatedTitle(sanitizedTitle, formatString, MaxTitleLength, ParameterUtils.GetParameter <bool>("GooglePlaFeedGenerator.TruncateTitle")); } var description = reader[dict["description"]].ToString(); description = string.IsNullOrWhiteSpace(description) ? sanitizedTitle : FeedUtils.SanitizeString(FeedUtils.RemoveHtmlTags(description)); var sanitizedDescription = string.IsNullOrWhiteSpace(description) ? sanitizedTitle : description; // Get the breadcrumb value var googleCategoryBreadcrumb = (string.IsNullOrWhiteSpace(defaultCategory.GoogleCategoryBreadcrumb)) ? dict["gGoogleProductCategory"] : defaultCategory.GoogleCategoryBreadcrumb; var allBreadcrumbs = (productData.BrowseCategoryIds.Any() && !productData.BrowseCategoryIds.Contains(-1)) ? new List <string>() : new List <string> { defaultCategory.Breadcrumb }; foreach (var browseCategoryId in productData.BrowseCategoryIds.Distinct()) { var categories = _feedGeneratorCategoryService.GetIndigoBreadcrumbCategories(browseCategoryId); if (categories != null) { foreach (var category in categories) { if (category.Crumbs[0].Equals(defaultCategory.Crumbs[0], StringComparison.OrdinalIgnoreCase)) { allBreadcrumbs.Add(category.Breadcrumb); } } } } decimal?salePrice = null; decimal parsedSalePrice; var regularPrice = decimal.Parse(reader[dict["price"]].ToString()); if (!string.IsNullOrEmpty(dict["adjustedPrice"]) && decimal.TryParse(reader[dict["adjustedPrice"]].ToString(), out parsedSalePrice)) { if (parsedSalePrice != regularPrice) { if (parsedSalePrice > regularPrice) { regularPrice = parsedSalePrice; } else { salePrice = parsedSalePrice; } } } var gtin = reader[dict["gGtin"]].ToString(); var isDefaultCpcValue = false; if (!isEntryExcluded) { var customLabel0 = PlaRelatedFeedUtils.GetCustomLabelValue(_runnerFeed, productData, FeedRuleType.Custom_Label_0); var customLabel1 = PlaRelatedFeedUtils.GetCustomLabelValue(_runnerFeed, productData, FeedRuleType.Custom_Label_1); var customLabel2 = PlaRelatedFeedUtils.GetCustomLabelValue(_runnerFeed, productData, FeedRuleType.Custom_Label_2); var customLabel3 = PlaRelatedFeedUtils.GetCustomLabelValue(_runnerFeed, productData, FeedRuleType.Custom_Label_3); var customLabel4 = PlaRelatedFeedUtils.GetCustomLabel4Value(_runnerFeed, productData, dict, reader); var cpcValue = PlaRelatedFeedUtils.GetCpcValue(_runnerFeed, productData, out isDefaultCpcValue); var entry = EntryAttribute(reader , id , sanitizedTitle, sanitizedDescription, googleCategoryBreadcrumb , linkCatalog, linkSku , regularPrice , salePrice , gAvailability//, availability , gtin , brandName, dict.ContainsKey("gBrand"), defaultCategory.Breadcrumb, allBreadcrumbs, cpcValue , customLabel0, customLabel1, customLabel2, customLabel3, customLabel4 , _runnerFeed.GetRunFeedType()); entry.WriteTo(xmlWriter); } if (hasTwoGenerators && !isEntrySecondaryExcluded) { var cpcValue = PlaRelatedFeedUtils.GetCpcValue(_runnerFeedSecondary, productData, out isDefaultCpcValue); var customLabel0 = PlaRelatedFeedUtils.GetCustomLabelValue(_runnerFeed, productData, FeedRuleType.Custom_Label_0); var customLabel1 = PlaRelatedFeedUtils.GetCustomLabelValue(_runnerFeed, productData, FeedRuleType.Custom_Label_1); var customLabel2 = PlaRelatedFeedUtils.GetCustomLabelValue(_runnerFeed, productData, FeedRuleType.Custom_Label_2); var customLabel3 = PlaRelatedFeedUtils.GetCustomLabelValue(_runnerFeed, productData, FeedRuleType.Custom_Label_3); var customLabel4 = PlaRelatedFeedUtils.GetCustomLabel4Value(_runnerFeed, productData, dict, reader); var entry = EntryAttribute(reader , id , sanitizedTitle, sanitizedDescription, googleCategoryBreadcrumb , linkCatalog, linkSku , regularPrice , salePrice , gAvailability , gtin , brandName, dict.ContainsKey("gBrand"), defaultCategory.Breadcrumb, allBreadcrumbs, cpcValue , customLabel0, customLabel1, customLabel2, customLabel3, customLabel4 , _runnerFeedSecondary.GetRunFeedType()); entry.WriteTo(xmlWriterSecondary); } if (isDefaultCpcValue) { _defaultCpcProductInfos.Add(new DefaultCpcProductInfo { Breadcrumb = defaultCategory.Breadcrumb, Sku = linkSku, Title = sanitizedTitle }); } countProcessed++; } catch (Exception e) { countError++; Log.ErrorFormat("Can't process the item. Id:{0};title:{1}", id, title); Log.DebugFormat("Error stack trace: {0}", e); _executionLogLogger.AddCustomMessage(string.Format("Can't process the item. Id: {0};title: {1}, file identifier: {2}", id, title, identifier)); if (HasTwoGenerators()) { _executionLogLoggerSecondary.AddCustomMessage(string.Format("Can't process the item. Id: {0};title: {1}, file identifier: {2}", id, title, identifier)); } if (!AllowItemErrorsInFiles) { _hasError = true; } } } PlaRelatedFeedUtils.EndXmlDocument(xmlWriter); if (hasTwoGenerators) { PlaRelatedFeedUtils.EndXmlDocument(xmlWriterSecondary); } counter = new Tuple <int, int, int>(countProcessed, countError, countSkipped); Log.InfoFormat("[WriteFeedFile] {0} completed Processed record count: {1}, Error record count: {2}, skipped record count: {3}", feedFilePath, countProcessed, countError, countSkipped); }
private void FeedXmlElement(XmlWriter xmlWriter, IDataReader reader, StringDictionary dict, string catalog, string identifier, string feedFilePath) { var counter = new ProcessingCounters(); var time = _effectiveFromTime.HasValue ? _effectiveFromTime.Value : DateTime.Now; PlaRelatedFeedUtils.StartXmlDocument(xmlWriter, GoogleRunFeedType.Google, time); //<entry> while (reader.Read()) { Log.DebugFormat("{0}::Processing record [{1}]: {2}", identifier, (counter.GetTotalProcessed()), reader["PID"]); var id = reader[dict["gId"]].ToString(); var title = reader[dict["title"]].ToString(); try { var haveExclusionRulesChanged = _runnerFeedRulesHelper.HaveExclusionRulesChanged(); var sku = reader[dict["sku"]].ToString(); var brandName = !dict.ContainsKey("gBrand") ? string.Empty : reader[dict["gBrand"]].ToString(); var cContributors = PlaRelatedFeedUtils.ContributorAttributes(dict, reader, id); var contributor = cContributors ?? null; var defaultCategory = FeedUtils.GetFeedGeneratorIndigoCategory(_feedGeneratorCategoryService, reader, dict, catalog, Log); var productData = FeedUtils.GetProductData(dict, reader, sku, catalog, brandName, contributor, defaultCategory); var sanitizedTitle = (string.IsNullOrWhiteSpace(title)) ? string.Empty : FeedUtils.SanitizeString(title); var gAvailability = !dict.ContainsKey("gAvailability") ? FeedUtils.GetGoogleAvailability(1) : FeedUtils.GetGoogleAvailability(int.Parse(reader[dict["gAvailability"]].ToString())); var availability = !dict.ContainsKey("gAvailability") ? 1 : int.Parse(reader[dict["gAvailability"]].ToString()); var recordType = !dict.ContainsKey("recordType") ? string.Empty : reader[dict["recordType"]].ToString(); var hasImage = true; if (!SkipHasImageCheck) { hasImage = (!dict.ContainsKey("hasImage")) || int.Parse(reader[dict["hasImage"]].ToString()) > 0; } string message; var isExcluded = false; if (_isIncrementalRun) { var statusId = int.Parse(reader["StatusId"].ToString()); switch (statusId) { // New product case 1: counter.NumberOfNew++; continue; // Unchanged product case 3: if (!haveExclusionRulesChanged) { Log.DebugFormat("Product with id {0} is skipped in incremental mode as it wasn't modified and the rules haven't changed.", id); counter.NumberOfUnchanged++; continue; } var oldExclusionResult = _runnerFeedRulesHelper.IsExcludedFromFeed(productData, true); var currentExclusionResult = _runnerFeedRulesHelper.IsExcludedFromFeed(productData, false); if (oldExclusionResult == currentExclusionResult) { Log.DebugFormat("Product with id {0} is skipped in incremental mode as it wasn't modified, rules had changed but exclusion rule evaluation's result remained the same.", id); counter.NumberOfUnchanged++; continue; } // If the product is excluded at the moment, then perform the "exclusion logic" per business requirements, // otherwise (i.e. product is included at the moment, treat it as "new") if (!currentExclusionResult) { Log.DebugFormat("Product with id {0} is marked as new and skipped in incremental mode as it wasn't modified, rules had changed but exclusion rule evaluation's result changed and currently product isn't excluded.", id); counter.NumberOfNew++; continue; } Log.DebugFormat("Product with id {0} is marked as excluded in incremental mode as it wasn't modified, rules had changed but exclusion rule evaluation's result changed and currently product is being excluded.", id); isExcluded = true; break; // Modified product case 2: var isEntryExcluded = IndigoBreadcrumbRepositoryUtils.IsExcludedDueToData(GooglePlaFeedId, sanitizedTitle, hasImage, availability, recordType, false, out message); if (isEntryExcluded) { Log.DebugFormat("Product with id {0} is marked as excluded in incremental mode as it was modified, and it failed the data requirements for inclusion.", id); isExcluded = true; break; } // If product was excluded from the feed due to rules, then mark it as excluded if (_runnerFeedRulesHelper.IsExcludedFromFeed(productData, false)) { Log.DebugFormat("Product with id {0} is marked as excluded in incremental mode as it was modified, and it's matching one of the exclusion rules.", id); isExcluded = true; } break; default: throw new ApplicationException("Invalid StatusId during an incremental run."); } } else { isExcluded = IndigoBreadcrumbRepositoryUtils.IsExcludedDueToData(GooglePlaFeedId, sanitizedTitle, hasImage, availability, recordType, false, out message); if (isExcluded) { Log.DebugFormat("Product with id {0} is marked as excluded in full mode as it failed the data requirements for inclusion.", id); } if (!isExcluded) { isExcluded = _runnerFeedRulesHelper.IsExcludedFromFeed(productData, false); } if (isExcluded) { Log.DebugFormat("Product with id {0} is marked as excluded in full mode as it's matching one of the exclusion rules.", id); } } // At this point, we know if the product is excluded or not, regardless of which type of run is being executed. // If we aren't supposed to be sending excluded product data, then update the skipped counter and exit if (isExcluded) { counter.NumberOfExcluded++; if (!SendExcludedProductData) { Log.Debug("Skipped the product because it was excluded."); continue; } gAvailability = ExcludedProductGoogleAvailabilityText; } var regularPrice = (decimal)reader[dict["price"]]; var adjustedPrice = string.IsNullOrEmpty(dict["adjustedPrice"]) ? "" : reader[dict["adjustedPrice"]].ToString(); decimal?salePrice = null; if (!string.IsNullOrWhiteSpace(adjustedPrice)) { var salePriceFromDatabase = Decimal.Parse(adjustedPrice); if (salePriceFromDatabase != regularPrice) { if (salePriceFromDatabase > regularPrice) { regularPrice = salePriceFromDatabase; salePrice = null; } else { salePrice = salePriceFromDatabase; } } } var entry = EntryAttribute(id, regularPrice, salePrice, gAvailability); entry.WriteTo(xmlWriter); counter.NumberOfProcessed++; } catch (Exception e) { counter.NumberOfErrored++; var errorMessage = string.Format("Can't process the item. Id:{0};title:{1},catalog:{2},Message:{3}", id, title, catalog, e.Message); Log.Error(errorMessage); Log.DebugFormat("Error stack trace: {0}", e); _executionLogLogger.AddCustomMessage(string.Format("Can't process the item. Id: {0};title: {1}, file identifier: {2}", id, title, identifier)); if (_isIncrementalRun && !AllowItemErrorsInFiles) { _hasError = true; } } } // If the setting for sending deleted products is set to true in an incremental run, then get the deleted products since the last run // and send them as the "special" deleted products, i.e. pid + availability of "out of stock" if (_isIncrementalRun && SendExcludedProductData) { AddDeletedProducts(xmlWriter, identifier, ref counter); } PlaRelatedFeedUtils.EndXmlDocument(xmlWriter); var infoLogMessage = string.Format("[WriteFeedFile] {0} completed processed record count: {1}, error record count: {2}, unchanged record count: {3}, " + "new record count: {4}, excluded record count: {5}, deleted record count: {6}. ", feedFilePath, counter.NumberOfProcessed, counter.NumberOfErrored, counter.NumberOfUnchanged, counter.NumberOfNew, counter.NumberOfExcluded, counter.NumberOfDeleted); if (SendExcludedProductData) { infoLogMessage += "Excluded produces were included in processed count."; } Log.Info(infoLogMessage); }
private void FeedXmlElement(XmlWriter xmlWriter, IDataReader reader, StringDictionary dict, string catalog, string identifier, string feedFilePath, ref Tuple <int, int, int> counter) { if (counter == null) { throw new ArgumentNullException("counter"); } var countProcessed = 0; var countError = 0; var countSkipped = 0; var dynamicMerchLabelProductCount = 0; XmlDataUtils.StartXmlDocument(xmlWriter); //<entry> while (reader.Read()) { Log.DebugFormat("{0}::Processing record [{1}]: {2}", identifier, (countProcessed + countError), reader["PID"]); var id = reader[dict["gId"]].ToString(); var title = reader[dict["title"]].ToString(); try { var sku = reader[dict["sku"]].ToString(); // Interesting business choice: for books, first author. For GM, real brand. var marinBrand = reader[dict["gMarinBrand"]].ToString(); var brandName = !dict.ContainsKey("gBrand") ? string.Empty : reader[dict["gBrand"]].ToString(); var cContributors = PlaRelatedFeedUtils.ContributorAttributes(dict, reader, id); var contributor = cContributors ?? null; var defaultCategory = FeedUtils.GetFeedGeneratorIndigoCategory(_feedGeneratorCategoryService, reader, dict, catalog, Log); var productData = FeedUtils.GetProductData(dict, reader, sku, catalog, brandName, contributor, defaultCategory); marinBrand = XmlDataUtils.GetBrand(marinBrand, catalog); var formatString = dict.ContainsKey("Format") ? FeedUtils.GetFormat(reader[dict["Format"]].ToString(), id) : string.Empty; var sanitizedTitle = (string.IsNullOrWhiteSpace(title)) ? string.Empty : FeedUtils.SanitizeString(string.IsNullOrWhiteSpace(formatString) ? title : $"{title}-{formatString}"); var quantity = reader[dict["quantity"]].ToString(); var gAvailability = !dict.ContainsKey("gAvailability") ? FeedUtils.GetGoogleAvailability(1) : FeedUtils.GetGoogleAvailability((int)reader[dict["gAvailability"]]); var availability = !dict.ContainsKey("gAvailability") ? 1 : (int)reader[dict["gAvailability"]]; var recordType = !dict.ContainsKey("recordType") ? string.Empty : reader[dict["recordType"]].ToString(); var hasImage = true; if (!SkipHasImageCheck) { hasImage = (!dict.ContainsKey("hasImage")) || int.Parse(reader[dict["hasImage"]].ToString()) > 0; } var size = !dict.ContainsKey("size") ? string.Empty : reader[dict["size"]].ToString(); var colour = !dict.ContainsKey("colour") ? string.Empty : reader[dict["colour"]].ToString(); var style = !dict.ContainsKey("style") ? string.Empty : reader[dict["style"]].ToString(); var scent = !dict.ContainsKey("scent") ? string.Empty : reader[dict["scent"]].ToString(); var flavour = !dict.ContainsKey("flavour") ? string.Empty : reader[dict["flavour"]].ToString(); var bindingType = !dict.ContainsKey("Format") ? string.Empty : FeedUtils.GetFormat(reader[dict["Format"]].ToString(), ""); var familyId = !dict.ContainsKey("FamilyId") ? string.Empty : reader[dict["FamilyId"]].ToString(); var dynamicMerchLabel = string.Empty; var hasLoadedDynamicMerchLabelData = false; string message; var isEntryExcluded = IndigoBreadcrumbRepositoryUtils.IsExcludedDueToData(GooglePlaFeedId, sanitizedTitle, hasImage, availability, recordType, false, out message); // If entry is excluded, then there is no need to process anything further if (isEntryExcluded) { // Business requested that any product that has a dynamic merch label value will be included in the feed, regardless of its data dynamicMerchLabel = PlaRelatedFeedUtils.GetDynamicMerchLabelValue(_runnerFeed, productData); hasLoadedDynamicMerchLabelData = true; if (string.IsNullOrWhiteSpace(dynamicMerchLabel)) { countSkipped++; continue; } } // At this point, check if the product is excluded from the feed due to an exclusion rule, if so, skip if (_runnerFeed.IsExcludedFromFeed(productData, false)) { countSkipped++; continue; } if (!hasLoadedDynamicMerchLabelData) { dynamicMerchLabel = PlaRelatedFeedUtils.GetDynamicMerchLabelValue(_runnerFeed, productData); } if (!string.IsNullOrWhiteSpace(dynamicMerchLabel)) { dynamicMerchLabelProductCount++; } var googleCategoryBreadcrumb = (string.IsNullOrWhiteSpace(defaultCategory.GoogleCategoryBreadcrumb)) ? dict["gGoogleProductCategory"] : defaultCategory.GoogleCategoryBreadcrumb; var customLabel0 = PlaRelatedFeedUtils.GetCustomLabelValue(_runnerFeed, productData, FeedRuleType.Custom_Label_0); var customLabel1 = PlaRelatedFeedUtils.GetCustomLabelValue(_runnerFeed, productData, FeedRuleType.Custom_Label_1); var customLabel2 = PlaRelatedFeedUtils.GetCustomLabelValue(_runnerFeed, productData, FeedRuleType.Custom_Label_2); var customLabel3 = PlaRelatedFeedUtils.GetCustomLabelValue(_runnerFeed, productData, FeedRuleType.Custom_Label_3); var customLabel4 = PlaRelatedFeedUtils.GetCustomLabel4Value(_runnerFeed, productData, dict, reader); var regularPrice = (decimal)reader[dict["price"]]; var effectivePrice = regularPrice; decimal?gSavingDollars = null; decimal?gSavingsPercentage = null; var adjustedPrice = string.IsNullOrEmpty(dict["adjustedPrice"]) ? "" : reader[dict["adjustedPrice"]].ToString(); var isOnSale = false; if (!string.IsNullOrWhiteSpace(adjustedPrice)) { decimal salePrice = Decimal.Parse(adjustedPrice); if (salePrice != regularPrice) { if (salePrice > regularPrice) { regularPrice = salePrice; } else { isOnSale = true; } effectivePrice = salePrice; } } // finally calculate the sale-related values if (isOnSale) { gSavingDollars = regularPrice - effectivePrice; gSavingsPercentage = 100 - effectivePrice * 100 / regularPrice; } var linkCatalog = dict["linkCatalog"]; var entry = EntryAttribute(reader , id , sanitizedTitle , sku , effectivePrice , gSavingDollars , gSavingsPercentage , gAvailability , marinBrand , quantity , linkCatalog, googleCategoryBreadcrumb, defaultCategory.Breadcrumb, customLabel0, customLabel1, customLabel2, customLabel3, customLabel4, dynamicMerchLabel , size, colour, style, scent, flavour, bindingType, familyId ); entry.WriteTo(xmlWriter); countProcessed++; } catch (Exception e) { countError++; var errorMessage = string.Format("Can't process the item. Id:{0};title:{1},catalog:{2},Message:{3}", id, title, catalog, e.Message); Log.Error(errorMessage); Log.DebugFormat("Error stack trace: {0}", e); _executionLogLogger.AddCustomMessage(string.Format("Can't process the item. Id: {0};title: {1}, file identifier: {2}", id, title, identifier)); if (!AllowItemErrorsInFiles) { _hasError = true; } } } XmlDataUtils.EndXmlDocument(xmlWriter); counter = new Tuple <int, int, int>(countProcessed, countError, countSkipped); AddToDynamicMerchProductCount(dynamicMerchLabelProductCount); Log.InfoFormat("[WriteFeedFile] {0} completed Processed record count: {1}, Error record count: {2}, skipped record count: {3}", feedFilePath, countProcessed, countError, countSkipped); }
private bool IsZeroCommissionElement(string sanitizedTitle, int feedId, bool hasImage, int?availabilityId, string recordType) { string message; return(IndigoBreadcrumbRepositoryUtils.IsExcludedDueToData(feedId, sanitizedTitle, hasImage, availabilityId, recordType, false, out message)); }