private XE GetContributors(StringDictionary dict, IDataRecord reader)
{
if (!dict.ContainsKey("contributors"))
{
return(null);
}
var contributors = FeedUtils.SanitizeString(reader[dict["contributors"]].ToString());
var dictC = GetContributors(contributors);
if (dictC.IsNullOrEmpty())
{
return(null);
}
// If there are multiple keys in the dictionary, log an info message and pick the first one and move on
if (dictC.Count > 1)
{
Log.DebugFormat("Item {0} contained multiple contributor types.", reader[dict["gId"]].ToString());
}
const string star = "star";
bool dictContainsStar = dictC.ContainsKey(star);
return(new XE(dictC.First().Key, string.Join(", ", dictC.First(keyValuePair =>
{
if (dictContainsStar && keyValuePair.Key.Equals(star))
{
return true;
}
if (!dictC.ContainsKey(star))
{
return true;
}
return false;
}).Value)));
}
public void SignFeed()
{
var feed = new Feed {
Name = "Test"
};
const string passphrase = "passphrase123";
var signature = new byte[] { 1, 2, 3 };
var secretKey = new OpenPgpSecretKey(keyID: 123, fingerprint: new byte[] { 1, 2, 3 }, userID: "user");
var openPgpMock = CreateMock <IOpenPgp>();
openPgpMock.Setup(x => x.Sign(It.IsAny <ArraySegment <byte> >(), secretKey, passphrase))
.Returns(signature);
string signedFeed;
using (var stream = new MemoryStream())
{
feed.SaveXml(stream);
FeedUtils.SignFeed(stream, secretKey, passphrase, openPgpMock.Object);
signedFeed = stream.ReadToString();
}
string expectedFeed;
using (var stream = new MemoryStream())
{
feed.SaveXml(stream);
expectedFeed = stream.ReadToString()
+ Store.Feeds.FeedUtils.SignatureBlockStart
+ Convert.ToBase64String(signature) + "\n"
+ Store.Feeds.FeedUtils.SignatureBlockEnd;
}
signedFeed.Should().Be(expectedFeed, because: "Feed should remain unchanged except for appended XML signatre");
}
private void FeedXmlElement(XmlWriter xmlWriter, XmlWriter xmlWriterSecondary, IDataReader reader, StringDictionary dict, string catalog, string identifier, string feedFilePath, ref Tuple <int, int, int> counter)
{
if (counter == null)
{
throw new ArgumentNullException("counter");
}
var countProcessed = 0;
var countError = 0;
var countSkipped = 0;
var hasTwoGenerators = HasTwoGenerators();
var time = DateTime.Now;
PlaRelatedFeedUtils.StartXmlDocument(xmlWriter, _runnerFeed.GetRunFeedType(), time);
if (hasTwoGenerators)
{
PlaRelatedFeedUtils.StartXmlDocument(xmlWriterSecondary, _runnerFeedSecondary.GetRunFeedType(), time);
}
//<entry>
while (reader.Read())
{
Log.DebugFormat("{0}::Processing record [{1}]: {2}", identifier, (countProcessed + countError), reader["PID"]);
var id = reader[dict["gId"]].ToString();
var title = reader[dict["title"]].ToString();
try
{
// First check if the product is a sensitive product, and if so, exclude
if (dict.ContainsKey("isSensitiveProduct") && int.Parse(reader[dict["isSensitiveProduct"]].ToString()) > 0)
{
countSkipped++;
continue;
}
var linkSku = reader[dict["linkSku"]].ToString();
var brandName = !dict.ContainsKey("gBrand") ? string.Empty : reader[dict["gBrand"]].ToString();
var cContributors = PlaRelatedFeedUtils.ContributorAttributes(dict, reader, id);
var contributor = cContributors ?? null;
var defaultCategory = FeedUtils.GetFeedGeneratorIndigoCategory(_feedGeneratorCategoryService, reader, dict, catalog, Log);
var productData = FeedUtils.GetProductData(dict, reader, linkSku, catalog, brandName, contributor, defaultCategory);
var formatString = dict.ContainsKey("bisacbindingtypeid") ? FeedUtils.GetFormat(reader[dict["bisacbindingtypeid"]].ToString(), id) : string.Empty;
var sanitizedTitle = string.IsNullOrWhiteSpace(title) ? string.Empty : FeedUtils.SanitizeString(title);
var gAvailability = !dict.ContainsKey("gAvailability") ? FeedUtils.GetGoogleAvailability(1) : FeedUtils.GetGoogleAvailability((int)reader[dict["gAvailability"]]);
var availability = !dict.ContainsKey("gAvailability") ? 1 : (int)reader[dict["gAvailability"]];
var recordType = !dict.ContainsKey("recordType") ? string.Empty : reader[dict["recordType"]].ToString();
var hasImage = true;
if (!SkipHasImageCheck)
{
hasImage = (!dict.ContainsKey("hasImage")) || int.Parse(reader[dict["hasImage"]].ToString()) > 0;
}
string message;
var isExcludedDueToData = IndigoBreadcrumbRepositoryUtils.IsExcludedDueToData(_feedId, sanitizedTitle, hasImage, availability, recordType, false, out message);
// First determine if the entries are to be excluded or not
var isEntryExcluded = isExcludedDueToData || _runnerFeed.IsExcludedFromFeed(productData, false);
var isEntrySecondaryExcluded = isExcludedDueToData;
if (!isEntrySecondaryExcluded && hasTwoGenerators)
{
isEntrySecondaryExcluded = _runnerFeedSecondary.IsExcludedFromFeed(productData, false);
}
// If both entries are excluded, then there is no need to process anything further
if ((isEntryExcluded && isEntrySecondaryExcluded) || (isEntryExcluded && !hasTwoGenerators))
{
countSkipped++;
continue;
}
var linkCatalog = dict["linkCatalog"];
if (MaxTitleLength > 0)
{
sanitizedTitle = FeedUtils.GetTruncatedTitle(sanitizedTitle, formatString, MaxTitleLength, ParameterUtils.GetParameter <bool>("GooglePlaFeedGenerator.TruncateTitle"));
}
var description = reader[dict["description"]].ToString();
description = string.IsNullOrWhiteSpace(description) ? sanitizedTitle : FeedUtils.SanitizeString(FeedUtils.RemoveHtmlTags(description));
var sanitizedDescription = string.IsNullOrWhiteSpace(description) ? sanitizedTitle : description;
// Get the breadcrumb value
var googleCategoryBreadcrumb = (string.IsNullOrWhiteSpace(defaultCategory.GoogleCategoryBreadcrumb)) ? dict["gGoogleProductCategory"] : defaultCategory.GoogleCategoryBreadcrumb;
var allBreadcrumbs = (productData.BrowseCategoryIds.Any() && !productData.BrowseCategoryIds.Contains(-1)) ? new List <string>() : new List <string> {
defaultCategory.Breadcrumb
};
foreach (var browseCategoryId in productData.BrowseCategoryIds.Distinct())
{
var categories = _feedGeneratorCategoryService.GetIndigoBreadcrumbCategories(browseCategoryId);
if (categories != null)
{
foreach (var category in categories)
{
if (category.Crumbs[0].Equals(defaultCategory.Crumbs[0], StringComparison.OrdinalIgnoreCase))
{
allBreadcrumbs.Add(category.Breadcrumb);
}
}
}
}
decimal?salePrice = null;
decimal parsedSalePrice;
var regularPrice = decimal.Parse(reader[dict["price"]].ToString());
if (!string.IsNullOrEmpty(dict["adjustedPrice"]) && decimal.TryParse(reader[dict["adjustedPrice"]].ToString(), out parsedSalePrice))
{
if (parsedSalePrice != regularPrice)
{
if (parsedSalePrice > regularPrice)
{
regularPrice = parsedSalePrice;
}
else
{
salePrice = parsedSalePrice;
}
}
}
var gtin = reader[dict["gGtin"]].ToString();
var isDefaultCpcValue = false;
if (!isEntryExcluded)
{
var customLabel0 = PlaRelatedFeedUtils.GetCustomLabelValue(_runnerFeed, productData, FeedRuleType.Custom_Label_0);
var customLabel1 = PlaRelatedFeedUtils.GetCustomLabelValue(_runnerFeed, productData, FeedRuleType.Custom_Label_1);
var customLabel2 = PlaRelatedFeedUtils.GetCustomLabelValue(_runnerFeed, productData, FeedRuleType.Custom_Label_2);
var customLabel3 = PlaRelatedFeedUtils.GetCustomLabelValue(_runnerFeed, productData, FeedRuleType.Custom_Label_3);
var customLabel4 = PlaRelatedFeedUtils.GetCustomLabel4Value(_runnerFeed, productData, dict, reader);
var cpcValue = PlaRelatedFeedUtils.GetCpcValue(_runnerFeed, productData, out isDefaultCpcValue);
var entry = EntryAttribute(reader
, id
, sanitizedTitle, sanitizedDescription, googleCategoryBreadcrumb
, linkCatalog, linkSku
, regularPrice
, salePrice
, gAvailability//, availability
, gtin
, brandName, dict.ContainsKey("gBrand"), defaultCategory.Breadcrumb, allBreadcrumbs, cpcValue
, customLabel0, customLabel1, customLabel2, customLabel3, customLabel4
, _runnerFeed.GetRunFeedType());
entry.WriteTo(xmlWriter);
}
if (hasTwoGenerators && !isEntrySecondaryExcluded)
{
var cpcValue = PlaRelatedFeedUtils.GetCpcValue(_runnerFeedSecondary, productData, out isDefaultCpcValue);
var customLabel0 = PlaRelatedFeedUtils.GetCustomLabelValue(_runnerFeed, productData, FeedRuleType.Custom_Label_0);
var customLabel1 = PlaRelatedFeedUtils.GetCustomLabelValue(_runnerFeed, productData, FeedRuleType.Custom_Label_1);
var customLabel2 = PlaRelatedFeedUtils.GetCustomLabelValue(_runnerFeed, productData, FeedRuleType.Custom_Label_2);
var customLabel3 = PlaRelatedFeedUtils.GetCustomLabelValue(_runnerFeed, productData, FeedRuleType.Custom_Label_3);
var customLabel4 = PlaRelatedFeedUtils.GetCustomLabel4Value(_runnerFeed, productData, dict, reader);
var entry = EntryAttribute(reader
, id
, sanitizedTitle, sanitizedDescription, googleCategoryBreadcrumb
, linkCatalog, linkSku
, regularPrice
, salePrice
, gAvailability
, gtin
, brandName, dict.ContainsKey("gBrand"), defaultCategory.Breadcrumb, allBreadcrumbs, cpcValue
, customLabel0, customLabel1, customLabel2, customLabel3, customLabel4
, _runnerFeedSecondary.GetRunFeedType());
entry.WriteTo(xmlWriterSecondary);
}
if (isDefaultCpcValue)
{
_defaultCpcProductInfos.Add(new DefaultCpcProductInfo {
Breadcrumb = defaultCategory.Breadcrumb, Sku = linkSku, Title = sanitizedTitle
});
}
countProcessed++;
}
catch (Exception e)
{
countError++;
Log.ErrorFormat("Can't process the item. Id:{0};title:{1}", id, title);
Log.DebugFormat("Error stack trace: {0}", e);
_executionLogLogger.AddCustomMessage(string.Format("Can't process the item. Id: {0};title: {1}, file identifier: {2}", id, title, identifier));
if (HasTwoGenerators())
{
_executionLogLoggerSecondary.AddCustomMessage(string.Format("Can't process the item. Id: {0};title: {1}, file identifier: {2}", id, title, identifier));
}
if (!AllowItemErrorsInFiles)
{
_hasError = true;
}
}
}
PlaRelatedFeedUtils.EndXmlDocument(xmlWriter);
if (hasTwoGenerators)
{
PlaRelatedFeedUtils.EndXmlDocument(xmlWriterSecondary);
}
counter = new Tuple <int, int, int>(countProcessed, countError, countSkipped);
Log.InfoFormat("[WriteFeedFile] {0} completed Processed record count: {1}, Error record count: {2}, skipped record count: {3}", feedFilePath, countProcessed, countError, countSkipped);
}
private void FeedXmlElement(XmlWriter xmlWriter, IDataReader reader, StringDictionary dict, string catalog, string identifier, string feedFilePath, ref Tuple <int, int, int> counter)
{
if (counter == null)
{
throw new ArgumentNullException("counter");
}
var countProcessed = 0;
var countError = 0;
var countSkipped = 0;
var dynamicMerchLabelProductCount = 0;
XmlDataUtils.StartXmlDocument(xmlWriter);
//<entry>
while (reader.Read())
{
Log.DebugFormat("{0}::Processing record [{1}]: {2}", identifier, (countProcessed + countError), reader["PID"]);
var id = reader[dict["gId"]].ToString();
var title = reader[dict["title"]].ToString();
try
{
var sku = reader[dict["sku"]].ToString();
// Interesting business choice: for books, first author. For GM, real brand.
var marinBrand = reader[dict["gMarinBrand"]].ToString();
var brandName = !dict.ContainsKey("gBrand") ? string.Empty : reader[dict["gBrand"]].ToString();
var cContributors = PlaRelatedFeedUtils.ContributorAttributes(dict, reader, id);
var contributor = cContributors ?? null;
var defaultCategory = FeedUtils.GetFeedGeneratorIndigoCategory(_feedGeneratorCategoryService, reader, dict, catalog, Log);
var productData = FeedUtils.GetProductData(dict, reader, sku, catalog, brandName, contributor, defaultCategory);
marinBrand = XmlDataUtils.GetBrand(marinBrand, catalog);
var formatString = dict.ContainsKey("Format") ? FeedUtils.GetFormat(reader[dict["Format"]].ToString(), id) : string.Empty;
var sanitizedTitle = (string.IsNullOrWhiteSpace(title)) ? string.Empty : FeedUtils.SanitizeString(string.IsNullOrWhiteSpace(formatString) ? title : $"{title}-{formatString}");
var quantity = reader[dict["quantity"]].ToString();
var gAvailability = !dict.ContainsKey("gAvailability") ? FeedUtils.GetGoogleAvailability(1) : FeedUtils.GetGoogleAvailability((int)reader[dict["gAvailability"]]);
var availability = !dict.ContainsKey("gAvailability") ? 1 : (int)reader[dict["gAvailability"]];
var recordType = !dict.ContainsKey("recordType") ? string.Empty : reader[dict["recordType"]].ToString();
var hasImage = true;
if (!SkipHasImageCheck)
{
hasImage = (!dict.ContainsKey("hasImage")) || int.Parse(reader[dict["hasImage"]].ToString()) > 0;
}
var size = !dict.ContainsKey("size") ? string.Empty : reader[dict["size"]].ToString();
var colour = !dict.ContainsKey("colour") ? string.Empty : reader[dict["colour"]].ToString();
var style = !dict.ContainsKey("style") ? string.Empty : reader[dict["style"]].ToString();
var scent = !dict.ContainsKey("scent") ? string.Empty : reader[dict["scent"]].ToString();
var flavour = !dict.ContainsKey("flavour") ? string.Empty : reader[dict["flavour"]].ToString();
var bindingType = !dict.ContainsKey("Format") ? string.Empty : FeedUtils.GetFormat(reader[dict["Format"]].ToString(), "");
var familyId = !dict.ContainsKey("FamilyId") ? string.Empty : reader[dict["FamilyId"]].ToString();
var dynamicMerchLabel = string.Empty;
var hasLoadedDynamicMerchLabelData = false;
string message;
var isEntryExcluded = IndigoBreadcrumbRepositoryUtils.IsExcludedDueToData(GooglePlaFeedId, sanitizedTitle, hasImage, availability, recordType, false, out message);
// If entry is excluded, then there is no need to process anything further
if (isEntryExcluded)
{
// Business requested that any product that has a dynamic merch label value will be included in the feed, regardless of its data
dynamicMerchLabel = PlaRelatedFeedUtils.GetDynamicMerchLabelValue(_runnerFeed, productData);
hasLoadedDynamicMerchLabelData = true;
if (string.IsNullOrWhiteSpace(dynamicMerchLabel))
{
countSkipped++;
continue;
}
}
// At this point, check if the product is excluded from the feed due to an exclusion rule, if so, skip
if (_runnerFeed.IsExcludedFromFeed(productData, false))
{
countSkipped++;
continue;
}
if (!hasLoadedDynamicMerchLabelData)
{
dynamicMerchLabel = PlaRelatedFeedUtils.GetDynamicMerchLabelValue(_runnerFeed, productData);
}
if (!string.IsNullOrWhiteSpace(dynamicMerchLabel))
{
dynamicMerchLabelProductCount++;
}
var googleCategoryBreadcrumb = (string.IsNullOrWhiteSpace(defaultCategory.GoogleCategoryBreadcrumb)) ? dict["gGoogleProductCategory"] : defaultCategory.GoogleCategoryBreadcrumb;
var customLabel0 = PlaRelatedFeedUtils.GetCustomLabelValue(_runnerFeed, productData, FeedRuleType.Custom_Label_0);
var customLabel1 = PlaRelatedFeedUtils.GetCustomLabelValue(_runnerFeed, productData, FeedRuleType.Custom_Label_1);
var customLabel2 = PlaRelatedFeedUtils.GetCustomLabelValue(_runnerFeed, productData, FeedRuleType.Custom_Label_2);
var customLabel3 = PlaRelatedFeedUtils.GetCustomLabelValue(_runnerFeed, productData, FeedRuleType.Custom_Label_3);
var customLabel4 = PlaRelatedFeedUtils.GetCustomLabel4Value(_runnerFeed, productData, dict, reader);
var regularPrice = (decimal)reader[dict["price"]];
var effectivePrice = regularPrice;
decimal?gSavingDollars = null;
decimal?gSavingsPercentage = null;
var adjustedPrice = string.IsNullOrEmpty(dict["adjustedPrice"]) ? "" : reader[dict["adjustedPrice"]].ToString();
var isOnSale = false;
if (!string.IsNullOrWhiteSpace(adjustedPrice))
{
decimal salePrice = Decimal.Parse(adjustedPrice);
if (salePrice != regularPrice)
{
if (salePrice > regularPrice)
{
regularPrice = salePrice;
}
else
{
isOnSale = true;
}
effectivePrice = salePrice;
}
}
// finally calculate the sale-related values
if (isOnSale)
{
gSavingDollars = regularPrice - effectivePrice;
gSavingsPercentage = 100 - effectivePrice * 100 / regularPrice;
}
var linkCatalog = dict["linkCatalog"];
var entry = EntryAttribute(reader
, id
, sanitizedTitle
, sku
, effectivePrice
, gSavingDollars
, gSavingsPercentage
, gAvailability
, marinBrand
, quantity
, linkCatalog, googleCategoryBreadcrumb, defaultCategory.Breadcrumb, customLabel0, customLabel1, customLabel2, customLabel3, customLabel4, dynamicMerchLabel
, size, colour, style, scent, flavour, bindingType, familyId
);
entry.WriteTo(xmlWriter);
countProcessed++;
}
catch (Exception e)
{
countError++;
var errorMessage = string.Format("Can't process the item. Id:{0};title:{1},catalog:{2},Message:{3}", id, title, catalog, e.Message);
Log.Error(errorMessage);
Log.DebugFormat("Error stack trace: {0}", e);
_executionLogLogger.AddCustomMessage(string.Format("Can't process the item. Id: {0};title: {1}, file identifier: {2}", id, title, identifier));
if (!AllowItemErrorsInFiles)
{
_hasError = true;
}
}
}
XmlDataUtils.EndXmlDocument(xmlWriter);
counter = new Tuple <int, int, int>(countProcessed, countError, countSkipped);
AddToDynamicMerchProductCount(dynamicMerchLabelProductCount);
Log.InfoFormat("[WriteFeedFile] {0} completed Processed record count: {1}, Error record count: {2}, skipped record count: {3}", feedFilePath, countProcessed, countError, countSkipped);
}
internal static string GetFeedEntryLinkValue(IDataReader reader, string catalog, string sku)
{
return(FeedUtils.GetProductUrl(BaseUrl, catalog, sku));
}
private void FeedXmlElement(XmlWriter xmlWriter, IDataReader reader, StringDictionary dict, string catalog, string identifier, string feedFilePath)
{
var counter = new ProcessingCounters();
var time = _effectiveFromTime.HasValue ? _effectiveFromTime.Value : DateTime.Now;
PlaRelatedFeedUtils.StartXmlDocument(xmlWriter, GoogleRunFeedType.Google, time);
//<entry>
while (reader.Read())
{
Log.DebugFormat("{0}::Processing record [{1}]: {2}", identifier, (counter.GetTotalProcessed()), reader["PID"]);
var id = reader[dict["gId"]].ToString();
var title = reader[dict["title"]].ToString();
try
{
var haveExclusionRulesChanged = _runnerFeedRulesHelper.HaveExclusionRulesChanged();
var sku = reader[dict["sku"]].ToString();
var brandName = !dict.ContainsKey("gBrand") ? string.Empty : reader[dict["gBrand"]].ToString();
var cContributors = PlaRelatedFeedUtils.ContributorAttributes(dict, reader, id);
var contributor = cContributors ?? null;
var defaultCategory = FeedUtils.GetFeedGeneratorIndigoCategory(_feedGeneratorCategoryService, reader, dict, catalog, Log);
var productData = FeedUtils.GetProductData(dict, reader, sku, catalog, brandName, contributor, defaultCategory);
var sanitizedTitle = (string.IsNullOrWhiteSpace(title)) ? string.Empty : FeedUtils.SanitizeString(title);
var gAvailability = !dict.ContainsKey("gAvailability") ? FeedUtils.GetGoogleAvailability(1)
: FeedUtils.GetGoogleAvailability(int.Parse(reader[dict["gAvailability"]].ToString()));
var availability = !dict.ContainsKey("gAvailability") ? 1 : int.Parse(reader[dict["gAvailability"]].ToString());
var recordType = !dict.ContainsKey("recordType") ? string.Empty : reader[dict["recordType"]].ToString();
var hasImage = true;
if (!SkipHasImageCheck)
{
hasImage = (!dict.ContainsKey("hasImage")) || int.Parse(reader[dict["hasImage"]].ToString()) > 0;
}
string message;
var isExcluded = false;
if (_isIncrementalRun)
{
var statusId = int.Parse(reader["StatusId"].ToString());
switch (statusId)
{
// New product
case 1:
counter.NumberOfNew++;
continue;
// Unchanged product
case 3:
if (!haveExclusionRulesChanged)
{
Log.DebugFormat("Product with id {0} is skipped in incremental mode as it wasn't modified and the rules haven't changed.", id);
counter.NumberOfUnchanged++;
continue;
}
var oldExclusionResult = _runnerFeedRulesHelper.IsExcludedFromFeed(productData, true);
var currentExclusionResult = _runnerFeedRulesHelper.IsExcludedFromFeed(productData, false);
if (oldExclusionResult == currentExclusionResult)
{
Log.DebugFormat("Product with id {0} is skipped in incremental mode as it wasn't modified, rules had changed but exclusion rule evaluation's result remained the same.", id);
counter.NumberOfUnchanged++;
continue;
}
// If the product is excluded at the moment, then perform the "exclusion logic" per business requirements,
// otherwise (i.e. product is included at the moment, treat it as "new")
if (!currentExclusionResult)
{
Log.DebugFormat("Product with id {0} is marked as new and skipped in incremental mode as it wasn't modified, rules had changed but exclusion rule evaluation's result changed and currently product isn't excluded.", id);
counter.NumberOfNew++;
continue;
}
Log.DebugFormat("Product with id {0} is marked as excluded in incremental mode as it wasn't modified, rules had changed but exclusion rule evaluation's result changed and currently product is being excluded.", id);
isExcluded = true;
break;
// Modified product
case 2:
var isEntryExcluded = IndigoBreadcrumbRepositoryUtils.IsExcludedDueToData(GooglePlaFeedId, sanitizedTitle, hasImage, availability, recordType, false, out message);
if (isEntryExcluded)
{
Log.DebugFormat("Product with id {0} is marked as excluded in incremental mode as it was modified, and it failed the data requirements for inclusion.", id);
isExcluded = true;
break;
}
// If product was excluded from the feed due to rules, then mark it as excluded
if (_runnerFeedRulesHelper.IsExcludedFromFeed(productData, false))
{
Log.DebugFormat("Product with id {0} is marked as excluded in incremental mode as it was modified, and it's matching one of the exclusion rules.", id);
isExcluded = true;
}
break;
default:
throw new ApplicationException("Invalid StatusId during an incremental run.");
}
}
else
{
isExcluded = IndigoBreadcrumbRepositoryUtils.IsExcludedDueToData(GooglePlaFeedId, sanitizedTitle, hasImage, availability, recordType, false, out message);
if (isExcluded)
{
Log.DebugFormat("Product with id {0} is marked as excluded in full mode as it failed the data requirements for inclusion.", id);
}
if (!isExcluded)
{
isExcluded = _runnerFeedRulesHelper.IsExcludedFromFeed(productData, false);
}
if (isExcluded)
{
Log.DebugFormat("Product with id {0} is marked as excluded in full mode as it's matching one of the exclusion rules.", id);
}
}
// At this point, we know if the product is excluded or not, regardless of which type of run is being executed.
// If we aren't supposed to be sending excluded product data, then update the skipped counter and exit
if (isExcluded)
{
counter.NumberOfExcluded++;
if (!SendExcludedProductData)
{
Log.Debug("Skipped the product because it was excluded.");
continue;
}
gAvailability = ExcludedProductGoogleAvailabilityText;
}
var regularPrice = (decimal)reader[dict["price"]];
var adjustedPrice = string.IsNullOrEmpty(dict["adjustedPrice"]) ? "" : reader[dict["adjustedPrice"]].ToString();
decimal?salePrice = null;
if (!string.IsNullOrWhiteSpace(adjustedPrice))
{
var salePriceFromDatabase = Decimal.Parse(adjustedPrice);
if (salePriceFromDatabase != regularPrice)
{
if (salePriceFromDatabase > regularPrice)
{
regularPrice = salePriceFromDatabase;
salePrice = null;
}
else
{
salePrice = salePriceFromDatabase;
}
}
}
var entry = EntryAttribute(id, regularPrice, salePrice, gAvailability);
entry.WriteTo(xmlWriter);
counter.NumberOfProcessed++;
}
catch (Exception e)
{
counter.NumberOfErrored++;
var errorMessage = string.Format("Can't process the item. Id:{0};title:{1},catalog:{2},Message:{3}", id, title, catalog, e.Message);
Log.Error(errorMessage);
Log.DebugFormat("Error stack trace: {0}", e);
_executionLogLogger.AddCustomMessage(string.Format("Can't process the item. Id: {0};title: {1}, file identifier: {2}", id, title, identifier));
if (_isIncrementalRun && !AllowItemErrorsInFiles)
{
_hasError = true;
}
}
}
// If the setting for sending deleted products is set to true in an incremental run, then get the deleted products since the last run
// and send them as the "special" deleted products, i.e. pid + availability of "out of stock"
if (_isIncrementalRun && SendExcludedProductData)
{
AddDeletedProducts(xmlWriter, identifier, ref counter);
}
PlaRelatedFeedUtils.EndXmlDocument(xmlWriter);
var infoLogMessage = string.Format("[WriteFeedFile] {0} completed processed record count: {1}, error record count: {2}, unchanged record count: {3}, " +
"new record count: {4}, excluded record count: {5}, deleted record count: {6}. ",
feedFilePath, counter.NumberOfProcessed, counter.NumberOfErrored, counter.NumberOfUnchanged, counter.NumberOfNew, counter.NumberOfExcluded, counter.NumberOfDeleted);
if (SendExcludedProductData)
{
infoLogMessage += "Excluded produces were included in processed count.";
}
Log.Info(infoLogMessage);
}
private static string GetFeedEntryLinkValue(IDataReader reader, string catalog, string sku)
{
return(catalog == "ebooks" ? reader["KoboItemPageURL"].ToString() : FeedUtils.GetProductUrl(BaseUrl, catalog, sku));
}
private XE EntryAttribute(string catalog, IDataReader reader, StringDictionary dict
, string gId
, string sanitizedTitle
, string linkCatalog
, string linkSku
, string gPrice
, string gAdjustedPrice
, string gAvailability
, string gBrandName,
string publisherName,
XE contributors,
string breadcrumb, bool isZeroCommissionProduct, string merchandiseType, IRuleEvaluationResult promotionalTextEvaluationResult, string description)
{
//entry
var entry = new XE("product");
sanitizedTitle = GetCjString(sanitizedTitle, MaxTitleLength);
var name = new XE("name", sanitizedTitle);
entry.Add(name);
var keywordsValue = string.Empty;
// First get the major contributor text
if (!string.IsNullOrWhiteSpace(gBrandName))
{
keywordsValue = gBrandName;
}
else if (contributors != null)
{
keywordsValue = contributors.Value;
}
// Now add the breadcrumb
if (!string.IsNullOrWhiteSpace(keywordsValue))
{
keywordsValue += ", ";
}
keywordsValue += string.Join(", ", breadcrumb.Split(new[] { BreadcrumbTrailSplitter }, StringSplitOptions.RemoveEmptyEntries)) + ", ";
keywordsValue += sanitizedTitle;
var keywords = new XE("keywords", GetCjString(keywordsValue, MaxKeywordLength));
entry.Add(keywords);
var descriptionValue = GetCjString(FeedUtils.RemoveHtmlTags(description), MaxDescriptionLength);
if (string.IsNullOrWhiteSpace(descriptionValue))
{
descriptionValue = sanitizedTitle;
}
var descriptionXe = new XE("description", descriptionValue);
entry.Add(descriptionXe);
var gaId = new XE("sku", gId);
entry.Add(gaId);
var aLink = FeedEntryLink(GetFeedEntryLinkValue(reader, linkCatalog, linkSku));
entry.Add(aLink);
var available = new XE("available", "Yes");
entry.Add(available);
var aImgLink = EntryImageLink(reader, linkSku);
entry.Add(aImgLink);
Decimal price;
bool isSpecial = false;
if (DisplaySalePriceInfo)
{
if (Decimal.TryParse(gPrice, out price))
{
var gaPrice = new XE("price", price.ToString("F", CultureInfo.InvariantCulture));
entry.Add(gaPrice);
Decimal salePrice;
if (!string.IsNullOrWhiteSpace(gAdjustedPrice) && Decimal.TryParse(gAdjustedPrice, out salePrice) && salePrice != price)
{
isSpecial = true;
var gaSalePrice = new XE("saleprice", salePrice.ToString("F", CultureInfo.InvariantCulture));
entry.Add(gaSalePrice);
}
}
}
else
{
price = Decimal.Parse(gAdjustedPrice);
var gaPrice = new XE("price", price.ToString("F", CultureInfo.InvariantCulture));
entry.Add(gaPrice);
}
entry.Add(new XE("currency", "CAD"));
// CJ expects the optional xml nodes in a certain order
var isBook = catalog.Equals("books", StringComparison.OrdinalIgnoreCase);
if (!isBook && (linkSku.Length == 12 || linkSku.Length == 13))
{
entry.Add(new XE("upc", linkSku));
}
if (promotionalTextEvaluationResult.HasMatch)
{
if (!promotionalTextEvaluationResult.IsDefaultMatch)
{
isSpecial = true;
}
var promoTextValue = GetCjString(promotionalTextEvaluationResult.MatchingRulePayLoads.First(), MaxPromotionalTextLength);
entry.Add(new XE("promotionaltext", promoTextValue));
}
var advertiserCategoryText = breadcrumb.Replace(BreadcrumbTrailSplitter, ">");
entry.Add(new XE("advertisercategory", GetCjString(advertiserCategoryText, MaxAdvertiserCategoryLength)));
if (!string.IsNullOrWhiteSpace(gBrandName))
{
var manufacturer = GetCjString(gBrandName, MaxManufacturerLength);
entry.Add(new XE("manufacturer", manufacturer));
entry.Add(new XE("manufacturerid", linkSku));
}
if (isBook)
{
entry.Add(new XE("isbn", linkSku));
if (contributors != null)
{
entry.Add(new XE("author", GetCjString(contributors.Value, MaxManufacturerLength)));
}
if (!string.IsNullOrWhiteSpace(publisherName))
{
entry.Add(new XE("publisher", GetCjString(publisherName, MaxManufacturerLength)));
}
}
else
{
if (contributors != null)
{
entry.Add(new XE("artist", GetCjString(contributors.Value, MaxManufacturerLength)));
}
}
entry.Add(new XE("title", sanitizedTitle));
if (!isBook)
{
if (!string.IsNullOrWhiteSpace(publisherName))
{
entry.Add(new XE("label", GetCjString(publisherName, MaxManufacturerLength)));
}
if (dict.ContainsKey("format"))
{
var format = reader[dict["format"]].ToString();
if (!string.IsNullOrWhiteSpace(format))
{
entry.Add(new XE("format", format));
}
}
}
entry.Add(new XE("special", (isSpecial) ? "Yes" : "No"));
if (catalog.Equals("generalmerchandise", StringComparison.OrdinalIgnoreCase))
{
entry.Add(new XE("gift", "Yes"));
}
entry.Add(new XE("instock", (string.IsNullOrWhiteSpace(gAvailability)) ? "No" : "Yes"));
entry.Add(new XE("condition", "New"));
if (!string.IsNullOrWhiteSpace(DefaultShippingCost))
{
entry.Add(new XE("standardshippingcost", DefaultShippingCost));
}
if (string.IsNullOrWhiteSpace(merchandiseType))
{
entry.Add(new XE("merchandisetype", ZeroCommissionListName));
}
else
{
if (isZeroCommissionProduct)
{
entry.Add(new XE("merchandisetype", ZeroCommissionListName));
// If the merchandise type has a different value and it's not equal to zero commission list
// and if we're putting the item on the zero commission list, add it to the list of items to be
// updated inside the
if (!merchandiseType.Equals(ZeroCommissionListName, StringComparison.OrdinalIgnoreCase))
{
_updatedMerchandiseTypeProductPids.Add(decimal.Parse(gId));
}
Log.DebugFormat("Product {0} was put in zero commission list.", gId);
}
else
{
entry.Add(new XE("merchandisetype", merchandiseType));
}
}
return(entry);
}
private void WriteFileComponentContent(XmlWriter xmlWriter, FeedGenerationFileLineItem fileComponent, IDataReader reader, string identifier, ref Tuple <int, int, int> countRec)
{
var attributesDictionary = ConfigurationManager.GetSection(fileComponent.Catalogattributesection) as StringDictionary;
if (countRec == null)
{
throw new ArgumentNullException("countRec");
}
var countProcessed = 0;
var countDeleted = 0;
var countError = 0;
//<entry>
while (reader.Read())
{
Log.DebugFormat("{0}::Processing record [{1}]: {2}", identifier, (countProcessed + countError + countDeleted), reader["PID"]);
var id = reader[attributesDictionary["gId"]].ToString();
var title = reader[attributesDictionary["title"]].ToString();
try
{
var haveRulesChanged = _runnerHelper.HaveRulesChanged;
var linkSku = reader[attributesDictionary["linkSku"]].ToString();
var brandName = !attributesDictionary.ContainsKey("gBrand") ? string.Empty : reader[attributesDictionary["gBrand"]].ToString();
var cContributors = GetContributors(attributesDictionary, reader);
// Get the breadcrumb value
var defaultCategory = FeedUtils.GetFeedGeneratorIndigoCategory(_feedGeneratorCategoryService, reader, attributesDictionary, fileComponent.Catalog, Log);
var productData = FeedUtils.GetProductData(attributesDictionary, reader, linkSku, fileComponent.Catalog, brandName, cContributors, defaultCategory);
var sanitizedTitle = (string.IsNullOrWhiteSpace(title)) ? string.Empty : FeedUtils.SanitizeString(title);
var gAvailability = !attributesDictionary.ContainsKey("gAvailability") ? FeedUtils.GetGoogleAvailability(1) : FeedUtils.GetGoogleAvailability((int)reader[attributesDictionary["gAvailability"]]);
var availability = !attributesDictionary.ContainsKey("gAvailability") ? 1 : (int)reader[attributesDictionary["gAvailability"]];
var recordType = !attributesDictionary.ContainsKey("recordType") ? string.Empty : reader[attributesDictionary["recordType"]].ToString();
var merchandiseType = !attributesDictionary.ContainsKey("merchandiseType") ? string.Empty : reader[attributesDictionary["merchandiseType"]].ToString();
if (string.IsNullOrWhiteSpace(merchandiseType))
{
_missingMerchandiseTypeProductInfos.Add(new MissingMerchandiseTypeProductInfo {
Breadcrumb = defaultCategory.Breadcrumb, Sku = linkSku, Title = sanitizedTitle
});
}
IRuleEvaluationResult newZeroCommissionResult = null;
IRuleEvaluationResult newPromotionalTextResult = null;
if (_isIncrementalRun)
{
var isModifiedData = int.Parse(reader["IsModified"].ToString());
if (isModifiedData == 0)
{
if (!haveRulesChanged)
{
Log.DebugFormat(
"Product with pid {0} is skipped in incremental mode as it wasn't modified and the rules haven't changed.",
id);
continue;
}
var oldZeroCommissionResult = _runnerHelper.GetZeroCommissionRuleResult(productData, true);
newZeroCommissionResult = _runnerHelper.GetZeroCommissionRuleResult(productData, false);
var oldPromotionalTextResult = _runnerHelper.GetPromotionalTextRuleResult(productData, true);
newPromotionalTextResult = _runnerHelper.GetPromotionalTextRuleResult(productData, false);
if (oldZeroCommissionResult.HasMatch == newZeroCommissionResult.HasMatch &&
oldPromotionalTextResult.HasMatch == newPromotionalTextResult.HasMatch &&
oldPromotionalTextResult.MatchingRulePayLoads.First()
.Equals(newPromotionalTextResult.MatchingRulePayLoads.First()))
{
countDeleted++;
Log.DebugFormat(
"Product with pid {0} is skipped in incremental mode as it wasn't modified and rule evaluations yielded same results.",
id);
continue;
}
// At this point, we know that rules have changed, which means we need to resend this product as modified
}
else
{
newZeroCommissionResult = _runnerHelper.GetZeroCommissionRuleResult(productData, false);
newPromotionalTextResult = _runnerHelper.GetPromotionalTextRuleResult(productData, false);
}
}
else
{
newZeroCommissionResult = _runnerHelper.GetZeroCommissionRuleResult(productData, false);
newPromotionalTextResult = _runnerHelper.GetPromotionalTextRuleResult(productData, false);
}
var isZeroCommissionElement = newZeroCommissionResult.HasMatch || IsZeroCommissionElement(sanitizedTitle, _feedId, true, availability, recordType);
if (isZeroCommissionElement)
{
Log.DebugFormat("Product with pid {0} is being placed in zero commission list due to either data issues or zero commission rules.)", id);
}
var linkCatalog = attributesDictionary["linkCatalog"];
if (string.IsNullOrWhiteSpace(sanitizedTitle))
{
sanitizedTitle = "(No Title)";
}
var description = reader[attributesDictionary["description"]].ToString();
description = string.IsNullOrWhiteSpace(description) ? sanitizedTitle : FeedUtils.SanitizeString(description);
// Get the breadcrumb value
var breadcrumb = defaultCategory.Breadcrumb;
var gPrice = string.IsNullOrEmpty(attributesDictionary["price"]) ? "" : reader[attributesDictionary["price"]].ToString();
var gAdjustedPrice = string.IsNullOrEmpty(attributesDictionary["adjustedPrice"]) ? string.Empty : reader[attributesDictionary["adjustedPrice"]].ToString();
var publisherName = !attributesDictionary.ContainsKey("publisherName") ? string.Empty : reader[attributesDictionary["publisherName"]].ToString();
if (!string.IsNullOrWhiteSpace(recordType) &&
recordType.Equals("GCard_Electronic", StringComparison.OrdinalIgnoreCase))
{
gPrice = DefaultEGiftCardPrice;
gAdjustedPrice = DefaultEGiftCardPrice;
}
var entry = EntryAttribute(fileComponent.Catalog, reader, attributesDictionary
, id
, sanitizedTitle
, linkCatalog, linkSku
, gPrice
, gAdjustedPrice
, gAvailability
, brandName, publisherName, cContributors, breadcrumb, isZeroCommissionElement, merchandiseType, newPromotionalTextResult, description);
countProcessed++;
//out of memory exception will be thrown if we keep the xml entries in memory
entry.WriteTo(xmlWriter);
}
catch (Exception e)
{
countError++;
Log.ErrorFormat("Can't process the item. Id:{0};title:{1}", id, title);
Log.DebugFormat("Error stack trace: {0}", e);
_executionLogLogger.AddCustomMessage(string.Format("Can't process the item. Id: {0};title: {1}, file identifier: {2}", id, title, identifier));
if (!AllowItemErrorsInFiles || _isIncrementalRun)
{
_hasError = true;
}
}
}
// Now process the deleted items for the range & producttypeid and put them in the xml file as "deleted" items
if (_isIncrementalRun)
{
// Don't do it for gift card file component
if (!fileComponent.StoredProcedureName.Equals("uspCJFeedGeneralMerchandiseGiftCard",
StringComparison.OrdinalIgnoreCase))
{
foreach (var deletedProductInfo in GetDeletedProductInfos(identifier))
{
GetDeletedElement(deletedProductInfo).WriteTo(xmlWriter);
countDeleted++;
}
}
}
Log.InfoFormat("{0} completed. Processed record count: {1}, Error record count: {2}, deleted/skipped record count {3}", identifier, countProcessed, countError, countDeleted);
countRec = new Tuple <int, int, int>(countRec.Item1 + countProcessed, countRec.Item2 + countError, countRec.Item3 + countDeleted);
}
