private string MergeWorkingFiles(MergeData mergeData, string fileInstructionsKey, Language language)
{
var workingPath = ParameterUtils.GetParameter <string>(mergeData.WorkingFilePathKey) + "/" + GetWorkingPathLanguageString(language);
Log.Info("MergeWorkingFiles working path: " + workingPath);
var fileList = Directory.EnumerateFiles(workingPath);
fileList = fileList.OrderBy(fileName => fileName, FileNameComparerObject);
string languageString = GetLanguageString(language);
var outputPath = OutputFolderPath + "/" + ParameterUtils.GetParameter <string>(mergeData.OutputFileBaseNameKey) + "_" + fileInstructionsKey + "_" + languageString +
DateTime.Now.ToString("yyyy_MM_dd") + ".txt";
using (var writer = new StreamWriter(outputPath))
{
writer.WriteLine(mergeData.FeedTypeFileHeader);
foreach (string filePath in fileList)
{
using (var reader = new StreamReader(filePath))
{
string line;
while ((line = reader.ReadLine()) != null)
{
writer.WriteLine(line);
}
}
}
}
return(outputPath);
}
private void RemoveExecutionFiles()
{
try
{
if (!ParameterUtils.GetParameter <bool>("DeleteWorkingFiles"))
{
return;
}
Directory.Delete(ParameterUtils.GetParameter <string>("WorkingDirectory"), recursive: true);
if (!ZipFiles)
{
return;
}
IEnumerable <string> outputTextFiles = Directory.EnumerateFileSystemEntries(OutputFolderPath,
"*.txt");
outputTextFiles.ForEach(File.Delete);
}
catch (Exception ex)
{
const string message = "Error during cleanup working file deletion.";
Log.Error(message, ex);
_executionLogLogger.AddCustomMessage(message);
}
}
public void Build(string[] args)
{
var startTime = DateTime.Now;
Log.Info("Execution started.");
if (ParameterUtils.GetParameter <bool>("EnableCategoryUpdates"))
{
try
{
DoCategoryUpdates();
}
catch (Exception ex)
{
Log.Error("An error occurred during processing of categories.", ex);
}
}
if (ParameterUtils.GetParameter <bool>("EnableBrandUpdates"))
{
try
{
DoBrandUpdates();
}
catch (Exception ex)
{
Log.Error("An error occurred during processing of brands.", ex);
}
}
if (ParameterUtils.GetParameter <bool>("EnableCmsProductListArchiving"))
{
try
{
DoCmsProductListArchiving();
}
catch (Exception ex)
{
Log.Error("An error occurred during processing of CMS Product Lists.", ex);
}
}
if (ParameterUtils.GetParameter <bool>("EnableDefaultRecosUpdates"))
{
try
{
DoDefaultRecosUpdates();
}
catch (Exception ex)
{
Log.Error("An error occurred while generation default recommendations.", ex);
}
}
var elapsedTime = DateTime.Now - startTime;
Log.InfoFormat("Execution completed. Elapsed time: {0}", elapsedTime.ToString(@"dd\.hh\:mm\:ss"));
}
/// <summary>
/// Gets the CMS content IDs of the product lists to use for default recommendations.
/// </summary>
private long[] GetNewProductListIds(int allowedDefaultRecommendationLanguageId)
{
// Load Rewards Centre config from CMS
// It contains a setting specifying the product lists to use for default recos
IDictionary <string, string> rewardsCentreSettings;
try
{
var cmsRewardsCentreConfigId = ParameterUtils.GetParameter <int>("CmsRewardsCentreConfigId");
Log.InfoFormat("Loading Rewards Centre config from CMS using content ID {0} for language id of {1}.", cmsRewardsCentreConfigId, allowedDefaultRecommendationLanguageId);
using (var merchandisingService = _merchandisingServiceFactory.Create())
{
rewardsCentreSettings = merchandisingService.ServiceClient.GetSettingsNoCache(cmsRewardsCentreConfigId, false);
}
}
catch (Exception ex)
{
throw new Exception("Failed to load Rewards Centre config from CMS", ex);
}
// Get the product list IDs setting
var setting = GetProductListSetting(allowedDefaultRecommendationLanguageId);
string productListIdsSetting;
rewardsCentreSettings.TryGetValue(setting, out productListIdsSetting);
if (string.IsNullOrWhiteSpace(productListIdsSetting))
{
// All list ids have been removed from CMS config. log a message and delete all the entries in recosdb
Log.InfoFormat("All product list ids have been removed from CMS configuration for language id of {0}. Removing all entries in recosdb as a result.", allowedDefaultRecommendationLanguageId);
return(new long[0]);
}
Log.InfoFormat("Found setting of {0}: {1} for language id of {2}", setting, productListIdsSetting, allowedDefaultRecommendationLanguageId);
// Parse ints
try
{
var productListIds = productListIdsSetting
.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries)
.Select(long.Parse)
.ToArray();
Log.InfoFormat("Parsed product list IDs: {0} for {1}", string.Join(", ", productListIds), allowedDefaultRecommendationLanguageId);
return(productListIds);
}
catch (Exception ex)
{
throw new Exception(string.Format("Failed to parse product list IDs ({0})", productListIdsSetting), ex);
}
}
private HttpContent GetContent(SerializerResolver serializer)
{
switch (BodyType)
{
case BodyType.Encoded:
return(new FormUrlEncodedContent(ParameterUtils.GetParameter <FormParameter>(this, RequestCulture)));
case BodyType.Serialized:
return(serializer.Resolve(GetType(), DataDirection.Out).Serialize(this));
case BodyType.SerializedProperty:
var body = serializer.Resolve(GetType(), DataDirection.Out).Serialize(ParameterUtils.GetSingleParameterObject <RequestBody>(this));
return(body);
case BodyType.Custom:
return(BodyContent);
default:
//todo custom exception - there should have been a datatype specified
throw new ArgumentOutOfRangeException();
}
}
private void FeedXmlElement(XmlWriter xmlWriter, XmlWriter xmlWriterSecondary, IDataReader reader, StringDictionary dict, string catalog, string identifier, string feedFilePath, ref Tuple <int, int, int> counter)
{
if (counter == null)
{
throw new ArgumentNullException("counter");
}
var countProcessed = 0;
var countError = 0;
var countSkipped = 0;
var hasTwoGenerators = HasTwoGenerators();
var time = DateTime.Now;
PlaRelatedFeedUtils.StartXmlDocument(xmlWriter, _runnerFeed.GetRunFeedType(), time);
if (hasTwoGenerators)
{
PlaRelatedFeedUtils.StartXmlDocument(xmlWriterSecondary, _runnerFeedSecondary.GetRunFeedType(), time);
}
//<entry>
while (reader.Read())
{
Log.DebugFormat("{0}::Processing record [{1}]: {2}", identifier, (countProcessed + countError), reader["PID"]);
var id = reader[dict["gId"]].ToString();
var title = reader[dict["title"]].ToString();
try
{
// First check if the product is a sensitive product, and if so, exclude
if (dict.ContainsKey("isSensitiveProduct") && int.Parse(reader[dict["isSensitiveProduct"]].ToString()) > 0)
{
countSkipped++;
continue;
}
var linkSku = reader[dict["linkSku"]].ToString();
var brandName = !dict.ContainsKey("gBrand") ? string.Empty : reader[dict["gBrand"]].ToString();
var cContributors = PlaRelatedFeedUtils.ContributorAttributes(dict, reader, id);
var contributor = cContributors ?? null;
var defaultCategory = FeedUtils.GetFeedGeneratorIndigoCategory(_feedGeneratorCategoryService, reader, dict, catalog, Log);
var productData = FeedUtils.GetProductData(dict, reader, linkSku, catalog, brandName, contributor, defaultCategory);
var formatString = dict.ContainsKey("bisacbindingtypeid") ? FeedUtils.GetFormat(reader[dict["bisacbindingtypeid"]].ToString(), id) : string.Empty;
var sanitizedTitle = string.IsNullOrWhiteSpace(title) ? string.Empty : FeedUtils.SanitizeString(title);
var gAvailability = !dict.ContainsKey("gAvailability") ? FeedUtils.GetGoogleAvailability(1) : FeedUtils.GetGoogleAvailability((int)reader[dict["gAvailability"]]);
var availability = !dict.ContainsKey("gAvailability") ? 1 : (int)reader[dict["gAvailability"]];
var recordType = !dict.ContainsKey("recordType") ? string.Empty : reader[dict["recordType"]].ToString();
var hasImage = true;
if (!SkipHasImageCheck)
{
hasImage = (!dict.ContainsKey("hasImage")) || int.Parse(reader[dict["hasImage"]].ToString()) > 0;
}
string message;
var isExcludedDueToData = IndigoBreadcrumbRepositoryUtils.IsExcludedDueToData(_feedId, sanitizedTitle, hasImage, availability, recordType, false, out message);
// First determine if the entries are to be excluded or not
var isEntryExcluded = isExcludedDueToData || _runnerFeed.IsExcludedFromFeed(productData, false);
var isEntrySecondaryExcluded = isExcludedDueToData;
if (!isEntrySecondaryExcluded && hasTwoGenerators)
{
isEntrySecondaryExcluded = _runnerFeedSecondary.IsExcludedFromFeed(productData, false);
}
// If both entries are excluded, then there is no need to process anything further
if ((isEntryExcluded && isEntrySecondaryExcluded) || (isEntryExcluded && !hasTwoGenerators))
{
countSkipped++;
continue;
}
var linkCatalog = dict["linkCatalog"];
if (MaxTitleLength > 0)
{
sanitizedTitle = FeedUtils.GetTruncatedTitle(sanitizedTitle, formatString, MaxTitleLength, ParameterUtils.GetParameter <bool>("GooglePlaFeedGenerator.TruncateTitle"));
}
var description = reader[dict["description"]].ToString();
description = string.IsNullOrWhiteSpace(description) ? sanitizedTitle : FeedUtils.SanitizeString(FeedUtils.RemoveHtmlTags(description));
var sanitizedDescription = string.IsNullOrWhiteSpace(description) ? sanitizedTitle : description;
// Get the breadcrumb value
var googleCategoryBreadcrumb = (string.IsNullOrWhiteSpace(defaultCategory.GoogleCategoryBreadcrumb)) ? dict["gGoogleProductCategory"] : defaultCategory.GoogleCategoryBreadcrumb;
var allBreadcrumbs = (productData.BrowseCategoryIds.Any() && !productData.BrowseCategoryIds.Contains(-1)) ? new List <string>() : new List <string> {
defaultCategory.Breadcrumb
};
foreach (var browseCategoryId in productData.BrowseCategoryIds.Distinct())
{
var categories = _feedGeneratorCategoryService.GetIndigoBreadcrumbCategories(browseCategoryId);
if (categories != null)
{
foreach (var category in categories)
{
if (category.Crumbs[0].Equals(defaultCategory.Crumbs[0], StringComparison.OrdinalIgnoreCase))
{
allBreadcrumbs.Add(category.Breadcrumb);
}
}
}
}
decimal?salePrice = null;
decimal parsedSalePrice;
var regularPrice = decimal.Parse(reader[dict["price"]].ToString());
if (!string.IsNullOrEmpty(dict["adjustedPrice"]) && decimal.TryParse(reader[dict["adjustedPrice"]].ToString(), out parsedSalePrice))
{
if (parsedSalePrice != regularPrice)
{
if (parsedSalePrice > regularPrice)
{
regularPrice = parsedSalePrice;
}
else
{
salePrice = parsedSalePrice;
}
}
}
var gtin = reader[dict["gGtin"]].ToString();
var isDefaultCpcValue = false;
if (!isEntryExcluded)
{
var customLabel0 = PlaRelatedFeedUtils.GetCustomLabelValue(_runnerFeed, productData, FeedRuleType.Custom_Label_0);
var customLabel1 = PlaRelatedFeedUtils.GetCustomLabelValue(_runnerFeed, productData, FeedRuleType.Custom_Label_1);
var customLabel2 = PlaRelatedFeedUtils.GetCustomLabelValue(_runnerFeed, productData, FeedRuleType.Custom_Label_2);
var customLabel3 = PlaRelatedFeedUtils.GetCustomLabelValue(_runnerFeed, productData, FeedRuleType.Custom_Label_3);
var customLabel4 = PlaRelatedFeedUtils.GetCustomLabel4Value(_runnerFeed, productData, dict, reader);
var cpcValue = PlaRelatedFeedUtils.GetCpcValue(_runnerFeed, productData, out isDefaultCpcValue);
var entry = EntryAttribute(reader
, id
, sanitizedTitle, sanitizedDescription, googleCategoryBreadcrumb
, linkCatalog, linkSku
, regularPrice
, salePrice
, gAvailability//, availability
, gtin
, brandName, dict.ContainsKey("gBrand"), defaultCategory.Breadcrumb, allBreadcrumbs, cpcValue
, customLabel0, customLabel1, customLabel2, customLabel3, customLabel4
, _runnerFeed.GetRunFeedType());
entry.WriteTo(xmlWriter);
}
if (hasTwoGenerators && !isEntrySecondaryExcluded)
{
var cpcValue = PlaRelatedFeedUtils.GetCpcValue(_runnerFeedSecondary, productData, out isDefaultCpcValue);
var customLabel0 = PlaRelatedFeedUtils.GetCustomLabelValue(_runnerFeed, productData, FeedRuleType.Custom_Label_0);
var customLabel1 = PlaRelatedFeedUtils.GetCustomLabelValue(_runnerFeed, productData, FeedRuleType.Custom_Label_1);
var customLabel2 = PlaRelatedFeedUtils.GetCustomLabelValue(_runnerFeed, productData, FeedRuleType.Custom_Label_2);
var customLabel3 = PlaRelatedFeedUtils.GetCustomLabelValue(_runnerFeed, productData, FeedRuleType.Custom_Label_3);
var customLabel4 = PlaRelatedFeedUtils.GetCustomLabel4Value(_runnerFeed, productData, dict, reader);
var entry = EntryAttribute(reader
, id
, sanitizedTitle, sanitizedDescription, googleCategoryBreadcrumb
, linkCatalog, linkSku
, regularPrice
, salePrice
, gAvailability
, gtin
, brandName, dict.ContainsKey("gBrand"), defaultCategory.Breadcrumb, allBreadcrumbs, cpcValue
, customLabel0, customLabel1, customLabel2, customLabel3, customLabel4
, _runnerFeedSecondary.GetRunFeedType());
entry.WriteTo(xmlWriterSecondary);
}
if (isDefaultCpcValue)
{
_defaultCpcProductInfos.Add(new DefaultCpcProductInfo {
Breadcrumb = defaultCategory.Breadcrumb, Sku = linkSku, Title = sanitizedTitle
});
}
countProcessed++;
}
catch (Exception e)
{
countError++;
Log.ErrorFormat("Can't process the item. Id:{0};title:{1}", id, title);
Log.DebugFormat("Error stack trace: {0}", e);
_executionLogLogger.AddCustomMessage(string.Format("Can't process the item. Id: {0};title: {1}, file identifier: {2}", id, title, identifier));
if (HasTwoGenerators())
{
_executionLogLoggerSecondary.AddCustomMessage(string.Format("Can't process the item. Id: {0};title: {1}, file identifier: {2}", id, title, identifier));
}
if (!AllowItemErrorsInFiles)
{
_hasError = true;
}
}
}
PlaRelatedFeedUtils.EndXmlDocument(xmlWriter);
if (hasTwoGenerators)
{
PlaRelatedFeedUtils.EndXmlDocument(xmlWriterSecondary);
}
counter = new Tuple <int, int, int>(countProcessed, countError, countSkipped);
Log.InfoFormat("[WriteFeedFile] {0} completed Processed record count: {1}, Error record count: {2}, skipped record count: {3}", feedFilePath, countProcessed, countError, countSkipped);
}