public override ActionTaskResult Process(ActionContext context)
{
bool canProcess = false;
lock (Locker)
{
if (!IsProcessing)
{
IsProcessing = true;
canProcess = true;
}
}
if (canProcess)
{
try
{
int chunkSize = GetValue(context, "UpdateChunkSize", DefaultChunkSize);
int maxDegreeOfParallelism = GetValue(context, "MaxDegreeOfParallelism", DefaultMaxDegreeOfParallelism);
var report = _validationService.ValidateAndUpdate(chunkSize, maxDegreeOfParallelism, TaskContext);
return(ActionTaskResult.Success(new ActionTaskResultMessage()
{
ResourceClass = nameof(TaskStrings),
ResourceName = nameof(TaskStrings.ValidationResult),
Parameters = new object[]
{
report.TotalProductsCount,
report.UpdatedProductsCount,
report.ValidatedProductsCount,
report.InvalidProductsCount,
report.ValidationErrorsCount
}
}));
}
finally
{
IsProcessing = false;
}
}
return(ActionTaskResult.Error(new ActionTaskResultMessage()
{
ResourceClass = nameof(TaskStrings),
ResourceName = nameof(TaskStrings.ActionRunning),
Parameters = new object[] { GetType().Name }
}));
}
public async Task ImportAsync(ITaskExecutionContext executionContext, string language, string state, Dictionary <string, IProductStore> stores)
{
if (executionContext.IsCancellationRequested)
{
executionContext.IsCancelled = true;
return;
}
var url = _configuration.GetReindexUrl(language, state);
_logger.LogInformation("Starting import...");
var ids = await GetIds(url);
_logger.LogInformation($"Product list received. Length: {ids.Length}. Splitting products into chunks by {_options.ChunkSize}...");
var chunks = ids.Chunk(_options.ChunkSize).ToArray();
var index = 1;
float progress = 0;
foreach (var chunk in chunks)
{
if (executionContext.IsCancellationRequested)
{
executionContext.IsCancelled = true;
return;
}
var enumerable = chunk as int[] ?? chunk.ToArray();
_logger.LogInformation($"Chunk {index} with ids ({string.Join(",", enumerable)}) requested...");
var dataTasks = enumerable.Select(n => GetProductById(url, n));
ProductPostProcessorData[] data;
try
{
data = await Task.WhenAll(dataTasks);
}
catch (Exception ex)
{
string message = $"An error occurs while receiving products for chunk {index}";
_logger.LogError(ex, message);
executionContext.Result = ActionTaskResult.Error(message);
throw;
}
_logger.LogInformation($"Products from chunk {index} received. Starting bulk import...");
var result = await _manager.BulkCreateAsync(data, language, state, stores);
if (result.Succeeded)
{
_logger.LogInformation($"Bulk import for chunk {index} succeeded.");
index++;
}
else
{
string message = $"Cannot proceed bulk import for chunk {index}: {result}";
_logger.LogError(message);
executionContext.Result = ActionTaskResult.Error(message);
throw result.GetException();
}
progress += (float)100 / chunks.Length;
executionContext.SetProgress((byte)progress);
}
executionContext.Result = ActionTaskResult.Success("Import completed");
}
public override ActionTaskResult Process(ActionContext context)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
if (context.ContentItemIds == null || context.ContentItemIds.Length == 0)
{
throw new ArgumentException("ContentItemIds cant be empty", "context.ContentItemIds");
}
try
{
string adapter = GetAdapter(context);
var marketingProducts = _articleService.List(context.ContentId, context.ContentItemIds).ToArray();
string ignoredStatus = GetIgnoredStatus(context);
string[] ignoredStatuses = (ignoredStatus == null) ? Enumerable.Empty <string>().ToArray() : ignoredStatus.Split(new[] { ',' });
string productsFieldName = _settingsService.GetSetting(SettingsTitles.MARKETING_PRODUCT_PRODUCTS_FIELD_NAME);
int backRelationFieldId = marketingProducts[0].FieldValues.Single(fv => fv.Field.Name == productsFieldName).Field.BackRelationId.Value;
var backRelationField = _fieldService.Read(backRelationFieldId);
int productContentId = backRelationField.ContentId;
int[] productIds = DoWithLogging(
() => Helpers.GetProductIdsFromMarketingProducts(context.ContentItemIds, _articleService, _settingsService),
"Receiving regional products from marketing product ids {ids} ", context.ContentItemIds
);
var filteredProductIds = _articleService.List(productContentId, productIds)
.Where(a => !ignoredStatuses.Contains(a.Status.Name) && !a.Archived && a.Visible)
.Select(a => a.Id)
.ToArray();
ActionTaskResult result;
if (filteredProductIds.Any())
{
var publishAction = ObjectFactoryBase.Resolve <PublishAction>();
var productContext = new ActionContext
{
BackendSid = context.BackendSid,
CustomerCode = context.CustomerCode,
ContentId = productContentId,
ContentItemIds = filteredProductIds,
ActionCode = context.ActionCode,
Parameters = context.Parameters,
UserId = context.UserId,
UserName = context.UserName
};
result = DoWithLogging(
() => publishAction.Process(productContext),
"Calling Publish Action for products {ids}",
filteredProductIds
);
}
else
{
result = ActionTaskResult.Error(new ActionTaskResultMessage()
{
ResourceClass = nameof(TaskStrings),
ResourceName = nameof(TaskStrings.NoProductsToPublish)
}, context.ContentItemIds);
}
var excludedProductIds = productIds.Except(filteredProductIds).ToArray();
if (excludedProductIds.Any())
{
result.Messages.Add(new ActionTaskResultMessage()
{
ResourceClass = nameof(TaskStrings),
ResourceName = nameof(TaskStrings.FilteredProducts),
Parameters = new [] { string.Join(", ", excludedProductIds), ignoredStatus }
});
}
return(result);
}
catch (ActionException)
{
throw;
}
catch (Exception ex) {
Logger.Error().Message(LoggerErrorMessage).Exception(ex).Write();
throw new ActionException(
TaskStrings.ActionErrorMessage,
context.ContentItemIds.Select(
id => new ProductException(id, nameof(TaskStrings.ServerError), ex)
),
context
);
}
}
public override ActionTaskResult Process(ActionContext context)
{
int bundleSize = GetBundleSize();
int maxDegreeOfParallelism = GetMaxDegreeOfParallelism();
string[] channels = context.Parameters.GetChannels();
bool localize = context.Parameters.GetLocalize();
int marketingProductContentId = int.Parse(_settingsService.GetSetting(SettingsTitles.MARKETING_PRODUCT_CONTENT_ID));
string productsFieldName = _settingsService.GetSetting(SettingsTitles.MARKETING_PRODUCT_PRODUCTS_FIELD_NAME);
Dictionary <int, int[]> articleIdsToCheckRelationsByContentId;
int[] productIds;
if (context.ContentItemIds == null || context.ContentItemIds.Length == 0)
{
productIds = DoWithLogging(
() => Helpers.GetAllProductIds(int.Parse(context.Parameters["site_id"]), context.ContentId, _provider.GetCustomer()),
"Getting all products from content {contentId}", context.ContentId
);
articleIdsToCheckRelationsByContentId = new Dictionary <int, int[]>
{
{ context.ContentId, productIds }
};
}
else
{
productIds = DoWithLogging(
() => Helpers.ExtractRegionalProductIdsFromMarketing(context.ContentItemIds, _articleService, marketingProductContentId, productsFieldName),
"Getting regional product ids from marketing products content {contentId} using field {fieldName} and ids {ids}",
marketingProductContentId, productsFieldName, context.ContentItemIds
);
articleIdsToCheckRelationsByContentId = Helpers.GetContentIds(productIds, _provider.GetCustomer());
}
if (productIds.Length == 0)
{
return(ActionTaskResult.Error(new ActionTaskResultMessage()
{
ResourceClass = nameof(SendProductActionStrings),
ResourceName = nameof(SendProductActionStrings.NotFound),
Extra = string.Join(", ", context.ContentItemIds)
}, context.ContentItemIds));
}
foreach (var articleIdsWithContentId in articleIdsToCheckRelationsByContentId)
{
var checkResult =
DoWithLogging(
() => _articleService.CheckRelationSecurity(articleIdsWithContentId.Key, articleIdsWithContentId.Value, false),
"Checking relation security in content {contentId} for articles {ids}",
articleIdsWithContentId.Key, articleIdsWithContentId.Value
);
string idsstr = string.Join(", ", checkResult.Where(n => !n.Value));
if (!string.IsNullOrEmpty(idsstr))
{
return(ActionTaskResult.Error(new ActionTaskResultMessage()
{
ResourceClass = nameof(SendProductActionStrings),
ResourceName = nameof(SendProductActionStrings.NoRelationAccess),
Extra = idsstr
}, context.ContentItemIds));
}
}
const string skipPublishingKey = "skipPublishing";
bool skipPublishing = context.Parameters.ContainsKey(skipPublishingKey) && bool.Parse(context.Parameters[skipPublishingKey]);
const string skipLiveKey = "skipLive";
bool skipLive = context.Parameters.ContainsKey(skipLiveKey) && bool.Parse(context.Parameters[skipLiveKey]);
const string ignoredStatusKey = "IgnoredStatus";
string ignoredStatus = (context.Parameters.ContainsKey(ignoredStatusKey)) ? context.Parameters[ignoredStatusKey] : null;
float currentPercent = 0;
object percentLocker = new object();
var parts = productIds.Section(bundleSize).ToArray();
var filteredInStage = new ConcurrentBag <int>();
var filteredInLive = new ConcurrentBag <int>();
var failed = new ConcurrentDictionary <int, object>();
var missing = new ConcurrentBag <int>();
var excluded = new ConcurrentBag <int>();
var frozen = new ConcurrentBag <int>();
var invisibleOrArchivedIds = new ConcurrentBag <int>();
var errors = new ConcurrentBag <Exception>();
var validationErrors = new ConcurrentDictionary <int, ActionTaskResult>();
var validationErrorsSerialized = new ConcurrentDictionary <int, string>();
Parallel.ForEach(parts, new ParallelOptions {
MaxDegreeOfParallelism = maxDegreeOfParallelism
},
() =>
{
HttpContextUserProvider.ForcedUserId = context.UserId;
return(new Local
{
ProductService = ObjectFactoryBase.Resolve <IProductService>(),
QpNotificationService = ObjectFactoryBase.Resolve <IQPNotificationService>(),
XmlProductService = ObjectFactoryBase.Resolve <IXmlProductService>()
});
},
(idsToProcess, ps, tl) =>
{
try
{
if (TaskContext.IsCancellationRequested)
{
TaskContext.IsCancelled = true;
return(tl);
}
var localInvisibleOrArchivedIds = new HashSet <int>();
Article[] prodsStage = DoWithLogging(
() => tl.ProductService.GetProductsByIds(idsToProcess.ToArray()),
"Getting products {ids}", idsToProcess.ToArray()
);
IEnumerable <string> ignoredStatuses = ignoredStatus?.Split(',') ??
Enumerable.Empty <string>().ToArray();
var excludedStage = prodsStage.Where(n => ignoredStatuses.Contains(n.Status)).ToArray();
foreach (var item in excludedStage)
{
excluded.Add(item.Id);
}
prodsStage = prodsStage.Except(excludedStage).ToArray();
var frozenIds = new int[0];
if (!skipLive)
{
var idsToCheck = prodsStage.Select(p => p.Id).ToArray();
frozenIds = DoWithLogging(
() => _freezeService.GetFrozenProductIds(idsToCheck),
"Getting freezing state for products {ids}", idsToCheck
);
}
prodsStage = prodsStage.Where(p => !frozenIds.Contains(p.Id)).ToArray();
foreach (int id in frozenIds)
{
frozen.Add(id);
}
if (TaskContext.IsCancellationRequested)
{
TaskContext.IsCancelled = true;
return(tl);
}
//Валидация продуктов
foreach (int id in prodsStage.Where(w => !w.Archived && w.Visible).Select(s => s.Id))
{
var xamlValidationErrors = DoWithLogging(
() => _articleService.XamlValidationById(id, true),
"Validating XAML for product {id}", id
);
var validationResult = ActionTaskResult.FromRulesException(xamlValidationErrors, id);
if (!validationResult.IsSuccess)
{
validationErrors.TryAdd(id, validationResult);
validationErrorsSerialized.TryAdd(id, validationResult.ToString());
}
}
prodsStage = prodsStage.Where(w => !validationErrors.Keys.Contains(w.Id)).ToArray();
var prodsLive = new Article[] { };
if (!skipLive)
{
if (!skipPublishing)
{
prodsLive = prodsStage;
}
else
{
prodsLive = DoWithLogging(
() => tl.ProductService.GetProductsByIds(idsToProcess.ToArray(), true),
"Getting separate live products {ids}", idsToProcess.ToArray()
);
}
}
if (TaskContext.IsCancellationRequested)
{
TaskContext.IsCancelled = true;
return(tl);
}
lock (percentLocker)
{
currentPercent += (float)50 / parts.Length;
}
TaskContext.SetProgress((byte)currentPercent);
// архивные или невидимые продукты следует удалить с витрин
foreach (var product in prodsLive.Where(product => product.Archived || !product.Visible))
{
localInvisibleOrArchivedIds.Add(product.Id);
}
if (!skipLive)
{
foreach (var item in idsToProcess.Except(prodsLive.Select(y => y.Id)))
{
missing.Add(item);
}
}
//неопубликованные или расщепленные публикуем сразу
int[] prodsToPublishIds = null;
if (!skipPublishing)
{
prodsToPublishIds = prodsLive
.Where(x =>
!x.Archived &&
x.Visible &&
(
!x.IsPublished ||
PublishAction.GetAllArticlesToCheck(x).Any(p => !p.IsPublished) ||
_freezeService.GetFreezeState(x.Id) == FreezeState.Unfrosen
)
)
.Select(x => x.Id)
.ToArray();
if (TaskContext.IsCancellationRequested)
{
TaskContext.IsCancelled = true;
return(tl);
}
// удалим требующие публикации или удаления продукты
prodsLive = prodsLive
.Where(p => !prodsToPublishIds.Contains(p.Id) && !localInvisibleOrArchivedIds.Contains(p.Id))
.ToArray();
}
if (TaskContext.IsCancellationRequested)
{
TaskContext.IsCancelled = true;
return(tl);
}
lock (percentLocker)
{
currentPercent += (float)30 / parts.Length;
}
TaskContext.SetProgress((byte)currentPercent);
foreach (var id in localInvisibleOrArchivedIds)
{
invisibleOrArchivedIds.Add(id);
}
int sectionSize = Math.Min(bundleSize, 5);
var tasks =
ArticleFilter.LiveFilter.Filter(prodsLive)
.Section(sectionSize)
.Select(z => tl.QpNotificationService
.SendProductsAsync(z.ToArray(), false, context.UserName, context.UserId,
localize, false, channels)
.ContinueWith(y => UpdateFilteredIds(filteredInLive,
y.IsFaulted ? null : y.Result, z, y.Exception, errors, failed)))
.Concat(ArticleFilter.DefaultFilter.Filter(prodsStage)
.Section(sectionSize)
.Select(z => tl.QpNotificationService.SendProductsAsync(z.ToArray(), true,
context.UserName, context.UserId, localize, false, channels)
.ContinueWith(y => UpdateFilteredIds(filteredInStage,
y.IsFaulted ? null : y.Result, z, y.Exception, errors, failed))))
.ToArray();
if (tasks.Length > 0)
{
float percentsPerTask = (float)10 / parts.Length / tasks.Length;
tasks = tasks
.Select(x => x.ContinueWith(y =>
{
lock (percentLocker)
{
currentPercent += percentsPerTask;
}
TaskContext.SetProgress((byte)currentPercent);
}))
.ToArray();
DoWithLogging(() => Task.WaitAll(tasks),
"Sending notifications for live ({liveIds}) and stage ({stageIds}) products",
ArticleFilter.LiveFilter.Filter(prodsLive).Select(n => n.Id).ToArray(),
ArticleFilter.DefaultFilter.Filter(prodsStage).Select(n => n.Id).ToArray()
);
}
else
{
lock (percentLocker)
{
currentPercent += (float)10 / parts.Length;
}
TaskContext.SetProgress((byte)currentPercent);
}
if (TaskContext.IsCancellationRequested)
{
TaskContext.IsCancelled = true;
return(tl);
}
// эти продукты имеют неопубликованные или расщепленные статьи
if (!skipPublishing && prodsToPublishIds.Length > 0)
{
var publishAction = ObjectFactoryBase.Resolve <PublishAction>();
var publishActionContext = new ActionContext
{
ContentItemIds = prodsToPublishIds,
Parameters = new Dictionary <string, string>()
{
{ ignoredStatusKey, ignoredStatus }
},
UserId = context.UserId,
UserName = context.UserName
};
try
{
DoWithLogging(
() => publishAction.Process(publishActionContext),
"Calling PublishAction for products {ids}",
prodsToPublishIds
);
}
catch (ActionException ex)
{
var ids = ex.InnerExceptions.OfType <ProductException>().Select(x => x.ProductId);
Logger.Error()
.Exception(ex)
.Message("Exception has been thrown while publishing products {ids}", prodsToPublishIds)
.Write();
foreach (var pID in ids)
{
failed.TryAdd(pID, null);
}
}
catch (Exception ex)
{
Logger.Error()
.Exception(ex)
.Message("Exception has been thrown while publishing products {ids}", prodsToPublishIds)
.Write();
}
}
lock (percentLocker)
{
currentPercent += (float)10 / parts.Length;
}
TaskContext.SetProgress((byte)currentPercent);
return(tl);
}
catch (Exception ex)
{
Logger.Error().Message("SendProductAction exception").Exception(ex).Write();
foreach (var item in idsToProcess)
{
failed.TryAdd(item, null);
}
errors.Add(ex);
}
HttpContextUserProvider.ForcedUserId = 0;
return(tl);
}, tt => { });
if (TaskContext.IsCancellationRequested)
{
TaskContext.IsCancelled = true;
return(ActionTaskResult.Error(new ActionTaskResultMessage()
{
ResourceClass = nameof(SendProductActionStrings),
ResourceName = nameof(SendProductActionStrings.Cancelled)
}, context.ContentItemIds));
}
var productsToRemove = new int[0];
var productsToRemoveCheck = missing
.Concat(invisibleOrArchivedIds)
.Except(excluded)
.Except(frozen)
.Except(validationErrors.Keys)
.ToArray();
if (productsToRemoveCheck.Length > 0)
{
// проверяем, какие из проблемных продуктов присутствуют на витрине
var cmService = ObjectFactoryBase.Resolve <IList <IConsumerMonitoringService> >();
productsToRemove = DoWithLogging(
() => cmService.SelectMany(s => s.FindExistingProducts(productsToRemoveCheck)).Distinct().ToArray(),
"Checking whether products {ids} are missing on fronts", productsToRemoveCheck
);
if (productsToRemove.Length > 0)
{
// эти продукты отсутствуют в DPC или не видны, но остались на витрине
// их надо удалить с витрин
var service = ObjectFactoryBase.Resolve <IQPNotificationService>();
var productService = ObjectFactoryBase.Resolve <IProductService>();
DoWithLogging(
() => Task.WhenAll(productsToRemove.Section(20).Select(
s => service.DeleteProductsAsync(
productService.GetSimpleProductsByIds(s.ToArray()),
context.UserName,
context.UserId,
false)
)
).Wait(),
"Removing missing products from fronts {ids}", productsToRemove
);
}
}
DoWithLogging(
() => _validationService.UpdateValidationInfo(productIds, validationErrorsSerialized),
"Updating validation info for products {ids}", productIds
);
int[] notFound = missing.Except(productsToRemove).Except(excluded).Except(frozen).Except(validationErrors.Keys).ToArray();
var notSucceeded = failed.Keys.Concat(notFound).Concat(excluded).Concat(frozen)
.Concat(validationErrors.Keys).ToArray();
var result = new ActionTaskResult()
{
FailedIds = notSucceeded
};
var msg = new ActionTaskResultMessage()
{
ResourceClass = nameof(SendProductActionStrings)
};
if (notSucceeded.Any())
{
msg.ResourceName = nameof(SendProductActionStrings.PartiallySucceededResult);
msg.Extra = string.Join(", ", notSucceeded);
msg.Parameters = new object[] { productIds.Length - notSucceeded.Length, productIds.Length };
}
else
{
msg.ResourceName = nameof(SendProductActionStrings.SucceededResult);
msg.Parameters = new object[] { productIds.Length };
}
result.Messages.Add(msg);
if (errors.Any())
{
result.Messages.Add(new ActionTaskResultMessage()
{
ResourceClass = nameof(SendProductActionStrings),
ResourceName = nameof(SendProductActionStrings.Errors),
Extra = string.Join(", ", errors.Select(x => x.Message).Distinct())
});
}
AddMessages(result, nameof(SendProductActionStrings.ExcludedByStatus), excluded.ToArray());
AddMessages(result, nameof(SendProductActionStrings.ExcludedWithFreezing), frozen.ToArray());
AddMessages(result, nameof(SendProductActionStrings.NotFoundInDpc), notFound.ToArray());
AddMessages(result, nameof(SendProductActionStrings.RemovedFromFronts), productsToRemove.ToArray());
AddMessages(result, nameof(SendProductActionStrings.NotPassedByStageFiltration), filteredInStage.ToArray());
AddMessages(result, nameof(SendProductActionStrings.NotPassedByLiveFiltration), filteredInLive.ToArray());
if (validationErrors.Any())
{
result.Messages.AddRange(validationErrors.SelectMany(v => v.Value.Messages));
}
TaskContext.Result = result;
return(result);
}
public override ActionTaskResult Process(ActionContext context)
{
bool canProcess = false;
lock (Locker)
{
if (!IsProcessing)
{
IsProcessing = true;
canProcess = true;
}
}
if (canProcess)
{
try
{
if (!int.TryParse(_settingsService.GetSetting(SettingsTitles.PRODUCTVERSION_CHUNK_SIZE), out int chunkSize))
{
chunkSize = DefaultChunkSize;
}
if (!int.TryParse(_settingsService.GetSetting(SettingsTitles.PRODUCTVERSION_TIMEOUT), out int timeout))
{
timeout = DefaultTimeout;
}
if (!int.TryParse(_settingsService.GetSetting(SettingsTitles.PRODUCTVERSION_CLEANUP_INTERVAL), out int cleanupInterval))
{
throw new Exception($"Setting {SettingsTitles.PRODUCTVERSION_CLEANUP_INTERVAL} is not provided or incorrect");
}
var date = DateTime.Now.AddDays(-cleanupInterval);
var expectedTotalCount = GetVersionsCount(date);
int processedCount = 0;
int currentCount = 0;
byte progress = 0;
Logger.Info($"Start CleanProductVersionAction cleanupInterval={cleanupInterval}, chunkSize={chunkSize}, expectedTotalCount={expectedTotalCount}");
do
{
currentCount = CleanVersions(date, chunkSize, timeout);
processedCount += currentCount;
Logger.Info($"Clean {currentCount} product versions");
progress = expectedTotalCount == 0 ? (byte)100 : Math.Min((byte)(processedCount * 100 / expectedTotalCount), (byte)100);
TaskContext.SetProgress(progress);
if (TaskContext.IsCancellationRequested)
{
TaskContext.IsCancelled = true;
break;
}
}while (currentCount > 0);
Logger.Info($"End CleanProductVersionAction processedCount={processedCount}");
return(ActionTaskResult.Success(new ActionTaskResultMessage()
{
ResourceClass = nameof(TaskStrings),
ResourceName = nameof(TaskStrings.VersionsCleaned),
Parameters = new object[] { processedCount, date, chunkSize }
}));
}
finally
{
IsProcessing = false;
}
}
return(ActionTaskResult.Error(new ActionTaskResultMessage()
{
ResourceClass = nameof(TaskStrings),
ResourceName = nameof(TaskStrings.ActionRunning),
Parameters = new object[] { GetType().Name }
}));
}