public async Task RunPriceListSynchronizationAsync()
{
var statistics = new PriceListSynchronizationStatistics();
try
{
await _ekSearchManagementClient.CreateProductIndexIfNotExistAsync();
var from = 0;
const int count = 1_000;
using (var searchIndexClient = _ekSearchManagementClient.CreateSearchIndexClient())
{
searchIndexClient.IndexName = _ekSearchManagementClient.ProductsIndexName;
var updateUtcTimestamp = TimestampHelper.GetCurrentUtcTotalMinutes();
while (true)
{
var isFinalPage = await RequestAndIndexPriceListPageAsync(searchIndexClient, from, count, updateUtcTimestamp, statistics);
if (isFinalPage)
{
break;
}
from += count;
}
// wait half of minute until changes are applied
await Task.Delay(TimeSpan.FromSeconds(30));
await PriceListHelper.CleanExpiredRecordsAsync(
searchIndexClient,
EkProductSourceEnum.OmegaAutoBiz,
updateUtcTimestamp,
statistics,
_logger);
}
}
catch (Exception ex)
{
_logger.LogError(LoggingEvents.UnhandledException, ex, "Omega price list synchronization failed.");
await _notificationManager.SendWorkerMessageAsync($"ERROR! Omega price list synchronization failed, {statistics}: {ex.Message}");
throw;
}
await _notificationManager.SendWorkerMessageAsync($"Omega price list synchronization succeed, {statistics}.");
}
public async Task RunPriceListSynchronizationAsync()
{
var cancellationToken = CancellationToken.None;
var statistics = new PriceListSynchronizationStatistics();
string priceListStatusMessage;
try
{
var lastAppliedPriceListId = await _persistentCache.GetValueAsync(LastAppliedElitPriceListIdKey, cancellationToken);
var priceList = await _elitUaClient.GetUnappliedPriceListAsync(lastAppliedPriceListId, cancellationToken);
if (!priceList.IsSuccess)
{
throw new InvalidOperationException(priceList.StatusMessage);
}
priceListStatusMessage = priceList.StatusMessage;
if (priceList.Records?.Count > 0)
{
await _ekSearchManagementClient.CreateProductIndexIfNotExistAsync();
using (var searchIndexClient = _ekSearchManagementClient.CreateSearchIndexClient())
{
searchIndexClient.IndexName = _ekSearchManagementClient.ProductsIndexName;
var updateUtcTimestamp = TimestampHelper.GetCurrentUtcTotalMinutes();
// max size of Azure Search batch
const int PageSize = 1_000;
var page = 0;
foreach (var priceListPage in priceList.Records.Batch(PageSize))
{
_logger.LogTrace(LoggingEvents.Synchronization, $"Processing {page*PageSize}+{PageSize}...");
await IndexPriceListPageAsync(searchIndexClient, priceListPage.ToArray(), updateUtcTimestamp, statistics);
page++;
}
// wait half of minute until changes are applied
await Task.Delay(TimeSpan.FromSeconds(30), cancellationToken);
await PriceListHelper.CleanExpiredRecordsAsync(
searchIndexClient,
EkProductSourceEnum.ElitUa,
updateUtcTimestamp,
statistics,
_logger);
}
}
await _persistentCache.SetValueAsync(LastAppliedElitPriceListIdKey, priceList.PriceListId, CancellationToken.None);
}
catch (Exception ex)
{
_logger.LogError(LoggingEvents.UnhandledException, ex, "Elit price list synchronization failed.");
await _notificationManager.SendWorkerMessageAsync($"ERROR! Elit price list synchronization failed, {statistics}: {ex.Message}");
throw;
}
await _notificationManager.SendWorkerMessageAsync($"Elit price list synchronization succeed, {statistics}, status: {priceListStatusMessage}");
}
public async Task SearchForAbsentThumbnailsAsync()
{
var totalProcessed = 0;
var totalThumbnailFound = 0;
try
{
const int BatchSize = 10;
using (var indexClient = _ekSearchManagementClient.CreateSearchIndexClient())
{
indexClient.IndexName = _ekSearchManagementClient.ProductsIndexName;
string lastProcessedKey = null;
while (true)
{
var filterBuilder = new StringBuilder();
filterBuilder.Append("isThumbnailSearchProvided eq null and thumbnailUrl eq null");
if (lastProcessedKey != null)
{
filterBuilder.Append($" and key gt '{lastProcessedKey}'");
}
var searchParameters = new SearchParameters()
{
Top = BatchSize,
Filter = filterBuilder.ToString(),
OrderBy = new List <string>()
{
"key asc",
},
};
var searchResult = await indexClient.Documents.SearchAsync <IndexProduct>(null, searchParameters);
var records = searchResult.Results
.Select(x => x.Document)
.ToArray();
if (records.Length > 0)
{
// search for thumbnails
var searchTasks = records.Select(record =>
Task.Run(async() =>
{
var document = new Document()
{
["key"] = record.Key,
};
var fileInfo = await GetPartNumberThumbnailAsync(
brandName: record.BrandName,
partNumber: record.PartNumber);
if (fileInfo?.Url != null)
{
document["thumbnailUrl"] = fileInfo.Url;
Interlocked.Increment(ref totalThumbnailFound);
}
document["isThumbnailSearchProvided"] = true;
Interlocked.Increment(ref totalProcessed);
return(document);
}));
var documents = await Task.WhenAll(searchTasks);
// index flags and thumbnail urls if found
var indexBatch = IndexBatch.New(documents.Select(x => IndexAction.Merge(x)));
try
{
await indexClient.Documents.IndexAsync(indexBatch);
}
catch (IndexBatchException ex)
{
var reasons = string.Join(", ", ex.IndexingResults
.Where(x => !x.Succeeded)
.Select(x => $"{x.Key}-{x.ErrorMessage}"));
var message = $"Failed to index products: {reasons}.";
throw new InvalidOperationException(message);
}
// use last processed key as additional filter since indexing takes some time
lastProcessedKey = records.Last().Key;
var foundPercentage = (decimal)totalThumbnailFound / totalProcessed;
_logger.LogInformation($"Processed: {totalProcessed}, found: {totalThumbnailFound}, {foundPercentage:P}.");
}
else
{
_logger.LogInformation("No more records.");
break;
}
}
}
}
catch (Exception ex)
{
_logger.LogError(LoggingEvents.UnhandledException, ex, null);
await _notificationManager.SendWorkerMessageAsync($"Search for absent thumbnails failed, processed: {totalProcessed}, found: {totalThumbnailFound}, {ex.Message}");
throw;
}
await _notificationManager.SendWorkerMessageAsync($"Search for absent thumbnails succeed, processed: {totalProcessed}, found: {totalThumbnailFound}.");
}