private async Task <bool> RequestAndIndexPriceListPageAsync(
SearchIndexClient searchIndexClient,
int from,
int count,
long updateUtcTimestamp,
PriceListSynchronizationStatistics statistics)
{
var tryNumber = 1;
while (true)
{
try
{
_logger.LogTrace(LoggingEvents.Synchronization, $"Processing {from}+{count}...");
var omegaResponse = await _omegaAutoBizClient.ProductPriceListAsync(from, count);
if (omegaResponse.Result == null ||
omegaResponse.Result.Length == 0)
{
return(true);
}
var indexActions = omegaResponse.Result
.Select(x => ConvertHelper.OmegaAutoBizApiModelToIndexAction(x, updateUtcTimestamp))
.ToArray();
var indexBatch = IndexBatch.New(indexActions);
try
{
await searchIndexClient.Documents.IndexAsync(indexBatch);
}
catch (IndexBatchException ex)
{
var reasons = string.Join(", ", ex.IndexingResults
.Where(x => !x.Succeeded)
.Select(x => $"{x.Key}-{x.ErrorMessage}"));
var message = $"Failed to index products: {reasons}.";
throw new InvalidOperationException(message);
}
_logger.LogTrace(LoggingEvents.Synchronization, "Success.");
statistics.Received += omegaResponse.Result.Length;
return(false);
}
catch (Exception ex)
{
_logger.LogError(LoggingEvents.UnhandledException, ex, $"Try {tryNumber}.");
if (tryNumber >= 3)
{
throw;
}
tryNumber++;
}
}
}
public async Task RunPriceListSynchronizationAsync()
{
var statistics = new PriceListSynchronizationStatistics();
try
{
await _ekSearchManagementClient.CreateProductIndexIfNotExistAsync();
var from = 0;
const int count = 1_000;
using (var searchIndexClient = _ekSearchManagementClient.CreateSearchIndexClient())
{
searchIndexClient.IndexName = _ekSearchManagementClient.ProductsIndexName;
var updateUtcTimestamp = TimestampHelper.GetCurrentUtcTotalMinutes();
while (true)
{
var isFinalPage = await RequestAndIndexPriceListPageAsync(searchIndexClient, from, count, updateUtcTimestamp, statistics);
if (isFinalPage)
{
break;
}
from += count;
}
// wait half of minute until changes are applied
await Task.Delay(TimeSpan.FromSeconds(30));
await PriceListHelper.CleanExpiredRecordsAsync(
searchIndexClient,
EkProductSourceEnum.OmegaAutoBiz,
updateUtcTimestamp,
statistics,
_logger);
}
}
catch (Exception ex)
{
_logger.LogError(LoggingEvents.UnhandledException, ex, "Omega price list synchronization failed.");
await _notificationManager.SendWorkerMessageAsync($"ERROR! Omega price list synchronization failed, {statistics}: {ex.Message}");
throw;
}
await _notificationManager.SendWorkerMessageAsync($"Omega price list synchronization succeed, {statistics}.");
}
private async Task IndexPriceListPageAsync(
SearchIndexClient searchIndexClient,
ElitPriceListRecord[] priceListPage,
long updateUtcTimestamp,
PriceListSynchronizationStatistics statistics)
{
var tryNumber = 1;
while (true)
{
try
{
var indexActions = priceListPage
.Select(x => ConvertHelper.ElitUaApiModelToIndexAction(x, updateUtcTimestamp))
.ToArray();
var indexBatch = IndexBatch.New(indexActions);
try
{
await searchIndexClient.Documents.IndexAsync(indexBatch);
}
catch (IndexBatchException ex)
{
var reasons = string.Join(", ", ex.IndexingResults
.Where(x => !x.Succeeded)
.Select(x => $"{x.Key}-{x.ErrorMessage}"));
var message = $"Failed to index products: {reasons}.";
throw new InvalidOperationException(message);
}
_logger.LogTrace(LoggingEvents.Synchronization, "Success.");
statistics.Received += priceListPage.Length;
return;
}
catch (Exception ex)
{
_logger.LogError(LoggingEvents.UnhandledException, ex, $"Try {tryNumber}.");
if (tryNumber >= 3)
{
throw;
}
tryNumber++;
}
}
}
public async Task RunPriceListSynchronizationAsync()
{
var cancellationToken = CancellationToken.None;
var statistics = new PriceListSynchronizationStatistics();
string priceListStatusMessage;
try
{
var lastAppliedPriceListId = await _persistentCache.GetValueAsync(LastAppliedElitPriceListIdKey, cancellationToken);
var priceList = await _elitUaClient.GetUnappliedPriceListAsync(lastAppliedPriceListId, cancellationToken);
if (!priceList.IsSuccess)
{
throw new InvalidOperationException(priceList.StatusMessage);
}
priceListStatusMessage = priceList.StatusMessage;
if (priceList.Records?.Count > 0)
{
await _ekSearchManagementClient.CreateProductIndexIfNotExistAsync();
using (var searchIndexClient = _ekSearchManagementClient.CreateSearchIndexClient())
{
searchIndexClient.IndexName = _ekSearchManagementClient.ProductsIndexName;
var updateUtcTimestamp = TimestampHelper.GetCurrentUtcTotalMinutes();
// max size of Azure Search batch
const int PageSize = 1_000;
var page = 0;
foreach (var priceListPage in priceList.Records.Batch(PageSize))
{
_logger.LogTrace(LoggingEvents.Synchronization, $"Processing {page*PageSize}+{PageSize}...");
await IndexPriceListPageAsync(searchIndexClient, priceListPage.ToArray(), updateUtcTimestamp, statistics);
page++;
}
// wait half of minute until changes are applied
await Task.Delay(TimeSpan.FromSeconds(30), cancellationToken);
await PriceListHelper.CleanExpiredRecordsAsync(
searchIndexClient,
EkProductSourceEnum.ElitUa,
updateUtcTimestamp,
statistics,
_logger);
}
}
await _persistentCache.SetValueAsync(LastAppliedElitPriceListIdKey, priceList.PriceListId, CancellationToken.None);
}
catch (Exception ex)
{
_logger.LogError(LoggingEvents.UnhandledException, ex, "Elit price list synchronization failed.");
await _notificationManager.SendWorkerMessageAsync($"ERROR! Elit price list synchronization failed, {statistics}: {ex.Message}");
throw;
}
await _notificationManager.SendWorkerMessageAsync($"Elit price list synchronization succeed, {statistics}, status: {priceListStatusMessage}");
}
public static async Task CleanExpiredRecordsAsync(
SearchIndexClient searchIndexClient,
EkProductSourceEnum productSource,
long lastUpdateUtcTimestamp,
PriceListSynchronizationStatistics statistics,
ILogger logger)
{
const int BatchSize = 1_000;
string lastProcessedKey = null;
while (true)
{
var filterBuilder = new StringBuilder();
filterBuilder.Append($"source eq {(int)productSource} and updatedOnUtcTimestamp ne {lastUpdateUtcTimestamp}");
if (lastProcessedKey != null)
{
filterBuilder.Append($" and key gt '{lastProcessedKey}'");
}
var searchParameters = new SearchParameters()
{
Top = BatchSize,
Filter = filterBuilder.ToString(),
Select = new List <string>()
{
"key"
},
OrderBy = new List <string>()
{
"key asc",
},
};
var searchResult = await searchIndexClient.Documents.SearchAsync(null, searchParameters);
var documents = searchResult.Results
.Select(x => x.Document)
.ToArray();
if (documents.Length > 0)
{
// remove records
var indexBatch = IndexBatch.New(documents.Select(x => IndexAction.Merge(x)));
try
{
await searchIndexClient.Documents.IndexAsync(indexBatch);
}
catch (IndexBatchException ex)
{
var reasons = string.Join(", ", ex.IndexingResults
.Where(x => !x.Succeeded)
.Select(x => $"{x.Key}-{x.ErrorMessage}"));
var message = $"Failed to index products: {reasons}.";
throw new InvalidOperationException(message);
}
// use last processed key as additional filter since indexing takes some time
lastProcessedKey = (string)documents.Last()["key"];
logger.LogInformation(LoggingEvents.Synchronization, $"Deleted: {documents.Length}.");
statistics.Removed += documents.Length;
}
else
{
logger.LogInformation(LoggingEvents.Synchronization, "No expired records.");
break;
}
}
}