private async Task <bool> RequestAndIndexPriceListPageAsync( SearchIndexClient searchIndexClient, int from, int count, long updateUtcTimestamp, PriceListSynchronizationStatistics statistics) { var tryNumber = 1; while (true) { try { _logger.LogTrace(LoggingEvents.Synchronization, $"Processing {from}+{count}..."); var omegaResponse = await _omegaAutoBizClient.ProductPriceListAsync(from, count); if (omegaResponse.Result == null || omegaResponse.Result.Length == 0) { return(true); } var indexActions = omegaResponse.Result .Select(x => ConvertHelper.OmegaAutoBizApiModelToIndexAction(x, updateUtcTimestamp)) .ToArray(); var indexBatch = IndexBatch.New(indexActions); try { await searchIndexClient.Documents.IndexAsync(indexBatch); } catch (IndexBatchException ex) { var reasons = string.Join(", ", ex.IndexingResults .Where(x => !x.Succeeded) .Select(x => $"{x.Key}-{x.ErrorMessage}")); var message = $"Failed to index products: {reasons}."; throw new InvalidOperationException(message); } _logger.LogTrace(LoggingEvents.Synchronization, "Success."); statistics.Received += omegaResponse.Result.Length; return(false); } catch (Exception ex) { _logger.LogError(LoggingEvents.UnhandledException, ex, $"Try {tryNumber}."); if (tryNumber >= 3) { throw; } tryNumber++; } } }
public async Task RunPriceListSynchronizationAsync() { var statistics = new PriceListSynchronizationStatistics(); try { await _ekSearchManagementClient.CreateProductIndexIfNotExistAsync(); var from = 0; const int count = 1_000; using (var searchIndexClient = _ekSearchManagementClient.CreateSearchIndexClient()) { searchIndexClient.IndexName = _ekSearchManagementClient.ProductsIndexName; var updateUtcTimestamp = TimestampHelper.GetCurrentUtcTotalMinutes(); while (true) { var isFinalPage = await RequestAndIndexPriceListPageAsync(searchIndexClient, from, count, updateUtcTimestamp, statistics); if (isFinalPage) { break; } from += count; } // wait half of minute until changes are applied await Task.Delay(TimeSpan.FromSeconds(30)); await PriceListHelper.CleanExpiredRecordsAsync( searchIndexClient, EkProductSourceEnum.OmegaAutoBiz, updateUtcTimestamp, statistics, _logger); } } catch (Exception ex) { _logger.LogError(LoggingEvents.UnhandledException, ex, "Omega price list synchronization failed."); await _notificationManager.SendWorkerMessageAsync($"ERROR! Omega price list synchronization failed, {statistics}: {ex.Message}"); throw; } await _notificationManager.SendWorkerMessageAsync($"Omega price list synchronization succeed, {statistics}."); }
private async Task IndexPriceListPageAsync( SearchIndexClient searchIndexClient, ElitPriceListRecord[] priceListPage, long updateUtcTimestamp, PriceListSynchronizationStatistics statistics) { var tryNumber = 1; while (true) { try { var indexActions = priceListPage .Select(x => ConvertHelper.ElitUaApiModelToIndexAction(x, updateUtcTimestamp)) .ToArray(); var indexBatch = IndexBatch.New(indexActions); try { await searchIndexClient.Documents.IndexAsync(indexBatch); } catch (IndexBatchException ex) { var reasons = string.Join(", ", ex.IndexingResults .Where(x => !x.Succeeded) .Select(x => $"{x.Key}-{x.ErrorMessage}")); var message = $"Failed to index products: {reasons}."; throw new InvalidOperationException(message); } _logger.LogTrace(LoggingEvents.Synchronization, "Success."); statistics.Received += priceListPage.Length; return; } catch (Exception ex) { _logger.LogError(LoggingEvents.UnhandledException, ex, $"Try {tryNumber}."); if (tryNumber >= 3) { throw; } tryNumber++; } } }
public async Task RunPriceListSynchronizationAsync() { var cancellationToken = CancellationToken.None; var statistics = new PriceListSynchronizationStatistics(); string priceListStatusMessage; try { var lastAppliedPriceListId = await _persistentCache.GetValueAsync(LastAppliedElitPriceListIdKey, cancellationToken); var priceList = await _elitUaClient.GetUnappliedPriceListAsync(lastAppliedPriceListId, cancellationToken); if (!priceList.IsSuccess) { throw new InvalidOperationException(priceList.StatusMessage); } priceListStatusMessage = priceList.StatusMessage; if (priceList.Records?.Count > 0) { await _ekSearchManagementClient.CreateProductIndexIfNotExistAsync(); using (var searchIndexClient = _ekSearchManagementClient.CreateSearchIndexClient()) { searchIndexClient.IndexName = _ekSearchManagementClient.ProductsIndexName; var updateUtcTimestamp = TimestampHelper.GetCurrentUtcTotalMinutes(); // max size of Azure Search batch const int PageSize = 1_000; var page = 0; foreach (var priceListPage in priceList.Records.Batch(PageSize)) { _logger.LogTrace(LoggingEvents.Synchronization, $"Processing {page*PageSize}+{PageSize}..."); await IndexPriceListPageAsync(searchIndexClient, priceListPage.ToArray(), updateUtcTimestamp, statistics); page++; } // wait half of minute until changes are applied await Task.Delay(TimeSpan.FromSeconds(30), cancellationToken); await PriceListHelper.CleanExpiredRecordsAsync( searchIndexClient, EkProductSourceEnum.ElitUa, updateUtcTimestamp, statistics, _logger); } } await _persistentCache.SetValueAsync(LastAppliedElitPriceListIdKey, priceList.PriceListId, CancellationToken.None); } catch (Exception ex) { _logger.LogError(LoggingEvents.UnhandledException, ex, "Elit price list synchronization failed."); await _notificationManager.SendWorkerMessageAsync($"ERROR! Elit price list synchronization failed, {statistics}: {ex.Message}"); throw; } await _notificationManager.SendWorkerMessageAsync($"Elit price list synchronization succeed, {statistics}, status: {priceListStatusMessage}"); }
public static async Task CleanExpiredRecordsAsync( SearchIndexClient searchIndexClient, EkProductSourceEnum productSource, long lastUpdateUtcTimestamp, PriceListSynchronizationStatistics statistics, ILogger logger) { const int BatchSize = 1_000; string lastProcessedKey = null; while (true) { var filterBuilder = new StringBuilder(); filterBuilder.Append($"source eq {(int)productSource} and updatedOnUtcTimestamp ne {lastUpdateUtcTimestamp}"); if (lastProcessedKey != null) { filterBuilder.Append($" and key gt '{lastProcessedKey}'"); } var searchParameters = new SearchParameters() { Top = BatchSize, Filter = filterBuilder.ToString(), Select = new List <string>() { "key" }, OrderBy = new List <string>() { "key asc", }, }; var searchResult = await searchIndexClient.Documents.SearchAsync(null, searchParameters); var documents = searchResult.Results .Select(x => x.Document) .ToArray(); if (documents.Length > 0) { // remove records var indexBatch = IndexBatch.New(documents.Select(x => IndexAction.Merge(x))); try { await searchIndexClient.Documents.IndexAsync(indexBatch); } catch (IndexBatchException ex) { var reasons = string.Join(", ", ex.IndexingResults .Where(x => !x.Succeeded) .Select(x => $"{x.Key}-{x.ErrorMessage}")); var message = $"Failed to index products: {reasons}."; throw new InvalidOperationException(message); } // use last processed key as additional filter since indexing takes some time lastProcessedKey = (string)documents.Last()["key"]; logger.LogInformation(LoggingEvents.Synchronization, $"Deleted: {documents.Length}."); statistics.Removed += documents.Length; } else { logger.LogInformation(LoggingEvents.Synchronization, "No expired records."); break; } } }