private async Task HandleFailureAsync(ReindexWorkItem workItem, BulkIndexByScrollFailure failure) { _logger.LogError("Error reindexing document {Index}/{Id}: [{Status}] {Message}", workItem.OldIndex, failure.Id, failure.Status, failure.Cause.Reason); var gr = await _client.GetAsync <object>(request : new GetRequest(workItem.OldIndex, failure.Id)).AnyContext(); if (!gr.IsValid) { _logger.LogErrorRequest(gr, "Error getting document {Index}/{Id}", workItem.OldIndex, failure.Id); return; } _logger.LogRequest(gr); string document = JsonConvert.SerializeObject(new { failure.Index, failure.Id, gr.Version, gr.Routing, gr.Source, failure.Cause, failure.Status, gr.Found, }); var indexResponse = await _client.LowLevel.IndexAsync <VoidResponse>(workItem.NewIndex + "-error", PostData.String(document)); if (indexResponse.Success) { _logger.LogRequest(indexResponse); } else { _logger.LogErrorRequest(indexResponse, "Error indexing document {Index}/{Id}", workItem.NewIndex + "-error", gr.Id); } }
private async Task HandleFailureAsync(ReindexWorkItem workItem, BulkIndexByScrollFailure failure) { _logger.LogError("Error reindexing document {Index}/{Type}/{Id}: [{Status}] {Message}", failure.Index, failure.Type, failure.Id, failure.Status, failure.Cause.Reason); var gr = await _client.GetAsync <object>(request : new GetRequest(failure.Index, failure.Type, failure.Id)).AnyContext(); if (!gr.IsValid) { _logger.LogError("Error getting document {Index}/{Type}/{Id}: {Message}", failure.Index, failure.Type, failure.Id, gr.GetErrorMessage()); return; } var document = new JObject(new { failure.Index, failure.Type, failure.Id, gr.Version, gr.Parent, gr.Source, failure.Cause, failure.Status, gr.Found, }); var indexResponse = await _client.IndexAsync(document, d => d.Index(workItem.NewIndex + "-error").Type("failures")).AnyContext(); if (!indexResponse.IsValid) { _logger.LogError("Error indexing document {Index}/{Type}/{Id}: {Message}", workItem.NewIndex + "-error", gr.Type, gr.Id, indexResponse.GetErrorMessage()); } }
private async Task <ReindexResult> InternalReindexAsync(ReindexWorkItem workItem, Func <int, string, Task> progressCallbackAsync, int startProgress = 0, int endProgress = 100, DateTime?startTime = null) { var query = await GetResumeQueryAsync(workItem.NewIndex, workItem.TimestampField, startTime).AnyContext(); var response = await _client.ReindexOnServerAsync(d => { d.Source(src => src .Index(workItem.OldIndex) .Query <object>(q => query) .Sort <object>(s => s.Ascending(new Field(workItem.TimestampField ?? ID_FIELD)))) .Destination(dest => dest.Index(workItem.NewIndex)) .Conflicts(Conflicts.Proceed); //NEST client emitting script if null, inline this when that's fixed if (!String.IsNullOrWhiteSpace(workItem.Script)) { d.Script(workItem.Script); } return(d); }).AnyContext(); if (_logger.IsEnabled(Microsoft.Extensions.Logging.LogLevel.Trace)) { _logger.LogInformation(response.GetRequest()); } long failures = 0; bool succeeded = true; if (!response.IsValid || response.ApiCall.HttpStatusCode != 200) { _logger.LogError(response.OriginalException, "Error while reindexing result: {Message}", response.GetErrorMessage()); if (await CreateFailureIndexAsync(workItem).AnyContext()) { foreach (var failure in response.Failures) { await HandleFailureAsync(workItem, failure).AnyContext(); failures++; } } succeeded = false; } long completed = response.Created + response.Updated + response.Noops; string message = $"Total: {response.Total:N0} Completed: {completed:N0} VersionConflicts: {response.VersionConflicts:N0}"; await progressCallbackAsync(CalculateProgress(response.Total, completed, startProgress, endProgress), message).AnyContext(); return(new ReindexResult { Total = response.Total, Completed = completed, Failures = failures, Succeeded = succeeded }); }
public ReindexWorkItem CreateReindexWorkItem(int currentVersion) { var reindexWorkItem = new ReindexWorkItem { OldIndex = String.Concat(Name, "-v", currentVersion), NewIndex = VersionedName, Alias = Name }; reindexWorkItem.DeleteOld = DiscardIndexesOnReindex && reindexWorkItem.OldIndex != reindexWorkItem.NewIndex; return(reindexWorkItem); }
public virtual Task ReindexAsync(Func <int, string, Task> progressCallbackAsync = null) { var reindexWorkItem = new ReindexWorkItem { OldIndex = Name, NewIndex = Name, DeleteOld = false }; var reindexer = new ElasticReindexer(Configuration.Client, Configuration.Cache, _logger); return(reindexer.ReindexAsync(reindexWorkItem, progressCallbackAsync)); }
public ReindexWorkItem CreateReindexWorkItem(int currentVersion) { var reindexWorkItem = new ReindexWorkItem { OldIndex = String.Concat(Name, "-v", currentVersion), NewIndex = VersionedName, Alias = Name, Script = GetReindexScripts(currentVersion), TimestampField = GetTimeStampField() }; reindexWorkItem.DeleteOld = DiscardIndexesOnReindex && reindexWorkItem.OldIndex != reindexWorkItem.NewIndex; return(reindexWorkItem); }
public override async Task ReindexAsync(Func <int, string, Task> progressCallbackAsync = null) { int currentVersion = await GetCurrentVersionAsync().AnyContext(); if (currentVersion < 0 || currentVersion >= Version) { return; } var indexes = await GetIndexesAsync(currentVersion).AnyContext(); if (indexes.Count == 0) { return; } var reindexer = new ElasticReindexer(Configuration.Client, _logger); foreach (var index in indexes) { if (SystemClock.UtcNow > GetIndexExpirationDate(index.DateUtc)) { continue; } if (index.CurrentVersion > Version) { continue; } var reindexWorkItem = new ReindexWorkItem { OldIndex = index.Index, NewIndex = GetVersionedIndex(GetIndexDate(index.Index), Version), Alias = Name, TimestampField = GetTimeStampField() }; reindexWorkItem.DeleteOld = DiscardIndexesOnReindex && reindexWorkItem.OldIndex != reindexWorkItem.NewIndex; // attempt to create the index. If it exists the index will not be created. await CreateIndexAsync(reindexWorkItem.NewIndex, ConfigureIndex).AnyContext(); // TODO: progress callback will report 0-100% multiple times... await reindexer.ReindexAsync(reindexWorkItem, progressCallbackAsync).AnyContext(); } }
private async Task <bool> CreateFailureIndexAsync(ReindexWorkItem workItem) { string errorIndex = workItem.NewIndex + "-error"; var existsResponse = await _client.IndexExistsAsync(errorIndex).AnyContext(); if (!existsResponse.IsValid || existsResponse.Exists) { return(true); } var createResponse = await _client.CreateIndexAsync(errorIndex, d => d.Mappings(m => m.Map("failures", md => md.Dynamic(false)))).AnyContext(); if (!createResponse.IsValid) { _logger.LogError(createResponse.OriginalException, "Unable to create error index: {Message}", createResponse.GetErrorMessage()); return(false); } return(true); }
private async Task <bool> CreateFailureIndexAsync(ReindexWorkItem workItem) { string errorIndex = workItem.NewIndex + "-error"; var existsResponse = await _client.IndexExistsAsync(errorIndex).AnyContext(); _logger.LogTraceRequest(existsResponse); if (!existsResponse.IsValid || existsResponse.Exists) { return(true); } var createResponse = await _client.CreateIndexAsync(errorIndex, d => d.Mappings(m => m.Map("failures", md => md.Dynamic(false)))).AnyContext(); if (!createResponse.IsValid) { _logger.LogErrorRequest(createResponse, "Unable to create error index"); return(false); } _logger.LogTraceRequest(createResponse); return(true); }
private async Task <bool> CreateFailureIndexAsync(ReindexWorkItem workItem) { string errorIndex = workItem.NewIndex + "-error"; var existsResponse = await _client.Indices.ExistsAsync(errorIndex).AnyContext(); _logger.LogRequest(existsResponse); if (existsResponse.ApiCall.Success && existsResponse.Exists) { return(true); } var createResponse = await _client.Indices.CreateAsync(errorIndex, d => d.Map(md => md.Dynamic(false))).AnyContext(); if (!createResponse.IsValid) { _logger.LogErrorRequest(createResponse, "Unable to create error index"); return(false); } _logger.LogRequest(createResponse); return(true); }
public async Task ReindexAsync(ReindexWorkItem workItem, Func <int, string, Task> progressCallbackAsync = null) { if (String.IsNullOrEmpty(workItem.OldIndex)) { throw new ArgumentNullException(nameof(workItem.OldIndex)); } if (String.IsNullOrEmpty(workItem.NewIndex)) { throw new ArgumentNullException(nameof(workItem.NewIndex)); } if (progressCallbackAsync == null) { progressCallbackAsync = (progress, message) => { _logger.LogInformation("Reindex Progress {Progress:F1}%: {Message}", progress, message); return(Task.CompletedTask); }; } _logger.LogInformation("Received reindex work item for new index: {NewIndex}", workItem.NewIndex); var startTime = SystemClock.UtcNow.AddSeconds(-1); await progressCallbackAsync(0, "Starting reindex...").AnyContext(); var firstPassResult = await InternalReindexAsync(workItem, progressCallbackAsync, 0, 90, workItem.StartUtc).AnyContext(); if (!firstPassResult.Succeeded) { return; } await progressCallbackAsync(91, $"Total: {firstPassResult.Total:N0} Completed: {firstPassResult.Completed:N0}").AnyContext(); // TODO: Check to make sure the docs have been added to the new index before changing alias if (workItem.OldIndex != workItem.NewIndex) { var aliases = await GetIndexAliasesAsync(workItem.OldIndex).AnyContext(); if (!String.IsNullOrEmpty(workItem.Alias) && !aliases.Contains(workItem.Alias)) { aliases.Add(workItem.Alias); } if (aliases.Count > 0) { var bulkResponse = await _client.AliasAsync(x => { foreach (string alias in aliases) { x = x.Remove(a => a.Alias(alias).Index(workItem.OldIndex)).Add(a => a.Alias(alias).Index(workItem.NewIndex)); } return(x); }).AnyContext(); if (_logger.IsEnabled(Microsoft.Extensions.Logging.LogLevel.Trace)) { _logger.LogTrace(bulkResponse.GetRequest()); } await progressCallbackAsync(92, $"Updated aliases: {String.Join(", ", aliases)} Remove: {workItem.OldIndex} Add: {workItem.NewIndex}").AnyContext(); } } await _client.RefreshAsync(Indices.All).AnyContext(); var secondPassResult = await InternalReindexAsync(workItem, progressCallbackAsync, 92, 96, startTime).AnyContext(); if (!secondPassResult.Succeeded) { return; } await progressCallbackAsync(97, $"Total: {secondPassResult.Total:N0} Completed: {secondPassResult.Completed:N0}").AnyContext(); bool hasFailures = (firstPassResult.Failures + secondPassResult.Failures) > 0; if (!hasFailures && workItem.DeleteOld && workItem.OldIndex != workItem.NewIndex) { await _client.RefreshAsync(Indices.All).AnyContext(); long newDocCount = (await _client.CountAsync <object>(d => d.Index(workItem.NewIndex)).AnyContext()).Count; long oldDocCount = (await _client.CountAsync <object>(d => d.Index(workItem.OldIndex)).AnyContext()).Count; await progressCallbackAsync(98, $"Old Docs: {oldDocCount} New Docs: {newDocCount}").AnyContext(); if (newDocCount >= oldDocCount) { await _client.DeleteIndexAsync(Indices.Index(workItem.OldIndex)).AnyContext(); await progressCallbackAsync(99, $"Deleted index: {workItem.OldIndex}").AnyContext(); } } await progressCallbackAsync(100, null).AnyContext(); }
public async Task ReindexAsync(ReindexWorkItem workItem, Func <int, string, Task> progressCallbackAsync = null) { if (progressCallbackAsync == null) { progressCallbackAsync = (progress, message) => { _logger.Info("Reindex Progress {0}%: {1}", progress, message); return(Task.CompletedTask); }; } _logger.Info("Received reindex work item for new index {0}", workItem.NewIndex); var startTime = SystemClock.UtcNow.AddSeconds(-1); await progressCallbackAsync(0, "Starting reindex...").AnyContext(); var result = await InternalReindexAsync(workItem, progressCallbackAsync, 0, 90, workItem.StartUtc).AnyContext(); await progressCallbackAsync(95, $"Total: {result.Total} Completed: {result.Completed}").AnyContext(); // TODO: Check to make sure the docs have been added to the new index before changing alias if (workItem.OldIndex != workItem.NewIndex) { var aliases = await GetIndexAliases(workItem.OldIndex).AnyContext(); if (!String.IsNullOrEmpty(workItem.Alias) && !aliases.Contains(workItem.Alias)) { aliases.Add(workItem.Alias); } if (aliases.Count > 0) { await _client.AliasAsync(x => { foreach (var alias in aliases) { x = x.Remove(a => a.Alias(alias).Index(workItem.OldIndex)).Add(a => a.Alias(alias).Index(workItem.NewIndex)); } return(x); }).AnyContext(); await progressCallbackAsync(98, $"Updated aliases: {String.Join(", ", aliases)} Remove: {workItem.OldIndex} Add: {workItem.NewIndex}").AnyContext(); } } await _client.RefreshAsync().AnyContext(); var secondPassResult = await InternalReindexAsync(workItem, progressCallbackAsync, 90, 98, startTime).AnyContext(); await progressCallbackAsync(98, $"Total: {secondPassResult.Total} Completed: {secondPassResult.Completed}").AnyContext(); if (workItem.DeleteOld && workItem.OldIndex != workItem.NewIndex) { await _client.RefreshAsync().AnyContext(); long newDocCount = (await _client.CountAsync(d => d.Index(workItem.NewIndex)).AnyContext()).Count; long oldDocCount = (await _client.CountAsync(d => d.Index(workItem.OldIndex)).AnyContext()).Count; await progressCallbackAsync(98, $"Old Docs: {oldDocCount} New Docs: {newDocCount}").AnyContext(); if (newDocCount >= oldDocCount) { await _client.DeleteIndexAsync(d => d.Index(workItem.OldIndex)).AnyContext(); await progressCallbackAsync(99, $"Deleted index: {workItem.OldIndex}").AnyContext(); } } await progressCallbackAsync(100, null).AnyContext(); }
private async Task <ReindexResult> InternalReindexAsync(ReindexWorkItem workItem, Func <int, string, Task> progressCallbackAsync, int startProgress = 0, int endProgress = 100, DateTime?startTime = null) { const string scroll = "5m"; bool errorIndexCreated = false; string timestampField = workItem.TimestampField ?? "_timestamp"; var scopedCacheClient = new ScopedCacheClient(_cache, workItem.GetHashCode().ToString()); var settingsResponse = await _client.GetIndexSettingsAsync(s => s.Index(workItem.OldIndex)).AnyContext(); if (!settingsResponse.IsValid) { throw new ApplicationException("Unable to retrieve index settings."); } int scrollSize = 500 / settingsResponse.IndexSettings.NumberOfShards ?? 50; var scanResults = await _client.SearchAsync <object>(s => s .Index(workItem.OldIndex) .AllTypes() .Query(q => q.Filtered(f => { if (startTime.HasValue) { f.Filter(f1 => f1.Range(r => r.OnField(timestampField).Greater(startTime.Value))); } })) .Fields("_source", "_parent") .Size(scrollSize) .SearchType(SearchType.Scan) .Scroll(scroll)).AnyContext(); _logger.Info(scanResults.GetRequest()); if (!scanResults.IsValid || scanResults.ScrollId == null) { _logger.Error().Exception(scanResults.ConnectionStatus.OriginalException).Message("Invalid search result: message={0}", scanResults.GetErrorMessage()).Write(); return(new ReindexResult()); } var results = await _client.ScrollAsync <JObject>("5m", scanResults.ScrollId).AnyContext(); if (!results.IsValid) { await scopedCacheClient.RemoveAsync("id").AnyContext(); return(await InternalReindexAsync(workItem, progressCallbackAsync, startProgress, endProgress, startTime).AnyContext()); } double completed = 0; long totalHits = results.Total; while (results.Hits.Any()) { ISearchResponse <JObject> results1 = results; IBulkResponse bulkResponse = null; try { bulkResponse = await Run.WithRetriesAsync(() => _client.BulkAsync(b => { foreach (var h in results1.Hits) { ConfigureIndexItem(b, h, workItem.NewIndex); } return(b); }), logger : _logger, maxAttempts : 2).AnyContext(); } catch (Exception ex) { _logger.Error(ex, $"Error trying to do bulk index: {ex.Message}"); } if (bulkResponse == null || !bulkResponse.IsValid || bulkResponse.ItemsWithErrors.Any()) { string message; if (bulkResponse != null) { message = $"Reindex bulk error: old={workItem.OldIndex} new={workItem.NewIndex} completed={completed} message={bulkResponse?.GetErrorMessage()}"; _logger.Warn(bulkResponse.ConnectionStatus.OriginalException, message); } // try each doc individually so we can see which doc is breaking us var hitsToRetry = bulkResponse?.ItemsWithErrors.Select(i => results1.Hits.First(hit => hit.Id == i.Id)) ?? results1.Hits; foreach (var itemToRetry in hitsToRetry) { IIndexResponse response; try { response = await _client.IndexAsync(itemToRetry.Source, d => ConfigureItem(d, itemToRetry, workItem.NewIndex)).AnyContext(); if (response.IsValid) { continue; } message = $"Reindex error: old={workItem.OldIndex} new={workItem.NewIndex} id={itemToRetry.Id} completed={completed} message={response.GetErrorMessage()}"; _logger.Error().Exception(response.ConnectionStatus.OriginalException).Message(message); var errorDoc = new JObject { ["Type"] = itemToRetry.Type, ["Content"] = itemToRetry.Source.ToString(Formatting.Indented) }; var errorIndex = workItem.NewIndex + "-error"; if (!errorIndexCreated && !(await _client.IndexExistsAsync(errorIndex).AnyContext()).Exists) { await _client.CreateIndexAsync(errorIndex).AnyContext(); errorIndexCreated = true; } // put the document into an error index response = await _client.IndexAsync(errorDoc, d => d.Index(errorIndex).Id(itemToRetry.Id)).AnyContext(); if (response.IsValid) { continue; } } catch { throw; } message = $"Reindex error: old={workItem.OldIndex} new={workItem.NewIndex} id={itemToRetry.Id} completed={completed} message={response.GetErrorMessage()}"; _logger.Error().Exception(response.ConnectionStatus.OriginalException).Message(message); throw new ReindexException(response.ConnectionStatus, message); } } completed += results.Hits.Count(); await progressCallbackAsync(CalculateProgress(totalHits, (long)completed, startProgress, endProgress), $"Total: {totalHits} Completed: {completed}").AnyContext(); results = await _client.ScrollAsync <JObject>("5m", results.ScrollId).AnyContext(); await scopedCacheClient.AddAsync("id", results.ScrollId, TimeSpan.FromHours(1)).AnyContext(); } await scopedCacheClient.RemoveAllAsync(new[] { "id" }).AnyContext(); return(new ReindexResult { Total = totalHits, Completed = (long)completed }); }
public virtual void ConfigureIndexes(IElasticClient client, IEnumerable <IElasticIndex> indexes = null) { if (indexes == null) { indexes = GetIndexes(); } foreach (var idx in indexes) { int currentVersion = GetAliasVersion(client, idx.AliasName); IIndicesOperationResponse response = null; var templatedIndex = idx as ITemplatedElasticIndex; if (templatedIndex != null) { response = client.PutTemplate(idx.VersionedName, template => templatedIndex.CreateTemplate(template).AddAlias(idx.AliasName)); } else if (!client.IndexExists(idx.VersionedName).Exists) { response = client.CreateIndex(idx.VersionedName, descriptor => idx.CreateIndex(descriptor).AddAlias(idx.AliasName)); } Debug.Assert(response == null || response.IsValid, response?.ServerError != null ? response.ServerError.Error : "An error occurred creating the index or template."); // Add existing indexes to the alias. if (!client.AliasExists(idx.AliasName).Exists) { if (templatedIndex != null) { var indices = client.IndicesStats().Indices.Where(kvp => kvp.Key.StartsWith(idx.VersionedName)).Select(kvp => kvp.Key).ToList(); if (indices.Count > 0) { var descriptor = new AliasDescriptor(); foreach (string name in indices) { descriptor.Add(add => add.Index(name).Alias(idx.AliasName)); } response = client.Alias(descriptor); } } else { response = client.Alias(a => a.Add(add => add.Index(idx.VersionedName).Alias(idx.AliasName))); } Debug.Assert(response != null && response.IsValid, response?.ServerError != null ? response.ServerError.Error : "An error occurred creating the alias."); } // already on current version if (currentVersion >= idx.Version || currentVersion < 1) { continue; } var reindexWorkItem = new ReindexWorkItem { OldIndex = String.Concat(idx.AliasName, "-v", currentVersion), NewIndex = idx.VersionedName, Alias = idx.AliasName, DeleteOld = true, ParentMaps = idx.GetIndexTypes() .Select(kvp => new ParentMap { Type = kvp.Value.Name, ParentPath = kvp.Value.ParentPath }) .Where(m => !String.IsNullOrEmpty(m.ParentPath)) .ToList() }; bool isReindexing = _lockProvider.IsLockedAsync(String.Concat("reindex:", reindexWorkItem.Alias, reindexWorkItem.OldIndex, reindexWorkItem.NewIndex)).Result; // already reindexing if (isReindexing) { continue; } // enqueue reindex to new version _lockProvider.TryUsingAsync("enqueue-reindex", () => _workItemQueue.EnqueueAsync(reindexWorkItem), TimeSpan.Zero, CancellationToken.None).Wait(); } }
private async Task <ReindexResult> ReindexAsync(ReindexWorkItem workItem, WorkItemContext context, int startProgress = 0, int endProgress = 100, DateTime?startTime = null) { const int pageSize = 100; const string scroll = "10s"; var scanResults = await _client.SearchAsync <JObject>(s => s.Index(workItem.OldIndex).AllTypes().Filter(f => startTime.HasValue ? f.Range(r => r.OnField(workItem.TimestampField ?? "_timestamp").Greater(startTime.Value)) : f.MatchAll()).From(0).Take(pageSize).SearchType(SearchType.Scan).Scroll(scroll)).AnyContext(); if (!scanResults.IsValid || scanResults.ScrollId == null) { Logger.Error().Message("Invalid search result: message={0}", scanResults.GetErrorMessage()).Write(); return(new ReindexResult()); } long totalHits = scanResults.Total; long completed = 0; int page = 0; var results = await _client.ScrollAsync <JObject>(scroll, scanResults.ScrollId).AnyContext(); while (results.Documents.Any()) { var bulkDescriptor = new BulkDescriptor(); foreach (var hit in results.Hits) { var h = hit; // TODO: Add support for doing JObject based schema migrations bulkDescriptor.Index <JObject>(idx => idx.Index(workItem.NewIndex).Type(h.Type).Id(h.Id).Document(h.Source)); } var bulkResponse = await _client.BulkAsync(bulkDescriptor).AnyContext(); if (!bulkResponse.IsValid) { string message = $"Reindex bulk error: old={workItem.OldIndex} new={workItem.NewIndex} page={page} message={bulkResponse.GetErrorMessage()}"; Logger.Warn().Message(message).Write(); // try each doc individually so we can see which doc is breaking us foreach (var hit in results.Hits) { var h = hit; var response = await _client.IndexAsync(h.Source, d => d.Index(workItem.NewIndex).Type(h.Type).Id(h.Id)).AnyContext(); if (response.IsValid) { continue; } message = $"Reindex error: old={workItem.OldIndex} new={workItem.NewIndex} id={hit.Id} page={page} message={response.GetErrorMessage()}"; Logger.Error().Message(message).Write(); throw new ReindexException(response.ConnectionStatus, message); } } completed += bulkResponse.Items.Count(); await context.ReportProgressAsync(CalculateProgress(totalHits, completed, startProgress, endProgress), $"Total: {totalHits} Completed: {completed}").AnyContext(); results = await _client.ScrollAsync <JObject>(scroll, results.ScrollId).AnyContext(); page++; } return(new ReindexResult { Total = totalHits, Completed = completed }); }
private async Task <ReindexResult> InternalReindexAsync(ReindexWorkItem workItem, Func <int, string, Task> progressCallbackAsync, int startProgress = 0, int endProgress = 100, DateTime?startTime = null, CancellationToken cancellationToken = default) { var query = await GetResumeQueryAsync(workItem.NewIndex, workItem.TimestampField, startTime).AnyContext(); var result = await Run.WithRetriesAsync(async() => { var response = await _client.ReindexOnServerAsync(d => { d.Source(src => src .Index(workItem.OldIndex) .Query <object>(q => query) .Sort <object>(s => s.Ascending(new Field(workItem.TimestampField ?? ID_FIELD)))) .Destination(dest => dest.Index(workItem.NewIndex)) .Conflicts(Conflicts.Proceed) .WaitForCompletion(false); //NEST client emitting script if null, inline this when that's fixed if (!String.IsNullOrWhiteSpace(workItem.Script)) { d.Script(workItem.Script); } return(d); }).AnyContext(); return(response); }, 5, TimeSpan.FromSeconds(10), cancellationToken, _logger).AnyContext(); _logger.LogInformation("Reindex Task Id: {TaskId}", result.Task.FullyQualifiedId); _logger.LogTraceRequest(result); bool taskSuccess = false; TaskReindexResult lastReindexResponse = null; int statusGetFails = 0; long lastProgress = 0; var sw = Stopwatch.StartNew(); do { await Task.Delay(TimeSpan.FromSeconds(10), cancellationToken).AnyContext(); var status = await _client.GetTaskAsync(result.Task, null, cancellationToken).AnyContext(); if (status.IsValid) { _logger.LogTraceRequest(status); } else { _logger.LogErrorRequest(status, "Error getting task status while reindexing: {OldIndex} -> {NewIndex}", workItem.OldIndex, workItem.NewIndex); statusGetFails++; if (statusGetFails > MAX_STATUS_FAILS) { _logger.LogError("Failed to get the status {FailureCount} times in a row", MAX_STATUS_FAILS); break; } continue; } statusGetFails = 0; var response = status.DeserializeRaw <TaskWithReindexResponse>(); if (response?.Error != null) { _logger.LogError("Error reindex: {Type}, {Reason}, Cause: {CausedBy} Stack: {Stack}", response.Error.Type, response.Error.Reason, response.Error.Caused_By?.Reason, String.Join("\r\n", response.Error.Script_Stack ?? new List <string>())); break; } lastReindexResponse = response?.Response; long lastCompleted = status.Task.Status.Created + status.Task.Status.Updated + status.Task.Status.Noops; // restart the stop watch if there was progress made if (lastCompleted > lastProgress) { sw.Restart(); } lastProgress = lastCompleted; string lastMessage = $"Total: {status.Task.Status.Total:N0} Completed: {lastCompleted:N0} VersionConflicts: {status.Task.Status.VersionConflicts:N0}"; await progressCallbackAsync(CalculateProgress(status.Task.Status.Total, lastCompleted, startProgress, endProgress), lastMessage).AnyContext(); if (status.Completed && response?.Error == null) { taskSuccess = true; break; } if (sw.Elapsed > TimeSpan.FromMinutes(10)) { _logger.LogError($"Timed out waiting for reindex {workItem.OldIndex} -> {workItem.NewIndex}."); break; } } while (!cancellationToken.IsCancellationRequested); sw.Stop(); long failures = 0; if (lastReindexResponse?.Failures != null && lastReindexResponse.Failures.Count > 0) { _logger.LogError("Error while reindexing result"); if (await CreateFailureIndexAsync(workItem).AnyContext()) { foreach (var failure in lastReindexResponse.Failures) { await HandleFailureAsync(workItem, failure).AnyContext(); failures++; } } taskSuccess = false; } long total = lastReindexResponse?.Total ?? 0; long versionConflicts = lastReindexResponse?.VersionConflicts ?? 0; long completed = (lastReindexResponse?.Created ?? 0) + (lastReindexResponse?.Updated ?? 0) + (lastReindexResponse?.Noops ?? 0); string message = $"Total: {total:N0} Completed: {completed:N0} VersionConflicts: {versionConflicts:N0}"; await progressCallbackAsync(CalculateProgress(total, completed, startProgress, endProgress), message).AnyContext(); return(new ReindexResult { Total = total, Completed = completed, Failures = failures, Succeeded = taskSuccess }); }