private async Task HandleFailureAsync(ReindexWorkItem workItem, BulkIndexByScrollFailure failure)
{
_logger.LogError("Error reindexing document {Index}/{Id}: [{Status}] {Message}", workItem.OldIndex, failure.Id, failure.Status, failure.Cause.Reason);
var gr = await _client.GetAsync <object>(request : new GetRequest(workItem.OldIndex, failure.Id)).AnyContext();
if (!gr.IsValid)
{
_logger.LogErrorRequest(gr, "Error getting document {Index}/{Id}", workItem.OldIndex, failure.Id);
return;
}
_logger.LogRequest(gr);
string document = JsonConvert.SerializeObject(new {
failure.Index,
failure.Id,
gr.Version,
gr.Routing,
gr.Source,
failure.Cause,
failure.Status,
gr.Found,
});
var indexResponse = await _client.LowLevel.IndexAsync <VoidResponse>(workItem.NewIndex + "-error", PostData.String(document));
if (indexResponse.Success)
{
_logger.LogRequest(indexResponse);
}
else
{
_logger.LogErrorRequest(indexResponse, "Error indexing document {Index}/{Id}", workItem.NewIndex + "-error", gr.Id);
}
}
private async Task HandleFailureAsync(ReindexWorkItem workItem, BulkIndexByScrollFailure failure)
{
_logger.LogError("Error reindexing document {Index}/{Type}/{Id}: [{Status}] {Message}", failure.Index, failure.Type, failure.Id, failure.Status, failure.Cause.Reason);
var gr = await _client.GetAsync <object>(request : new GetRequest(failure.Index, failure.Type, failure.Id)).AnyContext();
if (!gr.IsValid)
{
_logger.LogError("Error getting document {Index}/{Type}/{Id}: {Message}", failure.Index, failure.Type, failure.Id, gr.GetErrorMessage());
return;
}
var document = new JObject(new {
failure.Index,
failure.Type,
failure.Id,
gr.Version,
gr.Parent,
gr.Source,
failure.Cause,
failure.Status,
gr.Found,
});
var indexResponse = await _client.IndexAsync(document, d => d.Index(workItem.NewIndex + "-error").Type("failures")).AnyContext();
if (!indexResponse.IsValid)
{
_logger.LogError("Error indexing document {Index}/{Type}/{Id}: {Message}", workItem.NewIndex + "-error", gr.Type, gr.Id, indexResponse.GetErrorMessage());
}
}
private async Task <ReindexResult> InternalReindexAsync(ReindexWorkItem workItem, Func <int, string, Task> progressCallbackAsync, int startProgress = 0, int endProgress = 100, DateTime?startTime = null)
{
var query = await GetResumeQueryAsync(workItem.NewIndex, workItem.TimestampField, startTime).AnyContext();
var response = await _client.ReindexOnServerAsync(d => {
d.Source(src => src
.Index(workItem.OldIndex)
.Query <object>(q => query)
.Sort <object>(s => s.Ascending(new Field(workItem.TimestampField ?? ID_FIELD))))
.Destination(dest => dest.Index(workItem.NewIndex))
.Conflicts(Conflicts.Proceed);
//NEST client emitting script if null, inline this when that's fixed
if (!String.IsNullOrWhiteSpace(workItem.Script))
{
d.Script(workItem.Script);
}
return(d);
}).AnyContext();
if (_logger.IsEnabled(Microsoft.Extensions.Logging.LogLevel.Trace))
{
_logger.LogInformation(response.GetRequest());
}
long failures = 0;
bool succeeded = true;
if (!response.IsValid || response.ApiCall.HttpStatusCode != 200)
{
_logger.LogError(response.OriginalException, "Error while reindexing result: {Message}", response.GetErrorMessage());
if (await CreateFailureIndexAsync(workItem).AnyContext())
{
foreach (var failure in response.Failures)
{
await HandleFailureAsync(workItem, failure).AnyContext();
failures++;
}
}
succeeded = false;
}
long completed = response.Created + response.Updated + response.Noops;
string message = $"Total: {response.Total:N0} Completed: {completed:N0} VersionConflicts: {response.VersionConflicts:N0}";
await progressCallbackAsync(CalculateProgress(response.Total, completed, startProgress, endProgress), message).AnyContext();
return(new ReindexResult {
Total = response.Total, Completed = completed, Failures = failures, Succeeded = succeeded
});
}
public ReindexWorkItem CreateReindexWorkItem(int currentVersion)
{
var reindexWorkItem = new ReindexWorkItem {
OldIndex = String.Concat(Name, "-v", currentVersion),
NewIndex = VersionedName,
Alias = Name
};
reindexWorkItem.DeleteOld = DiscardIndexesOnReindex && reindexWorkItem.OldIndex != reindexWorkItem.NewIndex;
return(reindexWorkItem);
}
public virtual Task ReindexAsync(Func <int, string, Task> progressCallbackAsync = null)
{
var reindexWorkItem = new ReindexWorkItem {
OldIndex = Name,
NewIndex = Name,
DeleteOld = false
};
var reindexer = new ElasticReindexer(Configuration.Client, Configuration.Cache, _logger);
return(reindexer.ReindexAsync(reindexWorkItem, progressCallbackAsync));
}
public ReindexWorkItem CreateReindexWorkItem(int currentVersion)
{
var reindexWorkItem = new ReindexWorkItem {
OldIndex = String.Concat(Name, "-v", currentVersion),
NewIndex = VersionedName,
Alias = Name,
Script = GetReindexScripts(currentVersion),
TimestampField = GetTimeStampField()
};
reindexWorkItem.DeleteOld = DiscardIndexesOnReindex && reindexWorkItem.OldIndex != reindexWorkItem.NewIndex;
return(reindexWorkItem);
}
public override async Task ReindexAsync(Func <int, string, Task> progressCallbackAsync = null)
{
int currentVersion = await GetCurrentVersionAsync().AnyContext();
if (currentVersion < 0 || currentVersion >= Version)
{
return;
}
var indexes = await GetIndexesAsync(currentVersion).AnyContext();
if (indexes.Count == 0)
{
return;
}
var reindexer = new ElasticReindexer(Configuration.Client, _logger);
foreach (var index in indexes)
{
if (SystemClock.UtcNow > GetIndexExpirationDate(index.DateUtc))
{
continue;
}
if (index.CurrentVersion > Version)
{
continue;
}
var reindexWorkItem = new ReindexWorkItem {
OldIndex = index.Index,
NewIndex = GetVersionedIndex(GetIndexDate(index.Index), Version),
Alias = Name,
TimestampField = GetTimeStampField()
};
reindexWorkItem.DeleteOld = DiscardIndexesOnReindex && reindexWorkItem.OldIndex != reindexWorkItem.NewIndex;
// attempt to create the index. If it exists the index will not be created.
await CreateIndexAsync(reindexWorkItem.NewIndex, ConfigureIndex).AnyContext();
// TODO: progress callback will report 0-100% multiple times...
await reindexer.ReindexAsync(reindexWorkItem, progressCallbackAsync).AnyContext();
}
}
private async Task <bool> CreateFailureIndexAsync(ReindexWorkItem workItem)
{
string errorIndex = workItem.NewIndex + "-error";
var existsResponse = await _client.IndexExistsAsync(errorIndex).AnyContext();
if (!existsResponse.IsValid || existsResponse.Exists)
{
return(true);
}
var createResponse = await _client.CreateIndexAsync(errorIndex, d => d.Mappings(m => m.Map("failures", md => md.Dynamic(false)))).AnyContext();
if (!createResponse.IsValid)
{
_logger.LogError(createResponse.OriginalException, "Unable to create error index: {Message}", createResponse.GetErrorMessage());
return(false);
}
return(true);
}
private async Task <bool> CreateFailureIndexAsync(ReindexWorkItem workItem)
{
string errorIndex = workItem.NewIndex + "-error";
var existsResponse = await _client.IndexExistsAsync(errorIndex).AnyContext();
_logger.LogTraceRequest(existsResponse);
if (!existsResponse.IsValid || existsResponse.Exists)
{
return(true);
}
var createResponse = await _client.CreateIndexAsync(errorIndex, d => d.Mappings(m => m.Map("failures", md => md.Dynamic(false)))).AnyContext();
if (!createResponse.IsValid)
{
_logger.LogErrorRequest(createResponse, "Unable to create error index");
return(false);
}
_logger.LogTraceRequest(createResponse);
return(true);
}
private async Task <bool> CreateFailureIndexAsync(ReindexWorkItem workItem)
{
string errorIndex = workItem.NewIndex + "-error";
var existsResponse = await _client.Indices.ExistsAsync(errorIndex).AnyContext();
_logger.LogRequest(existsResponse);
if (existsResponse.ApiCall.Success && existsResponse.Exists)
{
return(true);
}
var createResponse = await _client.Indices.CreateAsync(errorIndex, d => d.Map(md => md.Dynamic(false))).AnyContext();
if (!createResponse.IsValid)
{
_logger.LogErrorRequest(createResponse, "Unable to create error index");
return(false);
}
_logger.LogRequest(createResponse);
return(true);
}
public async Task ReindexAsync(ReindexWorkItem workItem, Func <int, string, Task> progressCallbackAsync = null)
{
if (String.IsNullOrEmpty(workItem.OldIndex))
{
throw new ArgumentNullException(nameof(workItem.OldIndex));
}
if (String.IsNullOrEmpty(workItem.NewIndex))
{
throw new ArgumentNullException(nameof(workItem.NewIndex));
}
if (progressCallbackAsync == null)
{
progressCallbackAsync = (progress, message) => {
_logger.LogInformation("Reindex Progress {Progress:F1}%: {Message}", progress, message);
return(Task.CompletedTask);
};
}
_logger.LogInformation("Received reindex work item for new index: {NewIndex}", workItem.NewIndex);
var startTime = SystemClock.UtcNow.AddSeconds(-1);
await progressCallbackAsync(0, "Starting reindex...").AnyContext();
var firstPassResult = await InternalReindexAsync(workItem, progressCallbackAsync, 0, 90, workItem.StartUtc).AnyContext();
if (!firstPassResult.Succeeded)
{
return;
}
await progressCallbackAsync(91, $"Total: {firstPassResult.Total:N0} Completed: {firstPassResult.Completed:N0}").AnyContext();
// TODO: Check to make sure the docs have been added to the new index before changing alias
if (workItem.OldIndex != workItem.NewIndex)
{
var aliases = await GetIndexAliasesAsync(workItem.OldIndex).AnyContext();
if (!String.IsNullOrEmpty(workItem.Alias) && !aliases.Contains(workItem.Alias))
{
aliases.Add(workItem.Alias);
}
if (aliases.Count > 0)
{
var bulkResponse = await _client.AliasAsync(x => {
foreach (string alias in aliases)
{
x = x.Remove(a => a.Alias(alias).Index(workItem.OldIndex)).Add(a => a.Alias(alias).Index(workItem.NewIndex));
}
return(x);
}).AnyContext();
if (_logger.IsEnabled(Microsoft.Extensions.Logging.LogLevel.Trace))
{
_logger.LogTrace(bulkResponse.GetRequest());
}
await progressCallbackAsync(92, $"Updated aliases: {String.Join(", ", aliases)} Remove: {workItem.OldIndex} Add: {workItem.NewIndex}").AnyContext();
}
}
await _client.RefreshAsync(Indices.All).AnyContext();
var secondPassResult = await InternalReindexAsync(workItem, progressCallbackAsync, 92, 96, startTime).AnyContext();
if (!secondPassResult.Succeeded)
{
return;
}
await progressCallbackAsync(97, $"Total: {secondPassResult.Total:N0} Completed: {secondPassResult.Completed:N0}").AnyContext();
bool hasFailures = (firstPassResult.Failures + secondPassResult.Failures) > 0;
if (!hasFailures && workItem.DeleteOld && workItem.OldIndex != workItem.NewIndex)
{
await _client.RefreshAsync(Indices.All).AnyContext();
long newDocCount = (await _client.CountAsync <object>(d => d.Index(workItem.NewIndex)).AnyContext()).Count;
long oldDocCount = (await _client.CountAsync <object>(d => d.Index(workItem.OldIndex)).AnyContext()).Count;
await progressCallbackAsync(98, $"Old Docs: {oldDocCount} New Docs: {newDocCount}").AnyContext();
if (newDocCount >= oldDocCount)
{
await _client.DeleteIndexAsync(Indices.Index(workItem.OldIndex)).AnyContext();
await progressCallbackAsync(99, $"Deleted index: {workItem.OldIndex}").AnyContext();
}
}
await progressCallbackAsync(100, null).AnyContext();
}
public async Task ReindexAsync(ReindexWorkItem workItem, Func <int, string, Task> progressCallbackAsync = null)
{
if (progressCallbackAsync == null)
{
progressCallbackAsync = (progress, message) => {
_logger.Info("Reindex Progress {0}%: {1}", progress, message);
return(Task.CompletedTask);
};
}
_logger.Info("Received reindex work item for new index {0}", workItem.NewIndex);
var startTime = SystemClock.UtcNow.AddSeconds(-1);
await progressCallbackAsync(0, "Starting reindex...").AnyContext();
var result = await InternalReindexAsync(workItem, progressCallbackAsync, 0, 90, workItem.StartUtc).AnyContext();
await progressCallbackAsync(95, $"Total: {result.Total} Completed: {result.Completed}").AnyContext();
// TODO: Check to make sure the docs have been added to the new index before changing alias
if (workItem.OldIndex != workItem.NewIndex)
{
var aliases = await GetIndexAliases(workItem.OldIndex).AnyContext();
if (!String.IsNullOrEmpty(workItem.Alias) && !aliases.Contains(workItem.Alias))
{
aliases.Add(workItem.Alias);
}
if (aliases.Count > 0)
{
await _client.AliasAsync(x => {
foreach (var alias in aliases)
{
x = x.Remove(a => a.Alias(alias).Index(workItem.OldIndex)).Add(a => a.Alias(alias).Index(workItem.NewIndex));
}
return(x);
}).AnyContext();
await progressCallbackAsync(98, $"Updated aliases: {String.Join(", ", aliases)} Remove: {workItem.OldIndex} Add: {workItem.NewIndex}").AnyContext();
}
}
await _client.RefreshAsync().AnyContext();
var secondPassResult = await InternalReindexAsync(workItem, progressCallbackAsync, 90, 98, startTime).AnyContext();
await progressCallbackAsync(98, $"Total: {secondPassResult.Total} Completed: {secondPassResult.Completed}").AnyContext();
if (workItem.DeleteOld && workItem.OldIndex != workItem.NewIndex)
{
await _client.RefreshAsync().AnyContext();
long newDocCount = (await _client.CountAsync(d => d.Index(workItem.NewIndex)).AnyContext()).Count;
long oldDocCount = (await _client.CountAsync(d => d.Index(workItem.OldIndex)).AnyContext()).Count;
await progressCallbackAsync(98, $"Old Docs: {oldDocCount} New Docs: {newDocCount}").AnyContext();
if (newDocCount >= oldDocCount)
{
await _client.DeleteIndexAsync(d => d.Index(workItem.OldIndex)).AnyContext();
await progressCallbackAsync(99, $"Deleted index: {workItem.OldIndex}").AnyContext();
}
}
await progressCallbackAsync(100, null).AnyContext();
}
private async Task <ReindexResult> InternalReindexAsync(ReindexWorkItem workItem, Func <int, string, Task> progressCallbackAsync, int startProgress = 0, int endProgress = 100, DateTime?startTime = null)
{
const string scroll = "5m";
bool errorIndexCreated = false;
string timestampField = workItem.TimestampField ?? "_timestamp";
var scopedCacheClient = new ScopedCacheClient(_cache, workItem.GetHashCode().ToString());
var settingsResponse = await _client.GetIndexSettingsAsync(s => s.Index(workItem.OldIndex)).AnyContext();
if (!settingsResponse.IsValid)
{
throw new ApplicationException("Unable to retrieve index settings.");
}
int scrollSize = 500 / settingsResponse.IndexSettings.NumberOfShards ?? 50;
var scanResults = await _client.SearchAsync <object>(s => s
.Index(workItem.OldIndex)
.AllTypes()
.Query(q => q.Filtered(f => {
if (startTime.HasValue)
{
f.Filter(f1 => f1.Range(r => r.OnField(timestampField).Greater(startTime.Value)));
}
}))
.Fields("_source", "_parent")
.Size(scrollSize)
.SearchType(SearchType.Scan)
.Scroll(scroll)).AnyContext();
_logger.Info(scanResults.GetRequest());
if (!scanResults.IsValid || scanResults.ScrollId == null)
{
_logger.Error().Exception(scanResults.ConnectionStatus.OriginalException).Message("Invalid search result: message={0}", scanResults.GetErrorMessage()).Write();
return(new ReindexResult());
}
var results = await _client.ScrollAsync <JObject>("5m", scanResults.ScrollId).AnyContext();
if (!results.IsValid)
{
await scopedCacheClient.RemoveAsync("id").AnyContext();
return(await InternalReindexAsync(workItem, progressCallbackAsync, startProgress, endProgress, startTime).AnyContext());
}
double completed = 0;
long totalHits = results.Total;
while (results.Hits.Any())
{
ISearchResponse <JObject> results1 = results;
IBulkResponse bulkResponse = null;
try {
bulkResponse = await Run.WithRetriesAsync(() => _client.BulkAsync(b => {
foreach (var h in results1.Hits)
{
ConfigureIndexItem(b, h, workItem.NewIndex);
}
return(b);
}), logger : _logger, maxAttempts : 2).AnyContext();
} catch (Exception ex) {
_logger.Error(ex, $"Error trying to do bulk index: {ex.Message}");
}
if (bulkResponse == null || !bulkResponse.IsValid || bulkResponse.ItemsWithErrors.Any())
{
string message;
if (bulkResponse != null)
{
message = $"Reindex bulk error: old={workItem.OldIndex} new={workItem.NewIndex} completed={completed} message={bulkResponse?.GetErrorMessage()}";
_logger.Warn(bulkResponse.ConnectionStatus.OriginalException, message);
}
// try each doc individually so we can see which doc is breaking us
var hitsToRetry = bulkResponse?.ItemsWithErrors.Select(i => results1.Hits.First(hit => hit.Id == i.Id)) ?? results1.Hits;
foreach (var itemToRetry in hitsToRetry)
{
IIndexResponse response;
try {
response = await _client.IndexAsync(itemToRetry.Source, d => ConfigureItem(d, itemToRetry, workItem.NewIndex)).AnyContext();
if (response.IsValid)
{
continue;
}
message = $"Reindex error: old={workItem.OldIndex} new={workItem.NewIndex} id={itemToRetry.Id} completed={completed} message={response.GetErrorMessage()}";
_logger.Error().Exception(response.ConnectionStatus.OriginalException).Message(message);
var errorDoc = new JObject {
["Type"] = itemToRetry.Type,
["Content"] = itemToRetry.Source.ToString(Formatting.Indented)
};
var errorIndex = workItem.NewIndex + "-error";
if (!errorIndexCreated && !(await _client.IndexExistsAsync(errorIndex).AnyContext()).Exists)
{
await _client.CreateIndexAsync(errorIndex).AnyContext();
errorIndexCreated = true;
}
// put the document into an error index
response = await _client.IndexAsync(errorDoc, d => d.Index(errorIndex).Id(itemToRetry.Id)).AnyContext();
if (response.IsValid)
{
continue;
}
} catch {
throw;
}
message = $"Reindex error: old={workItem.OldIndex} new={workItem.NewIndex} id={itemToRetry.Id} completed={completed} message={response.GetErrorMessage()}";
_logger.Error().Exception(response.ConnectionStatus.OriginalException).Message(message);
throw new ReindexException(response.ConnectionStatus, message);
}
}
completed += results.Hits.Count();
await progressCallbackAsync(CalculateProgress(totalHits, (long)completed, startProgress, endProgress), $"Total: {totalHits} Completed: {completed}").AnyContext();
results = await _client.ScrollAsync <JObject>("5m", results.ScrollId).AnyContext();
await scopedCacheClient.AddAsync("id", results.ScrollId, TimeSpan.FromHours(1)).AnyContext();
}
await scopedCacheClient.RemoveAllAsync(new[] { "id" }).AnyContext();
return(new ReindexResult {
Total = totalHits, Completed = (long)completed
});
}
public virtual void ConfigureIndexes(IElasticClient client, IEnumerable <IElasticIndex> indexes = null)
{
if (indexes == null)
{
indexes = GetIndexes();
}
foreach (var idx in indexes)
{
int currentVersion = GetAliasVersion(client, idx.AliasName);
IIndicesOperationResponse response = null;
var templatedIndex = idx as ITemplatedElasticIndex;
if (templatedIndex != null)
{
response = client.PutTemplate(idx.VersionedName, template => templatedIndex.CreateTemplate(template).AddAlias(idx.AliasName));
}
else if (!client.IndexExists(idx.VersionedName).Exists)
{
response = client.CreateIndex(idx.VersionedName, descriptor => idx.CreateIndex(descriptor).AddAlias(idx.AliasName));
}
Debug.Assert(response == null || response.IsValid, response?.ServerError != null ? response.ServerError.Error : "An error occurred creating the index or template.");
// Add existing indexes to the alias.
if (!client.AliasExists(idx.AliasName).Exists)
{
if (templatedIndex != null)
{
var indices = client.IndicesStats().Indices.Where(kvp => kvp.Key.StartsWith(idx.VersionedName)).Select(kvp => kvp.Key).ToList();
if (indices.Count > 0)
{
var descriptor = new AliasDescriptor();
foreach (string name in indices)
{
descriptor.Add(add => add.Index(name).Alias(idx.AliasName));
}
response = client.Alias(descriptor);
}
}
else
{
response = client.Alias(a => a.Add(add => add.Index(idx.VersionedName).Alias(idx.AliasName)));
}
Debug.Assert(response != null && response.IsValid, response?.ServerError != null ? response.ServerError.Error : "An error occurred creating the alias.");
}
// already on current version
if (currentVersion >= idx.Version || currentVersion < 1)
{
continue;
}
var reindexWorkItem = new ReindexWorkItem {
OldIndex = String.Concat(idx.AliasName, "-v", currentVersion),
NewIndex = idx.VersionedName,
Alias = idx.AliasName,
DeleteOld = true,
ParentMaps = idx.GetIndexTypes()
.Select(kvp => new ParentMap {
Type = kvp.Value.Name, ParentPath = kvp.Value.ParentPath
})
.Where(m => !String.IsNullOrEmpty(m.ParentPath))
.ToList()
};
bool isReindexing = _lockProvider.IsLockedAsync(String.Concat("reindex:", reindexWorkItem.Alias, reindexWorkItem.OldIndex, reindexWorkItem.NewIndex)).Result;
// already reindexing
if (isReindexing)
{
continue;
}
// enqueue reindex to new version
_lockProvider.TryUsingAsync("enqueue-reindex", () => _workItemQueue.EnqueueAsync(reindexWorkItem), TimeSpan.Zero, CancellationToken.None).Wait();
}
}
private async Task <ReindexResult> ReindexAsync(ReindexWorkItem workItem, WorkItemContext context, int startProgress = 0, int endProgress = 100, DateTime?startTime = null)
{
const int pageSize = 100;
const string scroll = "10s";
var scanResults = await _client.SearchAsync <JObject>(s => s.Index(workItem.OldIndex).AllTypes().Filter(f => startTime.HasValue ? f.Range(r => r.OnField(workItem.TimestampField ?? "_timestamp").Greater(startTime.Value)) : f.MatchAll()).From(0).Take(pageSize).SearchType(SearchType.Scan).Scroll(scroll)).AnyContext();
if (!scanResults.IsValid || scanResults.ScrollId == null)
{
Logger.Error().Message("Invalid search result: message={0}", scanResults.GetErrorMessage()).Write();
return(new ReindexResult());
}
long totalHits = scanResults.Total;
long completed = 0;
int page = 0;
var results = await _client.ScrollAsync <JObject>(scroll, scanResults.ScrollId).AnyContext();
while (results.Documents.Any())
{
var bulkDescriptor = new BulkDescriptor();
foreach (var hit in results.Hits)
{
var h = hit;
// TODO: Add support for doing JObject based schema migrations
bulkDescriptor.Index <JObject>(idx => idx.Index(workItem.NewIndex).Type(h.Type).Id(h.Id).Document(h.Source));
}
var bulkResponse = await _client.BulkAsync(bulkDescriptor).AnyContext();
if (!bulkResponse.IsValid)
{
string message = $"Reindex bulk error: old={workItem.OldIndex} new={workItem.NewIndex} page={page} message={bulkResponse.GetErrorMessage()}";
Logger.Warn().Message(message).Write();
// try each doc individually so we can see which doc is breaking us
foreach (var hit in results.Hits)
{
var h = hit;
var response = await _client.IndexAsync(h.Source, d => d.Index(workItem.NewIndex).Type(h.Type).Id(h.Id)).AnyContext();
if (response.IsValid)
{
continue;
}
message = $"Reindex error: old={workItem.OldIndex} new={workItem.NewIndex} id={hit.Id} page={page} message={response.GetErrorMessage()}";
Logger.Error().Message(message).Write();
throw new ReindexException(response.ConnectionStatus, message);
}
}
completed += bulkResponse.Items.Count();
await context.ReportProgressAsync(CalculateProgress(totalHits, completed, startProgress, endProgress), $"Total: {totalHits} Completed: {completed}").AnyContext();
results = await _client.ScrollAsync <JObject>(scroll, results.ScrollId).AnyContext();
page++;
}
return(new ReindexResult {
Total = totalHits,
Completed = completed
});
}
private async Task <ReindexResult> InternalReindexAsync(ReindexWorkItem workItem, Func <int, string, Task> progressCallbackAsync, int startProgress = 0, int endProgress = 100, DateTime?startTime = null, CancellationToken cancellationToken = default)
{
var query = await GetResumeQueryAsync(workItem.NewIndex, workItem.TimestampField, startTime).AnyContext();
var result = await Run.WithRetriesAsync(async() => {
var response = await _client.ReindexOnServerAsync(d => {
d.Source(src => src
.Index(workItem.OldIndex)
.Query <object>(q => query)
.Sort <object>(s => s.Ascending(new Field(workItem.TimestampField ?? ID_FIELD))))
.Destination(dest => dest.Index(workItem.NewIndex))
.Conflicts(Conflicts.Proceed)
.WaitForCompletion(false);
//NEST client emitting script if null, inline this when that's fixed
if (!String.IsNullOrWhiteSpace(workItem.Script))
{
d.Script(workItem.Script);
}
return(d);
}).AnyContext();
return(response);
}, 5, TimeSpan.FromSeconds(10), cancellationToken, _logger).AnyContext();
_logger.LogInformation("Reindex Task Id: {TaskId}", result.Task.FullyQualifiedId);
_logger.LogTraceRequest(result);
bool taskSuccess = false;
TaskReindexResult lastReindexResponse = null;
int statusGetFails = 0;
long lastProgress = 0;
var sw = Stopwatch.StartNew();
do
{
await Task.Delay(TimeSpan.FromSeconds(10), cancellationToken).AnyContext();
var status = await _client.GetTaskAsync(result.Task, null, cancellationToken).AnyContext();
if (status.IsValid)
{
_logger.LogTraceRequest(status);
}
else
{
_logger.LogErrorRequest(status, "Error getting task status while reindexing: {OldIndex} -> {NewIndex}", workItem.OldIndex, workItem.NewIndex);
statusGetFails++;
if (statusGetFails > MAX_STATUS_FAILS)
{
_logger.LogError("Failed to get the status {FailureCount} times in a row", MAX_STATUS_FAILS);
break;
}
continue;
}
statusGetFails = 0;
var response = status.DeserializeRaw <TaskWithReindexResponse>();
if (response?.Error != null)
{
_logger.LogError("Error reindex: {Type}, {Reason}, Cause: {CausedBy} Stack: {Stack}", response.Error.Type, response.Error.Reason, response.Error.Caused_By?.Reason, String.Join("\r\n", response.Error.Script_Stack ?? new List <string>()));
break;
}
lastReindexResponse = response?.Response;
long lastCompleted = status.Task.Status.Created + status.Task.Status.Updated + status.Task.Status.Noops;
// restart the stop watch if there was progress made
if (lastCompleted > lastProgress)
{
sw.Restart();
}
lastProgress = lastCompleted;
string lastMessage = $"Total: {status.Task.Status.Total:N0} Completed: {lastCompleted:N0} VersionConflicts: {status.Task.Status.VersionConflicts:N0}";
await progressCallbackAsync(CalculateProgress(status.Task.Status.Total, lastCompleted, startProgress, endProgress), lastMessage).AnyContext();
if (status.Completed && response?.Error == null)
{
taskSuccess = true;
break;
}
if (sw.Elapsed > TimeSpan.FromMinutes(10))
{
_logger.LogError($"Timed out waiting for reindex {workItem.OldIndex} -> {workItem.NewIndex}.");
break;
}
} while (!cancellationToken.IsCancellationRequested);
sw.Stop();
long failures = 0;
if (lastReindexResponse?.Failures != null && lastReindexResponse.Failures.Count > 0)
{
_logger.LogError("Error while reindexing result");
if (await CreateFailureIndexAsync(workItem).AnyContext())
{
foreach (var failure in lastReindexResponse.Failures)
{
await HandleFailureAsync(workItem, failure).AnyContext();
failures++;
}
}
taskSuccess = false;
}
long total = lastReindexResponse?.Total ?? 0;
long versionConflicts = lastReindexResponse?.VersionConflicts ?? 0;
long completed = (lastReindexResponse?.Created ?? 0) + (lastReindexResponse?.Updated ?? 0) + (lastReindexResponse?.Noops ?? 0);
string message = $"Total: {total:N0} Completed: {completed:N0} VersionConflicts: {versionConflicts:N0}";
await progressCallbackAsync(CalculateProgress(total, completed, startProgress, endProgress), message).AnyContext();
return(new ReindexResult {
Total = total, Completed = completed, Failures = failures, Succeeded = taskSuccess
});
}
