public ReindexWorkItem CreateReindexWorkItem(int currentVersion) {
var reindexWorkItem = new ReindexWorkItem {
OldIndex = String.Concat(Name, "-v", currentVersion),
NewIndex = VersionedName,
Alias = Name
};
reindexWorkItem.DeleteOld = DiscardIndexesOnReindex && reindexWorkItem.OldIndex != reindexWorkItem.NewIndex;
return reindexWorkItem;
}
public async Task ReindexAsync(ReindexWorkItem workItem, Func<int, string, Task> progressCallbackAsync = null) {
if (progressCallbackAsync == null) {
progressCallbackAsync = (progress, message) => {
_logger.Info("Reindex Progress {0}%: {1}", progress, message);
return Task.CompletedTask;
};
}
_logger.Info("Received reindex work item for new index {0}", workItem.NewIndex);
var startTime = SystemClock.UtcNow.AddSeconds(-1);
await progressCallbackAsync(0, "Starting reindex...").AnyContext();
var result = await InternalReindexAsync(workItem, progressCallbackAsync, 0, 90, workItem.StartUtc).AnyContext();
await progressCallbackAsync(95, $"Total: {result.Total} Completed: {result.Completed}").AnyContext();
// TODO: Check to make sure the docs have been added to the new index before changing alias
if (workItem.OldIndex != workItem.NewIndex) {
var aliases = await GetIndexAliases(workItem.OldIndex).AnyContext();
if (!String.IsNullOrEmpty(workItem.Alias) && !aliases.Contains(workItem.Alias))
aliases.Add(workItem.Alias);
if (aliases.Count > 0) {
await _client.AliasAsync(x => {
foreach (var alias in aliases)
x = x.Remove(a => a.Alias(alias).Index(workItem.OldIndex)).Add(a => a.Alias(alias).Index(workItem.NewIndex));
return x;
}).AnyContext();
await progressCallbackAsync(98, $"Updated aliases: {String.Join(", ", aliases)} Remove: {workItem.OldIndex} Add: {workItem.NewIndex}").AnyContext();
}
}
await _client.RefreshAsync().AnyContext();
var secondPassResult = await InternalReindexAsync(workItem, progressCallbackAsync, 90, 98, startTime).AnyContext();
await progressCallbackAsync(98, $"Total: {secondPassResult.Total} Completed: {secondPassResult.Completed}").AnyContext();
if (workItem.DeleteOld && workItem.OldIndex != workItem.NewIndex) {
await _client.RefreshAsync().AnyContext();
long newDocCount = (await _client.CountAsync(d => d.Index(workItem.NewIndex)).AnyContext()).Count;
long oldDocCount = (await _client.CountAsync(d => d.Index(workItem.OldIndex)).AnyContext()).Count;
await progressCallbackAsync(98, $"Old Docs: {oldDocCount} New Docs: {newDocCount}").AnyContext();
if (newDocCount >= oldDocCount) {
await _client.DeleteIndexAsync(d => d.Index(workItem.OldIndex)).AnyContext();
await progressCallbackAsync(99, $"Deleted index: {workItem.OldIndex}").AnyContext();
}
}
await progressCallbackAsync(100, null).AnyContext();
}
private async Task<ReindexResult> InternalReindexAsync(ReindexWorkItem workItem, Func<int, string, Task> progressCallbackAsync, int startProgress = 0, int endProgress = 100, DateTime? startTime = null) {
const string scroll = "5m";
string timestampField = workItem.TimestampField ?? "_timestamp";
var scopedCacheClient = new ScopedCacheClient(_cache, workItem.GetHashCode().ToString());
var settingsResponse = await _client.GetIndexSettingsAsync(s => s.Index(workItem.OldIndex)).AnyContext();
if (!settingsResponse.IsValid)
throw new ApplicationException("Unable to retrieve index settings.");
int scrollSize = 500 / settingsResponse.IndexSettings.NumberOfShards ?? 50;
var scanResults = await _client.SearchAsync<object>(s => s
.Index(workItem.OldIndex)
.AllTypes()
.Query(q => q.Filtered(f => {
if (startTime.HasValue)
f.Filter(f1 => f1.Range(r => r.OnField(timestampField).Greater(startTime.Value)));
}))
.Fields("_source", "_parent")
.Size(scrollSize)
.SearchType(SearchType.Scan)
.Scroll(scroll)).AnyContext();
_logger.Info(scanResults.GetRequest());
if (!scanResults.IsValid || scanResults.ScrollId == null) {
_logger.Error().Exception(scanResults.ConnectionStatus.OriginalException).Message("Invalid search result: message={0}", scanResults.GetErrorMessage()).Write();
return new ReindexResult();
}
var results = await _client.ScrollAsync<JObject>("5m", scanResults.ScrollId).AnyContext();
if (!results.IsValid) {
await scopedCacheClient.RemoveAsync("id").AnyContext();
return await InternalReindexAsync(workItem, progressCallbackAsync, startProgress, endProgress, startTime).AnyContext();
}
double completed = 0;
long totalHits = results.Total;
while (results.Hits.Any()) {
ISearchResponse<JObject> results1 = results;
IBulkResponse bulkResponse = null;
try {
bulkResponse = await Run.WithRetriesAsync(() => _client.BulkAsync(b => {
foreach (var h in results1.Hits)
ConfigureIndexItem(b, h, workItem.NewIndex);
return b;
}), logger: _logger).AnyContext();
} catch (Exception ex) {
_logger.Error(ex, $"Error trying to do bulk index: {ex.Message}");
}
if (bulkResponse == null || !bulkResponse.IsValid || bulkResponse.ItemsWithErrors.Any()) {
string message = $"Reindex bulk error: old={workItem.OldIndex} new={workItem.NewIndex} completed={completed} message={bulkResponse.GetErrorMessage()}";
_logger.Warn(bulkResponse.ConnectionStatus.OriginalException, message);
// try each doc individually so we can see which doc is breaking us
var hitsToRetry = bulkResponse.ItemsWithErrors.Select(i => results.Hits.First(hit => hit.Id == i.Id));
foreach (var itemWithError in hitsToRetry) {
var response = await _client.IndexAsync(itemWithError.Source, d => ConfigureItem(d, itemWithError, workItem.NewIndex)).AnyContext();
if (response.IsValid)
continue;
message = $"Reindex error: old={workItem.OldIndex} new={workItem.NewIndex} id={itemWithError.Id} completed={completed} message={response.GetErrorMessage()}";
_logger.Error().Exception(response.ConnectionStatus.OriginalException).Message(message);
var errorDoc = new JObject(new {
itemWithError.Type,
Content = itemWithError.Source.ToString(Formatting.Indented)
});
// put the document into an error index
response = await _client.IndexAsync(errorDoc, d => d.Index(workItem.NewIndex + "-error").Id(itemWithError.Id)).AnyContext();
if (response.IsValid)
continue;
message = $"Reindex error: old={workItem.OldIndex} new={workItem.NewIndex} id={itemWithError.Id} completed={completed} message={response.GetErrorMessage()}";
_logger.Error().Exception(response.ConnectionStatus.OriginalException).Message(message);
throw new ReindexException(response.ConnectionStatus, message);
}
}
completed += results.Hits.Count();
await progressCallbackAsync(CalculateProgress(totalHits, (long)completed, startProgress, endProgress), $"Total: {totalHits} Completed: {completed}").AnyContext();
results = await _client.ScrollAsync<JObject>("5m", results.ScrollId).AnyContext();
await scopedCacheClient.AddAsync("id", results.ScrollId, TimeSpan.FromHours(1)).AnyContext();
}
await scopedCacheClient.RemoveAllAsync(new[] { "id" }).AnyContext();
return new ReindexResult { Total = totalHits, Completed = (long)completed };
}
protected bool Equals(ReindexWorkItem other)
{
return(string.Equals(OldIndex, other.OldIndex) && string.Equals(NewIndex, other.NewIndex) && string.Equals(Alias, other.Alias) && DeleteOld == other.DeleteOld && string.Equals(TimestampField, other.TimestampField) && StartUtc.Equals(other.StartUtc) && string.Equals(Script, other.Script));
}
protected bool Equals(ReindexWorkItem other) {
return string.Equals(OldIndex, other.OldIndex) && string.Equals(NewIndex, other.NewIndex) && string.Equals(Alias, other.Alias) && DeleteOld == other.DeleteOld && string.Equals(TimestampField, other.TimestampField) && StartUtc.Equals(other.StartUtc);
}