private IndexMeta getIndexMeta()
{
// TODO: all this needs cleaning.
var(index, aliased) = findOrCreateIndex(Name);
if (!AppSettings.IsRebuild && !aliased)
{
updateAlias(Name, index);
}
// look for any existing resume data.
var indexMeta = IndexMeta.GetByName(index) ?? new IndexMeta
{
Alias = Name,
Name = index,
};
indexMeta.LastId = ResumeFrom ?? indexMeta.LastId;
if (AppSettings.IsRebuild)
{
// Save the position the score processing queue should be reset to if rebuilding an index.
// If there is already an existing value, the processor is probabaly resuming from a previous run,
// so we likely want to preserve that value.
if (!indexMeta.ResetQueueTo.HasValue)
{
var mode = HighScore.GetRulesetId <T>();
indexMeta.ResetQueueTo = ScoreProcessQueue.GetLastProcessedQueueId(mode);
}
}
else
{
if (string.IsNullOrWhiteSpace(indexMeta.Schema))
{
throw new Exception("FATAL ERROR: attempting to process queue without a known version.");
}
if (indexMeta.Schema != AppSettings.Schema)
{
// A switchover is probably happening, so signal that this mode should be skipped.
throw new VersionMismatchException($"`{Name}` found version {indexMeta.Schema}, expecting {AppSettings.Schema}");
}
// process queue reset if any.
if (indexMeta.ResetQueueTo.HasValue)
{
Console.WriteLine($"Resetting queue_id > {indexMeta.ResetQueueTo}");
ScoreProcessQueue.UnCompleteQueued <T>(indexMeta.ResetQueueTo.Value);
indexMeta.ResetQueueTo = null;
}
}
indexMeta.UpdatedAt = DateTimeOffset.UtcNow;
IndexMeta.UpdateAsync(indexMeta);
IndexMeta.Refresh();
return(indexMeta);
}
/// <summary>
/// Reads a buffer and dispatches bulk index requests to Elasticsearch until the buffer
/// is marked as complete.
/// </summary>
internal void Run()
{
// custom partitioner and options to prevent Parallel.ForEach from going out of control.
var partitioner = Partitioner.Create(
readBuffer.GetConsumingEnumerable(),
EnumerablePartitionerOptions.NoBuffering // buffering causes spastic behaviour.
);
var options = new ParallelOptions {
MaxDegreeOfParallelism = AppSettings.Concurrency
};
Parallel.ForEach(partitioner, options, chunk =>
{
bool success;
while (true)
{
var bulkDescriptor = new BulkDescriptor().Index(index).IndexMany(chunk);
var response = elasticClient.Bulk(bulkDescriptor);
bool retry;
(success, retry) = retryOnResponse(response, chunk);
if (!retry)
{
break;
}
Task.Delay(AppSettings.BulkAllBackOffTimeDefault).Wait();
}
if (success)
{
// TODO: should probably aggregate responses and update to highest successful.
IndexMeta.UpdateAsync(new IndexMeta
{
Index = index,
Alias = alias,
LastId = chunk.Last().CursorValue,
UpdatedAt = DateTimeOffset.UtcNow
});
}
});
IndexMeta.Refresh();
}
public void Run()
{
if (!checkIfReady())
{
return;
}
var indexMeta = getIndexMeta();
var metaSchema = AppSettings.IsPrepMode ? null : AppSettings.Schema;
var indexCompletedArgs = new IndexCompletedArgs
{
Alias = Name,
Index = indexMeta.Name,
StartedAt = DateTime.Now
};
dispatcher = new BulkIndexingDispatcher <T>(indexMeta.Name);
if (AppSettings.IsRebuild)
{
dispatcher.BatchWithLastIdCompleted += handleBatchWithLastIdCompleted;
}
try
{
var readerTask = databaseReaderTask(indexMeta.LastId);
dispatcher.Run();
readerTask.Wait();
indexCompletedArgs.Count = readerTask.Result;
indexCompletedArgs.CompletedAt = DateTime.Now;
IndexMeta.Refresh();
// when preparing for schema changes, the alias update
// should be done by process waiting for the ready signal.
if (AppSettings.IsRebuild)
{
if (AppSettings.IsPrepMode)
{
IndexMeta.MarkAsReady(indexMeta.Name);
}
else
{
updateAlias(Name, indexMeta.Name);
}
}
IndexCompleted(this, indexCompletedArgs);
}
catch (AggregateException ae)
{
ae.Handle(handleAggregateException);
}
// Local function exception handler.
bool handleAggregateException(Exception ex)
{
if (!(ex is InvalidOperationException))
{
return(false);
}
Console.Error.WriteLine(ex.Message);
if (ex.InnerException != null)
{
Console.Error.WriteLine(ex.InnerException.Message);
}
return(true);
}
void handleBatchWithLastIdCompleted(object sender, ulong lastId)
{
// TODO: should probably aggregate responses and update to highest successful.
IndexMeta.UpdateAsync(new IndexMeta
{
Name = indexMeta.Name,
Alias = Name,
LastId = lastId,
ResetQueueTo = indexMeta.ResetQueueTo,
UpdatedAt = DateTimeOffset.UtcNow,
Schema = metaSchema
});
}
}