private IndexMeta getIndexMeta()
{
// TODO: all this needs cleaning.
var(index, aliased) = findOrCreateIndex(Name);
if (!AppSettings.IsRebuild && !aliased)
{
updateAlias(Name, index);
}
// look for any existing resume data.
var indexMeta = IndexMeta.GetByName(index) ?? new IndexMeta
{
Alias = Name,
Name = index,
};
indexMeta.LastId = ResumeFrom ?? indexMeta.LastId;
if (AppSettings.IsRebuild)
{
// Save the position the score processing queue should be reset to if rebuilding an index.
// If there is already an existing value, the processor is probabaly resuming from a previous run,
// so we likely want to preserve that value.
if (!indexMeta.ResetQueueTo.HasValue)
{
var mode = HighScore.GetRulesetId <T>();
indexMeta.ResetQueueTo = ScoreProcessQueue.GetLastProcessedQueueId(mode);
}
}
else
{
if (string.IsNullOrWhiteSpace(indexMeta.Schema))
{
throw new Exception("FATAL ERROR: attempting to process queue without a known version.");
}
if (indexMeta.Schema != AppSettings.Schema)
{
// A switchover is probably happening, so signal that this mode should be skipped.
throw new VersionMismatchException($"`{Name}` found version {indexMeta.Schema}, expecting {AppSettings.Schema}");
}
// process queue reset if any.
if (indexMeta.ResetQueueTo.HasValue)
{
Console.WriteLine($"Resetting queue_id > {indexMeta.ResetQueueTo}");
ScoreProcessQueue.UnCompleteQueued <T>(indexMeta.ResetQueueTo.Value);
indexMeta.ResetQueueTo = null;
}
}
indexMeta.UpdatedAt = DateTimeOffset.UtcNow;
IndexMeta.UpdateAsync(indexMeta);
IndexMeta.Refresh();
return(indexMeta);
}
public static void UnCompleteQueued <T>(ulong from) where T : HighScore
{
using (var dbConnection = new MySqlConnection(AppSettings.ConnectionString))
{
var mode = HighScore.GetRulesetId <T>();
dbConnection.Open();
dbConnection.Execute("UPDATE score_process_queue SET status = 1 WHERE queue_id > @from AND mode = @mode", new { from, mode });
}
}
/// <summary>
/// Self contained database reader task. Reads the database by cursoring through records
/// and adding chunks into readBuffer.
/// </summary>
/// <param name="resumeFrom">The cursor value to resume from;
/// use null to resume from the last known value.</param>
/// <returns>The database reader task.</returns>
private Task <long> databaseReaderTask(ulong resumeFrom) => Task.Factory.StartNew(() =>
{
long count = 0;
var mode = HighScore.GetRulesetId <T>();
while (true)
{
try
{
if (!AppSettings.IsRebuild)
{
Console.WriteLine("Reading from queue...");
var chunks = Model.Chunk <ScoreProcessQueue>($"status = 1 and mode = {mode}", AppSettings.ChunkSize);
foreach (var chunk in chunks)
{
var scoreIds = chunk.Select(x => x.ScoreId).ToList();
var scores = ScoreProcessQueue.FetchByScoreIds <T>(scoreIds).Where(x => x.ShouldIndex).ToList();
var removedScores = scoreIds
.Except(scores.Select(x => x.ScoreId))
.Select(scoreId => new T {
ScoreId = scoreId
})
.ToList();
Console.WriteLine($"Got {chunk.Count} items from queue, found {scores.Count} matching scores, {removedScores.Count} missing scores");
DogStatsd.Increment("indexed", scores.Count, tags: new[] { $"mode:{mode}", "result:success" });
DogStatsd.Increment("indexed", removedScores.Count, tags: new[] { $"mode:{mode}", "result:missing" });
dispatcher.Enqueue(add: scores, remove: removedScores);
ScoreProcessQueue.CompleteQueued(chunk);
count += scores.Count;
}
}
else
{
Console.WriteLine($"Rebuild from {resumeFrom}...");
var chunks = Model.Chunk <T>(AppSettings.ChunkSize, resumeFrom);
foreach (var chunk in chunks)
{
var scores = chunk.Where(x => x.ShouldIndex).ToList();
dispatcher.Enqueue(scores);
DogStatsd.Increment("indexed", scores.Count, tags: new[] { $"mode:{mode}", "result:success" });
count += chunk.Count;
// update resumeFrom in this scope to allow resuming from connection errors.
resumeFrom = chunk.Last().CursorValue;
}
}
break;
}
catch (DbException ex)
{
Console.Error.WriteLine(ex.Message);
Task.Delay(1000).Wait();
}
}
dispatcher.EnqueueEnd();
Console.WriteLine($"Finished reading database {count} records.");
return(count);
}, TaskCreationOptions.LongRunning | TaskCreationOptions.DenyChildAttach);