/// <summary>
/// Self contained database reader task. Reads the database by cursoring through records
/// and adding chunks into readBuffer.
/// </summary>
/// <param name="resumeFrom">The cursor value to resume from;
/// use null to resume from the last known value.</param>
/// <returns>The database reader task.</returns>
private Task <long> databaseReaderTask(long?resumeFrom)
{
return(Task.Factory.StartNew(() =>
{
long count = 0;
while (true)
{
try
{
var chunks = Model.Chunk <T>("pp is not null", AppSettings.ChunkSize, resumeFrom);
foreach (var chunk in chunks)
{
dispatcher.Enqueue(chunk);
count += chunk.Count;
// update resumeFrom in this scope to allow resuming from connection errors.
resumeFrom = chunk.Last().CursorValue;
}
break;
}
catch (DbException ex)
{
Console.Error.WriteLine(ex.Message);
Task.Delay(1000).Wait();
}
}
dispatcher.EnqueueEnd();
Console.WriteLine($"Finished reading database {count} records.");
return count;
}, TaskCreationOptions.LongRunning | TaskCreationOptions.DenyChildAttach));
}
/// <summary>
/// Self contained database reader task. Reads the database by cursoring through records
/// and adding chunks into readBuffer.
/// </summary>
/// <param name="resumeFrom">The cursor value to resume from;
/// use null to resume from the last known value.</param>
/// <returns>The database reader task.</returns>
private Task <long> databaseReaderTask(ulong resumeFrom) => Task.Factory.StartNew(() =>
{
long count = 0;
var mode = HighScore.GetRulesetId <T>();
while (true)
{
try
{
if (!AppSettings.IsRebuild)
{
Console.WriteLine("Reading from queue...");
var chunks = Model.Chunk <ScoreProcessQueue>($"status = 1 and mode = {mode}", AppSettings.ChunkSize);
foreach (var chunk in chunks)
{
var scoreIds = chunk.Select(x => x.ScoreId).ToList();
var scores = ScoreProcessQueue.FetchByScoreIds <T>(scoreIds).Where(x => x.ShouldIndex).ToList();
var removedScores = scoreIds
.Except(scores.Select(x => x.ScoreId))
.Select(scoreId => new T {
ScoreId = scoreId
})
.ToList();
Console.WriteLine($"Got {chunk.Count} items from queue, found {scores.Count} matching scores, {removedScores.Count} missing scores");
DogStatsd.Increment("indexed", scores.Count, tags: new[] { $"mode:{mode}", "result:success" });
DogStatsd.Increment("indexed", removedScores.Count, tags: new[] { $"mode:{mode}", "result:missing" });
dispatcher.Enqueue(add: scores, remove: removedScores);
ScoreProcessQueue.CompleteQueued(chunk);
count += scores.Count;
}
}
else
{
Console.WriteLine($"Rebuild from {resumeFrom}...");
var chunks = Model.Chunk <T>(AppSettings.ChunkSize, resumeFrom);
foreach (var chunk in chunks)
{
var scores = chunk.Where(x => x.ShouldIndex).ToList();
dispatcher.Enqueue(scores);
DogStatsd.Increment("indexed", scores.Count, tags: new[] { $"mode:{mode}", "result:success" });
count += chunk.Count;
// update resumeFrom in this scope to allow resuming from connection errors.
resumeFrom = chunk.Last().CursorValue;
}
}
break;
}
catch (DbException ex)
{
Console.Error.WriteLine(ex.Message);
Task.Delay(1000).Wait();
}
}
dispatcher.EnqueueEnd();
Console.WriteLine($"Finished reading database {count} records.");
return(count);
}, TaskCreationOptions.LongRunning | TaskCreationOptions.DenyChildAttach);