private IndexMeta getIndexMeta()
{
// TODO: all this needs cleaning.
var(index, aliased) = findOrCreateIndex(Name);
if (!AppSettings.IsRebuild && !aliased)
{
updateAlias(Name, index);
}
// look for any existing resume data.
var indexMeta = IndexMeta.GetByName(index) ?? new IndexMeta
{
Alias = Name,
Name = index,
};
indexMeta.LastId = ResumeFrom ?? indexMeta.LastId;
if (AppSettings.IsRebuild)
{
// Save the position the score processing queue should be reset to if rebuilding an index.
// If there is already an existing value, the processor is probabaly resuming from a previous run,
// so we likely want to preserve that value.
if (!indexMeta.ResetQueueTo.HasValue)
{
var mode = HighScore.GetRulesetId <T>();
indexMeta.ResetQueueTo = ScoreProcessQueue.GetLastProcessedQueueId(mode);
}
}
else
{
if (string.IsNullOrWhiteSpace(indexMeta.Schema))
{
throw new Exception("FATAL ERROR: attempting to process queue without a known version.");
}
if (indexMeta.Schema != AppSettings.Schema)
{
// A switchover is probably happening, so signal that this mode should be skipped.
throw new VersionMismatchException($"`{Name}` found version {indexMeta.Schema}, expecting {AppSettings.Schema}");
}
// process queue reset if any.
if (indexMeta.ResetQueueTo.HasValue)
{
Console.WriteLine($"Resetting queue_id > {indexMeta.ResetQueueTo}");
ScoreProcessQueue.UnCompleteQueued <T>(indexMeta.ResetQueueTo.Value);
indexMeta.ResetQueueTo = null;
}
}
indexMeta.UpdatedAt = DateTimeOffset.UtcNow;
IndexMeta.UpdateAsync(indexMeta);
IndexMeta.Refresh();
return(indexMeta);
}
public static void Main()
{
DogStatsd.Configure(new StatsdConfig
{
StatsdServerName = "127.0.0.1",
Prefix = "elasticsearch.scores"
});
if (AppSettings.UseDocker)
{
Console.WriteLine("Waiting for database...");
while (true)
{
try
{
using (var conn = new MySqlConnection(AppSettings.ConnectionString))
{
if (conn.QuerySingle <int>("SELECT `count` FROM `osu_counts` WHERE `name` = 'docker_db_step'") >= 3)
{
break;
}
}
}
catch
{
}
Thread.Sleep(1000);
}
}
DefaultTypeMap.MatchNamesWithUnderscores = true;
IndexMeta.CreateIndex();
Console.WriteLine($"Rebuilding index: `{AppSettings.IsRebuild}`");
if (AppSettings.IsWatching)
{
runWatchLoop();
}
else
{
runIndexing();
}
if (AppSettings.UseDocker)
{
using (var conn = new MySqlConnection(AppSettings.ConnectionString))
{
conn.Execute("INSERT INTO `osu_counts` (`name`, `count`) VALUES (@Name, @Count) ON DUPLICATE KEY UPDATE `count` = @Count", new
{
Name = "docker_db_step",
Count = 4
});
}
}
}
/// <summary>
/// Attempts to find the matching index or creates a new one.
/// </summary>
/// <param name="name">name of the index alias.</param>
/// <returns>Name of index found or created and any existing alias.</returns>
private (string index, bool aliased) findOrCreateIndex(string name)
{
Console.WriteLine();
var aliasedIndices = elasticClient.GetIndicesPointingToAlias(name);
var metas = (
AppSettings.IsRebuild
? IndexMeta.GetByAlias(name)
: IndexMeta.GetByAliasForCurrentVersion(name)
).ToList();
string index = null;
if (!AppSettings.IsNew)
{
// TODO: query ES that the index actually exists.
index = metas.FirstOrDefault(m => aliasedIndices.Contains(m.Name))?.Name;
// 3 cases are handled:
// 1. Index was already aliased and has tracking information; likely resuming from a completed job.
if (index != null)
{
Console.WriteLine($"Using alias `{index}`.");
return(index, aliased : true);
}
// 2. Index has not been aliased and has tracking information;
// likely resuming from an incomplete job or waiting to switch over.
index = metas.FirstOrDefault()?.Name;
if (index != null)
{
Console.WriteLine($"Using non-aliased `{index}`.");
return(index, aliased : false);
}
}
if (!AppSettings.IsRebuild && index == null)
{
throw new Exception("no existing index found");
}
// 3. Not aliased and no tracking information; likely starting from scratch
var suffix = Suffix ?? DateTimeOffset.UtcNow.ToUnixTimeSeconds().ToString();
index = $"{name}_{suffix}";
Console.WriteLine($"Creating `{index}` for `{name}`.");
// create by supplying the json file instead of the attributed class because we're not
// mapping every field but still want everything for _source.
var json = File.ReadAllText(Path.GetFullPath("schemas/high_scores.json"));
elasticClient.LowLevel.IndicesCreate <DynamicResponse>(index, json);
return(index, aliased : false);
// TODO: cases not covered should throw an Exception (aliased but not tracked, etc).
}
/// <summary>
/// Checks if the indexer should wait for the next pass or continue.
/// </summary>
/// <returns>true if ready; false, otherwise.</returns>
private bool checkIfReady()
{
if (AppSettings.IsRebuild || IndexMeta.GetByAliasForCurrentVersion(Name).FirstOrDefault() != null)
{
return(true);
}
Console.WriteLine($"`{Name}` for version {AppSettings.Schema} is not ready...");
return(false);
}
public static void Main()
{
DefaultTypeMap.MatchNamesWithUnderscores = true;
IndexMeta.CreateIndex();
if (AppSettings.IsWatching)
{
runWatchLoop();
}
else
{
// do a single run
runIndexing(AppSettings.ResumeFrom);
}
}
public void Run()
{
var index = findOrCreateIndex(Name);
// find out if we should be resuming
var resumeFrom = ResumeFrom ?? IndexMeta.GetByName(index)?.LastId;
Console.WriteLine();
Console.WriteLine($"{typeof(T)}, index `{index}`, chunkSize `{AppSettings.ChunkSize}`, resume `{resumeFrom}`");
Console.WriteLine();
var indexCompletedArgs = new IndexCompletedArgs
{
Alias = Name,
Index = index,
StartedAt = DateTime.Now
};
dispatcher = new BulkIndexingDispatcher <T>(Name, index);
try
{
var readerTask = databaseReaderTask(resumeFrom);
dispatcher.Run();
readerTask.Wait();
indexCompletedArgs.Count = readerTask.Result;
indexCompletedArgs.CompletedAt = DateTime.Now;
updateAlias(Name, index);
IndexCompleted(this, indexCompletedArgs);
}
catch (AggregateException ae)
{
ae.Handle(handleAggregateException);
}
// Local function exception handler.
bool handleAggregateException(Exception ex)
{
if (!(ex is InvalidOperationException))
{
return(false);
}
Console.WriteLine(ex.Message);
return(true);
}
}
/// <summary>
/// Reads a buffer and dispatches bulk index requests to Elasticsearch until the buffer
/// is marked as complete.
/// </summary>
internal void Run()
{
// custom partitioner and options to prevent Parallel.ForEach from going out of control.
var partitioner = Partitioner.Create(
readBuffer.GetConsumingEnumerable(),
EnumerablePartitionerOptions.NoBuffering // buffering causes spastic behaviour.
);
var options = new ParallelOptions {
MaxDegreeOfParallelism = AppSettings.Concurrency
};
Parallel.ForEach(partitioner, options, chunk =>
{
bool success;
while (true)
{
var bulkDescriptor = new BulkDescriptor().Index(index).IndexMany(chunk);
var response = elasticClient.Bulk(bulkDescriptor);
bool retry;
(success, retry) = retryOnResponse(response, chunk);
if (!retry)
{
break;
}
Task.Delay(AppSettings.BulkAllBackOffTimeDefault).Wait();
}
if (success)
{
// TODO: should probably aggregate responses and update to highest successful.
IndexMeta.UpdateAsync(new IndexMeta
{
Index = index,
Alias = alias,
LastId = chunk.Last().CursorValue,
UpdatedAt = DateTimeOffset.UtcNow
});
}
});
IndexMeta.Refresh();
}
public static void UpdateAsync(IndexMeta indexMeta)
{
client.IndexDocumentAsync(indexMeta);
}
public static void UpdateAsync(IndexMeta indexMeta)
{
ES_CLIENT.IndexDocumentAsync(indexMeta);
}
public void Run()
{
if (!checkIfReady())
{
return;
}
var indexMeta = getIndexMeta();
var metaSchema = AppSettings.IsPrepMode ? null : AppSettings.Schema;
var indexCompletedArgs = new IndexCompletedArgs
{
Alias = Name,
Index = indexMeta.Name,
StartedAt = DateTime.Now
};
dispatcher = new BulkIndexingDispatcher <T>(indexMeta.Name);
if (AppSettings.IsRebuild)
{
dispatcher.BatchWithLastIdCompleted += handleBatchWithLastIdCompleted;
}
try
{
var readerTask = databaseReaderTask(indexMeta.LastId);
dispatcher.Run();
readerTask.Wait();
indexCompletedArgs.Count = readerTask.Result;
indexCompletedArgs.CompletedAt = DateTime.Now;
IndexMeta.Refresh();
// when preparing for schema changes, the alias update
// should be done by process waiting for the ready signal.
if (AppSettings.IsRebuild)
{
if (AppSettings.IsPrepMode)
{
IndexMeta.MarkAsReady(indexMeta.Name);
}
else
{
updateAlias(Name, indexMeta.Name);
}
}
IndexCompleted(this, indexCompletedArgs);
}
catch (AggregateException ae)
{
ae.Handle(handleAggregateException);
}
// Local function exception handler.
bool handleAggregateException(Exception ex)
{
if (!(ex is InvalidOperationException))
{
return(false);
}
Console.Error.WriteLine(ex.Message);
if (ex.InnerException != null)
{
Console.Error.WriteLine(ex.InnerException.Message);
}
return(true);
}
void handleBatchWithLastIdCompleted(object sender, ulong lastId)
{
// TODO: should probably aggregate responses and update to highest successful.
IndexMeta.UpdateAsync(new IndexMeta
{
Name = indexMeta.Name,
Alias = Name,
LastId = lastId,
ResetQueueTo = indexMeta.ResetQueueTo,
UpdatedAt = DateTimeOffset.UtcNow,
Schema = metaSchema
});
}
}