private IndexMeta getIndexMeta() { // TODO: all this needs cleaning. var(index, aliased) = findOrCreateIndex(Name); if (!AppSettings.IsRebuild && !aliased) { updateAlias(Name, index); } // look for any existing resume data. var indexMeta = IndexMeta.GetByName(index) ?? new IndexMeta { Alias = Name, Name = index, }; indexMeta.LastId = ResumeFrom ?? indexMeta.LastId; if (AppSettings.IsRebuild) { // Save the position the score processing queue should be reset to if rebuilding an index. // If there is already an existing value, the processor is probabaly resuming from a previous run, // so we likely want to preserve that value. if (!indexMeta.ResetQueueTo.HasValue) { var mode = HighScore.GetRulesetId <T>(); indexMeta.ResetQueueTo = ScoreProcessQueue.GetLastProcessedQueueId(mode); } } else { if (string.IsNullOrWhiteSpace(indexMeta.Schema)) { throw new Exception("FATAL ERROR: attempting to process queue without a known version."); } if (indexMeta.Schema != AppSettings.Schema) { // A switchover is probably happening, so signal that this mode should be skipped. throw new VersionMismatchException($"`{Name}` found version {indexMeta.Schema}, expecting {AppSettings.Schema}"); } // process queue reset if any. if (indexMeta.ResetQueueTo.HasValue) { Console.WriteLine($"Resetting queue_id > {indexMeta.ResetQueueTo}"); ScoreProcessQueue.UnCompleteQueued <T>(indexMeta.ResetQueueTo.Value); indexMeta.ResetQueueTo = null; } } indexMeta.UpdatedAt = DateTimeOffset.UtcNow; IndexMeta.UpdateAsync(indexMeta); IndexMeta.Refresh(); return(indexMeta); }
public static void Main() { DogStatsd.Configure(new StatsdConfig { StatsdServerName = "127.0.0.1", Prefix = "elasticsearch.scores" }); if (AppSettings.UseDocker) { Console.WriteLine("Waiting for database..."); while (true) { try { using (var conn = new MySqlConnection(AppSettings.ConnectionString)) { if (conn.QuerySingle <int>("SELECT `count` FROM `osu_counts` WHERE `name` = 'docker_db_step'") >= 3) { break; } } } catch { } Thread.Sleep(1000); } } DefaultTypeMap.MatchNamesWithUnderscores = true; IndexMeta.CreateIndex(); Console.WriteLine($"Rebuilding index: `{AppSettings.IsRebuild}`"); if (AppSettings.IsWatching) { runWatchLoop(); } else { runIndexing(); } if (AppSettings.UseDocker) { using (var conn = new MySqlConnection(AppSettings.ConnectionString)) { conn.Execute("INSERT INTO `osu_counts` (`name`, `count`) VALUES (@Name, @Count) ON DUPLICATE KEY UPDATE `count` = @Count", new { Name = "docker_db_step", Count = 4 }); } } }
/// <summary> /// Attempts to find the matching index or creates a new one. /// </summary> /// <param name="name">name of the index alias.</param> /// <returns>Name of index found or created and any existing alias.</returns> private (string index, bool aliased) findOrCreateIndex(string name) { Console.WriteLine(); var aliasedIndices = elasticClient.GetIndicesPointingToAlias(name); var metas = ( AppSettings.IsRebuild ? IndexMeta.GetByAlias(name) : IndexMeta.GetByAliasForCurrentVersion(name) ).ToList(); string index = null; if (!AppSettings.IsNew) { // TODO: query ES that the index actually exists. index = metas.FirstOrDefault(m => aliasedIndices.Contains(m.Name))?.Name; // 3 cases are handled: // 1. Index was already aliased and has tracking information; likely resuming from a completed job. if (index != null) { Console.WriteLine($"Using alias `{index}`."); return(index, aliased : true); } // 2. Index has not been aliased and has tracking information; // likely resuming from an incomplete job or waiting to switch over. index = metas.FirstOrDefault()?.Name; if (index != null) { Console.WriteLine($"Using non-aliased `{index}`."); return(index, aliased : false); } } if (!AppSettings.IsRebuild && index == null) { throw new Exception("no existing index found"); } // 3. Not aliased and no tracking information; likely starting from scratch var suffix = Suffix ?? DateTimeOffset.UtcNow.ToUnixTimeSeconds().ToString(); index = $"{name}_{suffix}"; Console.WriteLine($"Creating `{index}` for `{name}`."); // create by supplying the json file instead of the attributed class because we're not // mapping every field but still want everything for _source. var json = File.ReadAllText(Path.GetFullPath("schemas/high_scores.json")); elasticClient.LowLevel.IndicesCreate <DynamicResponse>(index, json); return(index, aliased : false); // TODO: cases not covered should throw an Exception (aliased but not tracked, etc). }
/// <summary> /// Checks if the indexer should wait for the next pass or continue. /// </summary> /// <returns>true if ready; false, otherwise.</returns> private bool checkIfReady() { if (AppSettings.IsRebuild || IndexMeta.GetByAliasForCurrentVersion(Name).FirstOrDefault() != null) { return(true); } Console.WriteLine($"`{Name}` for version {AppSettings.Schema} is not ready..."); return(false); }
public static void Main() { DefaultTypeMap.MatchNamesWithUnderscores = true; IndexMeta.CreateIndex(); if (AppSettings.IsWatching) { runWatchLoop(); } else { // do a single run runIndexing(AppSettings.ResumeFrom); } }
public void Run() { var index = findOrCreateIndex(Name); // find out if we should be resuming var resumeFrom = ResumeFrom ?? IndexMeta.GetByName(index)?.LastId; Console.WriteLine(); Console.WriteLine($"{typeof(T)}, index `{index}`, chunkSize `{AppSettings.ChunkSize}`, resume `{resumeFrom}`"); Console.WriteLine(); var indexCompletedArgs = new IndexCompletedArgs { Alias = Name, Index = index, StartedAt = DateTime.Now }; dispatcher = new BulkIndexingDispatcher <T>(Name, index); try { var readerTask = databaseReaderTask(resumeFrom); dispatcher.Run(); readerTask.Wait(); indexCompletedArgs.Count = readerTask.Result; indexCompletedArgs.CompletedAt = DateTime.Now; updateAlias(Name, index); IndexCompleted(this, indexCompletedArgs); } catch (AggregateException ae) { ae.Handle(handleAggregateException); } // Local function exception handler. bool handleAggregateException(Exception ex) { if (!(ex is InvalidOperationException)) { return(false); } Console.WriteLine(ex.Message); return(true); } }
/// <summary> /// Reads a buffer and dispatches bulk index requests to Elasticsearch until the buffer /// is marked as complete. /// </summary> internal void Run() { // custom partitioner and options to prevent Parallel.ForEach from going out of control. var partitioner = Partitioner.Create( readBuffer.GetConsumingEnumerable(), EnumerablePartitionerOptions.NoBuffering // buffering causes spastic behaviour. ); var options = new ParallelOptions { MaxDegreeOfParallelism = AppSettings.Concurrency }; Parallel.ForEach(partitioner, options, chunk => { bool success; while (true) { var bulkDescriptor = new BulkDescriptor().Index(index).IndexMany(chunk); var response = elasticClient.Bulk(bulkDescriptor); bool retry; (success, retry) = retryOnResponse(response, chunk); if (!retry) { break; } Task.Delay(AppSettings.BulkAllBackOffTimeDefault).Wait(); } if (success) { // TODO: should probably aggregate responses and update to highest successful. IndexMeta.UpdateAsync(new IndexMeta { Index = index, Alias = alias, LastId = chunk.Last().CursorValue, UpdatedAt = DateTimeOffset.UtcNow }); } }); IndexMeta.Refresh(); }
public static void UpdateAsync(IndexMeta indexMeta) { client.IndexDocumentAsync(indexMeta); }
public static void UpdateAsync(IndexMeta indexMeta) { ES_CLIENT.IndexDocumentAsync(indexMeta); }
public void Run() { if (!checkIfReady()) { return; } var indexMeta = getIndexMeta(); var metaSchema = AppSettings.IsPrepMode ? null : AppSettings.Schema; var indexCompletedArgs = new IndexCompletedArgs { Alias = Name, Index = indexMeta.Name, StartedAt = DateTime.Now }; dispatcher = new BulkIndexingDispatcher <T>(indexMeta.Name); if (AppSettings.IsRebuild) { dispatcher.BatchWithLastIdCompleted += handleBatchWithLastIdCompleted; } try { var readerTask = databaseReaderTask(indexMeta.LastId); dispatcher.Run(); readerTask.Wait(); indexCompletedArgs.Count = readerTask.Result; indexCompletedArgs.CompletedAt = DateTime.Now; IndexMeta.Refresh(); // when preparing for schema changes, the alias update // should be done by process waiting for the ready signal. if (AppSettings.IsRebuild) { if (AppSettings.IsPrepMode) { IndexMeta.MarkAsReady(indexMeta.Name); } else { updateAlias(Name, indexMeta.Name); } } IndexCompleted(this, indexCompletedArgs); } catch (AggregateException ae) { ae.Handle(handleAggregateException); } // Local function exception handler. bool handleAggregateException(Exception ex) { if (!(ex is InvalidOperationException)) { return(false); } Console.Error.WriteLine(ex.Message); if (ex.InnerException != null) { Console.Error.WriteLine(ex.InnerException.Message); } return(true); } void handleBatchWithLastIdCompleted(object sender, ulong lastId) { // TODO: should probably aggregate responses and update to highest successful. IndexMeta.UpdateAsync(new IndexMeta { Name = indexMeta.Name, Alias = Name, LastId = lastId, ResetQueueTo = indexMeta.ResetQueueTo, UpdatedAt = DateTimeOffset.UtcNow, Schema = metaSchema }); } }