예제 #1
0
        private IndexMeta getIndexMeta()
        {
            // TODO: all this needs cleaning.
            var(index, aliased) = findOrCreateIndex(Name);

            if (!AppSettings.IsRebuild && !aliased)
            {
                updateAlias(Name, index);
            }

            // look for any existing resume data.
            var indexMeta = IndexMeta.GetByName(index) ?? new IndexMeta
            {
                Alias = Name,
                Name  = index,
            };

            indexMeta.LastId = ResumeFrom ?? indexMeta.LastId;

            if (AppSettings.IsRebuild)
            {
                // Save the position the score processing queue should be reset to if rebuilding an index.
                // If there is already an existing value, the processor is probabaly resuming from a previous run,
                // so we likely want to preserve that value.
                if (!indexMeta.ResetQueueTo.HasValue)
                {
                    var mode = HighScore.GetRulesetId <T>();
                    indexMeta.ResetQueueTo = ScoreProcessQueue.GetLastProcessedQueueId(mode);
                }
            }
            else
            {
                if (string.IsNullOrWhiteSpace(indexMeta.Schema))
                {
                    throw new Exception("FATAL ERROR: attempting to process queue without a known version.");
                }

                if (indexMeta.Schema != AppSettings.Schema)
                {
                    // A switchover is probably happening, so signal that this mode should be skipped.
                    throw new VersionMismatchException($"`{Name}` found version {indexMeta.Schema}, expecting {AppSettings.Schema}");
                }

                // process queue reset if any.
                if (indexMeta.ResetQueueTo.HasValue)
                {
                    Console.WriteLine($"Resetting queue_id > {indexMeta.ResetQueueTo}");
                    ScoreProcessQueue.UnCompleteQueued <T>(indexMeta.ResetQueueTo.Value);
                    indexMeta.ResetQueueTo = null;
                }
            }

            indexMeta.UpdatedAt = DateTimeOffset.UtcNow;
            IndexMeta.UpdateAsync(indexMeta);
            IndexMeta.Refresh();

            return(indexMeta);
        }
예제 #2
0
        public static void Main()
        {
            DogStatsd.Configure(new StatsdConfig
            {
                StatsdServerName = "127.0.0.1",
                Prefix           = "elasticsearch.scores"
            });

            if (AppSettings.UseDocker)
            {
                Console.WriteLine("Waiting for database...");

                while (true)
                {
                    try
                    {
                        using (var conn = new MySqlConnection(AppSettings.ConnectionString))
                        {
                            if (conn.QuerySingle <int>("SELECT `count` FROM `osu_counts` WHERE `name` = 'docker_db_step'") >= 3)
                            {
                                break;
                            }
                        }
                    }
                    catch
                    {
                    }

                    Thread.Sleep(1000);
                }
            }

            DefaultTypeMap.MatchNamesWithUnderscores = true;
            IndexMeta.CreateIndex();

            Console.WriteLine($"Rebuilding index: `{AppSettings.IsRebuild}`");
            if (AppSettings.IsWatching)
            {
                runWatchLoop();
            }
            else
            {
                runIndexing();
            }

            if (AppSettings.UseDocker)
            {
                using (var conn = new MySqlConnection(AppSettings.ConnectionString))
                {
                    conn.Execute("INSERT INTO `osu_counts` (`name`, `count`) VALUES (@Name, @Count) ON DUPLICATE KEY UPDATE `count` = @Count", new
                    {
                        Name  = "docker_db_step",
                        Count = 4
                    });
                }
            }
        }
예제 #3
0
        /// <summary>
        /// Attempts to find the matching index or creates a new one.
        /// </summary>
        /// <param name="name">name of the index alias.</param>
        /// <returns>Name of index found or created and any existing alias.</returns>
        private (string index, bool aliased) findOrCreateIndex(string name)
        {
            Console.WriteLine();

            var aliasedIndices = elasticClient.GetIndicesPointingToAlias(name);
            var metas          = (
                AppSettings.IsRebuild
                    ? IndexMeta.GetByAlias(name)
                    : IndexMeta.GetByAliasForCurrentVersion(name)
                ).ToList();

            string index = null;

            if (!AppSettings.IsNew)
            {
                // TODO: query ES that the index actually exists.

                index = metas.FirstOrDefault(m => aliasedIndices.Contains(m.Name))?.Name;
                // 3 cases are handled:
                // 1. Index was already aliased and has tracking information; likely resuming from a completed job.
                if (index != null)
                {
                    Console.WriteLine($"Using alias `{index}`.");
                    return(index, aliased : true);
                }

                // 2. Index has not been aliased and has tracking information;
                // likely resuming from an incomplete job or waiting to switch over.
                index = metas.FirstOrDefault()?.Name;
                if (index != null)
                {
                    Console.WriteLine($"Using non-aliased `{index}`.");
                    return(index, aliased : false);
                }
            }

            if (!AppSettings.IsRebuild && index == null)
            {
                throw new Exception("no existing index found");
            }

            // 3. Not aliased and no tracking information; likely starting from scratch
            var suffix = Suffix ?? DateTimeOffset.UtcNow.ToUnixTimeSeconds().ToString();

            index = $"{name}_{suffix}";

            Console.WriteLine($"Creating `{index}` for `{name}`.");
            // create by supplying the json file instead of the attributed class because we're not
            // mapping every field but still want everything for _source.
            var json = File.ReadAllText(Path.GetFullPath("schemas/high_scores.json"));

            elasticClient.LowLevel.IndicesCreate <DynamicResponse>(index, json);

            return(index, aliased : false);

            // TODO: cases not covered should throw an Exception (aliased but not tracked, etc).
        }
예제 #4
0
        /// <summary>
        /// Checks if the indexer should wait for the next pass or continue.
        /// </summary>
        /// <returns>true if ready; false, otherwise.</returns>
        private bool checkIfReady()
        {
            if (AppSettings.IsRebuild || IndexMeta.GetByAliasForCurrentVersion(Name).FirstOrDefault() != null)
            {
                return(true);
            }

            Console.WriteLine($"`{Name}` for version {AppSettings.Schema} is not ready...");
            return(false);
        }
예제 #5
0
        public static void Main()
        {
            DefaultTypeMap.MatchNamesWithUnderscores = true;
            IndexMeta.CreateIndex();

            if (AppSettings.IsWatching)
            {
                runWatchLoop();
            }
            else
            {
                // do a single run
                runIndexing(AppSettings.ResumeFrom);
            }
        }
예제 #6
0
        public void Run()
        {
            var index = findOrCreateIndex(Name);
            // find out if we should be resuming
            var resumeFrom = ResumeFrom ?? IndexMeta.GetByName(index)?.LastId;

            Console.WriteLine();
            Console.WriteLine($"{typeof(T)}, index `{index}`, chunkSize `{AppSettings.ChunkSize}`, resume `{resumeFrom}`");
            Console.WriteLine();

            var indexCompletedArgs = new IndexCompletedArgs
            {
                Alias     = Name,
                Index     = index,
                StartedAt = DateTime.Now
            };

            dispatcher = new BulkIndexingDispatcher <T>(Name, index);

            try
            {
                var readerTask = databaseReaderTask(resumeFrom);
                dispatcher.Run();
                readerTask.Wait();

                indexCompletedArgs.Count       = readerTask.Result;
                indexCompletedArgs.CompletedAt = DateTime.Now;

                updateAlias(Name, index);
                IndexCompleted(this, indexCompletedArgs);
            }
            catch (AggregateException ae)
            {
                ae.Handle(handleAggregateException);
            }

            // Local function exception handler.
            bool handleAggregateException(Exception ex)
            {
                if (!(ex is InvalidOperationException))
                {
                    return(false);
                }

                Console.WriteLine(ex.Message);
                return(true);
            }
        }
예제 #7
0
        /// <summary>
        /// Reads a buffer and dispatches bulk index requests to Elasticsearch until the buffer
        /// is marked as complete.
        /// </summary>
        internal void Run()
        {
            // custom partitioner and options to prevent Parallel.ForEach from going out of control.
            var partitioner = Partitioner.Create(
                readBuffer.GetConsumingEnumerable(),
                EnumerablePartitionerOptions.NoBuffering // buffering causes spastic behaviour.
                );

            var options = new ParallelOptions {
                MaxDegreeOfParallelism = AppSettings.Concurrency
            };

            Parallel.ForEach(partitioner, options, chunk =>
            {
                bool success;

                while (true)
                {
                    var bulkDescriptor = new BulkDescriptor().Index(index).IndexMany(chunk);
                    var response       = elasticClient.Bulk(bulkDescriptor);

                    bool retry;
                    (success, retry) = retryOnResponse(response, chunk);

                    if (!retry)
                    {
                        break;
                    }

                    Task.Delay(AppSettings.BulkAllBackOffTimeDefault).Wait();
                }

                if (success)
                {
                    // TODO: should probably aggregate responses and update to highest successful.
                    IndexMeta.UpdateAsync(new IndexMeta
                    {
                        Index     = index,
                        Alias     = alias,
                        LastId    = chunk.Last().CursorValue,
                        UpdatedAt = DateTimeOffset.UtcNow
                    });
                }
            });

            IndexMeta.Refresh();
        }
예제 #8
0
 public static void UpdateAsync(IndexMeta indexMeta)
 {
     client.IndexDocumentAsync(indexMeta);
 }
예제 #9
0
 public static void UpdateAsync(IndexMeta indexMeta)
 {
     ES_CLIENT.IndexDocumentAsync(indexMeta);
 }
예제 #10
0
        public void Run()
        {
            if (!checkIfReady())
            {
                return;
            }

            var indexMeta = getIndexMeta();

            var metaSchema = AppSettings.IsPrepMode ? null : AppSettings.Schema;

            var indexCompletedArgs = new IndexCompletedArgs
            {
                Alias     = Name,
                Index     = indexMeta.Name,
                StartedAt = DateTime.Now
            };

            dispatcher = new BulkIndexingDispatcher <T>(indexMeta.Name);

            if (AppSettings.IsRebuild)
            {
                dispatcher.BatchWithLastIdCompleted += handleBatchWithLastIdCompleted;
            }

            try
            {
                var readerTask = databaseReaderTask(indexMeta.LastId);
                dispatcher.Run();
                readerTask.Wait();

                indexCompletedArgs.Count       = readerTask.Result;
                indexCompletedArgs.CompletedAt = DateTime.Now;

                IndexMeta.Refresh();

                // when preparing for schema changes, the alias update
                // should be done by process waiting for the ready signal.
                if (AppSettings.IsRebuild)
                {
                    if (AppSettings.IsPrepMode)
                    {
                        IndexMeta.MarkAsReady(indexMeta.Name);
                    }
                    else
                    {
                        updateAlias(Name, indexMeta.Name);
                    }
                }

                IndexCompleted(this, indexCompletedArgs);
            }
            catch (AggregateException ae)
            {
                ae.Handle(handleAggregateException);
            }

            // Local function exception handler.
            bool handleAggregateException(Exception ex)
            {
                if (!(ex is InvalidOperationException))
                {
                    return(false);
                }

                Console.Error.WriteLine(ex.Message);
                if (ex.InnerException != null)
                {
                    Console.Error.WriteLine(ex.InnerException.Message);
                }

                return(true);
            }

            void handleBatchWithLastIdCompleted(object sender, ulong lastId)
            {
                // TODO: should probably aggregate responses and update to highest successful.
                IndexMeta.UpdateAsync(new IndexMeta
                {
                    Name         = indexMeta.Name,
                    Alias        = Name,
                    LastId       = lastId,
                    ResetQueueTo = indexMeta.ResetQueueTo,
                    UpdatedAt    = DateTimeOffset.UtcNow,
                    Schema       = metaSchema
                });
            }
        }