コード例 #1
0
        /// <summary>
        /// Self contained database reader task. Reads the database by cursoring through records
        /// and adding chunks into readBuffer.
        /// </summary>
        /// <param name="resumeFrom">The cursor value to resume from;
        /// use null to resume from the last known value.</param>
        /// <returns>The database reader task.</returns>
        private Task <long> databaseReaderTask(long?resumeFrom)
        {
            return(Task.Factory.StartNew(() =>
            {
                long count = 0;

                while (true)
                {
                    try
                    {
                        var chunks = Model.Chunk <T>("pp is not null", AppSettings.ChunkSize, resumeFrom);
                        foreach (var chunk in chunks)
                        {
                            dispatcher.Enqueue(chunk);
                            count += chunk.Count;
                            // update resumeFrom in this scope to allow resuming from connection errors.
                            resumeFrom = chunk.Last().CursorValue;
                        }

                        break;
                    }
                    catch (DbException ex)
                    {
                        Console.Error.WriteLine(ex.Message);
                        Task.Delay(1000).Wait();
                    }
                }

                dispatcher.EnqueueEnd();
                Console.WriteLine($"Finished reading database {count} records.");

                return count;
            }, TaskCreationOptions.LongRunning | TaskCreationOptions.DenyChildAttach));
        }
コード例 #2
0
        /// <summary>
        /// Self contained database reader task. Reads the database by cursoring through records
        /// and adding chunks into readBuffer.
        /// </summary>
        /// <param name="resumeFrom">The cursor value to resume from;
        /// use null to resume from the last known value.</param>
        /// <returns>The database reader task.</returns>
        private Task <long> databaseReaderTask(ulong resumeFrom) => Task.Factory.StartNew(() =>
        {
            long count = 0;
            var mode   = HighScore.GetRulesetId <T>();

            while (true)
            {
                try
                {
                    if (!AppSettings.IsRebuild)
                    {
                        Console.WriteLine("Reading from queue...");
                        var chunks = Model.Chunk <ScoreProcessQueue>($"status = 1 and mode = {mode}", AppSettings.ChunkSize);
                        foreach (var chunk in chunks)
                        {
                            var scoreIds      = chunk.Select(x => x.ScoreId).ToList();
                            var scores        = ScoreProcessQueue.FetchByScoreIds <T>(scoreIds).Where(x => x.ShouldIndex).ToList();
                            var removedScores = scoreIds
                                                .Except(scores.Select(x => x.ScoreId))
                                                .Select(scoreId => new T {
                                ScoreId = scoreId
                            })
                                                .ToList();
                            Console.WriteLine($"Got {chunk.Count} items from queue, found {scores.Count} matching scores, {removedScores.Count} missing scores");

                            DogStatsd.Increment("indexed", scores.Count, tags: new[] { $"mode:{mode}", "result:success" });
                            DogStatsd.Increment("indexed", removedScores.Count, tags: new[] { $"mode:{mode}", "result:missing" });

                            dispatcher.Enqueue(add: scores, remove: removedScores);
                            ScoreProcessQueue.CompleteQueued(chunk);
                            count += scores.Count;
                        }
                    }
                    else
                    {
                        Console.WriteLine($"Rebuild from {resumeFrom}...");
                        var chunks = Model.Chunk <T>(AppSettings.ChunkSize, resumeFrom);
                        foreach (var chunk in chunks)
                        {
                            var scores = chunk.Where(x => x.ShouldIndex).ToList();

                            dispatcher.Enqueue(scores);
                            DogStatsd.Increment("indexed", scores.Count, tags: new[] { $"mode:{mode}", "result:success" });
                            count += chunk.Count;
                            // update resumeFrom in this scope to allow resuming from connection errors.
                            resumeFrom = chunk.Last().CursorValue;
                        }
                    }

                    break;
                }
                catch (DbException ex)
                {
                    Console.Error.WriteLine(ex.Message);
                    Task.Delay(1000).Wait();
                }
            }

            dispatcher.EnqueueEnd();
            Console.WriteLine($"Finished reading database {count} records.");

            return(count);
        }, TaskCreationOptions.LongRunning | TaskCreationOptions.DenyChildAttach);