public async Task Run(ReplayOptions opts)
        {
            _logger.Information("Starting replay (type: {Type}, after: {After})", opts.Type, opts.After);

            using var hasher = new SibrHasher();
            var updates = _updateStore.ExportAllUpdatesRaw(UpdateType.Stream, new UpdateStore.EntityVersionQuery {
                After = opts.After
            });

            var sw = new Stopwatch();

            await using var conn = await _db.Obtain();

            await foreach (var chunk in updates.Buffer(200))
            {
                var extracted = chunk.SelectMany(streamUpdate =>
                {
                    var obj = JObject.Parse(streamUpdate.Data.GetRawText());
                    return(TgbUtils.ExtractUpdatesFromStreamRoot(streamUpdate.SourceId, streamUpdate.Timestamp, obj, hasher, opts.Type).EntityUpdates);
                }).ToList();

                sw.Restart();
                await using var tx = await conn.BeginTransactionAsync();

                var saved = await _updateStore.SaveUpdates(conn, extracted, false);

                await tx.CommitAsync();

                sw.Stop();

                var timestamp = chunk.Min(u => u.Timestamp);
                _logger.Information("@ {Timestamp}: Saved {NewUpdateCount}/{UpdateCount} updates from {StreamObjects} stream objects (took {Duration})",
                                    timestamp, saved, extracted.Count, chunk.Count, sw.Elapsed);
            }
        }
Exemple #2
0
        private async Task HandleStreamData(string obj)
        {
            var timestamp = _clock.GetCurrentInstant();
            var data      = JObject.Parse(obj);

            await using var conn = await _db.Obtain();

            await using (var tx = await conn.BeginTransactionAsync())
            {
                await _updateStore.SaveUpdate(conn, EntityUpdate.From(UpdateType.Stream, _sourceId, timestamp, data));

                await tx.CommitAsync();
            }

            await using (var tx = await conn.BeginTransactionAsync())
            {
                using var hasher = new SibrHasher();
                var extracted = TgbUtils.ExtractUpdatesFromStreamRoot(_sourceId, timestamp, data, hasher);
                var gameRes   = await _gameStore.SaveGameUpdates(conn, extracted.GameUpdates);

                var miscRes = await _updateStore.SaveUpdates(conn, extracted.EntityUpdates);

                var maxPlayCount = extracted.GameUpdates.Count > 0 ? extracted.GameUpdates.Max(gu => gu.PlayCount) : -1;

                _logger.Information(
                    "Received stream update, saved {GameUpdates} game updates, {MiscUpdates} updates, max PC {MaxPlayCount}",
                    gameRes, miscRes, maxPlayCount);

                await tx.CommitAsync();
            }
        }
        protected override async Task ProcessFile(string filename, IAsyncEnumerable <JToken> entries)
        {
            using var hasher = new SibrHasher();
            var streamUpdates = new List <EntityUpdate>();
            var miscUpdates   = new List <EntityUpdate>();
            var gameUpdates   = new List <GameUpdate>();
            var gameIds       = new HashSet <Guid>();

            var count = 0;

            await using var conn = await _db.Obtain();

            await foreach (var entry in entries)
            {
                var timestamp = ExtractTimestamp(entry);
                if (timestamp == null)
                {
                    continue;
                }

                var root = FindStreamRoot(entry as JObject);
                streamUpdates.Add(EntityUpdate.From(UpdateType.Stream, _sourceId, timestamp.Value, root));

                var updates = TgbUtils.ExtractUpdatesFromStreamRoot(_sourceId, timestamp.Value, root, hasher);
                gameUpdates.AddRange(updates.GameUpdates);
                miscUpdates.AddRange(updates.EntityUpdates);

                gameIds.UnionWith(updates.GameUpdates.Select(g => g.GameId));
                if (count++ % 100 == 0)
                {
                    await FlushUpdates(conn, streamUpdates, gameUpdates, miscUpdates);
                }
            }

            await FlushUpdates(conn, streamUpdates, gameUpdates, miscUpdates);

            await _gameStore.TryAddNewGameIds(conn, gameUpdates.Select(gu => gu.GameId));

            await _gameUpdateStore.UpdateSearchIndex(conn);
        }
Exemple #4
0
        public async Task Run(ReplayOptions opts)
        {
            _logger.Information("Starting replay (type: {Type}, start: {Start}, end: {End})",
                                opts.Type != null ? string.Join(",", opts.Type) : null, opts.Start, opts.End);

            using var hasher = new SibrHasher();

            var sw = new Stopwatch();

            await using var conn = await _db.Obtain();

            var page = opts.Start != null ? new PageToken(opts.Start.Value, default) : null;

            while (true)
            {
                var chunk = await _updateStore.ExportAllUpdatesChunked(conn, UpdateType.Stream,
                                                                       new UpdateStore.EntityVersionQuery
                {
                    Page   = page,
                    Before = opts.End,
                    Order  = SortOrder.Asc,
                    Count  = 500
                });

                if (chunk.Count == 0)
                {
                    break;
                }
                page = chunk.Last().NextPage;

                // if (opts.Type == UpdateType.Game)
                if (false) // todo: uhhh how do we do this
                {
                    var extractedGameUpdates = chunk.SelectMany(streamUpdate =>
                    {
                        var obj = JObject.Parse(streamUpdate.Data.GetRawText());
                        return(TgbUtils.ExtractUpdatesFromStreamRoot(streamUpdate.SourceId, streamUpdate.Timestamp, obj, hasher, opts.Type).GameUpdates);
                    }).ToList();

                    sw.Restart();
                    await using var tx = await conn.BeginTransactionAsync();

                    var savedGameUpdates = await _gameUpdateStore.SaveGameUpdates(conn, extractedGameUpdates, false);

                    await tx.CommitAsync();

                    sw.Stop();

                    var timestamp = chunk.Min(u => u.Timestamp);
                    _logger.Information("@ {Timestamp}: Saved {GameUpdateCount}/{UpdateCount} game updates from {StreamObjects} stream objects (took {Duration})",
                                        timestamp, savedGameUpdates, extractedGameUpdates.Count, chunk.Count, sw.Elapsed);
                }
                else
                {
                    var extractedUpdates = chunk.SelectMany(streamUpdate =>
                    {
                        var obj = JObject.Parse(streamUpdate.Data.GetRawText());
                        return(TgbUtils.ExtractUpdatesFromStreamRoot(streamUpdate.SourceId, streamUpdate.Timestamp, obj,
                                                                     hasher, opts.Type).EntityUpdates);
                    }).ToList();

                    sw.Restart();
                    await using var tx = await conn.BeginTransactionAsync();

                    var savedUpdates = await _updateStore.SaveUpdates(conn, extractedUpdates, false, append : false);

                    await tx.CommitAsync();

                    sw.Stop();

                    var timestamp = chunk.Min(u => u.Timestamp);
                    _logger.Information("@ {Timestamp}: Saved {NewUpdateCount}/{UpdateCount} updates from {StreamObjects} stream objects (took {Duration})",
                                        timestamp, savedUpdates, extractedUpdates.Count, chunk.Count, sw.Elapsed);
                }
            }
        }