public async Task Run(ReplayOptions opts) { _logger.Information("Starting replay (type: {Type}, after: {After})", opts.Type, opts.After); using var hasher = new SibrHasher(); var updates = _updateStore.ExportAllUpdatesRaw(UpdateType.Stream, new UpdateStore.EntityVersionQuery { After = opts.After }); var sw = new Stopwatch(); await using var conn = await _db.Obtain(); await foreach (var chunk in updates.Buffer(200)) { var extracted = chunk.SelectMany(streamUpdate => { var obj = JObject.Parse(streamUpdate.Data.GetRawText()); return(TgbUtils.ExtractUpdatesFromStreamRoot(streamUpdate.SourceId, streamUpdate.Timestamp, obj, hasher, opts.Type).EntityUpdates); }).ToList(); sw.Restart(); await using var tx = await conn.BeginTransactionAsync(); var saved = await _updateStore.SaveUpdates(conn, extracted, false); await tx.CommitAsync(); sw.Stop(); var timestamp = chunk.Min(u => u.Timestamp); _logger.Information("@ {Timestamp}: Saved {NewUpdateCount}/{UpdateCount} updates from {StreamObjects} stream objects (took {Duration})", timestamp, saved, extracted.Count, chunk.Count, sw.Elapsed); } }
private async Task HandleStreamData(string obj) { var timestamp = _clock.GetCurrentInstant(); var data = JObject.Parse(obj); await using var conn = await _db.Obtain(); await using (var tx = await conn.BeginTransactionAsync()) { await _updateStore.SaveUpdate(conn, EntityUpdate.From(UpdateType.Stream, _sourceId, timestamp, data)); await tx.CommitAsync(); } await using (var tx = await conn.BeginTransactionAsync()) { using var hasher = new SibrHasher(); var extracted = TgbUtils.ExtractUpdatesFromStreamRoot(_sourceId, timestamp, data, hasher); var gameRes = await _gameStore.SaveGameUpdates(conn, extracted.GameUpdates); var miscRes = await _updateStore.SaveUpdates(conn, extracted.EntityUpdates); var maxPlayCount = extracted.GameUpdates.Count > 0 ? extracted.GameUpdates.Max(gu => gu.PlayCount) : -1; _logger.Information( "Received stream update, saved {GameUpdates} game updates, {MiscUpdates} updates, max PC {MaxPlayCount}", gameRes, miscRes, maxPlayCount); await tx.CommitAsync(); } }
private async Task <IEnumerable <EntityUpdate> > GetUpdatesByIds(UpdateType type, string baseUrl, IEnumerable <string> ids, SibrHasher hasher) { var(timestamp, data) = await _client.GetJsonAsync($"{baseUrl}?ids={string.Join(',', ids)}"); return(data.Values <JObject>().Select(obj => { var id = TgbUtils.GenerateGuidFromString(obj.Value <string>("id")); return EntityUpdate.From(type, _sourceId, timestamp, obj, hasher, id); })); }
protected override async Task ProcessFile(string filename, IAsyncEnumerable <JToken> entries) { using var hasher = new SibrHasher(); var streamUpdates = new List <EntityUpdate>(); var miscUpdates = new List <EntityUpdate>(); var gameUpdates = new List <GameUpdate>(); var gameIds = new HashSet <Guid>(); var count = 0; await using var conn = await _db.Obtain(); await foreach (var entry in entries) { var timestamp = ExtractTimestamp(entry); if (timestamp == null) { continue; } var root = FindStreamRoot(entry as JObject); streamUpdates.Add(EntityUpdate.From(UpdateType.Stream, _sourceId, timestamp.Value, root)); var updates = TgbUtils.ExtractUpdatesFromStreamRoot(_sourceId, timestamp.Value, root, hasher); gameUpdates.AddRange(updates.GameUpdates); miscUpdates.AddRange(updates.EntityUpdates); gameIds.UnionWith(updates.GameUpdates.Select(g => g.GameId)); if (count++ % 100 == 0) { await FlushUpdates(conn, streamUpdates, gameUpdates, miscUpdates); } } await FlushUpdates(conn, streamUpdates, gameUpdates, miscUpdates); await _gameStore.TryAddNewGameIds(conn, gameUpdates.Select(gu => gu.GameId)); await _gameUpdateStore.UpdateSearchIndex(conn); }
private void GlobalHandler(string _, PusherEvent evt) { // Get timestamp earliest possible (outside async) var timestamp = _clock.GetCurrentInstant(); async Task Inner() { await Task.Yield(); try { var data = TgbUtils.TryDecodePusherData(evt.Data); await _pusherEventStore.SaveEvent(evt.ChannelName, evt.EventName, timestamp, data, evt.Data); } catch (Exception e) { _logger.Error(e, "Error handling Pusher event {ChannelName}/{EventName}", evt.ChannelName, evt.EventName); } } // Fork var __ = Inner(); }
private Action <PusherEvent> WrapHandler(Func <JToken, Task> inner) { async Task Inner(PusherEvent evt) { try { _logger.Verbose("Received Pusher event {EventName} on {ChannelName}: {Data}", evt.EventName, evt.ChannelName, evt.Data); var data = TgbUtils.TryDecodePusherData(evt.Data); await inner(data); } catch (Exception e) { _logger.Error(e, "Error handling Pusher event {EventName}", evt.EventName); } } return(evt => { // Fork inner task var _ = Inner(evt); }); }
public async Task Run(ReplayOptions opts) { _logger.Information("Starting replay (type: {Type}, start: {Start}, end: {End})", opts.Type != null ? string.Join(",", opts.Type) : null, opts.Start, opts.End); using var hasher = new SibrHasher(); var sw = new Stopwatch(); await using var conn = await _db.Obtain(); var page = opts.Start != null ? new PageToken(opts.Start.Value, default) : null; while (true) { var chunk = await _updateStore.ExportAllUpdatesChunked(conn, UpdateType.Stream, new UpdateStore.EntityVersionQuery { Page = page, Before = opts.End, Order = SortOrder.Asc, Count = 500 }); if (chunk.Count == 0) { break; } page = chunk.Last().NextPage; // if (opts.Type == UpdateType.Game) if (false) // todo: uhhh how do we do this { var extractedGameUpdates = chunk.SelectMany(streamUpdate => { var obj = JObject.Parse(streamUpdate.Data.GetRawText()); return(TgbUtils.ExtractUpdatesFromStreamRoot(streamUpdate.SourceId, streamUpdate.Timestamp, obj, hasher, opts.Type).GameUpdates); }).ToList(); sw.Restart(); await using var tx = await conn.BeginTransactionAsync(); var savedGameUpdates = await _gameUpdateStore.SaveGameUpdates(conn, extractedGameUpdates, false); await tx.CommitAsync(); sw.Stop(); var timestamp = chunk.Min(u => u.Timestamp); _logger.Information("@ {Timestamp}: Saved {GameUpdateCount}/{UpdateCount} game updates from {StreamObjects} stream objects (took {Duration})", timestamp, savedGameUpdates, extractedGameUpdates.Count, chunk.Count, sw.Elapsed); } else { var extractedUpdates = chunk.SelectMany(streamUpdate => { var obj = JObject.Parse(streamUpdate.Data.GetRawText()); return(TgbUtils.ExtractUpdatesFromStreamRoot(streamUpdate.SourceId, streamUpdate.Timestamp, obj, hasher, opts.Type).EntityUpdates); }).ToList(); sw.Restart(); await using var tx = await conn.BeginTransactionAsync(); var savedUpdates = await _updateStore.SaveUpdates(conn, extractedUpdates, false, append : false); await tx.CommitAsync(); sw.Stop(); var timestamp = chunk.Min(u => u.Timestamp); _logger.Information("@ {Timestamp}: Saved {NewUpdateCount}/{UpdateCount} updates from {StreamObjects} stream objects (took {Duration})", timestamp, savedUpdates, extractedUpdates.Count, chunk.Count, sw.Elapsed); } } }