public async Task Run(ReplayOptions opts)
{
_logger.Information("Starting replay (type: {Type}, after: {After})", opts.Type, opts.After);
using var hasher = new SibrHasher();
var updates = _updateStore.ExportAllUpdatesRaw(UpdateType.Stream, new UpdateStore.EntityVersionQuery {
After = opts.After
});
var sw = new Stopwatch();
await using var conn = await _db.Obtain();
await foreach (var chunk in updates.Buffer(200))
{
var extracted = chunk.SelectMany(streamUpdate =>
{
var obj = JObject.Parse(streamUpdate.Data.GetRawText());
return(TgbUtils.ExtractUpdatesFromStreamRoot(streamUpdate.SourceId, streamUpdate.Timestamp, obj, hasher, opts.Type).EntityUpdates);
}).ToList();
sw.Restart();
await using var tx = await conn.BeginTransactionAsync();
var saved = await _updateStore.SaveUpdates(conn, extracted, false);
await tx.CommitAsync();
sw.Stop();
var timestamp = chunk.Min(u => u.Timestamp);
_logger.Information("@ {Timestamp}: Saved {NewUpdateCount}/{UpdateCount} updates from {StreamObjects} stream objects (took {Duration})",
timestamp, saved, extracted.Count, chunk.Count, sw.Elapsed);
}
}
private async Task HandleStreamData(string obj)
{
var timestamp = _clock.GetCurrentInstant();
var data = JObject.Parse(obj);
await using var conn = await _db.Obtain();
await using (var tx = await conn.BeginTransactionAsync())
{
await _updateStore.SaveUpdate(conn, EntityUpdate.From(UpdateType.Stream, _sourceId, timestamp, data));
await tx.CommitAsync();
}
await using (var tx = await conn.BeginTransactionAsync())
{
using var hasher = new SibrHasher();
var extracted = TgbUtils.ExtractUpdatesFromStreamRoot(_sourceId, timestamp, data, hasher);
var gameRes = await _gameStore.SaveGameUpdates(conn, extracted.GameUpdates);
var miscRes = await _updateStore.SaveUpdates(conn, extracted.EntityUpdates);
var maxPlayCount = extracted.GameUpdates.Count > 0 ? extracted.GameUpdates.Max(gu => gu.PlayCount) : -1;
_logger.Information(
"Received stream update, saved {GameUpdates} game updates, {MiscUpdates} updates, max PC {MaxPlayCount}",
gameRes, miscRes, maxPlayCount);
await tx.CommitAsync();
}
}
private async Task <IEnumerable <EntityUpdate> > GetUpdatesByIds(UpdateType type, string baseUrl, IEnumerable <string> ids, SibrHasher hasher)
{
var(timestamp, data) = await _client.GetJsonAsync($"{baseUrl}?ids={string.Join(',', ids)}");
return(data.Values <JObject>().Select(obj =>
{
var id = TgbUtils.GenerateGuidFromString(obj.Value <string>("id"));
return EntityUpdate.From(type, _sourceId, timestamp, obj, hasher, id);
}));
}
protected override async Task ProcessFile(string filename, IAsyncEnumerable <JToken> entries)
{
using var hasher = new SibrHasher();
var streamUpdates = new List <EntityUpdate>();
var miscUpdates = new List <EntityUpdate>();
var gameUpdates = new List <GameUpdate>();
var gameIds = new HashSet <Guid>();
var count = 0;
await using var conn = await _db.Obtain();
await foreach (var entry in entries)
{
var timestamp = ExtractTimestamp(entry);
if (timestamp == null)
{
continue;
}
var root = FindStreamRoot(entry as JObject);
streamUpdates.Add(EntityUpdate.From(UpdateType.Stream, _sourceId, timestamp.Value, root));
var updates = TgbUtils.ExtractUpdatesFromStreamRoot(_sourceId, timestamp.Value, root, hasher);
gameUpdates.AddRange(updates.GameUpdates);
miscUpdates.AddRange(updates.EntityUpdates);
gameIds.UnionWith(updates.GameUpdates.Select(g => g.GameId));
if (count++ % 100 == 0)
{
await FlushUpdates(conn, streamUpdates, gameUpdates, miscUpdates);
}
}
await FlushUpdates(conn, streamUpdates, gameUpdates, miscUpdates);
await _gameStore.TryAddNewGameIds(conn, gameUpdates.Select(gu => gu.GameId));
await _gameUpdateStore.UpdateSearchIndex(conn);
}
private void GlobalHandler(string _, PusherEvent evt)
{
// Get timestamp earliest possible (outside async)
var timestamp = _clock.GetCurrentInstant();
async Task Inner()
{
await Task.Yield();
try
{
var data = TgbUtils.TryDecodePusherData(evt.Data);
await _pusherEventStore.SaveEvent(evt.ChannelName, evt.EventName, timestamp, data, evt.Data);
}
catch (Exception e)
{
_logger.Error(e, "Error handling Pusher event {ChannelName}/{EventName}", evt.ChannelName, evt.EventName);
}
}
// Fork
var __ = Inner();
}
private Action <PusherEvent> WrapHandler(Func <JToken, Task> inner)
{
async Task Inner(PusherEvent evt)
{
try
{
_logger.Verbose("Received Pusher event {EventName} on {ChannelName}: {Data}", evt.EventName,
evt.ChannelName, evt.Data);
var data = TgbUtils.TryDecodePusherData(evt.Data);
await inner(data);
}
catch (Exception e)
{
_logger.Error(e, "Error handling Pusher event {EventName}", evt.EventName);
}
}
return(evt =>
{
// Fork inner task
var _ = Inner(evt);
});
}
public async Task Run(ReplayOptions opts)
{
_logger.Information("Starting replay (type: {Type}, start: {Start}, end: {End})",
opts.Type != null ? string.Join(",", opts.Type) : null, opts.Start, opts.End);
using var hasher = new SibrHasher();
var sw = new Stopwatch();
await using var conn = await _db.Obtain();
var page = opts.Start != null ? new PageToken(opts.Start.Value, default) : null;
while (true)
{
var chunk = await _updateStore.ExportAllUpdatesChunked(conn, UpdateType.Stream,
new UpdateStore.EntityVersionQuery
{
Page = page,
Before = opts.End,
Order = SortOrder.Asc,
Count = 500
});
if (chunk.Count == 0)
{
break;
}
page = chunk.Last().NextPage;
// if (opts.Type == UpdateType.Game)
if (false) // todo: uhhh how do we do this
{
var extractedGameUpdates = chunk.SelectMany(streamUpdate =>
{
var obj = JObject.Parse(streamUpdate.Data.GetRawText());
return(TgbUtils.ExtractUpdatesFromStreamRoot(streamUpdate.SourceId, streamUpdate.Timestamp, obj, hasher, opts.Type).GameUpdates);
}).ToList();
sw.Restart();
await using var tx = await conn.BeginTransactionAsync();
var savedGameUpdates = await _gameUpdateStore.SaveGameUpdates(conn, extractedGameUpdates, false);
await tx.CommitAsync();
sw.Stop();
var timestamp = chunk.Min(u => u.Timestamp);
_logger.Information("@ {Timestamp}: Saved {GameUpdateCount}/{UpdateCount} game updates from {StreamObjects} stream objects (took {Duration})",
timestamp, savedGameUpdates, extractedGameUpdates.Count, chunk.Count, sw.Elapsed);
}
else
{
var extractedUpdates = chunk.SelectMany(streamUpdate =>
{
var obj = JObject.Parse(streamUpdate.Data.GetRawText());
return(TgbUtils.ExtractUpdatesFromStreamRoot(streamUpdate.SourceId, streamUpdate.Timestamp, obj,
hasher, opts.Type).EntityUpdates);
}).ToList();
sw.Restart();
await using var tx = await conn.BeginTransactionAsync();
var savedUpdates = await _updateStore.SaveUpdates(conn, extractedUpdates, false, append : false);
await tx.CommitAsync();
sw.Stop();
var timestamp = chunk.Min(u => u.Timestamp);
_logger.Information("@ {Timestamp}: Saved {NewUpdateCount}/{UpdateCount} updates from {StreamObjects} stream objects (took {Duration})",
timestamp, savedUpdates, extractedUpdates.Count, chunk.Count, sw.Elapsed);
}
}
}
