public async Task Run(ReplayOptions opts)
{
_logger.Information("Starting replay (type: {Type}, after: {After})", opts.Type, opts.After);
using var hasher = new SibrHasher();
var updates = _updateStore.ExportAllUpdatesRaw(UpdateType.Stream, new UpdateStore.EntityVersionQuery {
After = opts.After
});
var sw = new Stopwatch();
await using var conn = await _db.Obtain();
await foreach (var chunk in updates.Buffer(200))
{
var extracted = chunk.SelectMany(streamUpdate =>
{
var obj = JObject.Parse(streamUpdate.Data.GetRawText());
return(TgbUtils.ExtractUpdatesFromStreamRoot(streamUpdate.SourceId, streamUpdate.Timestamp, obj, hasher, opts.Type).EntityUpdates);
}).ToList();
sw.Restart();
await using var tx = await conn.BeginTransactionAsync();
var saved = await _updateStore.SaveUpdates(conn, extracted, false);
await tx.CommitAsync();
sw.Stop();
var timestamp = chunk.Min(u => u.Timestamp);
_logger.Information("@ {Timestamp}: Saved {NewUpdateCount}/{UpdateCount} updates from {StreamObjects} stream objects (took {Duration})",
timestamp, saved, extracted.Count, chunk.Count, sw.Elapsed);
}
}
// Make this an AsyncEnumerable so we can still get partial data saved if the servers break in the middle of a pull
private async IAsyncEnumerable <EntityUpdate> FetchElectionResults(int season)
{
using var hasher = new SibrHasher();
var recap = await _client.GetJsonAsync($"https://www.blaseball.com/database/offseasonRecap?season={season}");
yield return(EntityUpdate.From(UpdateType.OffseasonRecap, _sourceId, recap.Timestamp, recap.Data, hasher));
var decreeIds = recap.Data.Value <JArray>("decreeResults").Values <string>().ToList();
var decreeResults = await GetUpdatesByIds(UpdateType.DecreeResult, "https://www.blaseball.com/database/decreeResults", decreeIds, hasher);
_logger.Information("Fetched {DecreeCount} decree results", decreeIds.Count);
foreach (var result in decreeResults)
{
yield return(result);
}
var bonusIds = recap.Data.Value <JArray>("bonusResults").Values <string>().ToList();
var bonusResults = await GetUpdatesByIds(UpdateType.BonusResult, "https://www.blaseball.com/database/bonusResults", bonusIds, hasher);
_logger.Information("Fetched {BonusCount} bonus results", bonusIds.Count);
foreach (var result in bonusResults)
{
yield return(result);
}
var eventIds = recap.Data.Value <JArray>("eventResults").Values <string>().ToList();
var eventResults = await GetUpdatesByIds(UpdateType.EventResult, "https://www.blaseball.com/database/eventResults", eventIds, hasher);
_logger.Information("Fetched {EventCount} event results", eventIds.Count);
foreach (var result in eventResults)
{
yield return(result);
}
}
private async Task HandleStreamData(string obj)
{
var timestamp = _clock.GetCurrentInstant();
var data = JObject.Parse(obj);
await using var conn = await _db.Obtain();
await using (var tx = await conn.BeginTransactionAsync())
{
await _updateStore.SaveUpdate(conn, EntityUpdate.From(UpdateType.Stream, _sourceId, timestamp, data));
await tx.CommitAsync();
}
await using (var tx = await conn.BeginTransactionAsync())
{
using var hasher = new SibrHasher();
var extracted = TgbUtils.ExtractUpdatesFromStreamRoot(_sourceId, timestamp, data, hasher);
var gameRes = await _gameStore.SaveGameUpdates(conn, extracted.GameUpdates);
var miscRes = await _updateStore.SaveUpdates(conn, extracted.EntityUpdates);
var maxPlayCount = extracted.GameUpdates.Count > 0 ? extracted.GameUpdates.Max(gu => gu.PlayCount) : -1;
_logger.Information(
"Received stream update, saved {GameUpdates} game updates, {MiscUpdates} updates, max PC {MaxPlayCount}",
gameRes, miscRes, maxPlayCount);
await tx.CommitAsync();
}
}
protected override async Task ProcessFile(string filename, IAsyncEnumerable <JToken> entries)
{
using var hasher = new SibrHasher();
var streamUpdates = new List <EntityUpdate>();
var miscUpdates = new List <EntityUpdate>();
var gameUpdates = new List <GameUpdate>();
var gameIds = new HashSet <Guid>();
var count = 0;
await using var conn = await _db.Obtain();
await foreach (var entry in entries)
{
var timestamp = ExtractTimestamp(entry);
if (timestamp == null)
{
continue;
}
var root = FindStreamRoot(entry as JObject);
streamUpdates.Add(EntityUpdate.From(UpdateType.Stream, _sourceId, timestamp.Value, root));
var updates = TgbUtils.ExtractUpdatesFromStreamRoot(_sourceId, timestamp.Value, root, hasher);
gameUpdates.AddRange(updates.GameUpdates);
miscUpdates.AddRange(updates.EntityUpdates);
gameIds.UnionWith(updates.GameUpdates.Select(g => g.GameId));
if (count++ % 100 == 0)
{
await FlushUpdates(conn, streamUpdates, gameUpdates, miscUpdates);
}
}
await FlushUpdates(conn, streamUpdates, gameUpdates, miscUpdates);
await _gameStore.TryAddNewGameIds(conn, gameUpdates.Select(gu => gu.GameId));
await _gameUpdateStore.UpdateSearchIndex(conn);
}
public async Task Run(ReplayOptions opts)
{
_logger.Information("Starting replay (type: {Type}, start: {Start}, end: {End})",
opts.Type != null ? string.Join(",", opts.Type) : null, opts.Start, opts.End);
using var hasher = new SibrHasher();
var sw = new Stopwatch();
await using var conn = await _db.Obtain();
var page = opts.Start != null ? new PageToken(opts.Start.Value, default) : null;
while (true)
{
var chunk = await _updateStore.ExportAllUpdatesChunked(conn, UpdateType.Stream,
new UpdateStore.EntityVersionQuery
{
Page = page,
Before = opts.End,
Order = SortOrder.Asc,
Count = 500
});
if (chunk.Count == 0)
{
break;
}
page = chunk.Last().NextPage;
// if (opts.Type == UpdateType.Game)
if (false) // todo: uhhh how do we do this
{
var extractedGameUpdates = chunk.SelectMany(streamUpdate =>
{
var obj = JObject.Parse(streamUpdate.Data.GetRawText());
return(TgbUtils.ExtractUpdatesFromStreamRoot(streamUpdate.SourceId, streamUpdate.Timestamp, obj, hasher, opts.Type).GameUpdates);
}).ToList();
sw.Restart();
await using var tx = await conn.BeginTransactionAsync();
var savedGameUpdates = await _gameUpdateStore.SaveGameUpdates(conn, extractedGameUpdates, false);
await tx.CommitAsync();
sw.Stop();
var timestamp = chunk.Min(u => u.Timestamp);
_logger.Information("@ {Timestamp}: Saved {GameUpdateCount}/{UpdateCount} game updates from {StreamObjects} stream objects (took {Duration})",
timestamp, savedGameUpdates, extractedGameUpdates.Count, chunk.Count, sw.Elapsed);
}
else
{
var extractedUpdates = chunk.SelectMany(streamUpdate =>
{
var obj = JObject.Parse(streamUpdate.Data.GetRawText());
return(TgbUtils.ExtractUpdatesFromStreamRoot(streamUpdate.SourceId, streamUpdate.Timestamp, obj,
hasher, opts.Type).EntityUpdates);
}).ToList();
sw.Restart();
await using var tx = await conn.BeginTransactionAsync();
var savedUpdates = await _updateStore.SaveUpdates(conn, extractedUpdates, false, append : false);
await tx.CommitAsync();
sw.Stop();
var timestamp = chunk.Min(u => u.Timestamp);
_logger.Information("@ {Timestamp}: Saved {NewUpdateCount}/{UpdateCount} updates from {StreamObjects} stream objects (took {Duration})",
timestamp, savedUpdates, extractedUpdates.Count, chunk.Count, sw.Elapsed);
}
}
}
private async Task <IEnumerable <EntityUpdate> > GetUpdatesByIds(UpdateType type, string baseUrl, IEnumerable <string> ids, SibrHasher hasher)
{
var(timestamp, data) = await _client.GetJsonAsync($"{baseUrl}?ids={string.Join(',', ids)}");
return(data.Values <JObject>().Select(obj =>
{
var id = TgbUtils.GenerateGuidFromString(obj.Value <string>("id"));
return EntityUpdate.From(type, _sourceId, timestamp, obj, hasher, id);
}));
}
public static GameUpdate From(Guid sourceId, Instant timestamp, JToken data, SibrHasher hasher = null) =>
public static EntityUpdate From(UpdateType type, Guid sourceId, Instant timestamp, JToken data, SibrHasher hasher = null, Guid?idOverride = null) =>