private async Task HandleStreamData(string obj)
{
var timestamp = _clock.GetCurrentInstant();
var data = JObject.Parse(obj);
await using var conn = await _db.Obtain();
await using (var tx = await conn.BeginTransactionAsync())
{
await _updateStore.SaveUpdate(conn, EntityUpdate.From(UpdateType.Stream, _sourceId, timestamp, data));
await tx.CommitAsync();
}
await using (var tx = await conn.BeginTransactionAsync())
{
using var hasher = new SibrHasher();
var extracted = TgbUtils.ExtractUpdatesFromStreamRoot(_sourceId, timestamp, data, hasher);
var gameRes = await _gameStore.SaveGameUpdates(conn, extracted.GameUpdates);
var miscRes = await _updateStore.SaveUpdates(conn, extracted.EntityUpdates);
var maxPlayCount = extracted.GameUpdates.Count > 0 ? extracted.GameUpdates.Max(gu => gu.PlayCount) : -1;
_logger.Information(
"Received stream update, saved {GameUpdates} game updates, {MiscUpdates} updates, max PC {MaxPlayCount}",
gameRes, miscRes, maxPlayCount);
await tx.CommitAsync();
}
}
// Make this an AsyncEnumerable so we can still get partial data saved if the servers break in the middle of a pull
private async IAsyncEnumerable <EntityUpdate> FetchElectionResults(int season)
{
using var hasher = new SibrHasher();
var recap = await _client.GetJsonAsync($"https://www.blaseball.com/database/offseasonRecap?season={season}");
yield return(EntityUpdate.From(UpdateType.OffseasonRecap, _sourceId, recap.Timestamp, recap.Data, hasher));
var decreeIds = recap.Data.Value <JArray>("decreeResults").Values <string>().ToList();
var decreeResults = await GetUpdatesByIds(UpdateType.DecreeResult, "https://www.blaseball.com/database/decreeResults", decreeIds, hasher);
_logger.Information("Fetched {DecreeCount} decree results", decreeIds.Count);
foreach (var result in decreeResults)
{
yield return(result);
}
var bonusIds = recap.Data.Value <JArray>("bonusResults").Values <string>().ToList();
var bonusResults = await GetUpdatesByIds(UpdateType.BonusResult, "https://www.blaseball.com/database/bonusResults", bonusIds, hasher);
_logger.Information("Fetched {BonusCount} bonus results", bonusIds.Count);
foreach (var result in bonusResults)
{
yield return(result);
}
var eventIds = recap.Data.Value <JArray>("eventResults").Values <string>().ToList();
var eventResults = await GetUpdatesByIds(UpdateType.EventResult, "https://www.blaseball.com/database/eventResults", eventIds, hasher);
_logger.Information("Fetched {EventCount} event results", eventIds.Count);
foreach (var result in eventResults)
{
yield return(result);
}
}
protected override async Task ProcessFile(string filename, IAsyncEnumerable <JToken> entries,
ImportOptions options)
{
var timestamp = ExtractTimestampFromFilename(filename, @"idols-(\d+)\.json\.gz");
if (timestamp == null)
{
return;
}
await using var conn = await _db.Obtain();
await using var tx = await conn.BeginTransactionAsync();
var updates = await entries
.Select(entry => EntityUpdate.From(UpdateType.Idols, _sourceId, timestamp.Value, entry))
.ToListAsync();
var res = await _store.SaveUpdates(conn, updates);
if (res > 0)
{
_logger.Information("- Imported idols update at {Timestamp}", timestamp);
}
await tx.CommitAsync();
}
private async Task <EntityUpdate> FetchElectionDetails(Guid electionId)
{
var(electionTimestamp, electionData) =
await _client.GetJsonAsync($"https://api.blaseball.com/api/elections/{electionId}");
return(EntityUpdate.From(UpdateType.GammaElectionDetails, _sourceId, electionTimestamp,
electionData, idOverride: electionId));
}
private async Task <EntityUpdate> FetchElectionResults(Guid prizeId)
{
var(timestamp, data) =
await _client.GetJsonAsync($"https://api.blaseball.com/api/elections/{prizeId}/results");
return(EntityUpdate.From(UpdateType.GammaElectionResults, _sourceId, timestamp,
data, idOverride: prizeId));
}
private List <EntityUpdate> ExtractItems(List <EntityUpdate> playerUpdates)
{
return(playerUpdates.SelectMany(update =>
{
return update.Data.SelectTokens("items[*]")
.Select(data => EntityUpdate.From(UpdateType.Item, update.SourceId, update.Timestamp, data));
}).ToList());
}
private async Task <EntityUpdate> QueryRenovationProgress(Guid stadiumId)
{
var data = await _client.GetStringAsync($"https://www.blaseball.com/database/renovationProgress?id={stadiumId}");
var json = JToken.Parse(data);
var timestamp = _clock.GetCurrentInstant();
return(EntityUpdate.From(UpdateType.RenovationProgress, _sourceId, timestamp, json, idOverride: stadiumId));
}
private async Task <IEnumerable <EntityUpdate> > GetUpdatesByIds(UpdateType type, string baseUrl, IEnumerable <string> ids, SibrHasher hasher)
{
var(timestamp, data) = await _client.GetJsonAsync($"{baseUrl}?ids={string.Join(',', ids)}");
return(data.Values <JObject>().Select(obj =>
{
var id = TgbUtils.GenerateGuidFromString(obj.Value <string>("id"));
return EntityUpdate.From(type, _sourceId, timestamp, obj, hasher, id);
}));
}
private async Task TickerHandler(JToken obj)
{
var(timestamp, data) = await _client.GetJsonAsync("https://api.blaseball.com/database/globalEvents");
var update = EntityUpdate.From(UpdateType.GlobalEvents, _sourceId, timestamp, data);
await using var conn = await _db.Obtain();
await _updateStore.SaveUpdate(conn, update);
_logger.Information("Pulled GlobalEvents based on Pusher event");
}
private async Task SimDataHandler(JToken obj)
{
var(timestamp, data) = await _client.GetJsonAsync("https://api.blaseball.com/database/simulationData");
await using var conn = await _db.Obtain();
await _updateStore.SaveUpdate(conn, EntityUpdate.From(UpdateType.Sim, _sourceId, timestamp, data));
await _updateStore.SaveUpdate(conn, EntityUpdate.From(UpdateType.GammaSim, _sourceId, timestamp, obj));
_logger.Information("Pulled SimData based on Pusher event");
}
protected override async Task RunInterval()
{
var(timestamp, data) = await _client.GetJsonAsync("https://api.blaseball.com/championbets/availableBets");
var sortedData = new JArray(data.OrderBy(x => x["teamId"]));
var update = EntityUpdate.From(UpdateType.AvailableChampionBets, _sourceId, timestamp, sortedData);
await using var conn = await _db.Obtain();
await _updateStore.SaveUpdate(conn, update);
_logger.Information("Saved available champion bets");
}
private IEnumerable <EntityUpdate> ExtractUpdates(JToken entry, Instant timestamp)
{
var data = entry["data"];
var endpoint = entry["endpoint"] !.Value <string>();
return(endpoint switch
{
"allTeams" => EntityUpdate.FromArray(UpdateType.Team, _sourceId, timestamp, data),
"players" => EntityUpdate.FromArray(UpdateType.Player, _sourceId, timestamp, data),
"globalEvents" => new[] { EntityUpdate.From(UpdateType.GlobalEvents, _sourceId, timestamp, data) },
"offseasonSetup" => new[] { EntityUpdate.From(UpdateType.OffseasonSetup, _sourceId, timestamp, data) },
"idols" => new[] { EntityUpdate.From(UpdateType.Idols, _sourceId, timestamp, data) },
_ => ImmutableList <EntityUpdate> .Empty
});
protected override async Task ProcessFile(string filename, IAsyncEnumerable <JToken> entries)
{
await using var conn = await _db.Obtain();
await using var tx = await conn.BeginTransactionAsync();
await foreach (var entry in entries)
{
var timestamp = entry["firstSeen"]["$date"].ToObject <DateTime>().ToInstant();
var data = entry["payload"];
await _store.SaveUpdate(conn, EntityUpdate.From(UpdateType.Tributes, _sourceId, timestamp, data));
_logger.Information("Saved tributes update at {Timestamp}", timestamp);
}
await tx.CommitAsync();
}
private async Task <IEnumerable <EntityUpdate> > PollEndpoint(
IngestEndpoint endpoint)
{
try
{
var timestamp = _clock.GetCurrentInstant();
var json = await _client.GetStringAsync(endpoint.Url);
var token = JToken.Parse(json);
var update = EntityUpdate.From(endpoint.Type, _sourceId, timestamp, token);
_logger.Information("- Fetched endpoint {Endpoint} ({UpdateType}, hash {Hash})", endpoint.Url, endpoint.Type, update.Hash);
return(new[] { update });
}
catch (Exception e)
{
_logger.Error(e, "Error reading endpoint {Endpoint}", endpoint.Url);
}
return(ImmutableList <EntityUpdate> .Empty);
}
protected override async Task ProcessFile(string filename, IAsyncEnumerable <JToken> entries)
{
using var hasher = new SibrHasher();
var streamUpdates = new List <EntityUpdate>();
var miscUpdates = new List <EntityUpdate>();
var gameUpdates = new List <GameUpdate>();
var gameIds = new HashSet <Guid>();
var count = 0;
await using var conn = await _db.Obtain();
await foreach (var entry in entries)
{
var timestamp = ExtractTimestamp(entry);
if (timestamp == null)
{
continue;
}
var root = FindStreamRoot(entry as JObject);
streamUpdates.Add(EntityUpdate.From(UpdateType.Stream, _sourceId, timestamp.Value, root));
var updates = TgbUtils.ExtractUpdatesFromStreamRoot(_sourceId, timestamp.Value, root, hasher);
gameUpdates.AddRange(updates.GameUpdates);
miscUpdates.AddRange(updates.EntityUpdates);
gameIds.UnionWith(updates.GameUpdates.Select(g => g.GameId));
if (count++ % 100 == 0)
{
await FlushUpdates(conn, streamUpdates, gameUpdates, miscUpdates);
}
}
await FlushUpdates(conn, streamUpdates, gameUpdates, miscUpdates);
await _gameStore.TryAddNewGameIds(conn, gameUpdates.Select(gu => gu.GameId));
await _gameUpdateStore.UpdateSearchIndex(conn);
}
public async Task <IActionResult> SaveUpdates([FromQuery, Required] Guid source,
[FromBody] IEnumerable <IngestUpdate> updates)
{
if (!IsOnAllowedPort())
{
return(Unauthorized());
}
var entityUpdates = updates
.Select(u => EntityUpdate.From(u.Type, source, u.Timestamp, JToken.Parse(u.Data.GetRawText()), idOverride: u.IdOverride))
.ToList();
_logger.Information("Receiving {Count} updates from source {Source}", entityUpdates.Count, source);
await using var conn = await _db.Obtain();
await using var tx = await conn.BeginTransactionAsync();
var newUpdates = await _updateStore.SaveUpdates(conn, entityUpdates);
await tx.CommitAsync();
return(Ok($"Saved {newUpdates} updates."));
}
private async Task <EntityUpdate> GetTeamElectionStats(Guid teamId)
{
var(timestamp, data) = await _client.GetJsonAsync($"https://api.blaseball.com/database/teamElectionStats?id={teamId}");
return(EntityUpdate.From(UpdateType.TeamElectionStats, _sourceId, timestamp, data, idOverride: teamId));
}
private async Task <EntityUpdate> FetchStory(Guid id)
{
var(timestamp, json) = await _client.GetJsonAsync($"https://api.blaseball.com/database/feed/story?id={id}");
return(EntityUpdate.From(UpdateType.LibraryStory, _sourceId, timestamp, json, idOverride: id));
}
.SelectMany(item =>
root.SelectTokens(item.path)
.Where(token => token.First != null)
.Select(token => EntityUpdate.From(item.type, sourceId, timestamp, token))));
private async Task <EntityUpdate> FetchElectionInfo()
{
var(timestamp, data) = await _client.GetJsonAsync("https://api.blaseball.com/api/elections");
return(EntityUpdate.From(UpdateType.GammaElections, _sourceId, timestamp, data));
}
