예제 #1
0
        private async Task HandleStreamData(string obj)
        {
            var timestamp = _clock.GetCurrentInstant();
            var data      = JObject.Parse(obj);

            await using var conn = await _db.Obtain();

            await using (var tx = await conn.BeginTransactionAsync())
            {
                await _updateStore.SaveUpdate(conn, EntityUpdate.From(UpdateType.Stream, _sourceId, timestamp, data));

                await tx.CommitAsync();
            }

            await using (var tx = await conn.BeginTransactionAsync())
            {
                using var hasher = new SibrHasher();
                var extracted = TgbUtils.ExtractUpdatesFromStreamRoot(_sourceId, timestamp, data, hasher);
                var gameRes   = await _gameStore.SaveGameUpdates(conn, extracted.GameUpdates);

                var miscRes = await _updateStore.SaveUpdates(conn, extracted.EntityUpdates);

                var maxPlayCount = extracted.GameUpdates.Count > 0 ? extracted.GameUpdates.Max(gu => gu.PlayCount) : -1;

                _logger.Information(
                    "Received stream update, saved {GameUpdates} game updates, {MiscUpdates} updates, max PC {MaxPlayCount}",
                    gameRes, miscRes, maxPlayCount);

                await tx.CommitAsync();
            }
        }
예제 #2
0
        // Make this an AsyncEnumerable so we can still get partial data saved if the servers break in the middle of a pull
        private async IAsyncEnumerable <EntityUpdate> FetchElectionResults(int season)
        {
            using var hasher = new SibrHasher();

            var recap = await _client.GetJsonAsync($"https://www.blaseball.com/database/offseasonRecap?season={season}");

            yield return(EntityUpdate.From(UpdateType.OffseasonRecap, _sourceId, recap.Timestamp, recap.Data, hasher));

            var decreeIds     = recap.Data.Value <JArray>("decreeResults").Values <string>().ToList();
            var decreeResults = await GetUpdatesByIds(UpdateType.DecreeResult, "https://www.blaseball.com/database/decreeResults", decreeIds, hasher);

            _logger.Information("Fetched {DecreeCount} decree results", decreeIds.Count);
            foreach (var result in decreeResults)
            {
                yield return(result);
            }

            var bonusIds     = recap.Data.Value <JArray>("bonusResults").Values <string>().ToList();
            var bonusResults = await GetUpdatesByIds(UpdateType.BonusResult, "https://www.blaseball.com/database/bonusResults", bonusIds, hasher);

            _logger.Information("Fetched {BonusCount} bonus results", bonusIds.Count);
            foreach (var result in bonusResults)
            {
                yield return(result);
            }

            var eventIds     = recap.Data.Value <JArray>("eventResults").Values <string>().ToList();
            var eventResults = await GetUpdatesByIds(UpdateType.EventResult, "https://www.blaseball.com/database/eventResults", eventIds, hasher);

            _logger.Information("Fetched {EventCount} event results", eventIds.Count);
            foreach (var result in eventResults)
            {
                yield return(result);
            }
        }
예제 #3
0
        protected override async Task ProcessFile(string filename, IAsyncEnumerable <JToken> entries,
                                                  ImportOptions options)
        {
            var timestamp = ExtractTimestampFromFilename(filename, @"idols-(\d+)\.json\.gz");

            if (timestamp == null)
            {
                return;
            }

            await using var conn = await _db.Obtain();

            await using var tx = await conn.BeginTransactionAsync();

            var updates = await entries
                          .Select(entry => EntityUpdate.From(UpdateType.Idols, _sourceId, timestamp.Value, entry))
                          .ToListAsync();

            var res = await _store.SaveUpdates(conn, updates);

            if (res > 0)
            {
                _logger.Information("- Imported idols update at {Timestamp}", timestamp);
            }

            await tx.CommitAsync();
        }
예제 #4
0
        private async Task <EntityUpdate> FetchElectionDetails(Guid electionId)
        {
            var(electionTimestamp, electionData) =
                await _client.GetJsonAsync($"https://api.blaseball.com/api/elections/{electionId}");

            return(EntityUpdate.From(UpdateType.GammaElectionDetails, _sourceId, electionTimestamp,
                                     electionData, idOverride: electionId));
        }
예제 #5
0
        private async Task <EntityUpdate> FetchElectionResults(Guid prizeId)
        {
            var(timestamp, data) =
                await _client.GetJsonAsync($"https://api.blaseball.com/api/elections/{prizeId}/results");

            return(EntityUpdate.From(UpdateType.GammaElectionResults, _sourceId, timestamp,
                                     data, idOverride: prizeId));
        }
예제 #6
0
 private List <EntityUpdate> ExtractItems(List <EntityUpdate> playerUpdates)
 {
     return(playerUpdates.SelectMany(update =>
     {
         return update.Data.SelectTokens("items[*]")
         .Select(data => EntityUpdate.From(UpdateType.Item, update.SourceId, update.Timestamp, data));
     }).ToList());
 }
예제 #7
0
        private async Task <EntityUpdate> QueryRenovationProgress(Guid stadiumId)
        {
            var data = await _client.GetStringAsync($"https://www.blaseball.com/database/renovationProgress?id={stadiumId}");

            var json      = JToken.Parse(data);
            var timestamp = _clock.GetCurrentInstant();

            return(EntityUpdate.From(UpdateType.RenovationProgress, _sourceId, timestamp, json, idOverride: stadiumId));
        }
예제 #8
0
        private async Task <IEnumerable <EntityUpdate> > GetUpdatesByIds(UpdateType type, string baseUrl, IEnumerable <string> ids, SibrHasher hasher)
        {
            var(timestamp, data) = await _client.GetJsonAsync($"{baseUrl}?ids={string.Join(',', ids)}");

            return(data.Values <JObject>().Select(obj =>
            {
                var id = TgbUtils.GenerateGuidFromString(obj.Value <string>("id"));
                return EntityUpdate.From(type, _sourceId, timestamp, obj, hasher, id);
            }));
        }
예제 #9
0
        private async Task TickerHandler(JToken obj)
        {
            var(timestamp, data) = await _client.GetJsonAsync("https://api.blaseball.com/database/globalEvents");

            var update = EntityUpdate.From(UpdateType.GlobalEvents, _sourceId, timestamp, data);

            await using var conn = await _db.Obtain();

            await _updateStore.SaveUpdate(conn, update);

            _logger.Information("Pulled GlobalEvents based on Pusher event");
        }
예제 #10
0
        private async Task SimDataHandler(JToken obj)
        {
            var(timestamp, data) = await _client.GetJsonAsync("https://api.blaseball.com/database/simulationData");

            await using var conn = await _db.Obtain();

            await _updateStore.SaveUpdate(conn, EntityUpdate.From(UpdateType.Sim, _sourceId, timestamp, data));

            await _updateStore.SaveUpdate(conn, EntityUpdate.From(UpdateType.GammaSim, _sourceId, timestamp, obj));

            _logger.Information("Pulled SimData based on Pusher event");
        }
예제 #11
0
        protected override async Task RunInterval()
        {
            var(timestamp, data) = await _client.GetJsonAsync("https://api.blaseball.com/championbets/availableBets");

            var sortedData = new JArray(data.OrderBy(x => x["teamId"]));

            var update = EntityUpdate.From(UpdateType.AvailableChampionBets, _sourceId, timestamp, sortedData);

            await using var conn = await _db.Obtain();

            await _updateStore.SaveUpdate(conn, update);

            _logger.Information("Saved available champion bets");
        }
예제 #12
0
        private IEnumerable <EntityUpdate> ExtractUpdates(JToken entry, Instant timestamp)
        {
            var data     = entry["data"];
            var endpoint = entry["endpoint"] !.Value <string>();

            return(endpoint switch
            {
                "allTeams" => EntityUpdate.FromArray(UpdateType.Team, _sourceId, timestamp, data),
                "players" => EntityUpdate.FromArray(UpdateType.Player, _sourceId, timestamp, data),
                "globalEvents" => new[] { EntityUpdate.From(UpdateType.GlobalEvents, _sourceId, timestamp, data) },
                "offseasonSetup" => new[] { EntityUpdate.From(UpdateType.OffseasonSetup, _sourceId, timestamp, data) },
                "idols" => new[] { EntityUpdate.From(UpdateType.Idols, _sourceId, timestamp, data) },
                _ => ImmutableList <EntityUpdate> .Empty
            });
        protected override async Task ProcessFile(string filename, IAsyncEnumerable <JToken> entries)
        {
            await using var conn = await _db.Obtain();

            await using var tx = await conn.BeginTransactionAsync();

            await foreach (var entry in entries)
            {
                var timestamp = entry["firstSeen"]["$date"].ToObject <DateTime>().ToInstant();
                var data      = entry["payload"];
                await _store.SaveUpdate(conn, EntityUpdate.From(UpdateType.Tributes, _sourceId, timestamp, data));

                _logger.Information("Saved tributes update at {Timestamp}", timestamp);
            }

            await tx.CommitAsync();
        }
예제 #14
0
        private async Task <IEnumerable <EntityUpdate> > PollEndpoint(
            IngestEndpoint endpoint)
        {
            try
            {
                var timestamp = _clock.GetCurrentInstant();
                var json      = await _client.GetStringAsync(endpoint.Url);

                var token = JToken.Parse(json);

                var update = EntityUpdate.From(endpoint.Type, _sourceId, timestamp, token);
                _logger.Information("- Fetched endpoint {Endpoint} ({UpdateType}, hash {Hash})", endpoint.Url, endpoint.Type, update.Hash);
                return(new[] { update });
            }
            catch (Exception e)
            {
                _logger.Error(e, "Error reading endpoint {Endpoint}", endpoint.Url);
            }

            return(ImmutableList <EntityUpdate> .Empty);
        }
예제 #15
0
        protected override async Task ProcessFile(string filename, IAsyncEnumerable <JToken> entries)
        {
            using var hasher = new SibrHasher();
            var streamUpdates = new List <EntityUpdate>();
            var miscUpdates   = new List <EntityUpdate>();
            var gameUpdates   = new List <GameUpdate>();
            var gameIds       = new HashSet <Guid>();

            var count = 0;

            await using var conn = await _db.Obtain();

            await foreach (var entry in entries)
            {
                var timestamp = ExtractTimestamp(entry);
                if (timestamp == null)
                {
                    continue;
                }

                var root = FindStreamRoot(entry as JObject);
                streamUpdates.Add(EntityUpdate.From(UpdateType.Stream, _sourceId, timestamp.Value, root));

                var updates = TgbUtils.ExtractUpdatesFromStreamRoot(_sourceId, timestamp.Value, root, hasher);
                gameUpdates.AddRange(updates.GameUpdates);
                miscUpdates.AddRange(updates.EntityUpdates);

                gameIds.UnionWith(updates.GameUpdates.Select(g => g.GameId));
                if (count++ % 100 == 0)
                {
                    await FlushUpdates(conn, streamUpdates, gameUpdates, miscUpdates);
                }
            }

            await FlushUpdates(conn, streamUpdates, gameUpdates, miscUpdates);

            await _gameStore.TryAddNewGameIds(conn, gameUpdates.Select(gu => gu.GameId));

            await _gameUpdateStore.UpdateSearchIndex(conn);
        }
예제 #16
0
        public async Task <IActionResult> SaveUpdates([FromQuery, Required] Guid source,
                                                      [FromBody] IEnumerable <IngestUpdate> updates)
        {
            if (!IsOnAllowedPort())
            {
                return(Unauthorized());
            }

            var entityUpdates = updates
                                .Select(u => EntityUpdate.From(u.Type, source, u.Timestamp, JToken.Parse(u.Data.GetRawText()), idOverride: u.IdOverride))
                                .ToList();

            _logger.Information("Receiving {Count} updates from source {Source}", entityUpdates.Count, source);

            await using var conn = await _db.Obtain();

            await using var tx = await conn.BeginTransactionAsync();

            var newUpdates = await _updateStore.SaveUpdates(conn, entityUpdates);

            await tx.CommitAsync();

            return(Ok($"Saved {newUpdates} updates."));
        }
예제 #17
0
        private async Task <EntityUpdate> GetTeamElectionStats(Guid teamId)
        {
            var(timestamp, data) = await _client.GetJsonAsync($"https://api.blaseball.com/database/teamElectionStats?id={teamId}");

            return(EntityUpdate.From(UpdateType.TeamElectionStats, _sourceId, timestamp, data, idOverride: teamId));
        }
예제 #18
0
        private async Task <EntityUpdate> FetchStory(Guid id)
        {
            var(timestamp, json) = await _client.GetJsonAsync($"https://api.blaseball.com/database/feed/story?id={id}");

            return(EntityUpdate.From(UpdateType.LibraryStory, _sourceId, timestamp, json, idOverride: id));
        }
예제 #19
0
 .SelectMany(item =>
             root.SelectTokens(item.path)
             .Where(token => token.First != null)
             .Select(token => EntityUpdate.From(item.type, sourceId, timestamp, token))));
예제 #20
0
        private async Task <EntityUpdate> FetchElectionInfo()
        {
            var(timestamp, data) = await _client.GetJsonAsync("https://api.blaseball.com/api/elections");

            return(EntityUpdate.From(UpdateType.GammaElections, _sourceId, timestamp, data));
        }