private async Task ladderTask( CancellationToken stoppingToken, string currentLeagueName, int limit = 50, int offset = 0, string accountName = null) { var payloads = new List <PoeFetcher.CharFetchResultPayload> { }; await foreach (var payload in PoeFetcher.GetCharFetchResultPayloadFromLadderIterator(currentLeagueName, limit, offset, accountName)) { if (stoppingToken.IsCancellationRequested) { return; } payloads.Add(payload); } System.GC.Collect(); PobUtils.try_malloc_trim(0); // gc for run pob, need 500M+ memory. var pchars = new List <PoeCharacterModel> { }; using (var luaState = PobUtils.CreateLuaState()) { foreach (var payload in payloads) { if (stoppingToken.IsCancellationRequested) { break; } PoeCharacterModel pchar = payload.ToCharacterModel(luaState); if (pchar != null) { pchars.Add(pchar); } } } // force clear 500M+ memory System.GC.Collect(); PobUtils.try_malloc_trim(0); foreach (var pchar in pchars) { _logger.LogInformation($"Upserting {pchar.AccountName} / {pchar.CharacterName}"); await characterService.DefaultUpsert(pchar); } }
private async Task FetchAndUpsertCharacters(CancellationToken stoppingToken, string targetLeagueName) { int limit = 50; long limitCharRows = 9000; long numCharRows = await characterService.GetNumCharacters(); if (numCharRows >= limitCharRows) { _logger.LogInformation($"Full characters reach limit rows {limitCharRows}, current: {numCharRows} only update."); var entityType = PoeContext.Model.FindEntityType(typeof(PoeCharacterModel)); var schema = entityType.GetSchema(); var tableName = entityType.GetTableName(); var sql = $@" SELECT ""AccountName"", ""LeagueName"", ""CharacterId"", ""Account"" FROM ""{tableName}"" ORDER BY RANDOM() LIMIT {limit} "; var pchars = PoeContext.Database.GetDbConnection().Query <PoeCharacterModel>(sql).ToList(); var fetchResults = new List <PoeFetcher.CharFetchResultPayload> { }; foreach (PoeCharacterModel pchar in pchars) { // Not support specific character from ladder only by accountName. await foreach (var res in PoeFetcher.GetCharFetchResultPayloadFromLadderIterator(pchar.LeagueName, 5, 0, pchar.AccountName)) { if (res != null && res.ladderEntry != null && res.ladderEntry.Account == null) { res.ladderEntry.Account = pchar.Account; } fetchResults.Add(res); } await Task.Delay(500, stoppingToken); } fetchResults = fetchResults.Where(res => { return(pchars.Any(p => { return p.CharacterId == res.ladderEntry.Character.Id; })); } ).ToList(); System.GC.Collect(); PobUtils.try_malloc_trim(0); List <PoeCharacterModel> newPchars; using (var luaState = PobUtils.CreateLuaState()) { newPchars = fetchResults .Select(res => res.ToCharacterModel(luaState)) .Where(pchar => pchar != null) .ToList(); } System.GC.Collect(); PobUtils.try_malloc_trim(0); foreach (var pchar in newPchars) { _logger.LogInformation($"Upserting {pchar.AccountName} / {pchar.CharacterName}"); await characterService.DefaultUpsert(pchar); } } else { int currentMaxRank = await PoeContext.Characters.MaxAsync(c => (int?)c.Rank) ?? 0; int targetMax = 15000; int maxDelta = targetMax - currentMaxRank; string currentLeagueName = targetLeagueName; if (maxDelta <= limit) { System.Random rnd = new System.Random(); int offset = rnd.Next(0, targetMax - limit); await ladderTask(stoppingToken, currentLeagueName, limit, offset); } else { int offset = currentMaxRank; await ladderTask(stoppingToken, currentLeagueName, limit, offset); } } }