private async Task ladderTask( CancellationToken stoppingToken, string currentLeagueName, int limit = 50, int offset = 0, string accountName = null) { var payloads = new List <PoeFetcher.CharFetchResultPayload> { }; await foreach (var payload in PoeFetcher.GetCharFetchResultPayloadFromLadderIterator(currentLeagueName, limit, offset, accountName)) { if (stoppingToken.IsCancellationRequested) { return; } payloads.Add(payload); } System.GC.Collect(); PobUtils.try_malloc_trim(0); // gc for run pob, need 500M+ memory. var pchars = new List <PoeCharacterModel> { }; using (var luaState = PobUtils.CreateLuaState()) { foreach (var payload in payloads) { if (stoppingToken.IsCancellationRequested) { break; } PoeCharacterModel pchar = payload.ToCharacterModel(luaState); if (pchar != null) { pchars.Add(pchar); } } } // force clear 500M+ memory System.GC.Collect(); PobUtils.try_malloc_trim(0); foreach (var pchar in pchars) { _logger.LogInformation($"Upserting {pchar.AccountName} / {pchar.CharacterName}"); await characterService.DefaultUpsert(pchar); } }
public async Task DefaultUpsert(PoeCharacterModel poeChar) { async Task upsert(PoeDbContext PoeContext) { if (!String.IsNullOrWhiteSpace(poeChar.PobCode)) { await PoeContext.Upsert(poeChar). On(c => c.CharacterId). WhenMatched((oldChar, newChar) => new PoeCharacterModel() { Level = newChar.Level, Class = newChar.Class, LifeUnreserved = newChar.LifeUnreserved, EnergyShield = newChar.EnergyShield, Dead = newChar.Dead, Depth = newChar.Depth, Account = newChar.Account, Online = newChar.Online, Rank = newChar.Rank, Experience = newChar.Experience, PobCode = newChar.PobCode, Pob = newChar.Pob, Items = newChar.Items, UpdatedAt = newChar.PobCode != oldChar.PobCode ? System.DateTime.UtcNow : oldChar.UpdatedAt }).RunAsync(); } } using (var tran = PoeContext.Database.BeginTransaction()) { await upsert(PoeContext); var config = new GetCharactersConfig { characterId = poeChar.CharacterId }; var(sqlb, parameter) = LeagueService.GetCharactersSql(poeChar.LeagueName, config); var sql = sqlb.ToString(); CharacterAnalysis result = (await PoeContext.Database.GetDbConnection().QueryAsync <CharacterAnalysis>( sqlb.ToString(), parameter, tran.GetDbTransaction() )).FirstOrDefault(); if (result == null || result.Total != 1) { tran.Rollback(); } else { var config2 = new GetCharactersConfig { characterId = poeChar.CharacterId, mainSkills = result.MainSkillCountEntries != null?result.MainSkillCountEntries.Select(x => x.SkillId).ToList() : new List <string> { } }; var(sqlb2, parameter2) = LeagueService.GetCharactersSql(poeChar.LeagueName, config2); var sql2 = sqlb2.ToString(); CharacterAnalysis result2 = (await PoeContext.Database.GetDbConnection().QueryAsync <CharacterAnalysis>( sqlb2.ToString(), parameter2, tran.GetDbTransaction() )).FirstOrDefault(); if (result2 == null || result2.Total != 1) { tran.Rollback(); } else { CountAnalysis countAnalysis = new CountAnalysis() { ClassCountEntries = result2.ClassCountEntries ?? new ClassEntry[] { }, UniqueCountEntries = result2.UniqueCountEntries ?? new UniqueEntry[] { }, AllKeystoneCountEntries = result2.AllKeystoneCountEntries ?? new KeystoneEntry[] { }, MainSkillCountEntries = result2.MainSkillCountEntries ?? new SkillEntry[] { }, MainSkillSupportCountEntries = result2.MainSkillSupportCountEntries ?? new MainSkillSupportCountEntry[] { }, AllSkillCountEntries = result2.AllSkillCountEntries ?? new SkillEntry[] { }, WeaponTypeCountEntries = result2.WeaponTypeCountEntries ?? new WeaponTypeEntry[] { } }; var updateCountSql = $@"UPDATE ""Characters"" SET ""CountAnalysis"" = @CountAnalysis WHERE ""CharacterId"" = @CharacterId"; await PoeContext.Database.GetDbConnection().ExecuteAsync(updateCountSql, new { CharacterId = poeChar.CharacterId, CountAnalysis = new JsonbParameter(JsonSerializer.Serialize(countAnalysis)) }, tran.GetDbTransaction()); tran.Commit(); } } } }
public PoeCharacterModel ToCharacterModel(NLua.Lua luaState) { var entry = ladderEntry; string buildXml = PobUtils.GetBuildXmlByJsons(passivesJson, itemsJson, luaState); PathOfBuilding tryGetPob() { PathOfBuilding pob; try { pob = PobUtils.XmlToPob(buildXml); } catch (System.Exception e) { // TODO // Log entry and e if (e is System.InvalidOperationException || e is System.Xml.XmlException) { pob = null; System.Console.WriteLine(e.ToString()); } else { throw; } } return(pob); } PathOfBuilding pob = tryGetPob(); if (pob != null) { PoeItems items = JsonConvert.DeserializeObject <PoeItems>(itemsJson); PoePassives passives = JsonConvert.DeserializeObject <PoePassives>(passivesJson); string code = PobUtils.XmlToCode(buildXml); var poeChar = new PoeCharacterModel() { CharacterId = entry.Character.Id, CharacterName = entry.Character.Name, LeagueName = leagueName, Account = entry.Account, AccountName = entry.Account != null ? entry.Account.Name : null, Level = entry.Character.Level, Class = entry.Character.Class, Depth = entry.Character.Depth ?? new PoeDepth { Solo = 0, Default = 0 }, Dead = entry.Dead, Online = entry.Online, Rank = entry.Rank, Experience = entry.Character.Experience, LifeUnreserved = pob.GetStat("LifeUnreserved"), EnergyShield = pob.GetStat("EnergyShield"), Pob = pob, PobCode = code, Items = new List <PoeItem>() { }.Concat(items.items.Concat(passives.items.Select(i => i.ToPoeItem()))).ToList(), UpdatedAt = System.DateTime.UtcNow, }; return(poeChar); } else { return(null); } }