private async Task ladderTask(
CancellationToken stoppingToken,
string currentLeagueName, int limit = 50, int offset = 0, string accountName = null)
{
var payloads = new List <PoeFetcher.CharFetchResultPayload> {
};
await foreach (var payload in PoeFetcher.GetCharFetchResultPayloadFromLadderIterator(currentLeagueName, limit, offset, accountName))
{
if (stoppingToken.IsCancellationRequested)
{
return;
}
payloads.Add(payload);
}
System.GC.Collect();
PobUtils.try_malloc_trim(0);
// gc for run pob, need 500M+ memory.
var pchars = new List <PoeCharacterModel> {
};
using (var luaState = PobUtils.CreateLuaState())
{
foreach (var payload in payloads)
{
if (stoppingToken.IsCancellationRequested)
{
break;
}
PoeCharacterModel pchar = payload.ToCharacterModel(luaState);
if (pchar != null)
{
pchars.Add(pchar);
}
}
}
// force clear 500M+ memory
System.GC.Collect();
PobUtils.try_malloc_trim(0);
foreach (var pchar in pchars)
{
_logger.LogInformation($"Upserting {pchar.AccountName} / {pchar.CharacterName}");
await characterService.DefaultUpsert(pchar);
}
}
private void ForceUpdateCharactersPob(CancellationToken stoppingToken)
{
List <PoeCharacterModel> _chars =
PoeContext.Characters
.Select(c => new PoeCharacterModel {
CharacterId = c.CharacterId, PobCode = c.PobCode
})
.ToList();
foreach (var _c in _chars)
{
if (stoppingToken.IsCancellationRequested)
{
return;
}
try
{
string newPobXml = PobUtils.GetBuildXmlByXml(PobUtils.CodeToXml(_c.PobCode));
string nextPobCode = PobUtils.XmlToCode(newPobXml);
PathOfBuilding nextPob = PobUtils.XmlToPob(newPobXml);
_c.PobCode = nextPobCode;
_c.Pob = nextPob;
_c.Pob = PobUtils.XmlToPob(PobUtils.CodeToXml(_c.PobCode));
using (var db = new PoeDbContext())
{
db.Characters.Attach(_c);
db.Entry(_c).Property(x => x.PobCode).IsModified = true;
db.Entry(_c).Property(x => x.Pob).IsModified = true;
db.SaveChanges();
}
}
catch (System.Exception e)
{
_logger.LogError(e.ToString());
}
}
}
public PoeCharacterModel ToCharacterModel(NLua.Lua luaState)
{
var entry = ladderEntry;
string buildXml = PobUtils.GetBuildXmlByJsons(passivesJson, itemsJson, luaState);
PathOfBuilding tryGetPob()
{
PathOfBuilding pob;
try
{
pob = PobUtils.XmlToPob(buildXml);
}
catch (System.Exception e)
{
// TODO
// Log entry and e
if (e is System.InvalidOperationException || e is System.Xml.XmlException)
{
pob = null;
System.Console.WriteLine(e.ToString());
}
else
{
throw;
}
}
return(pob);
}
PathOfBuilding pob = tryGetPob();
if (pob != null)
{
PoeItems items = JsonConvert.DeserializeObject <PoeItems>(itemsJson);
PoePassives passives = JsonConvert.DeserializeObject <PoePassives>(passivesJson);
string code = PobUtils.XmlToCode(buildXml);
var poeChar = new PoeCharacterModel()
{
CharacterId = entry.Character.Id,
CharacterName = entry.Character.Name,
LeagueName = leagueName,
Account = entry.Account,
AccountName = entry.Account != null ? entry.Account.Name : null,
Level = entry.Character.Level,
Class = entry.Character.Class,
Depth = entry.Character.Depth ?? new PoeDepth {
Solo = 0, Default = 0
},
Dead = entry.Dead,
Online = entry.Online,
Rank = entry.Rank,
Experience = entry.Character.Experience,
LifeUnreserved = pob.GetStat("LifeUnreserved"),
EnergyShield = pob.GetStat("EnergyShield"),
Pob = pob,
PobCode = code,
Items = new List <PoeItem>()
{
}.Concat(items.items.Concat(passives.items.Select(i => i.ToPoeItem()))).ToList(),
UpdatedAt = System.DateTime.UtcNow,
};
return(poeChar);
}
else
{
return(null);
}
}
private async Task FetchAndUpsertCharacters(CancellationToken stoppingToken, string targetLeagueName)
{
int limit = 50;
long limitCharRows = 9000;
long numCharRows = await characterService.GetNumCharacters();
if (numCharRows >= limitCharRows)
{
_logger.LogInformation($"Full characters reach limit rows {limitCharRows}, current: {numCharRows} only update.");
var entityType = PoeContext.Model.FindEntityType(typeof(PoeCharacterModel));
var schema = entityType.GetSchema();
var tableName = entityType.GetTableName();
var sql = $@"
SELECT ""AccountName"", ""LeagueName"", ""CharacterId"", ""Account""
FROM ""{tableName}""
ORDER BY RANDOM()
LIMIT {limit}
";
var pchars = PoeContext.Database.GetDbConnection().Query <PoeCharacterModel>(sql).ToList();
var fetchResults = new List <PoeFetcher.CharFetchResultPayload> {
};
foreach (PoeCharacterModel pchar in pchars)
{
// Not support specific character from ladder only by accountName.
await foreach (var res in PoeFetcher.GetCharFetchResultPayloadFromLadderIterator(pchar.LeagueName, 5, 0, pchar.AccountName))
{
if (res != null && res.ladderEntry != null && res.ladderEntry.Account == null)
{
res.ladderEntry.Account = pchar.Account;
}
fetchResults.Add(res);
}
await Task.Delay(500, stoppingToken);
}
fetchResults = fetchResults.Where(res =>
{
return(pchars.Any(p =>
{
return p.CharacterId == res.ladderEntry.Character.Id;
}));
}
).ToList();
System.GC.Collect();
PobUtils.try_malloc_trim(0);
List <PoeCharacterModel> newPchars;
using (var luaState = PobUtils.CreateLuaState())
{
newPchars = fetchResults
.Select(res => res.ToCharacterModel(luaState))
.Where(pchar => pchar != null)
.ToList();
}
System.GC.Collect();
PobUtils.try_malloc_trim(0);
foreach (var pchar in newPchars)
{
_logger.LogInformation($"Upserting {pchar.AccountName} / {pchar.CharacterName}");
await characterService.DefaultUpsert(pchar);
}
}
else
{
int currentMaxRank = await PoeContext.Characters.MaxAsync(c => (int?)c.Rank) ?? 0;
int targetMax = 15000;
int maxDelta = targetMax - currentMaxRank;
string currentLeagueName = targetLeagueName;
if (maxDelta <= limit)
{
System.Random rnd = new System.Random();
int offset = rnd.Next(0, targetMax - limit);
await ladderTask(stoppingToken, currentLeagueName, limit, offset);
}
else
{
int offset = currentMaxRank;
await ladderTask(stoppingToken, currentLeagueName, limit, offset);
}
}
}