private async Task FetchAndUpsertLeagues()
{
List <PoeLeagueModel> leagues = await PoeFetcher.GetLeagues();
using (var PoeContext = new PoeDbContext())
{
await PoeContext.Leagues.UpsertRange(leagues.ToArray()).On(x => new { LeagueId = x.LeagueId }).RunAsync();
}
}
private async Task ladderTask(
CancellationToken stoppingToken,
string currentLeagueName, int limit = 50, int offset = 0, string accountName = null)
{
var payloads = new List <PoeFetcher.CharFetchResultPayload> {
};
await foreach (var payload in PoeFetcher.GetCharFetchResultPayloadFromLadderIterator(currentLeagueName, limit, offset, accountName))
{
if (stoppingToken.IsCancellationRequested)
{
return;
}
payloads.Add(payload);
}
System.GC.Collect();
PobUtils.try_malloc_trim(0);
// gc for run pob, need 500M+ memory.
var pchars = new List <PoeCharacterModel> {
};
using (var luaState = PobUtils.CreateLuaState())
{
foreach (var payload in payloads)
{
if (stoppingToken.IsCancellationRequested)
{
break;
}
PoeCharacterModel pchar = payload.ToCharacterModel(luaState);
if (pchar != null)
{
pchars.Add(pchar);
}
}
}
// force clear 500M+ memory
System.GC.Collect();
PobUtils.try_malloc_trim(0);
foreach (var pchar in pchars)
{
_logger.LogInformation($"Upserting {pchar.AccountName} / {pchar.CharacterName}");
await characterService.DefaultUpsert(pchar);
}
}
private async Task FetchAndUpsertCharacters(CancellationToken stoppingToken, string targetLeagueName)
{
int limit = 50;
long limitCharRows = 9000;
long numCharRows = await characterService.GetNumCharacters();
if (numCharRows >= limitCharRows)
{
_logger.LogInformation($"Full characters reach limit rows {limitCharRows}, current: {numCharRows} only update.");
var entityType = PoeContext.Model.FindEntityType(typeof(PoeCharacterModel));
var schema = entityType.GetSchema();
var tableName = entityType.GetTableName();
var sql = $@"
SELECT ""AccountName"", ""LeagueName"", ""CharacterId"", ""Account""
FROM ""{tableName}""
ORDER BY RANDOM()
LIMIT {limit}
";
var pchars = PoeContext.Database.GetDbConnection().Query <PoeCharacterModel>(sql).ToList();
var fetchResults = new List <PoeFetcher.CharFetchResultPayload> {
};
foreach (PoeCharacterModel pchar in pchars)
{
// Not support specific character from ladder only by accountName.
await foreach (var res in PoeFetcher.GetCharFetchResultPayloadFromLadderIterator(pchar.LeagueName, 5, 0, pchar.AccountName))
{
if (res != null && res.ladderEntry != null && res.ladderEntry.Account == null)
{
res.ladderEntry.Account = pchar.Account;
}
fetchResults.Add(res);
}
await Task.Delay(500, stoppingToken);
}
fetchResults = fetchResults.Where(res =>
{
return(pchars.Any(p =>
{
return p.CharacterId == res.ladderEntry.Character.Id;
}));
}
).ToList();
System.GC.Collect();
PobUtils.try_malloc_trim(0);
List <PoeCharacterModel> newPchars;
using (var luaState = PobUtils.CreateLuaState())
{
newPchars = fetchResults
.Select(res => res.ToCharacterModel(luaState))
.Where(pchar => pchar != null)
.ToList();
}
System.GC.Collect();
PobUtils.try_malloc_trim(0);
foreach (var pchar in newPchars)
{
_logger.LogInformation($"Upserting {pchar.AccountName} / {pchar.CharacterName}");
await characterService.DefaultUpsert(pchar);
}
}
else
{
int currentMaxRank = await PoeContext.Characters.MaxAsync(c => (int?)c.Rank) ?? 0;
int targetMax = 15000;
int maxDelta = targetMax - currentMaxRank;
string currentLeagueName = targetLeagueName;
if (maxDelta <= limit)
{
System.Random rnd = new System.Random();
int offset = rnd.Next(0, targetMax - limit);
await ladderTask(stoppingToken, currentLeagueName, limit, offset);
}
else
{
int offset = currentMaxRank;
await ladderTask(stoppingToken, currentLeagueName, limit, offset);
}
}
}