private static void RetrievePlayer(Player player, Fetcher fetcher,
DbProvider provider, DbRecorder recorder, IReadOnlyDictionary <long, Tank> allTanks,
Wn8ExpectedValues wn8Expected, Putter putter)
{
var tanks = fetcher.GetTanksForPlayer(player.Plataform, player.Id);
var validTanks = tanks.Where(t => allTanks.ContainsKey(t.TankId)).ToArray();
recorder.Set(validTanks);
var played = provider.GetWn8RawStatsForPlayer(player.Plataform, player.Id);
player.Performance = played;
player.Calculate(wn8Expected);
player.Moment = DateTime.UtcNow;
player.Origin = PlayerDataOrigin.Self;
var previous = provider.GetPlayer(player.Id, player.Date, true);
if (previous != null)
{
if (player.Check(previous, true))
{
Log.Warn($"Player {player.Name}.{player.Id}@{player.Plataform} was patched.");
}
}
if (player.CanSave())
{
recorder.Set(player);
if (!player.IsPatched)
{
_ = Task.Run(() =>
{
try
{
putter?.Put(player);
}
catch (Exception ex)
{
Log.Error($"Error putting player {player.Id} on the remote site.", ex);
}
});
}
}
else
{
Log.Warn($"Player {player.Name}.{player.Id}@{player.Plataform} has to much zero data.");
}
}
private static void RetrievePlayer(Player player, Fetcher fetcher, Fetcher externalFetcher,
DbProvider provider, DbRecorder recorder, IReadOnlyDictionary <long, Tank> allTanks, Wn8ExpectedValues wn8Expected, Putter putter)
{
Task <Player> externalTask = null;
if (externalFetcher != null)
{
externalTask = externalFetcher.GetPlayerWn8Async((Player)player.Clone());
}
var tanks = fetcher.GetTanksForPlayer(player.Plataform, player.Id);
var validTanks = tanks.Where(t => allTanks.ContainsKey(t.TankId)).ToArray();
recorder.Set(validTanks);
var played = provider.GetWn8RawStatsForPlayer(player.Plataform, player.Id);
player.Performance = played;
player.Calculate(wn8Expected);
player.Moment = DateTime.UtcNow;
player.Origin = PlayerDataOrigin.Self;
var previous = provider.GetPlayer(player.Id, player.Date, true);
if (previous != null)
{
if (player.Check(previous, true))
{
Log.Warn($"Player {player.Name}.{player.Id}@{player.Plataform} was patched.");
}
}
if (player.CanSave())
{
recorder.Set(player);
if (!player.IsPatched)
{
putter?.Put(player);
}
}
else
{
Log.Warn($"Player {player.Name}.{player.Id}@{player.Plataform} has to much zero data.");
}
// Se foi feita a chamada na API externa, espero o retorno, que será ignorado; mas serviu para popular o BD do parceiro.
externalTask?.Wait();
}
private static void HandleWn8ExpectedValues(Platform platform, DbProvider provider, string resultDirectory, FtpPutter ftpPutter, DbRecorder recorder, Fetcher fetcher)
{
// Aproveito e pego e salvo os dados de WN8
if ((recorder != null) && (fetcher != null))
{
Log.Info($"Pegando dados de WN8 para {platform}...");
recorder.Set(fetcher.GetXvmWn8ExpectedValuesAsync().Result);
Log.Info("Dados de WN8 obtidos e salvos.");
}
var wn8 = provider.GetWn8ExpectedValues(platform);
if (wn8 != null)
{
var json = JsonConvert.SerializeObject(wn8, Formatting.Indented);
var file = Path.Combine(resultDirectory, "MoE", $"{wn8.Date:yyyy-MM-dd}.WN8.json");
File.WriteAllText(file, json, Encoding.UTF8);
Log.DebugFormat("Salvo o WN8 Expected em '{0}'", file);
_ = Task.Run(() =>
{
try
{
ftpPutter.PutMoe(file);
}
catch (Exception ex)
{
Log.Error($"Error putting WN8 on {platform}", ex);
}
});
Log.Debug($"Feito upload do WN8 para {platform}");
}
}
private static void RunCalculations(bool calculateReference, bool calculateMoe, DateTime?moeLastDateXbox, DateTime?moeLastDatePs,
DateTime?lastReferencesXbox, DateTime?lastReferencesPs, DbProvider provider, DbRecorder recorder, MailSender mailSender,
string resultDirectory, string resultDirectoryPs, FtpPutter ftpPutterXbox, FtpPutter ftpPutterPs,
Fetcher fetcher, int utcShiftToCalculate)
{
Debug.Assert(mailSender != null);
// Obtém os valores esperados de WN8
if ((DateTime.UtcNow.Hour % 4) == 1)
{
HandleWn8ExpectedValues(Platform.XBOX, provider, resultDirectory, ftpPutterXbox, recorder, fetcher);
HandleWn8ExpectedValues(Platform.PS, provider, resultDirectoryPs, ftpPutterPs, recorder, fetcher);
}
// 2nd hour of every day retrieve PC Tanks
if (DateTime.UtcNow.Hour == 2)
{
Log.Info("Retrieving PC Tanks...");
recorder.Set(fetcher.GetTanks(Platform.PC));
Log.Debug("PC Tanks saved on Database");
}
if (calculateMoe)
{
CalculateMoE(moeLastDateXbox, moeLastDatePs, provider, recorder, mailSender, resultDirectory, resultDirectoryPs,
ftpPutterXbox, ftpPutterPs, utcShiftToCalculate);
}
if (calculateReference)
{
CalculateTanksReferences(lastReferencesXbox, lastReferencesPs, provider, recorder, mailSender, resultDirectory, resultDirectoryPs,
ftpPutterXbox, ftpPutterPs, utcShiftToCalculate);
}
}
public ValueTask ExecuteAsync(IConsole console)
{
Log.Info($"Starting {nameof(ImportMoe)}...");
_fetcher.WebCacheAge = WebCacheAge;
// Get Console Tanks on the API
var consoleTanks = _fetcher.GetTanks(Platform.Console).ToArray();
Log.Info($"{consoleTanks.Length} Console Tanks retrieved from the API.");
// Save on Database
_recorder.Set(Platform.Console, consoleTanks);
Log.Info("Console Tanks saved.");
// Get WoTConsole.ru MoE Values
var response = _fetcher.GetMoEFromWoTConsoleRu();
Log.Info("MoE values retrieved from WoTConsole.ru");
// Checking for bizarre ids..
var tanks = consoleTanks.Select(t => t.TankId).ToHashSet();
foreach (var t in response.data.Values.ToArray())
{
if (!tanks.Contains(t.TankId))
{
Log.Warn($"WoTConsole.ru reported tank Id {t.TankId} that is not a current Console tank.");
response.data.Remove(t.TankId);
}
}
// Save Data on DB
_recorder.Set(MoeMethod.WoTConsoleRu, response.moment, response.data.Values.ToArray());
Log.Info("MoE Values Saved.");
console.Output.WriteLine("Done!");
Log.Info($"Done {nameof(ImportMoe)}.");
return(default);
private static void GetAllTanks(Plataform plataform)
{
var cacheDirectory = ConfigurationManager.AppSettings["CacheDirectory"];
var fetcher = new Fetcher(cacheDirectory)
{
WebCacheAge = TimeSpan.FromMinutes(15),
WebFetchInterval = TimeSpan.FromSeconds(1),
ApplicationId = ConfigurationManager.AppSettings["WgApi"]
};
var tanks = fetcher.GetTanks(Plataform.PC).ToArray();
var connectionString = ConfigurationManager.ConnectionStrings["Main"].ConnectionString;
var recorder = new DbRecorder(connectionString);
recorder.Set(tanks);
}
/// <summary>
/// Retrieve and populate all medals in the game
/// </summary>
private static void PopulateAllMedals()
{
var cacheDirectory = ConfigurationManager.AppSettings["CacheDirectory"];
var fetcher = new Fetcher(cacheDirectory)
{
WebCacheAge = TimeSpan.FromMinutes(15),
WebFetchInterval = TimeSpan.FromSeconds(1),
ApplicationId = ConfigurationManager.AppSettings["WgApi"]
};
var gameMedals = fetcher.GetMedals(Platform.XBOX).ToDictionary(m => m.Code);
var maxCode = gameMedals.Values.Max(m => m.Code.Length);
var maxName = gameMedals.Values.Max(m => m.Name.Length);
var maxDescription = gameMedals.Values.Max(m => m.Description.Length);
var maxHeroInformation = gameMedals.Values.Max(m => m.HeroInformation?.Length ?? 0);
var maxCondition = gameMedals.Values.Max(m => m.Condition?.Length ?? 0);
var connectionString = ConfigurationManager.ConnectionStrings["Main"].ConnectionString;
var recorder = new DbRecorder(connectionString);
recorder.Set(gameMedals.Values);
}
private static void GetFromWoTStatConsole()
{
var connectionString = ConfigurationManager.ConnectionStrings["Main"].ConnectionString;
var provider = new DbProvider(connectionString);
var recorder = new DbRecorder(connectionString);
var webCacheAge = TimeSpan.FromMinutes(10);
var cacheDirectory = ConfigurationManager.AppSettings["CacheDirectory"];
var fetcher = new Fetcher(cacheDirectory)
{
WebCacheAge = webCacheAge,
WebFetchInterval = TimeSpan.FromSeconds(1),
ApplicationId = ConfigurationManager.AppSettings["WgApi"]
};
Log.Debug("Obtendo jogadores a atualizar.");
const int ageHours = 24;
var players = provider.GetPlayersUpdateOrder(1000, ageHours).Where(p => p.AdjustedAgeHours < 1000).Take(1000).ToArray();
if (players.Length <= 0)
{
Log.Warn("Fila vazia!");
return;
}
Log.Info($"{players.Length} na fila.");
var sw = Stopwatch.StartNew();
var cts = new CancellationTokenSource();
var po = new ParallelOptions
{
CancellationToken = cts.Token,
MaxDegreeOfParallelism = 2
};
bool isEnd = false;
int count = 0;
Parallel.For(0, players.Length, po, i =>
{
if (isEnd)
{
return;
}
if (sw.Elapsed.TotalMinutes > 10)
{
isEnd = true;
Log.Warn("Tempo Esgotado!");
Log.Info($"Completo! {count} em {sw.Elapsed.TotalSeconds}. {count / sw.Elapsed.TotalMinutes:N1} players/minute");
return;
}
var player = players[i];
var task = fetcher.GetPlayerWn8Async(player);
task.Wait(po.CancellationToken);
var completePlayer = task.Result;
if (completePlayer != null)
{
if (completePlayer.CanSave())
{
recorder.Set(completePlayer);
Log.Info("Salvo!");
}
else
{
Log.WarnFormat("Jogador {0}.{1}@{2} com muitos dados zerados não será salvo no BD.",
completePlayer.Id, completePlayer.Name, completePlayer.Plataform);
}
}
Interlocked.Increment(ref count);
});
sw.Stop();
Log.Info($"Completo! {count} em {sw.Elapsed.TotalSeconds}. {count / sw.Elapsed.TotalMinutes:N1} players/minute");
}
private static int Main(string[] args)
{
try
{
var sw = Stopwatch.StartNew();
ParseParans(args, out int ageHours, out int maxClans, out int maxRunMinutes, out bool noSaveOnDatabase,
out TimeSpan webFetchInterval, out int maxToAutoAdd, out int hourToAutoAdd,
out int autoMinNumberOfMembers);
var saveOnDatabase = !noSaveOnDatabase;
var cacheDirectory = ConfigurationManager.AppSettings["CacheDirectory"];
Log.Info("------------------------------------------------------------------------------------");
Log.Info("FetchClanMembership iniciando...");
Log.InfoFormat(
"ageHours: {0}; maxClans: {1}; maxRunMinutes: {2}; cacheDirectory: {3}; noSaveOnDatabase:{4}; webFetchInterval:{5}",
ageHours, maxClans, maxRunMinutes, cacheDirectory, noSaveOnDatabase, webFetchInterval);
var connectionString = ConfigurationManager.ConnectionStrings["Main"].ConnectionString;
var provider = new DbProvider(connectionString);
var fetcher = new Fetcher(cacheDirectory)
{
WebCacheAge = TimeSpan.FromHours(ageHours - 1),
WebFetchInterval = webFetchInterval,
ApplicationId = ConfigurationManager.AppSettings["WgApi"]
};
var recorder = new DbRecorder(connectionString);
var mailSender = new MailSender(ConfigurationManager.AppSettings["SmtpHost"],
int.Parse(ConfigurationManager.AppSettings["SmtpPort"]),
true, ConfigurationManager.AppSettings["SmtpUsername"],
ConfigurationManager.AppSettings["SmtpPassword"])
{
From = new MailAddress(ConfigurationManager.AppSettings["SmtpUsername"], "Membership Service")
};
// Verifica a necessidade de adicionar novos clãs ao sistema
var clansToAdd = provider.GetClansToAdd().ToArray();
foreach (var clanToAdd in clansToAdd)
{
var clanWithId = fetcher.FindClan(clanToAdd.Plataform, clanToAdd.ClanTag);
if (clanWithId != null)
{
var existingClan = provider.GetClan(clanWithId.Plataform, clanWithId.ClanId);
if (existingClan != null)
{
if (existingClan.Enabled)
{
Log.WarnFormat(
"Foi requerido adicionar o clã {0}@{1}, mas já existe no sistema o clã {2}.{3}@{4}.",
clanToAdd.ClanTag, clanToAdd.Plataform, existingClan.ClanId, existingClan.ClanTag,
existingClan.Plataform);
mailSender.Send($"Clã {clanWithId.ClanTag} já existe e está habilitado",
$"Id: {clanWithId.ClanId}; Plataforma: {clanWithId.Plataform}; URL: {GetClanUrl(clanWithId.Plataform, clanWithId.ClanTag)}");
}
else
{
// Reabilitar o cla
recorder.EnableClan(clanWithId.Plataform, clanWithId.ClanId);
mailSender.Send($"Habilitado manualmente o clã {clanWithId.ClanTag}",
$"Id: {clanWithId.ClanId}; Plataforma: {clanWithId.Plataform}; URL: {GetClanUrl(clanWithId.Plataform, clanWithId.ClanTag)}");
}
}
else
{
clanWithId.Country = clanToAdd.Country;
recorder.Add(clanWithId);
mailSender.Send($"Adicionado manualmente o clã {clanWithId.ClanTag}",
$"Id: {clanWithId.ClanId}; Plataforma: {clanWithId.Plataform}; URL: {GetClanUrl(clanWithId.Plataform, clanWithId.ClanTag)}");
}
}
else
{
Log.WarnFormat(
"Foi requerido adicionar o clã {0}@{1}, mas ele não foi encontrado, ou é muito pequeno.",
clanToAdd.ClanTag, clanToAdd.Plataform);
mailSender.Send($"Clã {clanToAdd.ClanTag}@{clanToAdd.Plataform} não adicionado",
$"Foi requerido adicionar o clã {clanToAdd.ClanTag}@{clanToAdd.Plataform}, mas ele não foi encontrado, ou é muito pequeno.");
}
}
recorder.ClearClansToAddQueue();
// Autocadastro de clãs
if (maxToAutoAdd > 0 && DateTime.UtcNow.Hour == hourToAutoAdd)
{
var toAdd = fetcher.GetClans(Platform.XBOX, autoMinNumberOfMembers)
.Where(c => !provider.ClanExists(c.Plataform, c.ClanId))
.Concat(fetcher.GetClans(Platform.PS, autoMinNumberOfMembers)
.Where(c => !provider.ClanExists(c.Plataform, c.ClanId)))
.OrderByDescending(c => c.AllMembersCount).ThenBy(c => c.ClanId).ToArray();
foreach (var c in toAdd.Take(maxToAutoAdd))
{
recorder.Add(c);
mailSender.Send($"Adicionado automaticamente o clã {c.ClanTag}@{c.Plataform}",
$"Id: {c.ClanId}, Plataforma: {c.Plataform}; Membros: {c.AllMembersCount}; URL: {GetClanUrl(c.Plataform, c.ClanTag)}");
}
// Readição de clãs pequenos ou que ficaram inativos
recorder.ReAddClans(maxToAutoAdd);
}
var clans = provider.GetClanMembershipUpdateOrder(maxClans, ageHours).ToArray();
Log.InfoFormat("{0} clãs devem ser atualizados.", clans.Length);
if (clans.Length == 0)
{
Log.Info("Nenhum clã para atualizar.");
return(0);
}
var clansToRename = new List <Clan>();
var clansToUpdate = fetcher.GetClans(clans).ToArray();
// Clãs debandados
foreach (var clan in clansToUpdate.Where(c => c.IsDisbanded))
{
var disbandedClan = provider.GetClan(clan.Plataform, clan.ClanId);
if (saveOnDatabase)
{
recorder.DisableClan(disbandedClan.Plataform, disbandedClan.ClanId, DisabledReason.Disbanded);
var putter = new Putter(disbandedClan.Plataform, ConfigurationManager.AppSettings["ApiAdminKey"]);
putter.DeleteClan(disbandedClan.ClanTag);
}
var sb = new StringBuilder();
sb.AppendLine(
$"Clã {disbandedClan.ClanTag}, id {disbandedClan.ClanId}, no {disbandedClan.Plataform}, com {disbandedClan.Count} jogadores, foi debandado.");
sb.AppendLine($"WN8t15: {disbandedClan.Top15Wn8:N0} em {disbandedClan.Moment:yyyy-MM-dd HH:mm:ss}");
sb.AppendLine($"URL: {GetClanUrl(disbandedClan.Plataform, disbandedClan.ClanTag)}");
sb.AppendLine();
sb.AppendLine("Jogadores:");
foreach (var player in disbandedClan.Players)
{
sb.AppendLine(
$"{player.Name}: WN8 recente de {player.MonthWn8:N0} em {player.MonthBattles:N0}");
}
mailSender.Send($"Clã {disbandedClan.ClanTag}@{disbandedClan.Plataform} foi debandado",
sb.ToString());
}
// Clãs que ficaram pequenos demais
foreach (var clan in clansToUpdate.Where(c => !c.IsDisbanded && c.Count < 4))
{
if (saveOnDatabase)
{
recorder.DisableClan(clan.Plataform, clan.ClanId, DisabledReason.TooSmall);
}
mailSender.Send($"Clã {clan.ClanTag}@{clan.Plataform} foi suspenso",
$"Clã {clan.ClanTag}@{clan.Plataform} com {clan.Count} jogadores foi suspenso de atualizações futuras. " +
$"URL: {GetClanUrl(clan.Plataform, clan.ClanTag)}");
}
foreach (var clan in clansToUpdate.Where(c => !c.IsDisbanded && c.Count >= 4))
{
Log.DebugFormat("Clã {0}.{1}@{2}...", clan.ClanId, clan.ClanTag, clan.Plataform);
if (saveOnDatabase)
{
recorder.Set(clan, true);
if (clan.HasChangedTag)
{
clansToRename.Add(clan);
}
}
}
if (clansToRename.Any(c => c.Plataform == Platform.XBOX))
{
var putter = new FtpPutter(ConfigurationManager.AppSettings["FtpFolder"],
ConfigurationManager.AppSettings["FtpUser"],
ConfigurationManager.AppSettings["FtpPassworld"]);
var resultDirectory = Path.Combine(ConfigurationManager.AppSettings["ResultDirectory"], "Clans");
foreach (var clan in clansToRename.Where(c => c.Plataform == Platform.XBOX))
{
Log.InfoFormat("O clã {0}.{1}@{2} teve o tag trocado a partir de {3}.", clan.ClanId,
clan.ClanTag, clan.Plataform, clan.OldTag);
// Faz copia do arquivo local, e o upload com o novo nome
string oldFile = Path.Combine(resultDirectory, $"clan.{clan.OldTag}.json");
if (File.Exists(oldFile))
{
string newFile = Path.Combine(resultDirectory, $"clan.{clan.ClanTag}.json");
File.Copy(oldFile, newFile, true);
_ = Task.Run(() =>
{
try
{
putter.PutClan(newFile);
putter.DeleteFile($"Clans/clan.{clan.OldTag}.json");
putter.SetRenameFile(clan.OldTag, clan.ClanTag);
}
catch (Exception ex)
{
Log.Error($"Error on XBOX renaming clan {clan.OldTag} to {clan.ClanTag}", ex);
}
});
}
mailSender.Send($"Clã Renomeado: {clan.OldTag} -> {clan.ClanTag} em {clan.Plataform}",
$"URL: {GetClanUrl(clan.Plataform, clan.ClanTag)}");
}
}
if (clansToRename.Any(c => c.Plataform == Platform.PS))
{
var putter = new FtpPutter(ConfigurationManager.AppSettings["PsFtpFolder"],
ConfigurationManager.AppSettings["PsFtpUser"],
ConfigurationManager.AppSettings["PsFtpPassworld"]);
var resultDirectory = ConfigurationManager.AppSettings["PsResultDirectory"];
foreach (var clan in clansToRename.Where(c => c.Plataform == Platform.PS))
{
Log.InfoFormat("O clã {0}.{1}@{2} teve o tag trocado a partir de {3}.", clan.ClanId,
clan.ClanTag, clan.Plataform, clan.OldTag);
// Faz copia do arquivo local, e o upload com o novo nome
string oldFile = Path.Combine(resultDirectory, $"clan.{clan.OldTag}.json");
if (File.Exists(oldFile))
{
string newFile = Path.Combine(resultDirectory, $"clan.{clan.ClanTag}.json");
File.Copy(oldFile, newFile, true);
_ = Task.Run(() =>
{
try
{
putter.PutClan(newFile);
putter.DeleteFile($"Clans/clan.{clan.OldTag}.json");
putter.SetRenameFile(clan.OldTag, clan.ClanTag);
}
catch (Exception ex)
{
Log.Error($"Error on PS renaming clan {clan.OldTag} to {clan.ClanTag}", ex);
}
});
}
mailSender.Send($"Clã Renomeado: {clan.OldTag} -> {clan.ClanTag} em {clan.Plataform}",
$"URL: {GetClanUrl(clan.Plataform, clan.ClanTag)}");
}
}
// Tempo extra para os e-mails terminarem de serem enviados
Log.Info("Esperando 2 minutos antes de encerrar para terminar de enviar os e-mails...");
Thread.Sleep(TimeSpan.FromMinutes(2));
Log.InfoFormat("FetchClanMembership terminando normalmente em {0}.", sw.Elapsed);
return(0);
}
catch (Exception ex)
{
Log.Fatal(ex);
Console.WriteLine(ex);
return(1);
}
}
/// <summary>
/// Retrieves a player from database, or only WG API
/// </summary>
/// <param name="gamerTag"></param>
/// <returns></returns>
public async Task <Player> GetPlayer(CommandContext ctx, string gamerTag)
{
try
{
var cfg = GuildConfiguration.FromGuild(ctx.Guild);
var plataform = GetPlataform(gamerTag, cfg.Plataform, out gamerTag);
var provider = new DbProvider(_connectionString);
var recorder = new DbRecorder(_connectionString);
DiscordMessage willTryApiMessage = null;
long?playerId = null;
if (gamerTag.EqualsCiAi("me"))
{
playerId = provider.GetPlayerIdByDiscordId((long)ctx.User.Id);
}
playerId = playerId ?? provider.GetPlayerIdByName(plataform, gamerTag);
if (playerId == null)
{
Log.Debug($"Could not find player {plataform}.{gamerTag} on the database... trying the API...");
willTryApiMessage = await ctx.RespondAsync($"I could not find a player on `{plataform}` " +
$"with the Gamer Tag `{gamerTag}` on the Database, {ctx.User.Mention}. I will try the Wargaming API... it may take some time...");
}
var cacheDirectory = ConfigurationManager.AppSettings["CacheDir"] ?? Path.GetTempPath();
var webCacheAge = TimeSpan.FromHours(4);
var appId = ConfigurationManager.AppSettings["WgAppId"] ?? "demo";
var fetcher = new Fetcher(cacheDirectory)
{
ApplicationId = appId,
WebCacheAge = webCacheAge,
WebFetchInterval = TimeSpan.FromSeconds(1)
};
Player apiPlayer = null;
if (playerId == null)
{
await ctx.TriggerTypingAsync();
apiPlayer = fetcher.GetPlayerByGamerTag(plataform, gamerTag);
if (apiPlayer == null)
{
Log.Debug($"Could not find player {plataform}.{gamerTag} on the WG API.");
if (willTryApiMessage != null)
{
await willTryApiMessage.DeleteAsync("Information updated.");
}
await ctx.RespondAsync($"Sorry, {ctx.User.Mention}. I could not find a player on `{plataform}` " +
$"with the Gamer Tag `{gamerTag}` on the Wargaming API. Are you sure about the **exact** gamer tag?.");
return(null);
}
playerId = apiPlayer.Id;
}
var player = provider.GetPlayer(playerId.Value, true);
var wn8Expected = provider.GetWn8ExpectedValues(player?.Plataform ?? apiPlayer.Plataform);
if ((player == null) && (apiPlayer != null))
{
// Not on my database, but on the API Let's work with overall data!
IEnumerable <WgApi.TankPlayer> tanks = fetcher.GetTanksForPlayer(apiPlayer.Plataform, apiPlayer.Id);
foreach (var t in tanks)
{
t.All.TreesCut = t.TreesCut;
t.All.BattleLifeTimeSeconds = t.BattleLifeTimeSeconds;
t.All.MarkOfMastery = t.MarkOfMastery;
t.All.MaxFrags = t.MaxFrags;
t.All.LastBattle = t.LastBattle;
}
apiPlayer.Performance = new Tanks.TankPlayerPeriods()
{
All = tanks.ToDictionary(t => t.TankId, t => t.All)
};
apiPlayer.Calculate(wn8Expected);
if (willTryApiMessage != null)
{
await willTryApiMessage.DeleteAsync("Information updated.");
}
return(apiPlayer);
}
if (player.Age.TotalHours > 4)
{
willTryApiMessage = await ctx.RespondAsync($"Data for `{player.Name}` on `{player.Plataform}` " +
$"is more than {player.Age.TotalHours:N0}h old, {ctx.User.Mention}. Retrieving fresh data, please wait...");
var tanks = fetcher.GetTanksForPlayer(player.Plataform, player.Id);
var allTanks = provider.GetTanks(player.Plataform).ToDictionary(t => t.TankId);
var validTanks = tanks.Where(t => allTanks.ContainsKey(t.TankId)).ToArray();
recorder.Set(validTanks);
var played = provider.GetWn8RawStatsForPlayer(player.Plataform, player.Id);
player.Performance = played;
player.Calculate(wn8Expected);
player.Moment = DateTime.UtcNow;
player.Origin = PlayerDataOrigin.Self;
var previous = provider.GetPlayer(player.Id, player.Date, true);
if (previous != null)
{
if (player.Check(previous, true))
{
Log.Warn($"Player {player.Name}.{player.Id}@{player.Plataform} was patched.");
}
}
if (player.CanSave())
{
recorder.Set(player);
if (!player.IsPatched)
{
var putter = new Putter(player.Plataform, ConfigurationManager.AppSettings["ApiAdminKey"]);
putter.Put(player);
}
}
else
{
Log.Warn($"Player {player.Name}.{player.Id}@{player.Plataform} has to much zero data.");
}
}
else
{
player.Calculate(wn8Expected);
}
if (willTryApiMessage != null)
{
await willTryApiMessage.DeleteAsync("Information updated.");
}
return(player);
}
catch (Exception ex)
{
Log.Error($"{nameof(GetPlayer)}({gamerTag})", ex);
await ctx.RespondAsync($"Sorry, {ctx.User.Mention}. There was an error... the *Coder* will be notified of `{ex.Message}`.");
return(null);
}
}
private static int Main(string[] args)
{
try
{
var sw = Stopwatch.StartNew();
ParseParans(args, out var ageHours, out var maxPlayers, out var maxRunMinutes,
out var webCacheAge, out var kp, out var ki, out var kd, out var maxParallel, out var queryWotStatConsoleApi,
out var putPlayers);
var cacheDirectory = ConfigurationManager.AppSettings["CacheDirectory"];
Log.Info("------------------------------------------------------------------------------------");
Log.Info("FetchPlayers iniciando...");
Log.Info($"Compilado em {RetrieveLinkerTimestamp():yyyy-MM-dd HH:mm}; executando no diretorio {Environment.CurrentDirectory}");
Log.InfoFormat("ageHours: {0}; cacheDirectory: {1}, maxParallel: {2}, queryWotStatConsoleApi: {3}", ageHours, cacheDirectory, maxParallel, queryWotStatConsoleApi);
Log.Info($"kp: {kp:R}; ki: {ki:R}; kd: {kd:R}");
var connectionString = ConfigurationManager.ConnectionStrings["Main"].ConnectionString;
var provider = new DbProvider(connectionString);
var recorder = new DbRecorder(connectionString);
if (maxRunMinutes == null)
{
maxRunMinutes = 60 - DateTime.Now.Minute;
if (maxRunMinutes < 5)
{
maxRunMinutes = 5;
}
}
var dbInfo = provider.GetDataDiagnostic();
int originalMaxPlayers = maxPlayers ?? 0;
if (maxPlayers == null)
{
maxPlayers = (int?)(dbInfo.ScheduledPlayersPerHour / 60.0 * maxRunMinutes.Value);
originalMaxPlayers = maxPlayers.Value;
maxPlayers = (int?)(maxPlayers.Value * 1.20);
}
Log.Info($"maxRunMinutes: {maxRunMinutes}; maxPlayers: {maxPlayers}; originalMaxPlayers: {originalMaxPlayers}");
var players = provider.GetPlayersUpdateOrder(maxPlayers.Value, ageHours).ToArray();
if (players.Length <= 0)
{
Log.Warn("Fila vazia!");
return(3);
}
double playersPerMinute = players.Length * 1.0 / ((double)maxRunMinutes);
Log.Debug("Avg playersPerMinute: {playersPerMinute:N0}");
double externalApiCallFactor = 0.0;
if (queryWotStatConsoleApi > 0)
{
externalApiCallFactor = queryWotStatConsoleApi / playersPerMinute;
if (externalApiCallFactor > 1.0)
{
externalApiCallFactor = 1.0;
}
Log.Debug("externalApiCallFactor: {externalApiCallFactor}");
}
var externalCallRandom = new Random(69);
var recentlyUpdatedPlayers = players.Select(p => p.AdjustedAgeHours).Where(a => a < 6000).ToArray();
if (recentlyUpdatedPlayers.Any())
{
var averageAgeHours = recentlyUpdatedPlayers.Take(originalMaxPlayers).Average();
var modeAgeHours = recentlyUpdatedPlayers
.Take(originalMaxPlayers).Select(a => (decimal)(Math.Floor(a * 100.0) / 100.0)).GroupBy(a => a)
.OrderByDescending(g => g.Count()).ThenBy(g => g.Key).Select(g => g.Key).FirstOrDefault();
Log.Info($"Idade média dos dados: {averageAgeHours:N2}");
Log.Info($"Idade moda dos dados: {modeAgeHours:N2}");
}
Log.Info($"Delay últimas 48h: {dbInfo.Last48HDelay:N2}");
if (dbInfo.Last48HDelay <= ageHours)
{
// Pontual, faz apenas o planejado
Log.InfoFormat("Não está atrasado.");
players = players.Take(originalMaxPlayers).ToArray();
}
var fetchers = new ConcurrentQueue <Fetcher>();
for (int i = 0; i < maxParallel * 8; ++i)
{
var fetcher = new Fetcher(cacheDirectory)
{
WebCacheAge = webCacheAge,
WebFetchInterval = TimeSpan.FromSeconds(1),
ApplicationId = ConfigurationManager.AppSettings["WgApi"]
};
fetchers.Enqueue(fetcher);
}
Log.Debug("Obtendo todos os tanques em XBOX...");
var f = fetchers.Dequeue();
recorder.Set(f.GetTanks(Plataform.XBOX));
var allTanksXbox = provider.GetTanks(Plataform.XBOX).ToDictionary(t => t.TankId);
fetchers.Enqueue(f);
Log.InfoFormat("Obtidos {0} tanques para XBOX.", allTanksXbox.Count);
Log.Debug("Obtendo todos os tanques em PS...");
f = fetchers.Dequeue();
recorder.Set(f.GetTanks(Plataform.PS));
var allTanksPs = provider.GetTanks(Plataform.PS).ToDictionary(t => t.TankId);
fetchers.Enqueue(f);
Log.InfoFormat("Obtidos {0} tanques para PS.", allTanksPs.Count);
// Ambas as plataformas usam os mesmos valores de referência
var wn8Expected = provider.GetWn8ExpectedValues(Plataform.XBOX);
var idealInterval = (maxRunMinutes.Value * 60.0 - sw.Elapsed.TotalSeconds) / players.Length;
double threadInterval = idealInterval * maxParallel * 0.80;
var lockObject = new object();
// To save on players on the remote server (Same remote DB, so the plataform doesn't matter)
var putter = putPlayers ? new Putter(Plataform.PS, ConfigurationManager.AppSettings["ApiAdminKey"]) : null;
var cts = new CancellationTokenSource();
var po = new ParallelOptions
{
CancellationToken = cts.Token,
MaxDegreeOfParallelism = maxParallel
};
// A velocidade do loop é controlada por um sistema PID. Veja exemplo em https://www.codeproject.com/Articles/36459/PID-process-control-a-Cruise-Control-example
double integral = 0.0;
double previousError = 0.0, sumErrorSq = 0.0;
DateTime previousTime = DateTime.UtcNow;
int count = 0, controlledLoops = 0;
try
{
Parallel.For(0, players.Length, po, i =>
{
if (po.CancellationToken.IsCancellationRequested)
{
return;
}
if (sw.Elapsed.TotalMinutes > maxRunMinutes)
{
cts.Cancel();
}
var wg = fetchers.Dequeue();
Fetcher ext = null;
if (externalCallRandom.NextDouble() < externalApiCallFactor)
{
ext = fetchers.Dequeue();
}
var player = players[i];
var swPlayer = Stopwatch.StartNew();
var allTanks = player.Plataform == Plataform.XBOX ? allTanksXbox : allTanksPs;
RetrievePlayer(player, wg, ext, provider, recorder, allTanks, wn8Expected, putter);
swPlayer.Stop();
if (ext != null)
{
fetchers.Enqueue(ext);
}
fetchers.Enqueue(wg);
Thread.Sleep(TimeSpan.FromSeconds(threadInterval));
lock (lockObject)
{
Interlocked.Increment(ref count);
var actualInterval = sw.Elapsed.TotalSeconds / count;
var error = idealInterval - actualInterval;
var time = DateTime.UtcNow;
var dt = (time - previousTime).TotalSeconds;
var remainingSeconds = maxRunMinutes.Value * 60.0 - sw.Elapsed.TotalSeconds;
var remainingPlayers = players.Length - count;
if ((count > 20) && (remainingSeconds > 60) && (remainingPlayers > 5) && (dt > 0.01))
{
integral += error * dt;
var derivative = (error - previousError) / dt;
var output = (error * kp) + (integral * ki) + (derivative * kd);
threadInterval += output;
controlledLoops++;
sumErrorSq += error * error;
if (threadInterval < 0.1)
{
threadInterval = 0.1;
}
else if (threadInterval > 30)
{
threadInterval = 30;
}
Log.Info($"i: {i:0000}; {player.Id:00000000000}@{player.Plataform.ToString().PadRight(4)}; Count: {count:0000}; " +
$"TI: {threadInterval:00.00}; Actual: {actualInterval:00.00}; Ideal: {idealInterval:00.00}; " +
$"out: {output:+00.000;-00.000}; dt: {dt:000.000}; rt: {swPlayer.Elapsed.TotalSeconds:000.000}; Target: {maxRunMinutes*60.0/actualInterval:0000}; " +
$"HadExt: {ext != null}");
idealInterval = remainingSeconds / remainingPlayers;
}
else
{
Log.Info($"i: {i:0000}; {player.Id:00000000000}@{player.Plataform.ToString().PadRight(4)}; Count: {count:0000}; " +
$"TI: {threadInterval:00.00}; Actual: {actualInterval:00.00}; Ideal: {idealInterval:00.00}; " +
$"out: {0.0:+00.000;-00.000}; dt: {dt:000.000}; rt: {swPlayer.Elapsed.TotalSeconds:000.000}; Target: {maxRunMinutes * 60.0 / actualInterval:0000}; " +
$"HadExt: {ext != null}");
}
previousError = error;
previousTime = time;
}
});
}
catch (OperationCanceledException)
{
Log.WarnFormat("Tempo esgotado antes de concluir a fila no {0} de {1}. Erro de Controle: {2:N4}",
count, players.Length, sumErrorSq / controlledLoops);
return(2);
}
Log.InfoFormat("FetchPlayers terminando normalmente em {0}. Feitos {1}. Erro de Controle: {2:N4}",
sw.Elapsed, players.Length, sumErrorSq / controlledLoops);
return(0);
}
catch (Exception ex)
{
Log.Fatal(ex);
Console.WriteLine(ex);
return(1);
}
}
public ValueTask ExecuteAsync(IConsole console)
{
Log.Info($"Starting {nameof(ImportWn8WotcStat)}...");
_fetcher.WebCacheAge = WebCacheAge;
// Get Tanks on the API
var apiTanks = _fetcher.GetTanks(Platform.Console).ToArray();
Log.Info($"{apiTanks.Length} Console Tanks retrieved from the API.");
// Save on Database
_recorder.Set(Platform.Console, apiTanks);
Log.Info("Console Tanks saved.");
// Get Expected Values
var expected = _fetcher.GetWotcStatWn8ExpectedValues();
Log.Info("Expected values retrieved from WotcStat");
// WotcStat may be returning some strange ids.., most likely it's my tanks database that's outdated (WG API may lag) or it's a Cold War tank
var tanks = apiTanks.Select(t => t.TankId).ToHashSet();
foreach (var t in expected.AllTanks)
{
if (!tanks.Contains(t.TankId))
{
Log.Warn($"WotcStat reported tank Id {t.TankId} that is not a current WW2 console tank or not yiey on the database.");
expected.Remove(t.TankId);
}
}
// WotcStat may be missing some tanks... just report. The reference values will come from XVM
tanks = expected.AllTanks.Select(t => t.TankId).ToHashSet();
foreach (var apiTank in apiTanks)
{
if (!tanks.Contains(apiTank.TankId))
{
Log.Warn($"WotcStat didn't have expected values for {apiTank.ShortName}, id {apiTank.TankId}, Tier {apiTank.Tier}, {apiTank.Type}.");
}
}
// Save WotcStat Data on DB and Calculate Console Values from it
_recorder.Set(expected, Compute);
Log.Info("Expected WotcStat Values saved.");
if (Compute)
{
Log.Info("Console values computed.");
}
if (PutOnSite)
{
// Get the new calculated values and save on
var wn8 = _provider.GetWn8ExpectedValues();
if (!_putter.Put(wn8))
{
throw new CommandException("Can't upload WN8 file!");
}
Log.Info("WN8 values saved to the site.");
}
console.Output.WriteLine("Done!");
Log.Info($"Done {nameof(ImportWn8WotcStat)}.");
return(default);
public ValueTask ExecuteAsync(IConsole console)
{
Log.Info($"Starting {nameof(ImportXvm)}...");
_fetcher.WebCacheAge = WebCacheAge;
// Get PC Tanks on the API
var pcTanks = _fetcher.GetTanks(Platform.PC).ToArray();
Log.Info($"{pcTanks.Length} PC Tanks retrieved from the API.");
// Save on Database
_recorder.Set(Platform.PC, pcTanks);
Log.Info("PC Tanks saved.");
// Get XVM Expected Values
var expected = _fetcher.GetXvmWn8ExpectedValues();
Log.Info("Expected PC values retrieved from XVM");
// XVM is returning some strange ids..
var tanks = pcTanks.Select(t => t.TankId).ToHashSet();
foreach (var t in expected.AllTanks)
{
if (!tanks.Contains(t.TankId))
{
Log.Warn($"XVM reported tank Id {t.TankId} that is not a current PC tank.");
expected.Remove(t.TankId);
}
}
// Save XVM Data on DB and Calculate Console Values from it
_recorder.Set(expected, Compute);
Log.Info("Expected XVM Values saved.");
if (Compute)
{
Log.Info("Console values computed.");
}
if (PutOnSite)
{
// Get the new calculated values and save on
var wn8 = _provider.GetWn8ExpectedValues();
if (wn8.Count <= 0)
{
throw new CommandException("No current source!");
}
if (!_putter.Put(wn8))
{
throw new CommandException("Can't upload WN8 file!");
}
Log.Info("WN8 values saved to the site.");
}
console.Output.WriteLine("Done!");
Log.Info($"Done {nameof(ImportXvm)}.");
return(default);