public async Task TestImputation()
{
var games = (await StorageHelper.GetAllApps(AppEntity.MeasuredFilter).ConfigureAwait(true)).ToArray();
await Imputer.ImputeByGenre(games).ConfigureAwait(false);
AssertValidTtbs(games);
}
private static void AssertTtbFixes(int main, int extras, int completionist, bool mainImputed, bool extrasImputed, bool completionistImputed)
{
int mainBefore = main;
int extrasBefore = extras;
int completionistBefore = completionist;
Imputer.FixInvalidTtbs(ref main, mainImputed, ref extras, extrasImputed, ref completionist, completionistImputed, new TtbRatios(0.7, 0.4, 0.3));
Assert.IsTrue(completionist >= extras && extras >= main,
Invariant($"Invalid TTBs not fixed: M{mainBefore}/E{extrasBefore}/C{completionistBefore} -> M{main}/E{extras}/C{completionist}"));
}
private static async Task UpdateMissingGames()
{
var tickCount = Environment.TickCount;
MissingUpdaterEventSource.Log.UpdateMissingGamesStart();
var allSteamAppsTask = GetAllSteamApps(s_client);
var allKnownAppsTask = StorageHelper.GetAllApps(null, StorageRetries);
await Task.WhenAll(allSteamAppsTask, allKnownAppsTask).ConfigureAwait(false);
var allSteamApps = allSteamAppsTask.Result;
var allKnownApps = allKnownAppsTask.Result;
var knownSteamIdsHash = new HashSet <int>(allKnownApps.Select(ae => ae.SteamAppId));
var missingApps = allSteamApps.Where(a => !knownSteamIdsHash.Contains(a.appid)).Take(UpdateLimit).ToArray();
MissingUpdaterEventSource.Log.MissingAppsDetermined(missingApps);
var updates = new ConcurrentBag <AppEntity>();
InvalidOperationException ioe = null;
try
{
await SteamStoreHelper
.GetStoreInformationUpdates(missingApps.Select(a => new BasicStoreInfo(a.appid, a.name, null)).ToArray(), s_client, updates)
.ConfigureAwait(false);
}
catch (Exception e)
{
ioe = new InvalidOperationException("Could not retrieve store information for all games", e);
}
var measuredUpdates = updates.Where(a => a.Measured).ToArray();
if (measuredUpdates.Length > 0)
{
await HltbScraper.ScrapeHltb(measuredUpdates).ConfigureAwait(false);
//re-impute for measured updates
await Imputer.Impute(allKnownApps.Where(a => a.Measured).Concat(measuredUpdates).ToArray()).ConfigureAwait(false);
}
//we're inserting new entries, no fear of collisions (even if two jobs overlap the next one will fix it)
await StorageHelper.Insert(updates, "updating missing games", StorageHelper.SteamToHltbTableName, StorageRetries).ConfigureAwait(false);
if (ioe != null)
{
throw ioe; //fail job
}
await SiteUtil.SendSuccessMail("Missing updater", updates.Count + " app(s) added", tickCount).ConfigureAwait(false);
MissingUpdaterEventSource.Log.UpdateMissingGamesStop();
}
private static async Task UpdateUnknownApps()
{
var ticks = Environment.TickCount;
UnknownUpdaterEventSource.Log.UpdateUnknownAppsStart();
var allUnknownApps = (await StorageHelper.GetAllApps(AppEntity.UnknownFilter, StorageRetries).ConfigureAwait(false)).Take(UpdateLimit).ToArray();
var updates = new ConcurrentBag <AppEntity>();
InvalidOperationException ioe = null;
try
{
await SteamStoreHelper.GetStoreInformationUpdates(
allUnknownApps.Select(ae => new BasicStoreInfo(ae.SteamAppId, ae.SteamName, ae.AppType)).ToArray(), Client, updates).ConfigureAwait(false);
}
catch (Exception e)
{
ioe = new InvalidOperationException("Could not retrieve store information for all apps", e);
}
UnknownUpdaterEventSource.Log.UpdateNewlyCategorizedApps(updates);
var measuredUpdates = updates.Where(a => a.Measured).ToArray();
if (measuredUpdates.Length > 0)
{
await HltbScraper.ScrapeHltb(measuredUpdates).ConfigureAwait(false);
//re-impute with new scraped values for updated games (Enumberable.Union() guarantees measuredUpdates will be enumerated before allApps!)
var allMeasuredApps = await StorageHelper.GetAllApps(AppEntity.MeasuredFilter).ConfigureAwait(false);
await Imputer.Impute(measuredUpdates.Union(allMeasuredApps, new AppEntitySteamIdComparer()).ToArray()).ConfigureAwait(false);
}
var unknownAppsMap = allUnknownApps.ToDictionary(ae => ae.SteamAppId);
await StorageHelper.ExecuteOperations(updates,
ae => new[] { TableOperation.Delete(unknownAppsMap[ae.SteamAppId]), TableOperation.Insert(ae) },
StorageHelper.SteamToHltbTableName, "updating previously unknown games", StorageRetries).ConfigureAwait(false);
if (ioe != null)
{
throw ioe; //fail job
}
await SiteUtil.SendSuccessMail("Unknown updater",
updates.Count + " previously unknown game(s) updated", ticks).ConfigureAwait(false);
UnknownUpdaterEventSource.Log.UpdateUnknownAppsStop();
}
private static void AssertSanitization(int main, int extras, int completionist, int mainExpected, int extrasExpected, int completionistExpected)
{
var game = new AppEntity(1, "test", AppEntity.GameTypeName);
game.SetMainTtb(main, main == 0);
game.SetExtrasTtb(extras, extras == 0);
game.SetCompletionistTtb(completionist, completionist == 0);
Imputer.Sanitize(new [] { game });
var ttbs = String.Format(CultureInfo.InvariantCulture, "{0}/{1}/{2}", main, extras, completionist);
Assert.AreEqual(mainExpected, game.MainTtb, "Invalid Main sanitization for TTBs: " + ttbs);
Assert.AreEqual(extrasExpected, game.ExtrasTtb, "Invalid Extras sanitization for TTBs: " + ttbs);
Assert.AreEqual(completionistExpected, game.CompletionistTtb, "Invalid Completionist sanitization for TTBs: " + ttbs);
}
private static async Task MainAsync()
{
var tickCount = Environment.TickCount;
SiteUtil.SetDefaultConnectionLimit();
var allMeasuredApps = (await StorageHelper.GetAllApps(AppEntity.MeasuredFilter, StorageRetries).ConfigureAwait(false)).Take(ScrapingLimit).ToArray();
await ScrapeHltb(allMeasuredApps).ConfigureAwait(false);
await Imputer.Impute(allMeasuredApps).ConfigureAwait(false);
//we're using Replace since the only other update to an existing game-typed entity would have to be manual which should take precedence
await StorageHelper.Replace(allMeasuredApps, "updating scraped gametimes", StorageHelper.SteamToHltbTableName, StorageRetries).ConfigureAwait(false);
await SiteUtil.SendSuccessMail("HLTB scraper", allMeasuredApps.Length + " game(s) scraped", tickCount).ConfigureAwait(false);
}
private static async Task ProcessDistinctSuggestions(
SuggestionEntity[] distinctSuggestions, IReadOnlyDictionary <int, AppEntity> allMeasuredAppsMap, Dictionary <int, AppEntity> allMeasuredAppsMapForImputation)
{
var validSuggestions = new ConcurrentDictionary <int, SuggestionInfo>(
GetSuggestionsForKnownAndPrepareApps(allMeasuredAppsMap, distinctSuggestions).ToDictionary(si => si.App.SteamAppId));
var invalidSuggestions = new ConcurrentBag <SuggestionInfo>(
distinctSuggestions.Where(s => !validSuggestions.ContainsKey(s.SteamAppId))
.Select(s => new SuggestionInfo(s, new AppEntity(-1, "[Unknown]", "[Unknown]"), -1, "[Unknown]")));
Console.WriteLine("Scraping HLTB info for suggestions...");
await HltbScraper.ScrapeHltb(validSuggestions.Values.Where(s => !s.Suggestion.IsRetype).Select(s => s.App).ToArray(),
(a, e) =>
{
Console.WriteLine("Can't parse HLTB ID {0} suggested for {1} ({2}): {3}", a.HltbId, a.SteamName, a.SteamAppId, e);
bool removed = validSuggestions.TryRemove(a.SteamAppId, out var invalidSuggestion);
Trace.Assert(removed, "Invalid validSuggestions state");
invalidSuggestions.Add(invalidSuggestion);
}).ConfigureAwait(false);
Console.WriteLine("Removing invalid suggestions...");
await RemoveInvalidSuggestions(invalidSuggestions);
Console.WriteLine("Processing suggestions...");
var acceptedHltbSuggestions = await ProcessSuggestionsByUserInput(validSuggestions.Values).ConfigureAwait(false);
if (acceptedHltbSuggestions.Count > 0)
{
foreach (var suggestion in acceptedHltbSuggestions)
{
allMeasuredAppsMapForImputation[suggestion.Suggestion.SteamAppId] = suggestion.App;
}
Console.WriteLine("Imputing missing TTBs from genre stats...");
await Imputer.Impute(allMeasuredAppsMapForImputation.Values.ToArray()).ConfigureAwait(false);
Console.Write("Updating HLTB suggestions... ");
foreach (var suggestion in acceptedHltbSuggestions)
{
await StorageHelper.AcceptSuggestion(suggestion.App, suggestion.Suggestion).ConfigureAwait(false);
}
}
}