private static async Task UpdateMissingGames()
{
var tickCount = Environment.TickCount;
MissingUpdaterEventSource.Log.UpdateMissingGamesStart();
var allSteamAppsTask = GetAllSteamApps(s_client);
var allKnownAppsTask = StorageHelper.GetAllApps(null, StorageRetries);
await Task.WhenAll(allSteamAppsTask, allKnownAppsTask).ConfigureAwait(false);
var allSteamApps = allSteamAppsTask.Result;
var allKnownApps = allKnownAppsTask.Result;
var knownSteamIdsHash = new HashSet <int>(allKnownApps.Select(ae => ae.SteamAppId));
var missingApps = allSteamApps.Where(a => !knownSteamIdsHash.Contains(a.appid)).Take(UpdateLimit).ToArray();
MissingUpdaterEventSource.Log.MissingAppsDetermined(missingApps);
var updates = new ConcurrentBag <AppEntity>();
InvalidOperationException ioe = null;
try
{
await SteamStoreHelper
.GetStoreInformationUpdates(missingApps.Select(a => new BasicStoreInfo(a.appid, a.name, null)).ToArray(), s_client, updates)
.ConfigureAwait(false);
}
catch (Exception e)
{
ioe = new InvalidOperationException("Could not retrieve store information for all games", e);
}
var measuredUpdates = updates.Where(a => a.Measured).ToArray();
if (measuredUpdates.Length > 0)
{
await HltbScraper.ScrapeHltb(measuredUpdates).ConfigureAwait(false);
//re-impute for measured updates
await Imputer.Impute(allKnownApps.Where(a => a.Measured).Concat(measuredUpdates).ToArray()).ConfigureAwait(false);
}
//we're inserting new entries, no fear of collisions (even if two jobs overlap the next one will fix it)
await StorageHelper.Insert(updates, "updating missing games", StorageHelper.SteamToHltbTableName, StorageRetries).ConfigureAwait(false);
if (ioe != null)
{
throw ioe; //fail job
}
await SiteUtil.SendSuccessMail("Missing updater", updates.Count + " app(s) added", tickCount).ConfigureAwait(false);
MissingUpdaterEventSource.Log.UpdateMissingGamesStop();
}
private static async Task UpdateUnknownApps()
{
var ticks = Environment.TickCount;
UnknownUpdaterEventSource.Log.UpdateUnknownAppsStart();
var allUnknownApps = (await StorageHelper.GetAllApps(AppEntity.UnknownFilter, StorageRetries).ConfigureAwait(false)).Take(UpdateLimit).ToArray();
var updates = new ConcurrentBag <AppEntity>();
InvalidOperationException ioe = null;
try
{
await SteamStoreHelper.GetStoreInformationUpdates(
allUnknownApps.Select(ae => new BasicStoreInfo(ae.SteamAppId, ae.SteamName, ae.AppType)).ToArray(), Client, updates).ConfigureAwait(false);
}
catch (Exception e)
{
ioe = new InvalidOperationException("Could not retrieve store information for all apps", e);
}
UnknownUpdaterEventSource.Log.UpdateNewlyCategorizedApps(updates);
var measuredUpdates = updates.Where(a => a.Measured).ToArray();
if (measuredUpdates.Length > 0)
{
await HltbScraper.ScrapeHltb(measuredUpdates).ConfigureAwait(false);
//re-impute with new scraped values for updated games (Enumberable.Union() guarantees measuredUpdates will be enumerated before allApps!)
var allMeasuredApps = await StorageHelper.GetAllApps(AppEntity.MeasuredFilter).ConfigureAwait(false);
await Imputer.Impute(measuredUpdates.Union(allMeasuredApps, new AppEntitySteamIdComparer()).ToArray()).ConfigureAwait(false);
}
var unknownAppsMap = allUnknownApps.ToDictionary(ae => ae.SteamAppId);
await StorageHelper.ExecuteOperations(updates,
ae => new[] { TableOperation.Delete(unknownAppsMap[ae.SteamAppId]), TableOperation.Insert(ae) },
StorageHelper.SteamToHltbTableName, "updating previously unknown games", StorageRetries).ConfigureAwait(false);
if (ioe != null)
{
throw ioe; //fail job
}
await SiteUtil.SendSuccessMail("Unknown updater",
updates.Count + " previously unknown game(s) updated", ticks).ConfigureAwait(false);
UnknownUpdaterEventSource.Log.UpdateUnknownAppsStop();
}
private static async Task MainAsync()
{
var tickCount = Environment.TickCount;
SiteUtil.SetDefaultConnectionLimit();
var allMeasuredApps = (await StorageHelper.GetAllApps(AppEntity.MeasuredFilter, StorageRetries).ConfigureAwait(false)).Take(ScrapingLimit).ToArray();
await ScrapeHltb(allMeasuredApps).ConfigureAwait(false);
await Imputer.Impute(allMeasuredApps).ConfigureAwait(false);
//we're using Replace since the only other update to an existing game-typed entity would have to be manual which should take precedence
await StorageHelper.Replace(allMeasuredApps, "updating scraped gametimes", StorageHelper.SteamToHltbTableName, StorageRetries).ConfigureAwait(false);
await SiteUtil.SendSuccessMail("HLTB scraper", allMeasuredApps.Length + " game(s) scraped", tickCount).ConfigureAwait(false);
}
private static async Task ProcessDistinctSuggestions(
SuggestionEntity[] distinctSuggestions, IReadOnlyDictionary <int, AppEntity> allMeasuredAppsMap, Dictionary <int, AppEntity> allMeasuredAppsMapForImputation)
{
var validSuggestions = new ConcurrentDictionary <int, SuggestionInfo>(
GetSuggestionsForKnownAndPrepareApps(allMeasuredAppsMap, distinctSuggestions).ToDictionary(si => si.App.SteamAppId));
var invalidSuggestions = new ConcurrentBag <SuggestionInfo>(
distinctSuggestions.Where(s => !validSuggestions.ContainsKey(s.SteamAppId))
.Select(s => new SuggestionInfo(s, new AppEntity(-1, "[Unknown]", "[Unknown]"), -1, "[Unknown]")));
Console.WriteLine("Scraping HLTB info for suggestions...");
await HltbScraper.ScrapeHltb(validSuggestions.Values.Where(s => !s.Suggestion.IsRetype).Select(s => s.App).ToArray(),
(a, e) =>
{
Console.WriteLine("Can't parse HLTB ID {0} suggested for {1} ({2}): {3}", a.HltbId, a.SteamName, a.SteamAppId, e);
bool removed = validSuggestions.TryRemove(a.SteamAppId, out var invalidSuggestion);
Trace.Assert(removed, "Invalid validSuggestions state");
invalidSuggestions.Add(invalidSuggestion);
}).ConfigureAwait(false);
Console.WriteLine("Removing invalid suggestions...");
await RemoveInvalidSuggestions(invalidSuggestions);
Console.WriteLine("Processing suggestions...");
var acceptedHltbSuggestions = await ProcessSuggestionsByUserInput(validSuggestions.Values).ConfigureAwait(false);
if (acceptedHltbSuggestions.Count > 0)
{
foreach (var suggestion in acceptedHltbSuggestions)
{
allMeasuredAppsMapForImputation[suggestion.Suggestion.SteamAppId] = suggestion.App;
}
Console.WriteLine("Imputing missing TTBs from genre stats...");
await Imputer.Impute(allMeasuredAppsMapForImputation.Values.ToArray()).ConfigureAwait(false);
Console.Write("Updating HLTB suggestions... ");
foreach (var suggestion in acceptedHltbSuggestions)
{
await StorageHelper.AcceptSuggestion(suggestion.App, suggestion.Suggestion).ConfigureAwait(false);
}
}
}