public static async Task ScrapeHltb(AppEntity[] allApps, Action <AppEntity, Exception> errorHandler = null)
{
using (s_client = new HttpRetryClient(ScrapingRetries))
{
await ScrapeHltbCore(allApps, errorHandler).ConfigureAwait(false);
}
}
public static async Task ImputeByGenre(IReadOnlyList <AppEntity> allApps)
{
Sanitize(allApps);
using (s_client = new HttpRetryClient(ImputationServiceRetries))
{
s_client.DefaultRequestAuthorization = new AuthenticationHeaderValue(HttpRetryClient.BearerAuthorizationScheme, ApiKey);
var gamesImputationTask = ImputeTypeGenres(allApps.Where(a => a.IsGame).ToArray(), GameTypeGames);
var dlcsImputationTask = ImputeTypeGenres(allApps.Where(a => !a.IsGame).ToArray(), GameTypeDlcsMods);
await Task.WhenAll(gamesImputationTask, dlcsImputationTask).ConfigureAwait(false);
}
}
public static async Task GetStoreInformationUpdates(
ICollection <BasicStoreInfo> missingApps,
HttpRetryClient client,
ConcurrentBag <AppEntity> updates)
{
int counter = 0;
CommonEventSource.Log.RetrieveMissingStoreInformationStart();
await missingApps.Partition(MaxSteamStoreIdsPerRequest).ForEachAsync(SiteUtil.MaxConcurrentHttpRequests, async partition =>
{
await GetStoreInfo(Interlocked.Add(ref counter, MaxSteamStoreIdsPerRequest), missingApps.Count, partition, updates, client)
.ConfigureAwait(false);
}).ConfigureAwait(false);
CommonEventSource.Log.RetrieveMissingStoreInformationStop();
}
private static void Main()
{
EventSource.SetCurrentThreadActivityId(Guid.NewGuid());
try
{
SiteUtil.KeepWebJobAlive();
SiteUtil.MockWebJobEnvironmentIfMissing("MissingUpdater");
SiteUtil.SetDefaultConnectionLimit();
using (s_client = new HttpRetryClient(SteamApiRetries))
{
UpdateMissingGames().Wait();
}
}
finally
{
EventSourceRegistrar.DisposeEventListeners();
}
}
internal static async Task <IList <App> > GetAllSteamApps(HttpRetryClient client)
{
MissingUpdaterEventSource.Log.RetrieveAllSteamAppsStart(GetSteamAppListUrl);
using (var allGamesRoot = await client.GetAsync <AllGamesRoot>(GetSteamAppListUrl).ConfigureAwait(false))
{
MissingUpdaterEventSource.Log.RetrieveAllSteamAppsStop(GetSteamAppListUrl);
if (allGamesRoot.Content?.applist?.apps?.app == null)
{
MissingUpdaterEventSource.Log.ErrorRetrievingAllSteamApps(GetSteamAppListUrl);
throw new InvalidOperationException("Invalid response from " + GetSteamAppListUrl);
}
var apps = allGamesRoot.Content.applist.apps.app;
MissingUpdaterEventSource.Log.RetrievedAllSteamApps(GetSteamAppListUrl, apps.Count);
return(apps);
}
}
public void TestStoreApi()
{
var updates = new ConcurrentBag <AppEntity>();
using (var client = new HttpRetryClient(200))
{
var steamApps = MissingUpdater.GetAllSteamApps(client).Result;
var portalApp = steamApps.FirstOrDefault(app => app.appid == 400 && String.Equals(app.name, "Portal", StringComparison.Ordinal));
Assert.IsNotNull(portalApp, "Could not find Portal in the steam library: {0}",
String.Join(", ", steamApps.Select(a => String.Format(CultureInfo.InvariantCulture, "{0}/{1}", a.appid, a.name))));
SteamStoreHelper.GetStoreInformationUpdates(new[] { new BasicStoreInfo(portalApp.appid, portalApp.name, null) }, client, updates).Wait();
}
Assert.AreEqual(1, updates.Count, "Expected exactly one update for requested app (Portal)");
var portal = updates.First();
Assert.IsTrue(portal.IsGame, "Portal is not classified as a game");
Assert.IsFalse(portal.IsDlc, "Portal is classified as a DLC");
Assert.IsFalse(portal.IsMod, "Portal is classified as a mod");
Assert.IsTrue(portal.Categories.Contains("Single-player", StringComparer.OrdinalIgnoreCase),
"Portal is not classified as single-player: {0}", portal.CategoriesFlat);
Assert.AreEqual("Valve", portal.Developers.SingleOrDefault(),
"Valve are not listed as the sole developers of Portal: {0}", portal.DevelopersFlat);
Assert.AreEqual("Valve", portal.Publishers.SingleOrDefault(),
"Valve are not listed as the sole publishers of Portal: {0}", portal.PublishersFlat);
Assert.IsTrue(portal.Genres.Contains("Action", StringComparer.OrdinalIgnoreCase),
"Portal is not classified as an action game: {0}", portal.GenresFlat);
Assert.IsTrue(portal.MetacriticScore > 85, "Portal is scored too low on Metacritic: {0}", portal.MetacriticScore);
Assert.IsTrue(portal.Platforms.HasFlag(Common.Entities.Platforms.Windows) &&
portal.Platforms.HasFlag(Common.Entities.Platforms.Linux) &&
portal.Platforms.HasFlag(Common.Entities.Platforms.Mac),
"Portal is not listed as supported on Windows, Mac, and Linux: {0}", portal.Platforms);
Assert.AreEqual(new DateTime(2007, 10, 10), portal.ReleaseDate, "Portal release date is incorrect");
}
private static async Task GetStoreInfo(int counter, int total, IList <BasicStoreInfo> apps, ConcurrentBag <AppEntity> updates, HttpRetryClient client)
{
var start = counter - MaxSteamStoreIdsPerRequest + 1;
var requestUrl = new Uri(String.Format(SteamStoreApiUrlTemplate, String.Join(",", apps.Select(si => si.AppId))));
CommonEventSource.Log.RetrieveStoreInformationStart(start, counter, total, requestUrl);
using (var jObject = await client.GetAsync <JObject>(requestUrl).ConfigureAwait(false))
{
foreach (var app in apps)
{
var appInfo = jObject.Content[app.AppId.ToString(CultureInfo.InvariantCulture)].ToObject <StoreAppInfo>();
string type = !appInfo.success || String.IsNullOrWhiteSpace(appInfo.data?.type)
? AppEntity.UnknownType
: appInfo.data.type;
if (app.AppType == type)
{
CommonEventSource.Log.SkippedPopulatedApp(app.AppId, app.Name, type);
continue;
}
if (type == AppEntity.UnknownType)
{
CommonEventSource.Log.PopulatingUnknownApp(app.AppId, app.Name);
updates.Add(new AppEntity(app.AppId, app.Name, AppEntity.UnknownType));
return;
}
PopulateApp(updates, appInfo, app, type);
}
}
CommonEventSource.Log.RetrieveStoreInformationStop(start, counter, total, requestUrl);
}
