public static void Cleanup() { // If threads is null, app was not yet initialized and there is nothing to cleanup if (Threads == null) { return; } Log.WriteInfo("Bootstrapper", "Exiting..."); ChangelistTimer.Stop(); Log.WriteInfo("Bootstrapper", "Disconnecting from Steam..."); try { Steam.Instance.IsRunning = false; Steam.Instance.Client.Disconnect(); } catch (Exception e) { ErrorReporter.Notify("Bootstrapper", e); } if (Settings.Current.IRC.Enabled) { Log.WriteInfo("Bootstrapper", "Closing IRC connection..."); RssReader.Timer.Stop(); IRC.Instance.Close(); } Log.WriteInfo("Bootstrapper", "Cancelling {0} tasks...", TaskManager.TasksCount); TaskManager.CancelAllTasks(); foreach (var thread in Threads.Where(thread => thread.ThreadState == ThreadState.Running)) { Log.WriteInfo("Bootstrapper", "Joining thread {0}...", thread.Name); thread.Join(TimeSpan.FromSeconds(5)); } Log.WriteInfo("Bootstrapper", "Saving local config..."); LocalConfig.Save(); try { Steam.Anonymous.IsRunning = false; Steam.Anonymous.Client.Disconnect(); } catch (Exception e) { ErrorReporter.Notify("Bootstrapper", e); } }
private void OnMachineAuth(SteamUser.UpdateMachineAuthCallback callback) { Log.WriteInfo("Steam", "Updating sentry file... {0}", callback.FileName); if (callback.Data.Length != callback.BytesToWrite) { ErrorReporter.Notify(new InvalidDataException(string.Format("Data.Length ({0}) != BytesToWrite ({1}) in OnMachineAuth", callback.Data.Length, callback.BytesToWrite))); } int fileSize; byte[] sentryHash; using (var stream = new MemoryStream(LocalConfig.Sentry ?? new byte[callback.BytesToWrite])) { stream.Seek(callback.Offset, SeekOrigin.Begin); stream.Write(callback.Data, 0, callback.BytesToWrite); stream.Seek(0, SeekOrigin.Begin); fileSize = (int)stream.Length; using (var sha = new SHA1CryptoServiceProvider()) { sentryHash = sha.ComputeHash(stream); } LocalConfig.Sentry = stream.ToArray(); } LocalConfig.SentryFileName = callback.FileName; LocalConfig.Save(); Steam.Instance.User.SendMachineAuthResponse(new SteamUser.MachineAuthDetails { JobID = callback.JobID, FileName = callback.FileName, BytesWritten = callback.BytesToWrite, FileSize = fileSize, Offset = callback.Offset, Result = EResult.OK, LastError = 0, OneTimePassword = callback.OneTimePassword, SentryFileHash = sentryHash }); }
private void OnFreeLicenseCallback(SteamApps.FreeLicenseCallback callback) { JobManager.TryRemoveJob(callback.JobID); var packageIDs = callback.GrantedPackages; var appIDs = callback.GrantedApps; Log.WriteDebug(nameof(FreeLicense), "Received free license: {0} ({1} apps: {2}, {3} packages: {4})", callback.Result, appIDs.Count, string.Join(", ", appIDs), packageIDs.Count, string.Join(", ", packageIDs)); if (appIDs.Count > 0) { JobManager.AddJob( () => Steam.Instance.Apps.PICSGetAccessTokens(appIDs, Enumerable.Empty <uint>()), new PICSTokens.RequestedTokens { Apps = appIDs.ToList() }); foreach (var appid in appIDs) { LocalConfig.Current.FreeLicensesToRequest.Remove(appid); } LocalConfig.Save(); } if (packageIDs.Count > 0) { JobManager.AddJob( () => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), packageIDs), new PICSTokens.RequestedTokens { Packages = packageIDs.ToList() }); TaskManager.RunAsync(async() => { await RefreshPackageNames(); foreach (var subID in packageIDs) { IRC.Instance.SendAnnounce("New free license granted: {0}{1}{2} -{3} {4}", Colors.BLUE, Steam.GetPackageName(subID), Colors.NORMAL, Colors.DARKBLUE, SteamDB.GetPackageUrl(subID) ); } }); } }
private static void AddToQueue(uint subId, uint appId) { if (!Settings.IsFullRun && !FreeLicenseTimer.Enabled) { FreeLicenseTimer.Start(); } if (LocalConfig.Current.FreeLicensesToRequest.ContainsKey(subId)) { return; } LocalConfig.Current.FreeLicensesToRequest.Add(subId, appId); LocalConfig.Save(); }
private static void OnTimer(object sender, ElapsedEventArgs e) { var list = LocalConfig.Current.FreeLicensesToRequest.Take(REQUEST_RATE_LIMIT).ToList(); var now = DateUtils.DateTimeToUnixTime(DateTime.UtcNow) - 60; Dictionary <uint, ulong> startTimes; using (var db = Database.Get()) { startTimes = db.Query( "SELECT `SubID`, `Value` FROM `SubsInfo` WHERE `Key` = @Key AND `SubID` IN @Ids", new { Key = KeyNameCache.GetSubKeyID("extended_starttime"), Ids = list.Select(x => x.Key) } ).ToDictionary(x => (uint)x.SubID, x => Convert.ToUInt64((string)x.Value)); } foreach (var(subId, _) in list) { if (startTimes.TryGetValue(subId, out var startTime) && startTime > now) { // If start time has not been reached yet, don't remove this app from the list and keep trying to activate it continue; } LocalConfig.Current.FreeLicensesToRequest.Remove(subId); } LocalConfig.Save(); var appids = list.Select(x => x.Value).Distinct(); AppsRequestedInHour = appids.Count(); Log.WriteDebug(nameof(FreeLicense), $"Requesting {AppsRequestedInHour} free apps as the rate limit timer ran: {string.Join(", ", appids)}"); JobManager.AddJob(() => Steam.Instance.Apps.RequestFreeLicense(appids)); if (LocalConfig.Current.FreeLicensesToRequest.Count > 0) { FreeLicenseTimer.Start(); } }
private static async void Tick(object sender, ElapsedEventArgs e) { DateTime lastPostDate; await using (var db = await Database.GetConnectionAsync()) { lastPostDate = db.ExecuteScalar <DateTime>("SELECT `Value` FROM `LocalConfig` WHERE `ConfigKey` = @Key", new { Key = "backend.lastrsspost" }); } var tasks = Settings.Current.RssFeeds.Select(uri => ProcessFeed(uri, lastPostDate)); var dates = await Task.WhenAll(tasks); var maxDate = dates.Max(); if (maxDate > lastPostDate) { await LocalConfig.Update("backend.lastrsspost", maxDate.ToString(CultureInfo.InvariantCulture)); } }
public static void Main() { Console.Title = "Steam Database"; var version = FileVersionInfo.GetVersionInfo(typeof(Steam).Assembly.Location); ProductVersion = version.ProductVersion; Log.WriteInfo("Bootstrapper", "Steam Database, built from commit: {0}", ProductVersion); Log.WriteInfo("Bootstrapper", "Copyright (c) 2013-2015, SteamDB. See LICENSE file for more information."); try { // Just create deepest folder we will use in the app string filesDir = Path.Combine(Application.Path, "files", ".support", "chunks"); Directory.CreateDirectory(filesDir); Settings.Load(); LocalConfig.Load(); } catch (Exception e) { Log.WriteError("Settings", "{0}", e.Message); return; } ErrorReporter.Init(Settings.Current.BugsnagApiKey); if (Settings.Current.SteamKitDebug) { DebugLog.AddListener(new Log.SteamKitLogger()); DebugLog.Enabled = true; } AppDomain.CurrentDomain.UnhandledException += OnSillyCrashHandler; Console.CancelKeyPress += OnCancelKey; Application.Init(); }
private static async Task RequestUpdateForList(List <uint> appIDs, List <uint> packageIDs) { Log.WriteInfo(nameof(FullUpdateProcessor), $"Requesting info for {appIDs.Count} apps and {packageIDs.Count} packages"); foreach (var list in appIDs.Split(200)) { JobManager.AddJob( () => Steam.Instance.Apps.PICSGetAccessTokens(list, Enumerable.Empty <uint>()), new PICSTokens.RequestedTokens { Apps = list.ToList() }); do { await Task.Delay(100); }while (IsBusy()); } if (Settings.Current.FullRun == FullRunState.WithForcedDepots) { return; } foreach (var list in packageIDs.Split(1000)) { JobManager.AddJob( () => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), list), new PICSTokens.RequestedTokens { Packages = list.ToList() }); do { await Task.Delay(100); }while (IsBusy()); } LocalConfig.Save(); }
public static void Cleanup() { Log.WriteInfo(nameof(Application), "Exiting..."); try { Steam.Instance.IsRunning = false; Steam.Instance.Client.Disconnect(); Steam.Instance.Dispose(); } catch (Exception e) { ErrorReporter.Notify(nameof(Application), e); } if (Settings.Current.BuiltInHttpServerPort > 0) { HttpServer.Dispose(); } if (Settings.Current.IRC.Enabled) { Log.WriteInfo(nameof(Application), "Closing IRC connection..."); RssReader.Timer.Stop(); RssReader.Dispose(); IRC.Instance.Close(); IrcThread.Join(TimeSpan.FromSeconds(5)); } TaskManager.CancelAllTasks(); if (!Settings.IsFullRun) { LocalConfig.Update("backend.changenumber", Steam.Instance.PICSChanges.PreviousChangeNumber.ToString()) .GetAwaiter().GetResult(); } }
private static void QueueRequest(uint appid) { if (Settings.IsFullRun || AppsRequestedInHour++ >= REQUEST_RATE_LIMIT) { if (LocalConfig.FreeLicensesToRequest.Contains(appid)) { return; } Log.WriteDebug("Free Packages", $"Adding app {appid} to queue as rate limit is reached"); LocalConfig.FreeLicensesToRequest.Add(appid); LocalConfig.Save(); return; } FreeLicenseTimer.Stop(); FreeLicenseTimer.Start(); JobManager.AddJob(() => Steam.Instance.Apps.RequestFreeLicense(appid)); }
private static void OnTimer(object sender, ElapsedEventArgs e) { var list = LocalConfig.FreeLicensesToRequest.Take(REQUEST_RATE_LIMIT).ToList(); foreach (var appid in list) { LocalConfig.FreeLicensesToRequest.Remove(appid); } LocalConfig.Save(); AppsRequestedInHour = list.Count; Log.WriteDebug("Free Packages", $"Requesting {AppsRequestedInHour} free apps as the rate limit timer ran"); JobManager.AddJob(() => Steam.Instance.Apps.RequestFreeLicense(list)); if (LocalConfig.FreeLicensesToRequest.Count > 0) { FreeLicenseTimer.Start(); } }
private async Task DownloadDepots(uint appID, List <ManifestJob> depots) { Log.WriteDebug("Depot Downloader", "Will process {0} depots ({1} depot locks left)", depots.Count(), DepotLocks.Count); var processTasks = new List <Task <EResult> >(); bool anyFilesDownloaded = false; foreach (var depot in depots) { var instance = depot.Anonymous ? Steam.Anonymous.Apps : Steam.Instance.Apps; depot.DepotKey = await GetDepotDecryptionKey(instance, depot.DepotID, appID); if (depot.DepotKey == null) { RemoveLock(depot.DepotID); continue; } var cdnToken = await GetCDNAuthToken(instance, appID, depot.DepotID); if (cdnToken == null) { RemoveLock(depot.DepotID); Log.WriteDebug("Depot Downloader", "Got a depot key for depot {0} but no cdn auth token", depot.DepotID); continue; } depot.CDNToken = cdnToken.Token; depot.Server = GetContentServer(); DepotManifest depotManifest = null; string lastError = string.Empty; for (var i = 0; i <= 5; i++) { try { depotManifest = await CDNClient.DownloadManifestAsync(depot.DepotID, depot.ManifestID, depot.Server, depot.CDNToken, depot.DepotKey); break; } catch (Exception e) { lastError = e.Message; Log.WriteError("Depot Processor", "Failed to download depot manifest for app {0} depot {1} ({2}: {3}) (#{4})", appID, depot.DepotID, depot.Server, lastError, i); } // TODO: get new auth key if auth fails depot.Server = GetContentServer(); if (depotManifest == null) { await Task.Delay(Utils.ExponentionalBackoff(i)); } } if (depotManifest == null) { LocalConfig.CDNAuthTokens.TryRemove(depot.DepotID, out _); RemoveLock(depot.DepotID); if (FileDownloader.IsImportantDepot(depot.DepotID)) { IRC.Instance.SendOps("{0}[{1}]{2} Failed to download manifest ({3})", Colors.OLIVE, depot.DepotName, Colors.NORMAL, lastError); } if (!Settings.IsFullRun) { JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null)); } continue; } var task = ProcessDepotAfterDownload(depot, depotManifest); processTasks.Add(task); if (!FileDownloader.IsImportantDepot(depot.DepotID)) { continue; } task = TaskManager.Run(async() => { var result = EResult.Fail; try { result = await FileDownloader.DownloadFilesFromDepot(depot, depotManifest); if (result == EResult.OK) { anyFilesDownloaded = true; } } catch (Exception e) { ErrorReporter.Notify($"Depot Processor {depot.DepotID}", e); } return(result); }).Unwrap(); processTasks.Add(task); } if (SaveLocalConfig) { SaveLocalConfig = false; LocalConfig.Save(); } await Task.WhenAll(processTasks).ConfigureAwait(false); Log.WriteDebug("Depot Downloader", "{0} depot downloads finished", depots.Count()); // TODO: use ContinueWith on tasks if (!anyFilesDownloaded) { foreach (var depot in depots) { RemoveLock(depot.DepotID); } return; } if (!File.Exists(UpdateScript)) { return; } lock (UpdateScriptLock) { foreach (var depot in depots) { if (depot.Result == EResult.OK) { RunUpdateScript(string.Format("{0} no-git", depot.DepotID)); } else if (depot.Result != EResult.Ignored) { Log.WriteWarn("Depot Processor", "Dropping stored token for {0} due to download failures", depot.DepotID); LocalConfig.CDNAuthTokens.TryRemove(depot.DepotID, out _); using (var db = Database.Get()) { // Mark this depot for redownload db.Execute("UPDATE `Depots` SET `LastManifestID` = 0 WHERE `DepotID` = @DepotID", new { depot.DepotID }); } } RemoveLock(depot.DepotID); } // Only commit changes if all depots downloaded if (processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored)) { if (!RunUpdateScript(appID, depots.First().BuildID)) { RunUpdateScript("0"); } } else { Log.WriteDebug("Depot Processor", "Reprocessing the app {0} because some files failed to download", appID); IRC.Instance.SendOps("{0}[{1}]{2} Reprocessing the app due to download failures", Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL ); JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null)); } } }
private async Task DownloadDepots(uint appID, List <ManifestJob> depots) { Log.WriteDebug("Depot Downloader", "Will process {0} depots ({1} depot locks left)", depots.Count, DepotLocks.Count); var processTasks = new List <Task <EResult> >(); var anyFilesDownloaded = false; var willDownloadFiles = false; foreach (var depot in depots) { if (depot.DepotKey == null) { await GetDepotDecryptionKey(Steam.Instance.Apps, depot, appID); if (depot.DepotKey == null && depot.LastManifestID == depot.ManifestID) { RemoveLock(depot.DepotID); continue; } } depot.Server = GetContentServer(); DepotManifest depotManifest = null; var lastError = string.Empty; for (var i = 0; i <= 5; i++) { try { depotManifest = await CDNClient.DownloadManifestAsync(depot.DepotID, depot.ManifestID, depot.Server, string.Empty, depot.DepotKey); break; } catch (Exception e) { lastError = e.Message; Log.WriteError("Depot Processor", "Failed to download depot manifest for app {0} depot {1} ({2}: {3}) (#{4})", appID, depot.DepotID, depot.Server, lastError, i); } if (depot.DepotKey != null) { RemoveErroredServer(depot.Server); } depot.Server = GetContentServer(); if (depotManifest == null) { await Task.Delay(Utils.ExponentionalBackoff(i)); } } if (depotManifest == null) { RemoveLock(depot.DepotID); if (FileDownloader.IsImportantDepot(depot.DepotID)) { IRC.Instance.SendOps("{0}[{1}]{2} Failed to download manifest ({3})", Colors.OLIVE, depot.DepotName, Colors.NORMAL, lastError); } if (!Settings.IsFullRun && depot.DepotKey != null) { JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null)); } continue; } var task = ProcessDepotAfterDownload(depot, depotManifest); processTasks.Add(task); if (!FileDownloader.IsImportantDepot(depot.DepotID) || depot.DepotKey == null) { continue; } willDownloadFiles = true; task = TaskManager.Run(async() => { var result = EResult.Fail; try { result = await FileDownloader.DownloadFilesFromDepot(depot, depotManifest); if (result == EResult.OK) { anyFilesDownloaded = true; } } catch (Exception e) { ErrorReporter.Notify($"Depot Processor {depot.DepotID}", e); } return(result); }).Unwrap(); processTasks.Add(task); } if (SaveLocalConfig) { SaveLocalConfig = false; LocalConfig.Save(); } await Task.WhenAll(processTasks).ConfigureAwait(false); Log.WriteDebug("Depot Downloader", $"{depots.Count} depot downloads finished for app {appID}"); // TODO: use ContinueWith on tasks if (!anyFilesDownloaded && !willDownloadFiles) { foreach (var depot in depots) { RemoveLock(depot.DepotID); } return; } lock (UpdateScriptLock) { foreach (var depot in depots) { if (depot.Result == EResult.OK) { RunUpdateScript(UpdateScript, string.Format("{0} no-git", depot.DepotID)); } else if (depot.Result != EResult.Ignored) { Log.WriteWarn("Depot Processor", $"Download failed for {depot.DepotID}"); // Mark this depot for redownload var db = Database.Get(); db.Execute("UPDATE `Depots` SET `LastManifestID` = 0 WHERE `DepotID` = @DepotID", new { depot.DepotID }); } RemoveLock(depot.DepotID); } // Only commit changes if all depots downloaded if (processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored)) { if (!RunUpdateScriptForApp(appID, depots[0].BuildID)) { RunUpdateScript(UpdateScript, "0"); } } else { Log.WriteDebug("Depot Processor", "Reprocessing the app {0} because some files failed to download", appID); IRC.Instance.SendOps("{0}[{1}]{2} Reprocessing the app due to download failures", Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL ); JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null)); } } }
/* * Here be dragons. */ public static async Task <EResult> DownloadFilesFromDepot(DepotProcessor.ManifestJob job, DepotManifest depotManifest) { var filesRegex = Files[job.DepotID]; var files = depotManifest.Files.Where(x => filesRegex.IsMatch(x.FileName.Replace('\\', '/'))).ToList(); var downloadState = EResult.Fail; ConcurrentDictionary <string, ExistingFileData> existingFileData; await using (var db = await Database.GetConnectionAsync()) { var data = db.ExecuteScalar <string>("SELECT `Value` FROM `LocalConfig` WHERE `ConfigKey` = @Key", new { Key = $"depot.{job.DepotID}" }); if (data != null) { existingFileData = JsonConvert.DeserializeObject <ConcurrentDictionary <string, ExistingFileData> >(data); } else { existingFileData = new ConcurrentDictionary <string, ExistingFileData>(); } } foreach (var file in existingFileData.Keys.Except(files.Select(x => x.FileName))) { Log.WriteWarn(nameof(FileDownloader), $"\"{file}\" no longer exists in manifest"); } Log.WriteInfo($"FileDownloader {job.DepotID}", $"Will download {files.Count} files"); var downloadedFiles = 0; var fileTasks = new Task[files.Count]; for (var i = 0; i < fileTasks.Length; i++) { var file = files[i]; fileTasks[i] = TaskManager.Run(async() => { var existingFile = existingFileData.GetOrAdd(file.FileName, _ => new ExistingFileData()); EResult fileState; try { await ChunkDownloadingSemaphore.WaitAsync().ConfigureAwait(false); fileState = await DownloadFile(job, file, existingFile); } finally { ChunkDownloadingSemaphore.Release(); } if (fileState == EResult.OK || fileState == EResult.SameAsPreviousValue) { existingFile.FileHash = file.FileHash; downloadedFiles++; } if (fileState != EResult.SameAsPreviousValue) { // Do not write progress info to log file Console.WriteLine($"{job.DepotName} [{downloadedFiles / (float) files.Count * 100.0f,6:#00.00}%] {files.Count - downloadedFiles} files left to download"); } if (downloadState == EResult.DataCorruption) { return; } if (fileState == EResult.OK || fileState == EResult.DataCorruption) { downloadState = fileState; } }); } await Task.WhenAll(fileTasks).ConfigureAwait(false); await LocalConfig.Update($"depot.{job.DepotID}", JsonConvert.SerializeObject(existingFileData)); job.Result = downloadState switch { EResult.OK => EResult.OK, EResult.DataCorruption => EResult.DataCorruption, _ => EResult.Ignored }; return(job.Result); }
private Task SaveBetas() { return(LocalConfig.Update("backend.beta.requests", JsonConvert.SerializeObject(BetasToRequest))); }
private void OnLoggedOn(SteamUser.LoggedOnCallback callback) { GameCoordinator.UpdateStatus(0, callback.Result.ToString()); if (callback.Result == EResult.AccountLogonDenied) { Console.Write("STEAM GUARD! Please enter the auth code sent to the email at {0}: ", callback.EmailDomain); AuthCode = Console.ReadLine(); if (AuthCode != null) { AuthCode = AuthCode.Trim(); } return; } if (callback.Result != EResult.OK) { Log.WriteInfo("Steam", "Failed to login: {0}", callback.Result); IRC.Instance.SendEmoteAnnounce("failed to log in: {0}", callback.Result); return; } var cellId = callback.CellID; if (LocalConfig.CellID != cellId) { Log.WriteDebug("Local Config", "CellID differs, {0} != {1}, forcing server refetch", LocalConfig.CellID, cellId); LocalConfig.CellID = cellId; // TODO: is this really needed? LocalConfig.LoadServers(); LocalConfig.Save(); } LastSuccessfulLogin = DateTime.Now; Log.WriteInfo("Steam", "Logged in, current Valve time is {0}", callback.ServerTime.ToString("R")); IRC.Instance.SendEmoteAnnounce("logged in. Valve time: {0}", callback.ServerTime.ToString("R")); if (Settings.IsFullRun) { if (Settings.Current.FullRun == FullRunState.ImportantOnly) { JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(Application.ImportantApps.Keys, Enumerable.Empty <uint>())); JobManager.AddJob(() => Steam.Instance.Apps.PICSGetProductInfo(Enumerable.Empty <SteamApps.PICSRequest>(), Application.ImportantSubs.Keys.Select(package => Utils.NewPICSRequest(package)))); } else if (Steam.Instance.PICSChanges.PreviousChangeNumber == 1) { Steam.Instance.PICSChanges.PerformSync(); //JobManager.AddJob(() => Steam.Instance.Apps.PICSGetChangesSince(1, true, true)); } } else { JobManager.RestartJobsIfAny(); Application.ChangelistTimer.Start(); } }
private Task Save() { return(LocalConfig.Update("backend.freelicense.requests", JsonConvert.SerializeObject(FreeLicensesToRequest))); }
private void OnPICSChanges(SteamApps.PICSChangesCallback callback) { if (PreviousChangeNumber == callback.CurrentChangeNumber) { return; } Log.WriteInfo(nameof(PICSChanges), $"Changelist {callback.LastChangeNumber} -> {callback.CurrentChangeNumber} ({callback.AppChanges.Count} apps, {callback.PackageChanges.Count} packages)"); PreviousChangeNumber = callback.CurrentChangeNumber; TaskManager.Run(async() => await HandleChangeNumbers(callback)); if (callback.RequiresFullAppUpdate || callback.RequiresFullPackageUpdate) { TaskManager.Run(async() => { if (callback.RequiresFullAppUpdate) { IRC.Instance.SendOps($"Changelist {callback.CurrentChangeNumber} has forced a full app update"); // When full update flag is set, presumably Steam client start hammering the servers // and the PICS service just does not return any data for a while until it clears up await FullUpdateProcessor.FullUpdateAppsMetadata(true); } if (callback.RequiresFullPackageUpdate) { IRC.Instance.SendOps($"Changelist {callback.CurrentChangeNumber} has forced a full package update"); await FullUpdateProcessor.FullUpdatePackagesMetadata(); } IRC.Instance.SendOps($"Changelist {callback.CurrentChangeNumber} full update has finished"); }); } if (callback.AppChanges.Count == 0 && callback.PackageChanges.Count == 0) { return; } const int appsPerJob = 50; if (callback.AppChanges.Count > appsPerJob) { foreach (var list in callback.AppChanges.Keys.Split(appsPerJob)) { JobManager.AddJob( () => Steam.Instance.Apps.PICSGetAccessTokens(list, Enumerable.Empty <uint>()), new PICSTokens.RequestedTokens { Apps = list.ToList() }); } } else if (callback.AppChanges.Count > 0) { JobManager.AddJob( () => Steam.Instance.Apps.PICSGetAccessTokens(callback.AppChanges.Keys, Enumerable.Empty <uint>()), new PICSTokens.RequestedTokens { Apps = callback.AppChanges.Keys.ToList() }); } if (callback.PackageChanges.Count > appsPerJob) { foreach (var list in callback.PackageChanges.Keys.Split(appsPerJob)) { JobManager.AddJob( () => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), list), new PICSTokens.RequestedTokens { Packages = list.ToList() }); } } else if (callback.PackageChanges.Count > 0) { JobManager.AddJob( () => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), callback.PackageChanges.Keys), new PICSTokens.RequestedTokens { Packages = callback.PackageChanges.Keys.ToList() }); } if (callback.AppChanges.Count > 0) { _ = TaskManager.Run(async() => await HandleApps(callback)); } if (callback.PackageChanges.Count > 0) { _ = TaskManager.Run(async() => await HandlePackages(callback)); _ = TaskManager.Run(async() => await HandlePackagesChangelists(callback)); } if (PreviousChangeNumber - LastStoredChangeNumber >= 1000) { LastStoredChangeNumber = PreviousChangeNumber; _ = TaskManager.Run(async() => await LocalConfig.Update("backend.changenumber", LastStoredChangeNumber.ToString())); } PrintImportants(callback); }
private async Task DownloadDepots(uint appID, List <ManifestJob> depots) { Log.WriteDebug("Depot Downloader", "Will process {0} depots ({1} depot locks left)", depots.Count(), DepotLocks.Count); var processTasks = new List <Task <EResult> >(); bool anyFilesDownloaded = false; foreach (var depot in depots) { var instance = depot.Anonymous ? Steam.Anonymous.Apps : Steam.Instance.Apps; depot.DepotKey = await GetDepotDecryptionKey(instance, depot.DepotID, appID); if (depot.DepotKey == null) { RemoveLock(depot.DepotID); continue; } var cdnToken = await GetCDNAuthToken(instance, appID, depot.DepotID); if (cdnToken == null) { RemoveLock(depot.DepotID); Log.WriteDebug("Depot Downloader", "Got a depot key for depot {0} but no cdn auth token", depot.DepotID); continue; } depot.CDNToken = cdnToken.Token; depot.Server = cdnToken.Server; DepotManifest depotManifest = null; string lastError = string.Empty; for (var i = 0; i <= 5; i++) { try { depotManifest = CDNClient.DownloadManifest(depot.DepotID, depot.ManifestID, depot.Server, depot.CDNToken, depot.DepotKey); break; } catch (Exception e) { Log.WriteWarn("Depot Downloader", "[{0}] Manifest download failed: {1} - {2}", depot.DepotID, e.GetType(), e.Message); lastError = e.Message; } // TODO: get new auth key if auth fails if (depotManifest == null) { await Task.Delay(Utils.ExponentionalBackoff(i)); } } if (depotManifest == null) { RemoveLock(depot.DepotID); Log.WriteError("Depot Processor", "Failed to download depot manifest for app {0} depot {1} ({2}: {3})", appID, depot.DepotID, depot.Server, lastError); if (FileDownloader.IsImportantDepot(depot.DepotID)) { IRC.Instance.SendOps("{0}[{1}]{2} Failed to download app {3} depot {4} manifest ({5}: {6})", Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL, appID, depot.DepotID, depot.Server, lastError); } continue; } var task = TaskManager.Run(() => { using (var db = Database.GetConnection()) { using (var transaction = db.BeginTransaction()) { var result = ProcessDepotAfterDownload(db, depot, depotManifest); transaction.Commit(); return(result); } } }); processTasks.Add(task); if (FileDownloader.IsImportantDepot(depot.DepotID)) { task = TaskManager.Run(() => { var result = FileDownloader.DownloadFilesFromDepot(appID, depot, depotManifest); if (result == EResult.OK) { anyFilesDownloaded = true; } return(result); }, TaskCreationOptions.LongRunning); TaskManager.RegisterErrorHandler(task); processTasks.Add(task); } } if (SaveLocalConfig) { SaveLocalConfig = false; LocalConfig.Save(); } await Task.WhenAll(processTasks); Log.WriteDebug("Depot Downloader", "{0} depot downloads finished", depots.Count()); // TODO: use ContinueWith on tasks if (!anyFilesDownloaded) { foreach (var depot in depots) { RemoveLock(depot.DepotID); } return; } if (!File.Exists(UpdateScript)) { return; } bool lockTaken = false; try { UpdateScriptLock.Enter(ref lockTaken); foreach (var depot in depots) { if (depot.Result == EResult.OK) { RunUpdateScript(string.Format("{0} no-git", depot.DepotID)); } RemoveLock(depot.DepotID); } // Only commit changes if all depots downloaded if (processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored)) { if (!RunUpdateScript(appID)) { RunUpdateScript("0"); } } else { Log.WriteDebug("Depot Processor", "Reprocessing the app {0} because some files failed to download", appID); IRC.Instance.SendOps("{0}[{1}]{2} Reprocessing the app due to download failures", Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL ); JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null)); } } finally { if (lockTaken) { UpdateScriptLock.Exit(); } } }
private async Task DownloadDepots(IEnumerable <ManifestJob> depots) { Log.WriteDebug("Depot Downloader", "Will process {0} depots ({1} depot locks left)", depots.Count(), DepotLocks.Count); var processTasks = new List <Task <EResult> >(); bool anyFilesDownloaded = false; foreach (var depot in depots) { depot.DepotKey = await GetDepotDecryptionKey(depot.DepotID, depot.ParentAppID); if (depot.DepotKey == null) { RemoveLock(depot.DepotID); continue; } var cdnToken = await GetCDNAuthToken(depot.DepotID); if (cdnToken == null) { RemoveLock(depot.DepotID); Log.WriteDebug("Depot Downloader", "Got a depot key for depot {0} but no cdn auth token", depot.DepotID); continue; } depot.CDNToken = cdnToken.Token; depot.Server = cdnToken.Server; DepotManifest depotManifest = null; string lastError = string.Empty; for (var i = 0; i <= 5; i++) { try { depotManifest = CDNClient.DownloadManifest(depot.DepotID, depot.ManifestID, depot.Server, depot.CDNToken, depot.DepotKey); break; } catch (Exception e) { Log.WriteWarn("Depot Downloader", "{0} Manifest download failed: {1} - {2}", depot.DepotID, e.GetType(), e.Message); lastError = e.Message; } // TODO: get new auth key if auth fails if (depotManifest == null) { await Task.Delay(Utils.ExponentionalBackoff(i)); } } if (depotManifest == null) { RemoveLock(depot.DepotID); Log.WriteError("Depot Processor", "Failed to download depot manifest for depot {0} ({1}: {2})", depot.DepotID, depot.Server, lastError); if (FileDownloader.IsImportantDepot(depot.DepotID)) { IRC.Instance.SendOps("{0}[{1}]{2} Failed to download depot {3} manifest ({4}: {5})", Colors.OLIVE, Steam.GetAppName(depot.ParentAppID), Colors.NORMAL, depot.DepotID, depot.Server, lastError); } continue; } var task = TaskManager.Run(() => { using (var db = Database.GetConnection()) { return(ProcessDepotAfterDownload(db, depot, depotManifest)); } }); processTasks.Add(task); if (FileDownloader.IsImportantDepot(depot.DepotID)) { task = TaskManager.Run(() => { var result = FileDownloader.DownloadFilesFromDepot(depot, depotManifest); if (result == EResult.OK) { anyFilesDownloaded = true; } return(result); }, TaskCreationOptions.LongRunning); TaskManager.RegisterErrorHandler(task); processTasks.Add(task); } } if (SaveLocalConfig) { SaveLocalConfig = false; LocalConfig.Save(); } await Task.WhenAll(processTasks); // TODO: use ContinueWith on tasks if (!anyFilesDownloaded) { Log.WriteDebug("Depot Downloader", "Tasks awaited for {0} depot downloads", depots.Count()); foreach (var depot in depots) { RemoveLock(depot.DepotID); } return; } var canUpdate = processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored) && File.Exists(UpdateScript); #if true Log.WriteDebug("Depot Downloader", "Tasks awaited for {0} depot downloads (will run script: {1})", depots.Count(), canUpdate); #endif bool lockTaken = false; try { UpdateScriptLock.Enter(ref lockTaken); foreach (var depot in depots) { // TODO: this only needs to run if any downloaded files changed if (canUpdate && FileDownloader.IsImportantDepot(depot.DepotID)) { RunUpdateScript(string.Format("{0} no-git", depot.DepotID)); } RemoveLock(depot.DepotID); } if (canUpdate) { RunUpdateScript("0"); } } finally { if (lockTaken) { UpdateScriptLock.Exit(); } } }