private async Task <byte[]> GetDepotDecryptionKey(SteamApps instance, uint depotID, uint appID) { using (var db = Database.Get()) { var currentDecryptionKey = await db.ExecuteScalarAsync <string>("SELECT `Key` FROM `DepotsKeys` WHERE `DepotID` = @DepotID", new { depotID }); if (currentDecryptionKey != null) { return(Utils.StringToByteArray(currentDecryptionKey)); } } var task = instance.GetDepotDecryptionKey(depotID, appID); task.Timeout = TimeSpan.FromMinutes(15); SteamApps.DepotKeyCallback callback; try { callback = await task; } catch (TaskCanceledException) { Log.WriteError("Depot Processor", "Decryption key timed out for {0}", depotID); return(null); } if (callback.Result != EResult.OK) { if (callback.Result != EResult.AccessDenied) { Log.WriteError("Depot Processor", "No access to depot {0} ({1})", depotID, callback.Result); } return(null); } Log.WriteDebug("Depot Downloader", "Got a new depot key for depot {0}", depotID); using (var db = Database.Get()) { await db.ExecuteAsync("INSERT INTO `DepotsKeys` (`DepotID`, `Key`) VALUES (@DepotID, @Key) ON DUPLICATE KEY UPDATE `Key` = VALUES(`Key`)", new { depotID, Key = Utils.ByteArrayToString(callback.DepotKey) }); } return(callback.DepotKey); }
public async Task Process(IDbConnection db, uint appID, uint changeNumber, KeyValue depots) { var requests = new List <ManifestJob>(); var dlcNames = new Dictionary <uint, string>(); // Get data in format we want first foreach (var depot in depots.Children) { var request = new ManifestJob { ChangeNumber = changeNumber, }; // Ignore keys that aren't integers, for example "branches" if (!uint.TryParse(depot.Name, out request.DepotID)) { continue; } // Ignore these for now, parent app should be updated too anyway if (depot["depotfromapp"].Value != null) { continue; } request.DepotName = depot["name"].AsString(); if (string.IsNullOrEmpty(request.DepotName)) { request.DepotName = $"SteamDB Unnamed Depot {request.DepotID}"; } else if (depot["dlcappid"].Value != null) { if (uint.TryParse(depot["dlcappid"].Value, out var dlcAppId)) { dlcNames[dlcAppId] = request.DepotName; } } // TODO: instead of locking we could wait for current process to finish if (DepotLocks.ContainsKey(request.DepotID)) { continue; } // SteamVR trickery if (appID == 250820 && depot["manifests"]["beta"].Value != null && depots["branches"]["beta"]["buildid"].AsInteger() > depots["branches"]["public"]["buildid"].AsInteger()) { request.BuildID = depots["branches"]["beta"]["buildid"].AsInteger(); request.ManifestID = ulong.Parse(depot["manifests"]["beta"].Value); } else if (depot["manifests"]["public"].Value == null || !ulong.TryParse(depot["manifests"]["public"].Value, out request.ManifestID)) { var branch = depot["manifests"].Children.Find(x => x.Name != "local"); if (branch == null || !ulong.TryParse(branch.Value, out request.ManifestID)) { await db.ExecuteAsync("INSERT INTO `Depots` (`DepotID`, `Name`) VALUES (@DepotID, @DepotName) ON DUPLICATE KEY UPDATE `DepotID` = VALUES(`DepotID`)", new { request.DepotID, request.DepotName }); continue; } Log.WriteDebug(nameof(DepotProcessor), $"Depot {request.DepotID} (from {appID}) has no public branch, but there is another one"); request.BuildID = depots["branches"][branch.Name]["buildid"].AsInteger(); } else { request.BuildID = depots["branches"]["public"]["buildid"].AsInteger(); } requests.Add(request); } if (dlcNames.Any()) { await ProcessDlcNames(appID, changeNumber, dlcNames); } foreach (var branch in depots["branches"].Children) { var buildId = branch["buildid"].AsInteger(); if (buildId < 1) { continue; } var isPublic = branch.Name != null && branch.Name.Equals("public", StringComparison.OrdinalIgnoreCase); await db.ExecuteAsync(isPublic? "INSERT INTO `Builds` (`BuildID`, `ChangeID`, `AppID`, `Public`) VALUES (@BuildID, @ChangeNumber, @AppID, 1) ON DUPLICATE KEY UPDATE `ChangeID` = IF(`Public` = 0, VALUES(`ChangeID`), `ChangeID`), `Public` = 1" : "INSERT INTO `Builds` (`BuildID`, `ChangeID`, `AppID`) VALUES (@BuildID, @ChangeNumber, @AppID) ON DUPLICATE KEY UPDATE `AppID` = `AppID`", new { BuildID = buildId, ChangeNumber = changeNumber, AppID = appID, }); } if (requests.Count == 0) { return; } var depotsToDownload = new List <ManifestJob>(); var depotIds = requests.Select(x => x.DepotID).ToList(); var dbDepots = (await db.QueryAsync <Depot>("SELECT `DepotID`, `Name`, `BuildID`, `ManifestID`, `LastManifestID`, `FilenamesEncrypted` FROM `Depots` WHERE `DepotID` IN @depotIds", new { depotIds })) .ToDictionary(x => x.DepotID, x => x); var decryptionKeys = (await db.QueryAsync <DepotKey>("SELECT `DepotID`, `Key` FROM `DepotsKeys` WHERE `DepotID` IN @depotIds", new { depotIds })) .ToDictionary(x => x.DepotID, x => Utils.StringToByteArray(x.Key)); foreach (var request in requests) { Depot dbDepot; decryptionKeys.TryGetValue(request.DepotID, out request.DepotKey); if (dbDepots.ContainsKey(request.DepotID)) { dbDepot = dbDepots[request.DepotID]; if (dbDepot.BuildID > request.BuildID) { // buildid went back in time? this either means a rollback, or a shared depot that isn't synced properly Log.WriteDebug(nameof(DepotProcessor), $"Skipping depot {request.DepotID} due to old buildid: {dbDepot.BuildID} > {request.BuildID}"); continue; } if (dbDepot.LastManifestID == request.ManifestID && dbDepot.ManifestID == request.ManifestID && Settings.FullRun != FullRunState.WithForcedDepots && !dbDepot.FilenamesEncrypted && request.DepotKey != null) { // Update depot name if changed if (request.DepotName != dbDepot.Name) { await db.ExecuteAsync("UPDATE `Depots` SET `Name` = @DepotName WHERE `DepotID` = @DepotID", new { request.DepotID, request.DepotName }); } continue; } request.StoredFilenamesEncrypted = dbDepot.FilenamesEncrypted; request.LastManifestID = dbDepot.LastManifestID; } else { dbDepot = new Depot(); } if (dbDepot.BuildID != request.BuildID || dbDepot.ManifestID != request.ManifestID || request.DepotName != dbDepot.Name) { await db.ExecuteAsync(@"INSERT INTO `Depots` (`DepotID`, `Name`, `BuildID`, `ManifestID`) VALUES (@DepotID, @DepotName, @BuildID, @ManifestID) ON DUPLICATE KEY UPDATE `LastUpdated` = CURRENT_TIMESTAMP(), `Name` = VALUES(`Name`), `BuildID` = VALUES(`BuildID`), `ManifestID` = VALUES(`ManifestID`)", new { request.DepotID, request.DepotName, request.BuildID, request.ManifestID }); } if (dbDepot.ManifestID != request.ManifestID) { await MakeHistory(db, null, request, string.Empty, "manifest_change", dbDepot.ManifestID, request.ManifestID); } if (Settings.Current.OnlyOwnedDepots && !LicenseList.OwnedDepots.ContainsKey(request.DepotID)) { continue; } lock (DepotLocks) { DepotLocks.Add(request.DepotID, 1); } depotsToDownload.Add(request); } if (depotsToDownload.Count > 0) { _ = TaskManager.Run(async() => await DownloadDepots(appID, depotsToDownload)).ContinueWith(task => { Log.WriteError(nameof(DepotProcessor), $"An exception occured when processing depots from app {appID}, removing locks"); foreach (var depot in depotsToDownload) { RemoveLock(depot.DepotID); } }, TaskContinuationOptions.OnlyOnFaulted); } }
private async void OnConnected(SteamClient.ConnectedCallback callback) { ReconnectionTimer.Stop(); await using var db = await Database.GetConnectionAsync(); var config = (await db.QueryAsync <(string, string)>( "SELECT `ConfigKey`, `Value` FROM `LocalConfig` WHERE `ConfigKey` IN ('backend.sentryhash', 'backend.loginkey')" )).ToDictionary(x => x.Item1, x => x.Item2); Log.WriteInfo(nameof(Steam), $"Connected, logging in..."); if (Settings.Current.Steam.Username == "anonymous") { Log.WriteInfo(nameof(Steam), "Using an anonymous account"); Steam.Instance.User.LogOnAnonymous(); return; } Steam.Instance.User.LogOn(new SteamUser.LogOnDetails { Username = Settings.Current.Steam.Username, Password = Settings.Current.Steam.Password, AuthCode = IsTwoFactor ? null : AuthCode, TwoFactorCode = IsTwoFactor ? AuthCode : null, ShouldRememberPassword = true, SentryFileHash = config.TryGetValue("backend.sentryhash", out var sentryHash) ? Utils.StringToByteArray(sentryHash) : null, LoginKey = config.TryGetValue("backend.loginkey", out var loginKey) ? loginKey : null, LoginID = 0x78_50_61_77, });
public async Task Process(uint appID, uint changeNumber, KeyValue depots) { var requests = new List <ManifestJob>(); // Get data in format we want first foreach (var depot in depots.Children) { // Ignore these for now, parent app should be updated too anyway if (depot["depotfromapp"].Value != null) { continue; } var request = new ManifestJob { ChangeNumber = changeNumber, }; // Ignore keys that aren't integers, for example "branches" if (!uint.TryParse(depot.Name, out request.DepotID)) { continue; } request.DepotName = depot["name"].AsString(); if (string.IsNullOrEmpty(request.DepotName)) { request.DepotName = $"SteamDB Unnamed Depot {request.DepotID}"; } // TODO: instead of locking we could wait for current process to finish if (DepotLocks.ContainsKey(request.DepotID)) { continue; } // SteamVR trickery if (appID == 250820 && depot["manifests"]["beta"].Value != null && depots["branches"]["beta"]["buildid"].AsInteger() > depots["branches"]["public"]["buildid"].AsInteger()) { request.BuildID = depots["branches"]["beta"]["buildid"].AsInteger(); request.ManifestID = ulong.Parse(depot["manifests"]["beta"].Value); } else if (depot["manifests"]["public"].Value == null || !ulong.TryParse(depot["manifests"]["public"].Value, out request.ManifestID)) { var branch = depot["manifests"].Children.Find(x => x.Name != "local"); if (branch == null || !ulong.TryParse(branch.Value, out request.ManifestID)) { await using var db = await Database.GetConnectionAsync(); await db.ExecuteAsync("INSERT INTO `Depots` (`DepotID`, `Name`) VALUES (@DepotID, @DepotName) ON DUPLICATE KEY UPDATE `DepotID` = VALUES(`DepotID`)", new { request.DepotID, request.DepotName }); continue; } Log.WriteDebug(nameof(DepotProcessor), $"Depot {request.DepotID} (from {appID}) has no public branch, but there is another one"); request.BuildID = depots["branches"][branch.Name]["buildid"].AsInteger(); } else { request.BuildID = depots["branches"]["public"]["buildid"].AsInteger(); } requests.Add(request); } if (requests.Count == 0) { return; } var depotsToDownload = new List <ManifestJob>(); await using (var db = await Database.GetConnectionAsync()) { await db.ExecuteAsync("INSERT INTO `Builds` (`BuildID`, `ChangeID`, `AppID`) VALUES (@BuildID, @ChangeNumber, @AppID) ON DUPLICATE KEY UPDATE `AppID` = VALUES(`AppID`)", new { requests[0].BuildID, requests[0].ChangeNumber, appID }); var depotIds = requests.Select(x => x.DepotID).ToList(); var dbDepots = (await db.QueryAsync <Depot>("SELECT `DepotID`, `Name`, `BuildID`, `ManifestID`, `LastManifestID`, `FilenamesEncrypted` FROM `Depots` WHERE `DepotID` IN @depotIds", new { depotIds })) .ToDictionary(x => x.DepotID, x => x); var decryptionKeys = (await db.QueryAsync <DepotKey>("SELECT `DepotID`, `Key` FROM `DepotsKeys` WHERE `DepotID` IN @depotIds", new { depotIds })) .ToDictionary(x => x.DepotID, x => Utils.StringToByteArray(x.Key)); foreach (var request in requests) { Depot dbDepot; decryptionKeys.TryGetValue(request.DepotID, out request.DepotKey); if (dbDepots.ContainsKey(request.DepotID)) { dbDepot = dbDepots[request.DepotID]; if (dbDepot.BuildID > request.BuildID) { // buildid went back in time? this either means a rollback, or a shared depot that isn't synced properly Log.WriteDebug(nameof(DepotProcessor), $"Skipping depot {request.DepotID} due to old buildid: {dbDepot.BuildID} > {request.BuildID}"); continue; } if (dbDepot.LastManifestID == request.ManifestID && dbDepot.ManifestID == request.ManifestID && Settings.Current.FullRun != FullRunState.WithForcedDepots && !dbDepot.FilenamesEncrypted && request.DepotKey != null) { // Update depot name if changed if (request.DepotName != dbDepot.Name) { await db.ExecuteAsync("UPDATE `Depots` SET `Name` = @DepotName WHERE `DepotID` = @DepotID", new { request.DepotID, request.DepotName }); } continue; } request.StoredFilenamesEncrypted = dbDepot.FilenamesEncrypted; request.LastManifestID = dbDepot.LastManifestID; } else { dbDepot = new Depot(); } if (dbDepot.BuildID != request.BuildID || dbDepot.ManifestID != request.ManifestID || request.DepotName != dbDepot.Name) { await db.ExecuteAsync(@"INSERT INTO `Depots` (`DepotID`, `Name`, `BuildID`, `ManifestID`) VALUES (@DepotID, @DepotName, @BuildID, @ManifestID) ON DUPLICATE KEY UPDATE `LastUpdated` = CURRENT_TIMESTAMP(), `Name` = VALUES(`Name`), `BuildID` = VALUES(`BuildID`), `ManifestID` = VALUES(`ManifestID`)", new { request.DepotID, request.DepotName, request.BuildID, request.ManifestID }); } if (dbDepot.ManifestID != request.ManifestID) { await MakeHistory(db, null, request, string.Empty, "manifest_change", dbDepot.ManifestID, request.ManifestID); } lock (DepotLocks) { // This doesn't really save us from concurrency issues if (DepotLocks.ContainsKey(request.DepotID)) { Log.WriteWarn(nameof(DepotProcessor), $"Depot {request.DepotID} was locked in another thread"); continue; } DepotLocks.Add(request.DepotID, 1); } depotsToDownload.Add(request); } } if (depotsToDownload.Count > 0) { _ = TaskManager.Run(async() => { try { await DownloadDepots(appID, depotsToDownload); } catch (Exception e) { ErrorReporter.Notify(nameof(DepotProcessor), e); } foreach (var depot in depotsToDownload) { RemoveLock(depot.DepotID); } }); } }