private async Task <byte[]> GetDepotDecryptionKey(SteamApps instance, uint depotID, uint appID)
{
using (var db = Database.Get())
{
var currentDecryptionKey = await db.ExecuteScalarAsync <string>("SELECT `Key` FROM `DepotsKeys` WHERE `DepotID` = @DepotID", new { depotID });
if (currentDecryptionKey != null)
{
return(Utils.StringToByteArray(currentDecryptionKey));
}
}
var task = instance.GetDepotDecryptionKey(depotID, appID);
task.Timeout = TimeSpan.FromMinutes(15);
SteamApps.DepotKeyCallback callback;
try
{
callback = await task;
}
catch (TaskCanceledException)
{
Log.WriteError("Depot Processor", "Decryption key timed out for {0}", depotID);
return(null);
}
if (callback.Result != EResult.OK)
{
if (callback.Result != EResult.AccessDenied)
{
Log.WriteError("Depot Processor", "No access to depot {0} ({1})", depotID, callback.Result);
}
return(null);
}
Log.WriteDebug("Depot Downloader", "Got a new depot key for depot {0}", depotID);
using (var db = Database.Get())
{
await db.ExecuteAsync("INSERT INTO `DepotsKeys` (`DepotID`, `Key`) VALUES (@DepotID, @Key) ON DUPLICATE KEY UPDATE `Key` = VALUES(`Key`)", new { depotID, Key = Utils.ByteArrayToString(callback.DepotKey) });
}
return(callback.DepotKey);
}
public async Task Process(IDbConnection db, uint appID, uint changeNumber, KeyValue depots)
{
var requests = new List <ManifestJob>();
var dlcNames = new Dictionary <uint, string>();
// Get data in format we want first
foreach (var depot in depots.Children)
{
var request = new ManifestJob
{
ChangeNumber = changeNumber,
};
// Ignore keys that aren't integers, for example "branches"
if (!uint.TryParse(depot.Name, out request.DepotID))
{
continue;
}
// Ignore these for now, parent app should be updated too anyway
if (depot["depotfromapp"].Value != null)
{
continue;
}
request.DepotName = depot["name"].AsString();
if (string.IsNullOrEmpty(request.DepotName))
{
request.DepotName = $"SteamDB Unnamed Depot {request.DepotID}";
}
else if (depot["dlcappid"].Value != null)
{
if (uint.TryParse(depot["dlcappid"].Value, out var dlcAppId))
{
dlcNames[dlcAppId] = request.DepotName;
}
}
// TODO: instead of locking we could wait for current process to finish
if (DepotLocks.ContainsKey(request.DepotID))
{
continue;
}
// SteamVR trickery
if (appID == 250820 &&
depot["manifests"]["beta"].Value != null &&
depots["branches"]["beta"]["buildid"].AsInteger() > depots["branches"]["public"]["buildid"].AsInteger())
{
request.BuildID = depots["branches"]["beta"]["buildid"].AsInteger();
request.ManifestID = ulong.Parse(depot["manifests"]["beta"].Value);
}
else
if (depot["manifests"]["public"].Value == null || !ulong.TryParse(depot["manifests"]["public"].Value, out request.ManifestID))
{
var branch = depot["manifests"].Children.Find(x => x.Name != "local");
if (branch == null || !ulong.TryParse(branch.Value, out request.ManifestID))
{
await db.ExecuteAsync("INSERT INTO `Depots` (`DepotID`, `Name`) VALUES (@DepotID, @DepotName) ON DUPLICATE KEY UPDATE `DepotID` = VALUES(`DepotID`)", new { request.DepotID, request.DepotName });
continue;
}
Log.WriteDebug(nameof(DepotProcessor), $"Depot {request.DepotID} (from {appID}) has no public branch, but there is another one");
request.BuildID = depots["branches"][branch.Name]["buildid"].AsInteger();
}
else
{
request.BuildID = depots["branches"]["public"]["buildid"].AsInteger();
}
requests.Add(request);
}
if (dlcNames.Any())
{
await ProcessDlcNames(appID, changeNumber, dlcNames);
}
foreach (var branch in depots["branches"].Children)
{
var buildId = branch["buildid"].AsInteger();
if (buildId < 1)
{
continue;
}
var isPublic = branch.Name != null && branch.Name.Equals("public", StringComparison.OrdinalIgnoreCase);
await db.ExecuteAsync(isPublic?
"INSERT INTO `Builds` (`BuildID`, `ChangeID`, `AppID`, `Public`) VALUES (@BuildID, @ChangeNumber, @AppID, 1) ON DUPLICATE KEY UPDATE `ChangeID` = IF(`Public` = 0, VALUES(`ChangeID`), `ChangeID`), `Public` = 1"
:
"INSERT INTO `Builds` (`BuildID`, `ChangeID`, `AppID`) VALUES (@BuildID, @ChangeNumber, @AppID) ON DUPLICATE KEY UPDATE `AppID` = `AppID`",
new
{
BuildID = buildId,
ChangeNumber = changeNumber,
AppID = appID,
});
}
if (requests.Count == 0)
{
return;
}
var depotsToDownload = new List <ManifestJob>();
var depotIds = requests.Select(x => x.DepotID).ToList();
var dbDepots = (await db.QueryAsync <Depot>("SELECT `DepotID`, `Name`, `BuildID`, `ManifestID`, `LastManifestID`, `FilenamesEncrypted` FROM `Depots` WHERE `DepotID` IN @depotIds", new { depotIds }))
.ToDictionary(x => x.DepotID, x => x);
var decryptionKeys = (await db.QueryAsync <DepotKey>("SELECT `DepotID`, `Key` FROM `DepotsKeys` WHERE `DepotID` IN @depotIds", new { depotIds }))
.ToDictionary(x => x.DepotID, x => Utils.StringToByteArray(x.Key));
foreach (var request in requests)
{
Depot dbDepot;
decryptionKeys.TryGetValue(request.DepotID, out request.DepotKey);
if (dbDepots.ContainsKey(request.DepotID))
{
dbDepot = dbDepots[request.DepotID];
if (dbDepot.BuildID > request.BuildID)
{
// buildid went back in time? this either means a rollback, or a shared depot that isn't synced properly
Log.WriteDebug(nameof(DepotProcessor), $"Skipping depot {request.DepotID} due to old buildid: {dbDepot.BuildID} > {request.BuildID}");
continue;
}
if (dbDepot.LastManifestID == request.ManifestID &&
dbDepot.ManifestID == request.ManifestID &&
Settings.FullRun != FullRunState.WithForcedDepots &&
!dbDepot.FilenamesEncrypted && request.DepotKey != null)
{
// Update depot name if changed
if (request.DepotName != dbDepot.Name)
{
await db.ExecuteAsync("UPDATE `Depots` SET `Name` = @DepotName WHERE `DepotID` = @DepotID", new { request.DepotID, request.DepotName });
}
continue;
}
request.StoredFilenamesEncrypted = dbDepot.FilenamesEncrypted;
request.LastManifestID = dbDepot.LastManifestID;
}
else
{
dbDepot = new Depot();
}
if (dbDepot.BuildID != request.BuildID || dbDepot.ManifestID != request.ManifestID || request.DepotName != dbDepot.Name)
{
await db.ExecuteAsync(@"INSERT INTO `Depots` (`DepotID`, `Name`, `BuildID`, `ManifestID`) VALUES (@DepotID, @DepotName, @BuildID, @ManifestID)
ON DUPLICATE KEY UPDATE `LastUpdated` = CURRENT_TIMESTAMP(), `Name` = VALUES(`Name`), `BuildID` = VALUES(`BuildID`), `ManifestID` = VALUES(`ManifestID`)",
new
{
request.DepotID,
request.DepotName,
request.BuildID,
request.ManifestID
});
}
if (dbDepot.ManifestID != request.ManifestID)
{
await MakeHistory(db, null, request, string.Empty, "manifest_change", dbDepot.ManifestID, request.ManifestID);
}
if (Settings.Current.OnlyOwnedDepots && !LicenseList.OwnedDepots.ContainsKey(request.DepotID))
{
continue;
}
lock (DepotLocks)
{
DepotLocks.Add(request.DepotID, 1);
}
depotsToDownload.Add(request);
}
if (depotsToDownload.Count > 0)
{
_ = TaskManager.Run(async() => await DownloadDepots(appID, depotsToDownload)).ContinueWith(task =>
{
Log.WriteError(nameof(DepotProcessor), $"An exception occured when processing depots from app {appID}, removing locks");
foreach (var depot in depotsToDownload)
{
RemoveLock(depot.DepotID);
}
}, TaskContinuationOptions.OnlyOnFaulted);
}
}
private async void OnConnected(SteamClient.ConnectedCallback callback)
{
ReconnectionTimer.Stop();
await using var db = await Database.GetConnectionAsync();
var config = (await db.QueryAsync <(string, string)>(
"SELECT `ConfigKey`, `Value` FROM `LocalConfig` WHERE `ConfigKey` IN ('backend.sentryhash', 'backend.loginkey')"
)).ToDictionary(x => x.Item1, x => x.Item2);
Log.WriteInfo(nameof(Steam), $"Connected, logging in...");
if (Settings.Current.Steam.Username == "anonymous")
{
Log.WriteInfo(nameof(Steam), "Using an anonymous account");
Steam.Instance.User.LogOnAnonymous();
return;
}
Steam.Instance.User.LogOn(new SteamUser.LogOnDetails
{
Username = Settings.Current.Steam.Username,
Password = Settings.Current.Steam.Password,
AuthCode = IsTwoFactor ? null : AuthCode,
TwoFactorCode = IsTwoFactor ? AuthCode : null,
ShouldRememberPassword = true,
SentryFileHash = config.TryGetValue("backend.sentryhash", out var sentryHash) ? Utils.StringToByteArray(sentryHash) : null,
LoginKey = config.TryGetValue("backend.loginkey", out var loginKey) ? loginKey : null,
LoginID = 0x78_50_61_77,
});
public async Task Process(uint appID, uint changeNumber, KeyValue depots)
{
var requests = new List <ManifestJob>();
// Get data in format we want first
foreach (var depot in depots.Children)
{
// Ignore these for now, parent app should be updated too anyway
if (depot["depotfromapp"].Value != null)
{
continue;
}
var request = new ManifestJob
{
ChangeNumber = changeNumber,
};
// Ignore keys that aren't integers, for example "branches"
if (!uint.TryParse(depot.Name, out request.DepotID))
{
continue;
}
request.DepotName = depot["name"].AsString();
if (string.IsNullOrEmpty(request.DepotName))
{
request.DepotName = $"SteamDB Unnamed Depot {request.DepotID}";
}
// TODO: instead of locking we could wait for current process to finish
if (DepotLocks.ContainsKey(request.DepotID))
{
continue;
}
// SteamVR trickery
if (appID == 250820 &&
depot["manifests"]["beta"].Value != null &&
depots["branches"]["beta"]["buildid"].AsInteger() > depots["branches"]["public"]["buildid"].AsInteger())
{
request.BuildID = depots["branches"]["beta"]["buildid"].AsInteger();
request.ManifestID = ulong.Parse(depot["manifests"]["beta"].Value);
}
else
if (depot["manifests"]["public"].Value == null || !ulong.TryParse(depot["manifests"]["public"].Value, out request.ManifestID))
{
var branch = depot["manifests"].Children.Find(x => x.Name != "local");
if (branch == null || !ulong.TryParse(branch.Value, out request.ManifestID))
{
await using var db = await Database.GetConnectionAsync();
await db.ExecuteAsync("INSERT INTO `Depots` (`DepotID`, `Name`) VALUES (@DepotID, @DepotName) ON DUPLICATE KEY UPDATE `DepotID` = VALUES(`DepotID`)", new { request.DepotID, request.DepotName });
continue;
}
Log.WriteDebug(nameof(DepotProcessor), $"Depot {request.DepotID} (from {appID}) has no public branch, but there is another one");
request.BuildID = depots["branches"][branch.Name]["buildid"].AsInteger();
}
else
{
request.BuildID = depots["branches"]["public"]["buildid"].AsInteger();
}
requests.Add(request);
}
if (requests.Count == 0)
{
return;
}
var depotsToDownload = new List <ManifestJob>();
await using (var db = await Database.GetConnectionAsync())
{
await db.ExecuteAsync("INSERT INTO `Builds` (`BuildID`, `ChangeID`, `AppID`) VALUES (@BuildID, @ChangeNumber, @AppID) ON DUPLICATE KEY UPDATE `AppID` = VALUES(`AppID`)",
new
{
requests[0].BuildID,
requests[0].ChangeNumber,
appID
});
var depotIds = requests.Select(x => x.DepotID).ToList();
var dbDepots = (await db.QueryAsync <Depot>("SELECT `DepotID`, `Name`, `BuildID`, `ManifestID`, `LastManifestID`, `FilenamesEncrypted` FROM `Depots` WHERE `DepotID` IN @depotIds", new { depotIds }))
.ToDictionary(x => x.DepotID, x => x);
var decryptionKeys = (await db.QueryAsync <DepotKey>("SELECT `DepotID`, `Key` FROM `DepotsKeys` WHERE `DepotID` IN @depotIds", new { depotIds }))
.ToDictionary(x => x.DepotID, x => Utils.StringToByteArray(x.Key));
foreach (var request in requests)
{
Depot dbDepot;
decryptionKeys.TryGetValue(request.DepotID, out request.DepotKey);
if (dbDepots.ContainsKey(request.DepotID))
{
dbDepot = dbDepots[request.DepotID];
if (dbDepot.BuildID > request.BuildID)
{
// buildid went back in time? this either means a rollback, or a shared depot that isn't synced properly
Log.WriteDebug(nameof(DepotProcessor), $"Skipping depot {request.DepotID} due to old buildid: {dbDepot.BuildID} > {request.BuildID}");
continue;
}
if (dbDepot.LastManifestID == request.ManifestID &&
dbDepot.ManifestID == request.ManifestID &&
Settings.Current.FullRun != FullRunState.WithForcedDepots &&
!dbDepot.FilenamesEncrypted && request.DepotKey != null)
{
// Update depot name if changed
if (request.DepotName != dbDepot.Name)
{
await db.ExecuteAsync("UPDATE `Depots` SET `Name` = @DepotName WHERE `DepotID` = @DepotID", new { request.DepotID, request.DepotName });
}
continue;
}
request.StoredFilenamesEncrypted = dbDepot.FilenamesEncrypted;
request.LastManifestID = dbDepot.LastManifestID;
}
else
{
dbDepot = new Depot();
}
if (dbDepot.BuildID != request.BuildID || dbDepot.ManifestID != request.ManifestID || request.DepotName != dbDepot.Name)
{
await db.ExecuteAsync(@"INSERT INTO `Depots` (`DepotID`, `Name`, `BuildID`, `ManifestID`) VALUES (@DepotID, @DepotName, @BuildID, @ManifestID)
ON DUPLICATE KEY UPDATE `LastUpdated` = CURRENT_TIMESTAMP(), `Name` = VALUES(`Name`), `BuildID` = VALUES(`BuildID`), `ManifestID` = VALUES(`ManifestID`)",
new
{
request.DepotID,
request.DepotName,
request.BuildID,
request.ManifestID
});
}
if (dbDepot.ManifestID != request.ManifestID)
{
await MakeHistory(db, null, request, string.Empty, "manifest_change", dbDepot.ManifestID, request.ManifestID);
}
lock (DepotLocks)
{
// This doesn't really save us from concurrency issues
if (DepotLocks.ContainsKey(request.DepotID))
{
Log.WriteWarn(nameof(DepotProcessor), $"Depot {request.DepotID} was locked in another thread");
continue;
}
DepotLocks.Add(request.DepotID, 1);
}
depotsToDownload.Add(request);
}
}
if (depotsToDownload.Count > 0)
{
_ = TaskManager.Run(async() =>
{
try
{
await DownloadDepots(appID, depotsToDownload);
}
catch (Exception e)
{
ErrorReporter.Notify(nameof(DepotProcessor), e);
}
foreach (var depot in depotsToDownload)
{
RemoveLock(depot.DepotID);
}
});
}
}
