private static void MakeHistory(ManifestJob request, string file, string action, ulong oldValue = 0, ulong newValue = 0)
{
DbWorker.ExecuteNonQuery("INSERT INTO `DepotsHistory` (`ChangeID`, `DepotID`, `File`, `Action`, `OldValue`, `NewValue`) VALUES (@ChangeID, @DepotID, @File, @Action, @OldValue, @NewValue)",
new MySqlParameter("@DepotID", request.DepotID),
new MySqlParameter("@ChangeID", request.ChangeNumber),
new MySqlParameter("@File", file),
new MySqlParameter("@Action", action),
new MySqlParameter("@OldValue", oldValue),
new MySqlParameter("@NewValue", newValue)
);
}
private async Task <EResult> ProcessDepotAfterDownload(ManifestJob request, DepotManifest depotManifest)
{
using (var db = await Database.GetConnectionAsync())
using (var transaction = await db.BeginTransactionAsync())
{
var result = await ProcessDepotAfterDownload(db, transaction, request, depotManifest);
await transaction.CommitAsync();
return(result);
}
}
private static void MakeHistory(IDbConnection db, ManifestJob request, string file, string action, ulong oldValue = 0, ulong newValue = 0)
{
db.Execute(GetHistoryQuery(),
new DepotHistory
{
DepotID = request.DepotID,
ChangeID = request.ChangeNumber,
Action = action,
File = file,
OldValue = oldValue,
NewValue = newValue
}
);
}
private static void TryDownloadManifest(ManifestJob request)
{
try
{
DownloadManifest(request);
}
catch (Exception e)
{
Log.WriteError("Depot Processor", "Caught exception while processing depot {0}: {1}\n{2}", request.DepotID, e.Message, e.StackTrace);
}
lock (ManifestJobs)
{
ManifestJobs.Remove(request);
}
}
private static async Task <(uint, EResult)> ProcessDepotAfterDownload(ManifestJob request, DepotManifest depotManifest)
{
if (depotManifest.FilenamesEncrypted && request.DepotKey != null)
{
Log.WriteError(nameof(DepotProcessor), $"Depot key for depot {request.DepotID} is invalid?");
IRC.Instance.SendOps($"[Tokens] Looks like the depot key for depot {request.DepotID} is invalid");
}
await using var db = await Database.GetConnectionAsync();
await using var transaction = await db.BeginTransactionAsync();
var result = await ProcessDepotAfterDownload(db, transaction, request, depotManifest);
await transaction.CommitAsync();
return(request.DepotID, result);
}
private static async Task GetDepotDecryptionKey(SteamApps instance, ManifestJob depot, uint appID)
{
if (!LicenseList.OwnedApps.ContainsKey(depot.DepotID))
{
return;
}
var task = instance.GetDepotDecryptionKey(depot.DepotID, appID);
task.Timeout = TimeSpan.FromMinutes(15);
SteamApps.DepotKeyCallback callback;
try
{
callback = await task;
}
catch (TaskCanceledException)
{
Log.WriteWarn(nameof(DepotProcessor), $"Decryption key timed out for {depot.DepotID}");
return;
}
if (callback.Result != EResult.OK)
{
if (callback.Result != EResult.AccessDenied)
{
Log.WriteWarn(nameof(DepotProcessor), $"No access to depot {depot.DepotID} ({callback.Result})");
}
return;
}
Log.WriteDebug(nameof(DepotProcessor), $"Got a new depot key for depot {depot.DepotID}");
await using (var db = await Database.GetConnectionAsync())
{
await db.ExecuteAsync("INSERT INTO `DepotsKeys` (`DepotID`, `Key`) VALUES (@DepotID, @Key) ON DUPLICATE KEY UPDATE `Key` = VALUES(`Key`)", new { depot.DepotID, Key = Utils.ByteArrayToString(callback.DepotKey) });
}
depot.DepotKey = callback.DepotKey;
}
public static void Process(uint appID, uint changeNumber, KeyValue depots)
{
foreach (KeyValue depot in depots.Children)
{
// Ignore these for now, parent app should be updated too anyway
if (depot["depotfromapp"].Value != null)
{
////Log.WriteDebug("Depot Processor", "Ignoring depot {0} with depotfromapp value {1} (parent {2})", depot.Name, depot["depotfromapp"].AsString(), AppID);
continue;
}
uint depotID;
if (!uint.TryParse(depot.Name, out depotID))
{
// Ignore keys that aren't integers, for example "branches"
continue;
}
lock (ManifestJobs)
{
if (ManifestJobs.Find(r => r.DepotID == depotID) != null)
{
// If we already have this depot in our job list, ignore it
continue;
}
}
ulong manifestID;
if (depot["manifests"]["public"].Value == null || !ulong.TryParse(depot["manifests"]["public"].Value, out manifestID))
{
#if false
Log.WriteDebug("Depot Processor", "Failed to public branch for depot {0} (parent {1}) - {2}", DepotID, AppID);
// If there is no public manifest for this depot, it still could have some sort of open beta
var branch = depot["manifests"].Children.SingleOrDefault(x => x.Name != "local");
if (branch == null || !ulong.TryParse(branch.Value, out ManifestID))
{
continue;
}
#endif
continue;
}
var request = new ManifestJob
{
ChangeNumber = changeNumber,
ParentAppID = appID,
DepotID = depotID,
ManifestID = manifestID,
DepotName = depot["name"].AsString()
};
// Check if manifestid in our database is equal
using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `ManifestID` FROM `Depots` WHERE `DepotID` = @DepotID AND `Files` != '' LIMIT 1", new MySqlParameter("DepotID", depotID)))
{
if (Reader.Read())
{
request.PreviousManifestID = Reader.GetUInt64("ManifestID");
if (Settings.Current.FullRun != 3 && request.PreviousManifestID == manifestID)
{
continue;
}
}
}
Log.WriteInfo("Depot Processor", "DepotID: {0}", depotID);
lock (ManifestJobs)
{
ManifestJobs.Add(request);
}
request.JobID = Steam.Instance.Apps.GetAppOwnershipTicket(depotID);
}
}
private static void DownloadManifest(ManifestJob request)
{
CDNClient cdnClient = new CDNClient(Steam.Instance.Client, request.DepotID, request.Ticket, request.DepotKey);
List<CDNClient.Server> cdnServers;
try
{
cdnServers = cdnClient.FetchServerList();
if (cdnServers.Count == 0)
{
throw new InvalidOperationException("No servers returned"); // Great programming!
}
}
catch
{
Log.WriteError("Depot Processor", "Failed to get server list for depot {0}", request.DepotID);
lock (ManifestJobs)
{
ManifestJobs.Remove(request);
}
return;
}
DepotManifest depotManifest = null;
foreach (var server in cdnServers)
{
try
{
cdnClient.Connect(server);
depotManifest = cdnClient.DownloadManifest(request.ManifestID);
break;
}
catch { }
}
lock (ManifestJobs)
{
ManifestJobs.Remove(request);
}
if (depotManifest == null)
{
Log.WriteError("Depot Processor", "Failed to download depot manifest for depot {0} (parent {1}) (jobs still in queue: {2})", request.DepotID, request.ParentAppID, ManifestJobs.Count);
if (SteamProxy.Instance.ImportantApps.Contains(request.ParentAppID))
{
IRC.SendMain("Important manifest update: {0}{1}{2} {3}(parent {4}){5} -{6} manifest download failed from {7} servers", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.DARK_GRAY, request.ParentAppID, Colors.NORMAL, Colors.RED, cdnServers.Count);
}
return;
}
if (SteamProxy.Instance.ImportantApps.Contains(request.ParentAppID))
{
IRC.SendMain("Important manifest update: {0}{1}{2} {3}(parent {4}){5} -{6} {7}", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.DARK_GRAY, request.ParentAppID, Colors.NORMAL, Colors.DARK_BLUE, SteamDB.GetDepotURL(request.DepotID, "history"));
}
var sortedFiles = depotManifest.Files.OrderBy(f => f.FileName, StringComparer.OrdinalIgnoreCase);
bool shouldHistorize = false;
List<DepotFile> filesNew = new List<DepotFile>();
List<DepotFile> filesOld = new List<DepotFile>();
foreach (var file in sortedFiles)
{
System.Text.Encoding.UTF8.GetString(file.FileHash);
filesNew.Add(new DepotFile
{
Name = file.FileName.Replace('\\', '/'),
Size = file.TotalSize,
Chunks = file.Chunks.Count,
Flags = (int)file.Flags
});
}
using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `Files` FROM `Depots` WHERE `DepotID` = @DepotID LIMIT 1", new MySqlParameter("DepotID", request.DepotID)))
{
if (Reader.Read())
{
string files = Reader.GetString("Files");
if (!string.IsNullOrEmpty(files))
{
shouldHistorize = true;
filesOld = JsonConvert.DeserializeObject<List<DepotFile>>(files);
}
}
}
DbWorker.ExecuteNonQuery("UPDATE `Depots` SET `Files` = @Files WHERE `DepotID` = @DepotID",
new MySqlParameter("@DepotID", request.DepotID),
new MySqlParameter("@Files", JsonConvert.SerializeObject(filesNew, new JsonSerializerSettings { DefaultValueHandling = DefaultValueHandling.Ignore }))
);
if (shouldHistorize)
{
List<string> filesAdded = new List<string>();
foreach (var file in filesNew)
{
var oldFile = filesOld.Find(x => x.Name == file.Name);
if (oldFile == null)
{
// We want to historize modifications first, and only then deletions and additions
filesAdded.Add(file.Name);
}
else
{
if (oldFile.Size != file.Size)
{
MakeHistory(request, file.Name, "modified", oldFile.Size, file.Size);
}
filesOld.Remove(oldFile);
}
}
foreach (var file in filesOld)
{
MakeHistory(request, file.Name, "removed");
}
foreach (string file in filesAdded)
{
MakeHistory(request, file, "added");
}
}
lock (ManifestJobs)
{
Log.WriteDebug("Depot Processor", "DepotID: Processed {0} (jobs still in queue: {1})", request.DepotID, ManifestJobs.Count);
}
// Our job is done here, now download important files if any
if (Settings.Current.ImportantFiles.ContainsKey(request.DepotID))
{
// TODO: Horribly inefficient
var importantFiles = sortedFiles.Where(x => Settings.Current.ImportantFiles[request.DepotID].Contains(x.FileName.Replace('\\', '/')));
ThreadPool.QueueWorkItem(DownloadFiles, importantFiles, request.DepotID, cdnClient);
}
}
public async Task Process(uint appID, uint changeNumber, KeyValue depots)
{
var requests = new List <ManifestJob>();
// Get data in format we want first
foreach (var depot in depots.Children)
{
// Ignore these for now, parent app should be updated too anyway
if (depot["depotfromapp"].Value != null)
{
continue;
}
var request = new ManifestJob
{
ChangeNumber = changeNumber,
};
// Ignore keys that aren't integers, for example "branches"
if (!uint.TryParse(depot.Name, out request.DepotID))
{
continue;
}
request.DepotName = depot["name"].AsString();
if (string.IsNullOrEmpty(request.DepotName))
{
request.DepotName = $"SteamDB Unnamed Depot {request.DepotID}";
}
// TODO: instead of locking we could wait for current process to finish
if (DepotLocks.ContainsKey(request.DepotID))
{
continue;
}
// SteamVR trickery
if (appID == 250820 &&
depot["manifests"]["beta"].Value != null &&
depots["branches"]["beta"]["buildid"].AsInteger() > depots["branches"]["public"]["buildid"].AsInteger())
{
request.BuildID = depots["branches"]["beta"]["buildid"].AsInteger();
request.ManifestID = ulong.Parse(depot["manifests"]["beta"].Value);
}
else
if (depot["manifests"]["public"].Value == null || !ulong.TryParse(depot["manifests"]["public"].Value, out request.ManifestID))
{
var branch = depot["manifests"].Children.Find(x => x.Name != "local");
if (branch == null || !ulong.TryParse(branch.Value, out request.ManifestID))
{
await using var db = await Database.GetConnectionAsync();
await db.ExecuteAsync("INSERT INTO `Depots` (`DepotID`, `Name`) VALUES (@DepotID, @DepotName) ON DUPLICATE KEY UPDATE `DepotID` = VALUES(`DepotID`)", new { request.DepotID, request.DepotName });
continue;
}
Log.WriteDebug(nameof(DepotProcessor), $"Depot {request.DepotID} (from {appID}) has no public branch, but there is another one");
request.BuildID = depots["branches"][branch.Name]["buildid"].AsInteger();
}
else
{
request.BuildID = depots["branches"]["public"]["buildid"].AsInteger();
}
requests.Add(request);
}
if (requests.Count == 0)
{
return;
}
var depotsToDownload = new List <ManifestJob>();
await using (var db = await Database.GetConnectionAsync())
{
await db.ExecuteAsync("INSERT INTO `Builds` (`BuildID`, `ChangeID`, `AppID`) VALUES (@BuildID, @ChangeNumber, @AppID) ON DUPLICATE KEY UPDATE `AppID` = VALUES(`AppID`)",
new
{
requests[0].BuildID,
requests[0].ChangeNumber,
appID
});
var depotIds = requests.Select(x => x.DepotID).ToList();
var dbDepots = (await db.QueryAsync <Depot>("SELECT `DepotID`, `Name`, `BuildID`, `ManifestID`, `LastManifestID`, `FilenamesEncrypted` FROM `Depots` WHERE `DepotID` IN @depotIds", new { depotIds }))
.ToDictionary(x => x.DepotID, x => x);
var decryptionKeys = (await db.QueryAsync <DepotKey>("SELECT `DepotID`, `Key` FROM `DepotsKeys` WHERE `DepotID` IN @depotIds", new { depotIds }))
.ToDictionary(x => x.DepotID, x => Utils.StringToByteArray(x.Key));
foreach (var request in requests)
{
Depot dbDepot;
decryptionKeys.TryGetValue(request.DepotID, out request.DepotKey);
if (dbDepots.ContainsKey(request.DepotID))
{
dbDepot = dbDepots[request.DepotID];
if (dbDepot.BuildID > request.BuildID)
{
// buildid went back in time? this either means a rollback, or a shared depot that isn't synced properly
Log.WriteDebug(nameof(DepotProcessor), $"Skipping depot {request.DepotID} due to old buildid: {dbDepot.BuildID} > {request.BuildID}");
continue;
}
if (dbDepot.LastManifestID == request.ManifestID &&
dbDepot.ManifestID == request.ManifestID &&
Settings.Current.FullRun != FullRunState.WithForcedDepots &&
!dbDepot.FilenamesEncrypted && request.DepotKey != null)
{
// Update depot name if changed
if (request.DepotName != dbDepot.Name)
{
await db.ExecuteAsync("UPDATE `Depots` SET `Name` = @DepotName WHERE `DepotID` = @DepotID", new { request.DepotID, request.DepotName });
}
continue;
}
request.StoredFilenamesEncrypted = dbDepot.FilenamesEncrypted;
request.LastManifestID = dbDepot.LastManifestID;
}
else
{
dbDepot = new Depot();
}
if (dbDepot.BuildID != request.BuildID || dbDepot.ManifestID != request.ManifestID || request.DepotName != dbDepot.Name)
{
await db.ExecuteAsync(@"INSERT INTO `Depots` (`DepotID`, `Name`, `BuildID`, `ManifestID`) VALUES (@DepotID, @DepotName, @BuildID, @ManifestID)
ON DUPLICATE KEY UPDATE `LastUpdated` = CURRENT_TIMESTAMP(), `Name` = VALUES(`Name`), `BuildID` = VALUES(`BuildID`), `ManifestID` = VALUES(`ManifestID`)",
new
{
request.DepotID,
request.DepotName,
request.BuildID,
request.ManifestID
});
}
if (dbDepot.ManifestID != request.ManifestID)
{
await MakeHistory(db, null, request, string.Empty, "manifest_change", dbDepot.ManifestID, request.ManifestID);
}
lock (DepotLocks)
{
// This doesn't really save us from concurrency issues
if (DepotLocks.ContainsKey(request.DepotID))
{
Log.WriteWarn(nameof(DepotProcessor), $"Depot {request.DepotID} was locked in another thread");
continue;
}
DepotLocks.Add(request.DepotID, 1);
}
depotsToDownload.Add(request);
}
}
if (depotsToDownload.Count > 0)
{
_ = TaskManager.Run(async() =>
{
try
{
await DownloadDepots(appID, depotsToDownload);
}
catch (Exception e)
{
ErrorReporter.Notify(nameof(DepotProcessor), e);
}
foreach (var depot in depotsToDownload)
{
RemoveLock(depot.DepotID);
}
});
}
}
private static Task MakeHistory(IDbConnection db, IDbTransaction transaction, ManifestJob request, string file, string action, ulong oldValue = 0, ulong newValue = 0)
{
return(db.ExecuteAsync(HistoryQuery,
new DepotHistory
{
DepotID = request.DepotID,
ChangeID = request.ChangeNumber,
Action = action,
File = file,
OldValue = oldValue,
NewValue = newValue
},
transaction: transaction
));
}
private void DownloadManifest(ManifestJob request)
{
Log.WriteInfo("Depot Processor", "DepotID: {0}", request.DepotID);
DepotManifest depotManifest = null;
string lastError = string.Empty;
// CDN is very random, just keep trying
for (var i = 0; i <= 5; i++)
{
try
{
depotManifest = CDNClient.DownloadManifest(request.DepotID, request.ManifestID, request.Server, request.CDNToken, request.DepotKey);
break;
}
catch (Exception e)
{
lastError = e.Message;
}
}
if (depotManifest == null)
{
Log.WriteError("Depot Processor", "Failed to download depot manifest for depot {0} ({1}: {2}) (#{3})", request.DepotID, request.Server, lastError, request.Tries);
if (--request.Tries >= 0)
{
request.Server = GetContentServer(request.Tries);
JobManager.AddJob(() => Steam.Instance.Apps.GetCDNAuthToken(request.DepotID, request.Server), request);
return;
}
RemoveLock(request.DepotID); // TODO: Remove this once task in OnCDNAuthTokenCallback is used
if (FileDownloader.IsImportantDepot(request.DepotID))
{
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download depot {3} manifest ({4}: {5})",
Colors.OLIVE, Steam.GetAppName(request.ParentAppID), Colors.NORMAL, request.DepotID, request.Server, lastError);
}
return;
}
if (FileDownloader.IsImportantDepot(request.DepotID))
{
TaskManager.Run(() => FileDownloader.DownloadFilesFromDepot(request, depotManifest));
}
// TODO: Task here instead of in OnCDNAuthTokenCallback due to mono's silly threadpool
TaskManager.Run(() =>
{
using(var db = Database.GetConnection())
{
ProcessDepotAfterDownload(db, request, depotManifest);
}
}).ContinueWith(task =>
{
RemoveLock(request.DepotID);
Log.WriteDebug("Depot Processor", "Processed depot {0} ({1} depot locks left)", request.DepotID, DepotLocks.Count);
});
}
private void MakeHistory(ManifestJob request, string file, string action, ulong oldValue = 0, ulong newValue = 0)
{
DbWorker.ExecuteNonQuery(
"INSERT INTO `DepotsHistory` (`ChangeID`, `DepotID`, `File`, `Action`, `OldValue`, `NewValue`) VALUES (@ChangeID, @DepotID, @File, @Action, @OldValue, @NewValue)",
new MySqlParameter("@DepotID", request.DepotID),
new MySqlParameter("@ChangeID", request.ChangeNumber),
new MySqlParameter("@File", file),
new MySqlParameter("@Action", action),
new MySqlParameter("@OldValue", oldValue),
new MySqlParameter("@NewValue", newValue)
);
}
private static void MakeHistory(IDbConnection db, ManifestJob request, string file, string action, ulong oldValue = 0, ulong newValue = 0)
{
db.Execute(GetHistoryQuery(),
new DepotHistory
{
DepotID = request.DepotID,
ChangeID = request.ChangeNumber,
Action = action,
File = file,
OldValue = oldValue,
NewValue = newValue
}
);
}
private void TryDownloadManifest(ManifestJob request)
{
try
{
DownloadManifest(request);
}
catch (Exception e)
{
Log.WriteError("Depot Processor", "Caught exception while processing depot {0}: {1}\n{2}", request.DepotID, e.Message, e.StackTrace);
}
RemoveLock(request.DepotID);
Log.WriteDebug("Depot Processor", "Processed depot {0} ({1} depot locks left)", request.DepotID, DepotLocks.Count);
}
public async Task Process(IDbConnection db, uint appID, uint changeNumber, KeyValue depots)
{
var requests = new List <ManifestJob>();
var dlcNames = new Dictionary <uint, string>();
// Get data in format we want first
foreach (var depot in depots.Children)
{
var request = new ManifestJob
{
ChangeNumber = changeNumber,
};
// Ignore keys that aren't integers, for example "branches"
if (!uint.TryParse(depot.Name, out request.DepotID))
{
continue;
}
// Ignore these for now, parent app should be updated too anyway
if (depot["depotfromapp"].Value != null)
{
continue;
}
request.DepotName = depot["name"].AsString();
if (string.IsNullOrEmpty(request.DepotName))
{
request.DepotName = $"SteamDB Unnamed Depot {request.DepotID}";
}
else if (depot["dlcappid"].Value != null)
{
if (uint.TryParse(depot["dlcappid"].Value, out var dlcAppId))
{
dlcNames[dlcAppId] = request.DepotName;
}
}
// TODO: instead of locking we could wait for current process to finish
if (DepotLocks.ContainsKey(request.DepotID))
{
continue;
}
// SteamVR trickery
if (appID == 250820 &&
depot["manifests"]["beta"].Value != null &&
depots["branches"]["beta"]["buildid"].AsInteger() > depots["branches"]["public"]["buildid"].AsInteger())
{
request.BuildID = depots["branches"]["beta"]["buildid"].AsInteger();
request.ManifestID = ulong.Parse(depot["manifests"]["beta"].Value);
}
else
if (depot["manifests"]["public"].Value == null || !ulong.TryParse(depot["manifests"]["public"].Value, out request.ManifestID))
{
var branch = depot["manifests"].Children.Find(x => x.Name != "local");
if (branch == null || !ulong.TryParse(branch.Value, out request.ManifestID))
{
await db.ExecuteAsync("INSERT INTO `Depots` (`DepotID`, `Name`) VALUES (@DepotID, @DepotName) ON DUPLICATE KEY UPDATE `DepotID` = VALUES(`DepotID`)", new { request.DepotID, request.DepotName });
continue;
}
Log.WriteDebug(nameof(DepotProcessor), $"Depot {request.DepotID} (from {appID}) has no public branch, but there is another one");
request.BuildID = depots["branches"][branch.Name]["buildid"].AsInteger();
}
else
{
request.BuildID = depots["branches"]["public"]["buildid"].AsInteger();
}
requests.Add(request);
}
if (dlcNames.Any())
{
await ProcessDlcNames(appID, changeNumber, dlcNames);
}
foreach (var branch in depots["branches"].Children)
{
var buildId = branch["buildid"].AsInteger();
if (buildId < 1)
{
continue;
}
var isPublic = branch.Name != null && branch.Name.Equals("public", StringComparison.OrdinalIgnoreCase);
await db.ExecuteAsync(isPublic?
"INSERT INTO `Builds` (`BuildID`, `ChangeID`, `AppID`, `Public`) VALUES (@BuildID, @ChangeNumber, @AppID, 1) ON DUPLICATE KEY UPDATE `ChangeID` = IF(`Public` = 0, VALUES(`ChangeID`), `ChangeID`), `Public` = 1"
:
"INSERT INTO `Builds` (`BuildID`, `ChangeID`, `AppID`) VALUES (@BuildID, @ChangeNumber, @AppID) ON DUPLICATE KEY UPDATE `AppID` = `AppID`",
new
{
BuildID = buildId,
ChangeNumber = changeNumber,
AppID = appID,
});
}
if (requests.Count == 0)
{
return;
}
var depotsToDownload = new List <ManifestJob>();
var depotIds = requests.Select(x => x.DepotID).ToList();
var dbDepots = (await db.QueryAsync <Depot>("SELECT `DepotID`, `Name`, `BuildID`, `ManifestID`, `LastManifestID`, `FilenamesEncrypted` FROM `Depots` WHERE `DepotID` IN @depotIds", new { depotIds }))
.ToDictionary(x => x.DepotID, x => x);
var decryptionKeys = (await db.QueryAsync <DepotKey>("SELECT `DepotID`, `Key` FROM `DepotsKeys` WHERE `DepotID` IN @depotIds", new { depotIds }))
.ToDictionary(x => x.DepotID, x => Utils.StringToByteArray(x.Key));
foreach (var request in requests)
{
Depot dbDepot;
decryptionKeys.TryGetValue(request.DepotID, out request.DepotKey);
if (dbDepots.ContainsKey(request.DepotID))
{
dbDepot = dbDepots[request.DepotID];
if (dbDepot.BuildID > request.BuildID)
{
// buildid went back in time? this either means a rollback, or a shared depot that isn't synced properly
Log.WriteDebug(nameof(DepotProcessor), $"Skipping depot {request.DepotID} due to old buildid: {dbDepot.BuildID} > {request.BuildID}");
continue;
}
if (dbDepot.LastManifestID == request.ManifestID &&
dbDepot.ManifestID == request.ManifestID &&
Settings.FullRun != FullRunState.WithForcedDepots &&
!dbDepot.FilenamesEncrypted && request.DepotKey != null)
{
// Update depot name if changed
if (request.DepotName != dbDepot.Name)
{
await db.ExecuteAsync("UPDATE `Depots` SET `Name` = @DepotName WHERE `DepotID` = @DepotID", new { request.DepotID, request.DepotName });
}
continue;
}
request.StoredFilenamesEncrypted = dbDepot.FilenamesEncrypted;
request.LastManifestID = dbDepot.LastManifestID;
}
else
{
dbDepot = new Depot();
}
if (dbDepot.BuildID != request.BuildID || dbDepot.ManifestID != request.ManifestID || request.DepotName != dbDepot.Name)
{
await db.ExecuteAsync(@"INSERT INTO `Depots` (`DepotID`, `Name`, `BuildID`, `ManifestID`) VALUES (@DepotID, @DepotName, @BuildID, @ManifestID)
ON DUPLICATE KEY UPDATE `LastUpdated` = CURRENT_TIMESTAMP(), `Name` = VALUES(`Name`), `BuildID` = VALUES(`BuildID`), `ManifestID` = VALUES(`ManifestID`)",
new
{
request.DepotID,
request.DepotName,
request.BuildID,
request.ManifestID
});
}
if (dbDepot.ManifestID != request.ManifestID)
{
await MakeHistory(db, null, request, string.Empty, "manifest_change", dbDepot.ManifestID, request.ManifestID);
}
if (Settings.Current.OnlyOwnedDepots && !LicenseList.OwnedDepots.ContainsKey(request.DepotID))
{
continue;
}
lock (DepotLocks)
{
DepotLocks.Add(request.DepotID, 1);
}
depotsToDownload.Add(request);
}
if (depotsToDownload.Count > 0)
{
_ = TaskManager.Run(async() => await DownloadDepots(appID, depotsToDownload)).ContinueWith(task =>
{
Log.WriteError(nameof(DepotProcessor), $"An exception occured when processing depots from app {appID}, removing locks");
foreach (var depot in depotsToDownload)
{
RemoveLock(depot.DepotID);
}
}, TaskContinuationOptions.OnlyOnFaulted);
}
}
private static void DownloadManifest(ManifestJob request)
{
var cdnClient = new CDNClient(Steam.Instance.Client, request.DepotID, request.Ticket, request.DepotKey);
List<CDNClient.Server> cdnServers = null;
for (var i = 0; i <= 5; i++)
{
try
{
cdnServers = cdnClient.FetchServerList();
if (cdnServers.Count > 0)
{
break;
}
}
catch { }
}
if (cdnServers == null || cdnServers.Count == 0)
{
Log.WriteError("Depot Processor", "Failed to get server list for depot {0}", request.DepotID);
if (SteamProxy.Instance.ImportantApps.Contains(request.ParentAppID))
{
IRC.SendMain("Important manifest update: {0}{1}{2} {3}(parent {4}){5} -{6} failed to fetch server list", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.DARK_GRAY, request.ParentAppID, Colors.NORMAL, Colors.RED);
}
lock (ManifestJobs)
{
ManifestJobs.Remove(request);
}
return;
}
DepotManifest depotManifest = null;
foreach (var server in cdnServers)
{
try
{
cdnClient.Connect(server);
depotManifest = cdnClient.DownloadManifest(request.ManifestID);
break;
}
catch { }
}
lock (ManifestJobs)
{
ManifestJobs.Remove(request);
}
if (depotManifest == null)
{
Log.WriteError("Depot Processor", "Failed to download depot manifest for depot {0} (parent {1}) (jobs still in queue: {2})", request.DepotID, request.ParentAppID, ManifestJobs.Count);
if (SteamProxy.Instance.ImportantApps.Contains(request.ParentAppID))
{
IRC.SendMain("Important manifest update: {0}{1}{2} {3}(parent {4}){5} -{6} failed to download depot manifest from {7} servers", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.DARK_GRAY, request.ParentAppID, Colors.NORMAL, Colors.RED, cdnServers.Count);
}
return;
}
if (SteamProxy.Instance.ImportantApps.Contains(request.ParentAppID))
{
IRC.SendMain("Important manifest update: {0}{1}{2} {3}(parent {4}){5} -{6} {7}", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.DARK_GRAY, request.ParentAppID, Colors.NORMAL, Colors.DARK_BLUE, SteamDB.GetDepotURL(request.DepotID, "history"));
}
var sortedFiles = depotManifest.Files.OrderBy(f => f.FileName, StringComparer.OrdinalIgnoreCase);
bool shouldHistorize = false;
var filesNew = new List<DepotFile>();
var filesOld = new List<DepotFile>();
foreach (var file in sortedFiles)
{
System.Text.Encoding.UTF8.GetString(file.FileHash);
var depotFile = new DepotFile
{
Name = file.FileName.Replace('\\', '/'),
Size = file.TotalSize,
Chunks = file.Chunks.Count,
Flags = (int)file.Flags
};
// TODO: Ideally we would check if filehash is not empty
if (!file.Flags.HasFlag(EDepotFileFlag.Directory))
{
depotFile.Hash = string.Concat(Array.ConvertAll(file.FileHash, x => x.ToString("X2")));
}
filesNew.Add(depotFile);
}
using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `Files` FROM `Depots` WHERE `DepotID` = @DepotID LIMIT 1", new MySqlParameter("DepotID", request.DepotID)))
{
if (Reader.Read())
{
string files = Reader.GetString("Files");
if (!string.IsNullOrEmpty(files))
{
shouldHistorize = true;
filesOld = JsonConvert.DeserializeObject<List<DepotFile>>(files);
}
}
}
DbWorker.ExecuteNonQuery("UPDATE `Depots` SET `Files` = @Files WHERE `DepotID` = @DepotID",
new MySqlParameter("@DepotID", request.DepotID),
new MySqlParameter("@Files", JsonConvert.SerializeObject(filesNew, new JsonSerializerSettings { DefaultValueHandling = DefaultValueHandling.Ignore }))
);
if (shouldHistorize)
{
var filesAdded = new List<string>();
foreach (var file in filesNew)
{
var oldFile = filesOld.Find(x => x.Name == file.Name);
if (oldFile == null)
{
// We want to historize modifications first, and only then deletions and additions
filesAdded.Add(file.Name);
}
else
{
if (oldFile.Size != file.Size)
{
MakeHistory(request, file.Name, "modified", oldFile.Size, file.Size);
}
else if (file.Hash != null && oldFile.Hash != null && !file.Hash.Equals(oldFile.Hash))
{
MakeHistory(request, file.Name, "modified", oldFile.Size, file.Size);
}
filesOld.Remove(oldFile);
}
}
foreach (var file in filesOld)
{
MakeHistory(request, file.Name, "removed");
}
foreach (string file in filesAdded)
{
MakeHistory(request, file, "added");
}
}
lock (ManifestJobs)
{
Log.WriteDebug("Depot Processor", "DepotID: Processed {0} (jobs still in queue: {1})", request.DepotID, ManifestJobs.Count);
}
}
public static void Process(uint appID, uint changeNumber, KeyValue depots)
{
foreach (KeyValue depot in depots.Children)
{
// Ignore these for now, parent app should be updated too anyway
if (depot["depotfromapp"].Value != null)
{
////Log.WriteDebug("Depot Processor", "Ignoring depot {0} with depotfromapp value {1} (parent {2})", depot.Name, depot["depotfromapp"].AsString(), AppID);
continue;
}
uint depotID;
if (!uint.TryParse(depot.Name, out depotID))
{
// Ignore keys that aren't integers, for example "branches"
continue;
}
lock (ManifestJobs)
{
if (ManifestJobs.Find(r => r.DepotID == depotID) != null)
{
// If we already have this depot in our job list, ignore it
continue;
}
}
ulong manifestID;
if (depot["manifests"]["public"].Value == null || !ulong.TryParse(depot["manifests"]["public"].Value, out manifestID))
{
#if false
Log.WriteDebug("Depot Processor", "Failed to public branch for depot {0} (parent {1}) - {2}", DepotID, AppID);
// If there is no public manifest for this depot, it still could have some sort of open beta
var branch = depot["manifests"].Children.SingleOrDefault(x => x.Name != "local");
if (branch == null || !ulong.TryParse(branch.Value, out ManifestID))
{
continue;
}
#endif
continue;
}
var request = new ManifestJob
{
ChangeNumber = changeNumber,
ParentAppID = appID,
DepotID = depotID,
ManifestID = manifestID,
DepotName = depot["name"].AsString()
};
// Check if manifestid in our database is equal
using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `ManifestID` FROM `Depots` WHERE `DepotID` = @DepotID AND `Files` != '' LIMIT 1", new MySqlParameter("DepotID", depotID)))
{
if (Reader.Read())
{
request.PreviousManifestID = Reader.GetUInt64("ManifestID");
if (request.PreviousManifestID == manifestID && Settings.Current.FullRun < 2)
{
continue;
}
}
}
lock (ManifestJobs)
{
ManifestJobs.Add(request);
}
request.Server = CDNServers[new Random().Next(CDNServers.Count)];
request.JobID = Steam.Instance.Apps.GetCDNAuthToken(depotID, request.Server);
}
}
private static void DownloadManifest(ManifestJob request)
{
DepotManifest depotManifest = null;
string lastError = string.Empty;
// CDN is very random, just keep trying
for (var i = 0; i <= 5; i++)
{
try
{
depotManifest = CDNClient.DownloadManifest(request.DepotID, request.ManifestID, request.Server, request.DepotKey, request.CDNToken);
break;
}
catch (Exception e)
{
lastError = e.Message;
}
}
if (depotManifest == null)
{
Log.WriteError("Depot Processor", "Failed to download depot manifest for depot {0} (jobs still in queue: {1}) ({2}: {3})", request.DepotID, ManifestJobs.Count - 1, request.Server, lastError);
if (SteamProxy.Instance.ImportantApps.Contains(request.ParentAppID))
{
IRC.SendMain("Important depot update: {0}{1}{2} -{3} failed to download depot manifest", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.RED);
}
return;
}
if (SteamProxy.Instance.ImportantApps.Contains(request.ParentAppID))
{
IRC.SendMain("Important depot update: {0}{1}{2} -{3} {4}", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.DARK_BLUE, SteamDB.GetDepotURL(request.DepotID, "history"));
}
var sortedFiles = depotManifest.Files.OrderBy(f => f.FileName, StringComparer.OrdinalIgnoreCase);
bool shouldHistorize = false;
var filesNew = new List <DepotFile>();
var filesOld = new Dictionary <string, DepotFile>();
foreach (var file in sortedFiles)
{
System.Text.Encoding.UTF8.GetString(file.FileHash);
var depotFile = new DepotFile
{
Name = file.FileName.Replace('\\', '/'),
Size = file.TotalSize,
Chunks = file.Chunks.Count,
Flags = (int)file.Flags
};
// TODO: Ideally we would check if filehash is not empty
if (!file.Flags.HasFlag(EDepotFileFlag.Directory))
{
depotFile.Hash = string.Concat(Array.ConvertAll(file.FileHash, x => x.ToString("X2")));
}
filesNew.Add(depotFile);
}
using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `Files` FROM `Depots` WHERE `DepotID` = @DepotID LIMIT 1", new MySqlParameter("DepotID", request.DepotID)))
{
if (Reader.Read())
{
string files = Reader.GetString("Files");
if (!string.IsNullOrEmpty(files))
{
shouldHistorize = true;
var _filesOld = JsonConvert.DeserializeObject <List <DepotFile> >(files);
filesOld = _filesOld.ToDictionary(x => x.Name);
}
}
}
DbWorker.ExecuteNonQuery("UPDATE `Depots` SET `Files` = @Files WHERE `DepotID` = @DepotID",
new MySqlParameter("@DepotID", request.DepotID),
new MySqlParameter("@Files", JsonConvert.SerializeObject(filesNew, new JsonSerializerSettings {
DefaultValueHandling = DefaultValueHandling.Ignore
}))
);
if (shouldHistorize)
{
var filesAdded = new List <string>();
foreach (var file in filesNew)
{
if (filesOld.ContainsKey(file.Name))
{
var oldFile = filesOld[file.Name];
if (oldFile.Size != file.Size)
{
MakeHistory(request, file.Name, "modified", oldFile.Size, file.Size);
}
else if (file.Hash != null && oldFile.Hash != null && !file.Hash.Equals(oldFile.Hash))
{
MakeHistory(request, file.Name, "modified", oldFile.Size, file.Size);
}
filesOld.Remove(file.Name);
}
else
{
// We want to historize modifications first, and only then deletions and additions
filesAdded.Add(file.Name);
}
}
foreach (var file in filesOld)
{
MakeHistory(request, file.Value.Name, "removed");
}
foreach (string file in filesAdded)
{
MakeHistory(request, file, "added");
}
}
lock (ManifestJobs)
{
Log.WriteDebug("Depot Processor", "DepotID: Processed {0} (jobs still in queue: {1})", request.DepotID, ManifestJobs.Count - 1);
}
}
private void DownloadManifest(ManifestJob request)
{
DepotManifest depotManifest = null;
string lastError = string.Empty;
// CDN is very random, just keep trying
for (var i = 0; i <= 5; i++)
{
try
{
depotManifest = CDNClient.DownloadManifest(request.DepotID, request.ManifestID, request.Server, request.CDNToken, request.DepotKey);
break;
}
catch (Exception e)
{
lastError = e.Message;
}
}
if (depotManifest == null)
{
Log.WriteError("Depot Processor", "Failed to download depot manifest for depot {0} ({1}: {2})", request.DepotID, request.Server, lastError);
if (Application.ImportantApps.ContainsKey(request.ParentAppID))
{
IRC.Instance.SendMain("Important depot update: {0}{1}{2} -{3} failed to download depot manifest", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.RED);
}
return;
}
if (Application.ImportantApps.ContainsKey(request.ParentAppID))
{
IRC.Instance.SendMain("Important depot update: {0}{1}{2} -{3} {4}", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.DARKBLUE, SteamDB.GetDepotURL(request.DepotID, "history"));
}
var sortedFiles = depotManifest.Files.OrderBy(f => f.FileName, StringComparer.OrdinalIgnoreCase);
bool shouldHistorize = false;
var filesNew = new List<DepotFile>();
var filesOld = new Dictionary<string, DepotFile>();
foreach (var file in sortedFiles)
{
System.Text.Encoding.UTF8.GetString(file.FileHash);
var depotFile = new DepotFile
{
Name = file.FileName.Replace('\\', '/'),
Size = file.TotalSize,
Chunks = file.Chunks.Count,
Flags = (int)file.Flags
};
// TODO: Ideally we would check if filehash is not empty
if (!file.Flags.HasFlag(EDepotFileFlag.Directory))
{
depotFile.Hash = string.Concat(Array.ConvertAll(file.FileHash, x => x.ToString("X2")));
}
filesNew.Add(depotFile);
}
using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `Files` FROM `Depots` WHERE `DepotID` = @DepotID LIMIT 1", new MySqlParameter("DepotID", request.DepotID)))
{
if (Reader.Read())
{
string files = Reader.GetString("Files");
if (!string.IsNullOrEmpty(files))
{
shouldHistorize = true;
filesOld = JsonConvert.DeserializeObject<List<DepotFile>>(files).ToDictionary(x => x.Name);
}
}
}
DbWorker.ExecuteNonQuery(
"UPDATE `Depots` SET `Files` = @Files WHERE `DepotID` = @DepotID",
new MySqlParameter("@DepotID", request.DepotID),
new MySqlParameter("@Files", JsonConvert.SerializeObject(filesNew, new JsonSerializerSettings { DefaultValueHandling = DefaultValueHandling.Ignore }))
);
if (shouldHistorize)
{
var filesAdded = new List<string>();
foreach (var file in filesNew)
{
if (filesOld.ContainsKey(file.Name))
{
var oldFile = filesOld[file.Name];
if (oldFile.Size != file.Size)
{
MakeHistory(request, file.Name, "modified", oldFile.Size, file.Size);
}
else if (file.Hash != null && oldFile.Hash != null && !file.Hash.Equals(oldFile.Hash))
{
MakeHistory(request, file.Name, "modified", oldFile.Size, file.Size);
}
filesOld.Remove(file.Name);
}
else
{
// We want to historize modifications first, and only then deletions and additions
filesAdded.Add(file.Name);
}
}
foreach (var file in filesOld)
{
MakeHistory(request, file.Value.Name, "removed");
}
foreach (string file in filesAdded)
{
MakeHistory(request, file, "added");
}
}
}
private static async Task <EResult> ProcessDepotAfterDownload(IDbConnection db, IDbTransaction transaction, ManifestJob request, DepotManifest depotManifest)
{
var filesOld = (await db.QueryAsync <DepotFile>("SELECT `File`, `Hash`, `Size`, `Flags` FROM `DepotsFiles` WHERE `DepotID` = @DepotID", new { request.DepotID }, transaction)).ToDictionary(x => x.File, x => x);
var filesAdded = new List <DepotFile>();
var shouldHistorize = filesOld.Count > 0 && !depotManifest.FilenamesEncrypted; // Don't historize file additions if we didn't have any data before
if (request.StoredFilenamesEncrypted && !depotManifest.FilenamesEncrypted)
{
Log.WriteInfo(nameof(DepotProcessor), $"Depot {request.DepotID} will decrypt stored filenames");
var decryptedFilesOld = new Dictionary <string, DepotFile>();
foreach (var file in filesOld.Values)
{
var oldFile = file.File;
file.File = DecryptFilename(oldFile, request.DepotKey);
decryptedFilesOld.Add(file.File, file);
await db.ExecuteAsync("UPDATE `DepotsFiles` SET `File` = @File WHERE `DepotID` = @DepotID AND `File` = @OldFile", new
{
request.DepotID,
file.File,
OldFile = oldFile
}, transaction);
}
await MakeHistory(db, transaction, request, string.Empty, "files_decrypted");
filesOld = decryptedFilesOld;
}
foreach (var file in depotManifest.Files.OrderByDescending(x => x.FileName))
{
var name = depotManifest.FilenamesEncrypted ? file.FileName.Replace("\n", "") : file.FileName.Replace('\\', '/');
byte[] hash = null;
// Store empty hashes as NULL (e.g. an empty file)
if ((file.Flags & EDepotFileFlag.Directory) == 0 && file.FileHash.Length > 0 && file.FileHash.Any(c => c != 0))
{
hash = file.FileHash;
}
// Limit path names to 260 characters (default windows max length)
// File column is varchar(260) and not higher to prevent reducing performance
// See https://stackoverflow.com/questions/1962310/importance-of-varchar-length-in-mysql-table/1962329#1962329
// Until 2019 there hasn't been a single file that went over this limit, so far there has been only one
// game with a big node_modules path, so we're safeguarding by limiting it.
if (name.Length > 260)
{
if (depotManifest.FilenamesEncrypted)
{
continue;
}
using var sha = SHA1.Create();
var nameHash = Utils.ByteArrayToString(sha.ComputeHash(Encoding.UTF8.GetBytes(name)));
name = $"{{SteamDB file name is too long}}/{nameHash}/...{name.Substring(name.Length - 150)}";
}
if (filesOld.ContainsKey(name))
{
var oldFile = filesOld[name];
var updateFile = false;
if (oldFile.Size != file.TotalSize || !Utils.ByteArrayEquals(hash, oldFile.Hash))
{
await MakeHistory(db, transaction, request, name, "modified", oldFile.Size, file.TotalSize);
updateFile = true;
}
if (oldFile.Flags != file.Flags)
{
await MakeHistory(db, transaction, request, name, "modified_flags", (ulong)oldFile.Flags, (ulong)file.Flags);
updateFile = true;
}
if (updateFile)
{
await db.ExecuteAsync("UPDATE `DepotsFiles` SET `Hash` = @Hash, `Size` = @Size, `Flags` = @Flags WHERE `DepotID` = @DepotID AND `File` = @File", new DepotFile
{
DepotID = request.DepotID,
File = name,
Hash = hash,
Size = file.TotalSize,
Flags = file.Flags
}, transaction);
}
filesOld.Remove(name);
}
else
{
// We want to historize modifications first, and only then deletions and additions
filesAdded.Add(new DepotFile
{
DepotID = request.DepotID,
Hash = hash,
File = name,
Size = file.TotalSize,
Flags = file.Flags
});
}
}
if (filesOld.Count > 0)
{
// Chunk file deletion queries so it doesn't go over max_allowed_packet
var filesOldChunks = filesOld.Select(x => x.Value.File).Split(1000);
foreach (var filesOldChunk in filesOldChunks)
{
await db.ExecuteAsync("DELETE FROM `DepotsFiles` WHERE `DepotID` = @DepotID AND `File` IN @Files",
new
{
request.DepotID,
Files = filesOldChunk,
}, transaction);
}
if (shouldHistorize)
{
await db.ExecuteAsync(HistoryQuery, filesOld.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ManifestID = request.ManifestID,
ChangeID = request.ChangeNumber,
Action = "removed",
File = x.Value.File,
OldValue = x.Value.Size
}), transaction);
}
}
if (filesAdded.Count > 0)
{
await db.ExecuteAsync("INSERT INTO `DepotsFiles` (`DepotID`, `File`, `Hash`, `Size`, `Flags`) VALUES (@DepotID, @File, @Hash, @Size, @Flags)", filesAdded, transaction);
if (shouldHistorize)
{
await db.ExecuteAsync(HistoryQuery, filesAdded.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ManifestID = request.ManifestID,
ChangeID = request.ChangeNumber,
Action = "added",
File = x.File,
NewValue = x.Size
}), transaction);
}
}
await db.ExecuteAsync(
request.LastManifestID == request.ManifestID?
"UPDATE `Depots` SET `LastManifestID` = @ManifestID, `ManifestDate` = @ManifestDate, `FilenamesEncrypted` = @FilenamesEncrypted, `SizeOriginal` = @SizeOriginal, `SizeCompressed` = @SizeCompressed WHERE `DepotID` = @DepotID" :
"UPDATE `Depots` SET `LastManifestID` = @ManifestID, `ManifestDate` = @ManifestDate, `FilenamesEncrypted` = @FilenamesEncrypted, `SizeOriginal` = @SizeOriginal, `SizeCompressed` = @SizeCompressed, `LastUpdated` = CURRENT_TIMESTAMP() WHERE `DepotID` = @DepotID",
new
{
request.DepotID,
request.ManifestID,
depotManifest.FilenamesEncrypted,
ManifestDate = depotManifest.CreationTime,
SizeOriginal = depotManifest.TotalUncompressedSize,
SizeCompressed = depotManifest.TotalCompressedSize,
}, transaction);
return(EResult.OK);
}
public void Process(uint appID, uint changeNumber, KeyValue depots)
{
var buildID = depots["branches"]["public"]["buildid"].AsInteger();
foreach (KeyValue depot in depots.Children)
{
// Ignore these for now, parent app should be updated too anyway
if (depot["depotfromapp"].Value != null)
{
////Log.WriteDebug("Depot Processor", "Ignoring depot {0} with depotfromapp value {1} (parent {2})", depot.Name, depot["depotfromapp"].AsString(), AppID);
continue;
}
uint depotID;
if (!uint.TryParse(depot.Name, out depotID))
{
// Ignore keys that aren't integers, for example "branches"
continue;
}
// TODO: instead of locking we could wait for current process to finish
if (DepotLocks.ContainsKey(depotID))
{
continue;
}
ulong manifestID;
var depotName = depot["name"].AsString();
if (depot["manifests"]["public"].Value == null || !ulong.TryParse(depot["manifests"]["public"].Value, out manifestID))
{
#if false
Log.WriteDebug("Depot Processor", "Failed to public branch for depot {0} (parent {1}) - {2}", DepotID, AppID);
// If there is no public manifest for this depot, it still could have some sort of open beta
var branch = depot["manifests"].Children.SingleOrDefault(x => x.Name != "local");
if (branch == null || !ulong.TryParse(branch.Value, out ManifestID))
{
continue;
}
#endif
DbWorker.ExecuteNonQuery("INSERT INTO `Depots` (`DepotID`, `Name`) VALUES (@DepotID, @Name) ON DUPLICATE KEY UPDATE `LastUpdated` = CURRENT_TIMESTAMP(), `Name` = @Name",
new MySqlParameter("@DepotID", depotID),
new MySqlParameter("@Name", depotName)
);
continue;
}
var request = new ManifestJob
{
ChangeNumber = changeNumber,
ParentAppID = appID,
DepotID = depotID,
ManifestID = manifestID,
DepotName = depotName
};
int currentBuildID = 0;
// Check if manifestid in our database is equal
using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `Name`, `ManifestID`, `BuildID` FROM `Depots` WHERE `DepotID` = @DepotID LIMIT 1", new MySqlParameter("DepotID", depotID)))
{
if (Reader.Read())
{
currentBuildID = Reader.GetInt32("buildID");
request.PreviousManifestID = Reader.GetUInt64("ManifestID");
if (request.PreviousManifestID == manifestID && Settings.Current.FullRun < 2)
{
// Update depot name if changed
if(!depotName.Equals(Reader.GetString("Name")))
{
DbWorker.ExecuteNonQuery("UPDATE `Depots` SET `Name` = @Name WHERE `DepotID` = @DepotID",
new MySqlParameter("@DepotID", request.DepotID),
new MySqlParameter("@Name", request.DepotName)
);
}
continue;
}
if (currentBuildID > buildID)
{
Log.WriteDebug("Depot Processor", "Skipping depot {0} due to old buildid: {1} > {2}", depotID, currentBuildID, buildID);
continue;
}
}
}
DepotLocks.TryAdd(depotID, 1);
// Update/insert depot information straight away
if (currentBuildID != buildID || request.PreviousManifestID != request.ManifestID)
{
DbWorker.ExecuteNonQuery("INSERT INTO `Depots` (`DepotID`, `Name`, `BuildID`, `ManifestID`) VALUES (@DepotID, @Name, @BuildID, @ManifestID) ON DUPLICATE KEY UPDATE `LastUpdated` = CURRENT_TIMESTAMP(), `Name` = @Name, `BuildID` = @BuildID, `ManifestID` = @ManifestID",
new MySqlParameter("@DepotID", request.DepotID),
new MySqlParameter("@BuildID", buildID),
new MySqlParameter("@ManifestID", request.ManifestID),
new MySqlParameter("@Name", request.DepotName)
);
MakeHistory(request, string.Empty, "manifest_change", request.PreviousManifestID, request.ManifestID);
}
request.Server = CDNServers[new Random().Next(CDNServers.Count)];
JobManager.AddJob(() => Steam.Instance.Apps.GetCDNAuthToken(depotID, request.Server), request);
}
}
private void DownloadManifest(ManifestJob request)
{
Log.WriteInfo("Depot Processor", "DepotID: {0}", request.DepotID);
DepotManifest depotManifest = null;
string lastError = string.Empty;
// CDN is very random, just keep trying
for (var i = 0; i <= 5; i++)
{
try
{
depotManifest = CDNClient.DownloadManifest(request.DepotID, request.ManifestID, request.Server, request.CDNToken, request.DepotKey);
break;
}
catch (Exception e)
{
lastError = e.Message;
}
}
if (depotManifest == null)
{
Log.WriteError("Depot Processor", "Failed to download depot manifest for depot {0} ({1}: {2}) (#{3})", request.DepotID, request.Server, lastError, request.Tries);
if (--request.Tries >= 0)
{
request.Server = GetContentServer(request.Tries);
JobManager.AddJob(() => Steam.Instance.Apps.GetCDNAuthToken(request.DepotID, request.Server), request);
return;
}
RemoveLock(request.DepotID); // TODO: Remove this once task in OnCDNAuthTokenCallback is used
if (FileDownloader.IsImportantDepot(request.DepotID))
{
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download depot {3} manifest ({4}: {5})",
Colors.OLIVE, Steam.GetAppName(request.ParentAppID), Colors.NORMAL, request.DepotID, request.Server, lastError);
}
return;
}
if (FileDownloader.IsImportantDepot(request.DepotID))
{
TaskManager.Run(() => FileDownloader.DownloadFilesFromDepot(request, depotManifest));
}
// TODO: Task here instead of in OnCDNAuthTokenCallback due to mono's silly threadpool
TaskManager.Run(() =>
{
using (var db = Database.GetConnection())
{
ProcessDepotAfterDownload(db, request, depotManifest);
}
}).ContinueWith(task =>
{
RemoveLock(request.DepotID);
Log.WriteDebug("Depot Processor", "Processed depot {0} ({1} depot locks left)", request.DepotID, DepotLocks.Count);
});
}
private static void ProcessDepotAfterDownload(IDbConnection db, ManifestJob request, DepotManifest depotManifest)
{
var filesOld = db.Query<DepotFile>("SELECT `ID`, `File`, `Hash`, `Size`, `Flags` FROM `DepotsFiles` WHERE `DepotID` = @DepotID", new { request.DepotID }).ToDictionary(x => x.File, x => x);
var filesNew = new List<DepotFile>();
var filesAdded = new List<DepotFile>();
var shouldHistorize = filesOld.Any(); // Don't historize file additions if we didn't have any data before
foreach (var file in depotManifest.Files)
{
var name = file.FileName.Replace('\\', '/');
// safe guard
if (name.Length > 255)
{
ErrorReporter.Notify(new OverflowException(string.Format("File \"{0}\" in depot {1} is too long", name, request.DepotID)));
continue;
}
var depotFile = new DepotFile
{
DepotID = request.DepotID,
File = name,
Size = file.TotalSize,
Flags = file.Flags
};
if (file.FileHash.Length > 0 && !file.Flags.HasFlag(EDepotFileFlag.Directory))
{
depotFile.Hash = Utils.ByteArrayToString(file.FileHash);
}
else
{
depotFile.Hash = "0000000000000000000000000000000000000000";
}
filesNew.Add(depotFile);
}
foreach (var file in filesNew)
{
if (filesOld.ContainsKey(file.File))
{
var oldFile = filesOld[file.File];
var updateFile = false;
if (oldFile.Size != file.Size || !file.Hash.Equals(oldFile.Hash))
{
MakeHistory(db, request, file.File, "modified", oldFile.Size, file.Size);
updateFile = true;
}
if (oldFile.Flags != file.Flags)
{
MakeHistory(db, request, file.File, "modified_flags", (ulong)oldFile.Flags, (ulong)file.Flags);
updateFile = true;
}
if (updateFile)
{
file.ID = oldFile.ID;
db.Execute("UPDATE `DepotsFiles` SET `Hash` = @Hash, `Size` = @Size, `Flags` = @Flags WHERE `DepotID` = @DepotID AND `ID` = @ID", file);
}
filesOld.Remove(file.File);
}
else
{
// We want to historize modifications first, and only then deletions and additions
filesAdded.Add(file);
}
}
if (filesOld.Any())
{
db.Execute("DELETE FROM `DepotsFiles` WHERE `DepotID` = @DepotID AND `ID` IN @Files", new { request.DepotID, Files = filesOld.Select(x => x.Value.ID) });
db.Execute(GetHistoryQuery(), filesOld.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ChangeID = request.ChangeNumber,
Action = "removed",
File = x.Value.File
}));
}
if (filesAdded.Any())
{
db.Execute("INSERT INTO `DepotsFiles` (`DepotID`, `File`, `Hash`, `Size`, `Flags`) VALUES (@DepotID, @File, @Hash, @Size, @Flags)", filesAdded);
if (shouldHistorize)
{
db.Execute(GetHistoryQuery(), filesAdded.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ChangeID = request.ChangeNumber,
Action = "added",
File = x.File
}));
}
}
db.Execute("UPDATE `Depots` SET `LastManifestID` = @ManifestID WHERE `DepotID` = @DepotID", new { request.DepotID, request.ManifestID });
}
public void Process(uint appID, uint changeNumber, KeyValue depots)
{
var requests = new List <ManifestJob>();
// Get data in format we want first
foreach (var depot in depots.Children)
{
// Ignore these for now, parent app should be updated too anyway
if (depot["depotfromapp"].Value != null)
{
continue;
}
var request = new ManifestJob
{
ChangeNumber = changeNumber,
ParentAppID = appID,
DepotName = depot["name"].AsString()
};
// Ignore keys that aren't integers, for example "branches"
if (!uint.TryParse(depot.Name, out request.DepotID))
{
continue;
}
// TODO: instead of locking we could wait for current process to finish
if (DepotLocks.ContainsKey(request.DepotID))
{
continue;
}
// If there is no public manifest for this depot, it still could have some sort of open beta
if (depot["manifests"]["public"].Value == null || !ulong.TryParse(depot["manifests"]["public"].Value, out request.ManifestID))
{
var branch = depot["manifests"].Children.FirstOrDefault(x => x.Name != "local");
if (branch == null || !ulong.TryParse(branch.Value, out request.ManifestID))
{
using (var db = Database.GetConnection())
{
db.Execute("INSERT INTO `Depots` (`DepotID`, `Name`) VALUES (@DepotID, @DepotName) ON DUPLICATE KEY UPDATE `Name` = @DepotName", new { request.DepotID, request.DepotName });
}
continue;
}
request.BuildID = branch["build"].AsInteger();
}
else
{
request.BuildID = depots["branches"]["public"]["buildid"].AsInteger();
}
requests.Add(request);
}
if (!requests.Any())
{
return;
}
using (var db = Database.GetConnection())
{
var dbDepots = db.Query <Depot>("SELECT `DepotID`, `Name`, `BuildID`, `ManifestID`, `LastManifestID` FROM `Depots` WHERE `DepotID` IN @Depots", new { Depots = requests.Select(x => x.DepotID) })
.ToDictionary(x => x.DepotID, x => x);
foreach (var request in requests)
{
Depot dbDepot;
if (dbDepots.ContainsKey(request.DepotID))
{
dbDepot = dbDepots[request.DepotID];
if (dbDepot.BuildID > request.BuildID)
{
// buildid went back in time? this either means a rollback, or a shared depot that isn't synced properly
Log.WriteDebug("Depot Processor", "Skipping depot {0} due to old buildid: {1} > {2}", request.DepotID, dbDepot.BuildID, request.BuildID);
continue;
}
if (dbDepot.LastManifestID == request.ManifestID && dbDepot.ManifestID == request.ManifestID && Settings.Current.FullRun < 2)
{
// Update depot name if changed
if (!request.DepotName.Equals(dbDepot.Name))
{
db.Execute("UPDATE `Depots` SET `Name` = @DepotName WHERE `DepotID` = @DepotID", new { request.DepotID, request.DepotName });
}
continue;
}
}
else
{
dbDepot = new Depot();
}
if (dbDepot.BuildID != request.BuildID || dbDepot.ManifestID != request.ManifestID || !request.DepotName.Equals(dbDepot.Name))
{
db.Execute(@"INSERT INTO `Depots` (`DepotID`, `Name`, `BuildID`, `ManifestID`) VALUES (@DepotID, @DepotName, @BuildID, @ManifestID)
ON DUPLICATE KEY UPDATE `LastUpdated` = CURRENT_TIMESTAMP(), `Name` = @DepotName, `BuildID` = @BuildID, `ManifestID` = @ManifestID",
new {
request.DepotID,
request.DepotName,
request.BuildID,
request.ManifestID
});
}
if (dbDepot.ManifestID != request.ManifestID)
{
MakeHistory(db, request, string.Empty, "manifest_change", dbDepot.ManifestID, request.ManifestID);
}
if (LicenseList.OwnedApps.ContainsKey(request.DepotID) || Settings.Current.FullRun > 1)
{
DepotLocks.TryAdd(request.DepotID, 1);
JobManager.AddJob(() => Steam.Instance.Apps.GetDepotDecryptionKey(request.DepotID, request.ParentAppID), request);
}
#if DEBUG
else
{
Log.WriteDebug("Depot Processor", "Skipping depot {0} from app {1} because we don't own it", request.DepotID, request.ParentAppID);
}
#endif
}
}
}
public void Process(uint appID, uint changeNumber, KeyValue depots)
{
var requests = new List<ManifestJob>();
// Get data in format we want first
foreach (var depot in depots.Children)
{
// Ignore these for now, parent app should be updated too anyway
if (depot["depotfromapp"].Value != null)
{
continue;
}
var request = new ManifestJob
{
ChangeNumber = changeNumber,
ParentAppID = appID,
DepotName = depot["name"].AsString()
};
// Ignore keys that aren't integers, for example "branches"
if (!uint.TryParse(depot.Name, out request.DepotID))
{
continue;
}
// TODO: instead of locking we could wait for current process to finish
if (DepotLocks.ContainsKey(request.DepotID))
{
continue;
}
// If there is no public manifest for this depot, it still could have some sort of open beta
if (depot["manifests"]["public"].Value == null || !ulong.TryParse(depot["manifests"]["public"].Value, out request.ManifestID))
{
var branch = depot["manifests"].Children.FirstOrDefault(x => x.Name != "local");
if (branch == null || !ulong.TryParse(branch.Value, out request.ManifestID))
{
using (var db = Database.GetConnection())
{
db.Execute("INSERT INTO `Depots` (`DepotID`, `Name`) VALUES (@DepotID, @DepotName) ON DUPLICATE KEY UPDATE `Name` = VALUES(`Name`)", new { request.DepotID, request.DepotName });
}
continue;
}
request.BuildID = branch["build"].AsInteger();
}
else
{
request.BuildID = depots["branches"]["public"]["buildid"].AsInteger();
}
requests.Add(request);
}
if (!requests.Any())
{
return;
}
using (var db = Database.GetConnection())
{
var dbDepots = db.Query<Depot>("SELECT `DepotID`, `Name`, `BuildID`, `ManifestID`, `LastManifestID` FROM `Depots` WHERE `DepotID` IN @Depots", new { Depots = requests.Select(x => x.DepotID) })
.ToDictionary(x => x.DepotID, x => x);
foreach (var request in requests)
{
Depot dbDepot;
if (dbDepots.ContainsKey(request.DepotID))
{
dbDepot = dbDepots[request.DepotID];
if (dbDepot.BuildID > request.BuildID)
{
// buildid went back in time? this either means a rollback, or a shared depot that isn't synced properly
Log.WriteDebug("Depot Processor", "Skipping depot {0} due to old buildid: {1} > {2}", request.DepotID, dbDepot.BuildID, request.BuildID);
continue;
}
if (dbDepot.LastManifestID == request.ManifestID && dbDepot.ManifestID == request.ManifestID && Settings.Current.FullRun < 2)
{
// Update depot name if changed
if (!request.DepotName.Equals(dbDepot.Name))
{
db.Execute("UPDATE `Depots` SET `Name` = @DepotName WHERE `DepotID` = @DepotID", new { request.DepotID, request.DepotName });
}
continue;
}
}
else
{
dbDepot = new Depot();
}
if (dbDepot.BuildID != request.BuildID || dbDepot.ManifestID != request.ManifestID || !request.DepotName.Equals(dbDepot.Name))
{
db.Execute(@"INSERT INTO `Depots` (`DepotID`, `Name`, `BuildID`, `ManifestID`) VALUES (@DepotID, @DepotName, @BuildID, @ManifestID)
ON DUPLICATE KEY UPDATE `LastUpdated` = CURRENT_TIMESTAMP(), `Name` = VALUES(`Name`), `BuildID` = VALUES(`BuildID`), `ManifestID` = VALUES(`ManifestID`)",
new {
request.DepotID,
request.DepotName,
request.BuildID,
request.ManifestID
});
}
if (dbDepot.ManifestID != request.ManifestID)
{
MakeHistory(db, request, string.Empty, "manifest_change", dbDepot.ManifestID, request.ManifestID);
}
if (LicenseList.OwnedApps.ContainsKey(request.DepotID) || Settings.Current.FullRun > 1)
{
DepotLocks.TryAdd(request.DepotID, 1);
JobManager.AddJob(() => Steam.Instance.Apps.GetDepotDecryptionKey(request.DepotID, request.ParentAppID), request);
}
#if DEBUG
else
{
Log.WriteDebug("Depot Processor", "Skipping depot {0} from app {1} because we don't own it", request.DepotID, request.ParentAppID);
}
#endif
}
}
}
private EResult ProcessDepotAfterDownload(IDbConnection db, ManifestJob request, DepotManifest depotManifest)
{
var filesOld = db.Query <DepotFile>("SELECT `ID`, `File`, `Hash`, `Size`, `Flags` FROM `DepotsFiles` WHERE `DepotID` = @DepotID", new { request.DepotID }).ToDictionary(x => x.File, x => x);
var filesNew = new List <DepotFile>();
var filesAdded = new List <DepotFile>();
var shouldHistorize = filesOld.Any(); // Don't historize file additions if we didn't have any data before
foreach (var file in depotManifest.Files)
{
var name = file.FileName.Replace('\\', '/');
// safe guard
if (name.Length > 255)
{
ErrorReporter.Notify(new OverflowException(string.Format("File \"{0}\" in depot {1} is too long", name, request.DepotID)));
continue;
}
var depotFile = new DepotFile
{
DepotID = request.DepotID,
File = name,
Size = file.TotalSize,
Flags = file.Flags
};
if (file.FileHash.Length > 0 && !file.Flags.HasFlag(EDepotFileFlag.Directory))
{
depotFile.Hash = Utils.ByteArrayToString(file.FileHash);
}
else
{
depotFile.Hash = "0000000000000000000000000000000000000000";
}
filesNew.Add(depotFile);
}
foreach (var file in filesNew)
{
if (filesOld.ContainsKey(file.File))
{
var oldFile = filesOld[file.File];
var updateFile = false;
if (oldFile.Size != file.Size || !file.Hash.Equals(oldFile.Hash))
{
MakeHistory(db, request, file.File, "modified", oldFile.Size, file.Size);
updateFile = true;
}
if (oldFile.Flags != file.Flags)
{
MakeHistory(db, request, file.File, "modified_flags", (ulong)oldFile.Flags, (ulong)file.Flags);
updateFile = true;
}
if (updateFile)
{
file.ID = oldFile.ID;
db.Execute("UPDATE `DepotsFiles` SET `Hash` = @Hash, `Size` = @Size, `Flags` = @Flags WHERE `DepotID` = @DepotID AND `ID` = @ID", file);
}
filesOld.Remove(file.File);
}
else
{
// We want to historize modifications first, and only then deletions and additions
filesAdded.Add(file);
}
}
if (filesOld.Any())
{
db.Execute("DELETE FROM `DepotsFiles` WHERE `DepotID` = @DepotID AND `ID` IN @Files", new { request.DepotID, Files = filesOld.Select(x => x.Value.ID) });
db.Execute(GetHistoryQuery(), filesOld.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ChangeID = request.ChangeNumber,
Action = "removed",
File = x.Value.File
}));
}
if (filesAdded.Any())
{
db.Execute("INSERT INTO `DepotsFiles` (`DepotID`, `File`, `Hash`, `Size`, `Flags`) VALUES (@DepotID, @File, @Hash, @Size, @Flags)", filesAdded);
if (shouldHistorize)
{
db.Execute(GetHistoryQuery(), filesAdded.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ChangeID = request.ChangeNumber,
Action = "added",
File = x.File
}));
}
}
db.Execute("UPDATE `Depots` SET `LastManifestID` = @ManifestID WHERE `DepotID` = @DepotID", new { request.DepotID, request.ManifestID });
return(EResult.OK);
}
private async Task <EResult> ProcessDepotAfterDownload(IDbConnection db, IDbTransaction transaction, ManifestJob request, DepotManifest depotManifest)
{
var filesOld = (await db.QueryAsync <DepotFile>("SELECT `ID`, `File`, `Hash`, `Size`, `Flags` FROM `DepotsFiles` WHERE `DepotID` = @DepotID", new { request.DepotID }, transaction: transaction)).ToDictionary(x => x.File, x => x);
var filesAdded = new List <DepotFile>();
var shouldHistorize = filesOld.Any(); // Don't historize file additions if we didn't have any data before
foreach (var file in depotManifest.Files)
{
var name = file.FileName.Replace('\\', '/');
byte[] hash = null;
// Store empty hashes as NULL (e.g. an empty file)
if (file.FileHash.Length > 0 && !file.Flags.HasFlag(EDepotFileFlag.Directory))
{
for (int i = 0; i < file.FileHash.Length; ++i)
{
if (file.FileHash[i] != 0)
{
hash = file.FileHash;
break;
}
}
}
// safe guard
if (name.Length > 255)
{
ErrorReporter.Notify("Depot Processor", new OverflowException(string.Format("File \"{0}\" in depot {1} is too long", name, request.DepotID)));
continue;
}
if (filesOld.ContainsKey(name))
{
var oldFile = filesOld[name];
var updateFile = false;
if (oldFile.Size != file.TotalSize || !Utils.IsEqualSHA1(hash, oldFile.Hash))
{
await MakeHistory(db, transaction, request, name, "modified", oldFile.Size, file.TotalSize);
updateFile = true;
}
if (oldFile.Flags != file.Flags)
{
await MakeHistory(db, transaction, request, name, "modified_flags", (ulong)oldFile.Flags, (ulong)file.Flags);
updateFile = true;
}
if (updateFile)
{
await db.ExecuteAsync("UPDATE `DepotsFiles` SET `Hash` = @Hash, `Size` = @Size, `Flags` = @Flags WHERE `DepotID` = @DepotID AND `ID` = @ID", new DepotFile
{
ID = oldFile.ID,
DepotID = request.DepotID,
Hash = hash,
Size = file.TotalSize,
Flags = file.Flags
}, transaction : transaction);
}
filesOld.Remove(name);
}
else
{
// We want to historize modifications first, and only then deletions and additions
filesAdded.Add(new DepotFile
{
DepotID = request.DepotID,
Hash = hash,
File = name,
Size = file.TotalSize,
Flags = file.Flags
});
}
}
if (filesOld.Any())
{
await db.ExecuteAsync("DELETE FROM `DepotsFiles` WHERE `DepotID` = @DepotID AND `ID` IN @Files", new { request.DepotID, Files = filesOld.Select(x => x.Value.ID) }, transaction : transaction);
await db.ExecuteAsync(HistoryQuery, filesOld.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ChangeID = request.ChangeNumber,
Action = "removed",
File = x.Value.File
}), transaction : transaction);
}
if (filesAdded.Any())
{
await db.ExecuteAsync("INSERT INTO `DepotsFiles` (`DepotID`, `File`, `Hash`, `Size`, `Flags`) VALUES (@DepotID, @File, @Hash, @Size, @Flags)", filesAdded, transaction : transaction);
if (shouldHistorize)
{
await db.ExecuteAsync(HistoryQuery, filesAdded.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ChangeID = request.ChangeNumber,
Action = "added",
File = x.File
}), transaction : transaction);
}
}
await db.ExecuteAsync("UPDATE `Depots` SET `LastManifestID` = @ManifestID, `LastUpdated` = CURRENT_TIMESTAMP() WHERE `DepotID` = @DepotID", new { request.DepotID, request.ManifestID }, transaction : transaction);
return(EResult.OK);
}
public void Process(uint appID, uint changeNumber, KeyValue depots)
{
var requests = new List <ManifestJob>();
// Get data in format we want first
foreach (var depot in depots.Children)
{
// Ignore these for now, parent app should be updated too anyway
if (depot["depotfromapp"].Value != null)
{
continue;
}
var request = new ManifestJob
{
ChangeNumber = changeNumber,
DepotName = depot["name"].AsString()
};
// Ignore keys that aren't integers, for example "branches"
if (!uint.TryParse(depot.Name, out request.DepotID))
{
continue;
}
// TODO: instead of locking we could wait for current process to finish
if (DepotLocks.ContainsKey(request.DepotID))
{
continue;
}
// If there is no public manifest for this depot, it still could have some sort of open beta
if (depot["manifests"]["public"].Value == null || !ulong.TryParse(depot["manifests"]["public"].Value, out request.ManifestID))
{
var branch = depot["manifests"].Children.FirstOrDefault(x => x.Name != "local");
if (branch == null || !ulong.TryParse(branch.Value, out request.ManifestID))
{
using (var db = Database.GetConnection())
{
db.Execute("INSERT INTO `Depots` (`DepotID`, `Name`) VALUES (@DepotID, @DepotName) ON DUPLICATE KEY UPDATE `Name` = VALUES(`Name`)", new { request.DepotID, request.DepotName });
}
continue;
}
request.BuildID = branch["build"].AsInteger();
}
else
{
request.BuildID = depots["branches"]["public"]["buildid"].AsInteger();
}
requests.Add(request);
}
if (!requests.Any())
{
return;
}
var depotsToDownload = new List <ManifestJob>();
using (var db = Database.GetConnection())
{
var dbDepots = db.Query <Depot>("SELECT `DepotID`, `Name`, `BuildID`, `ManifestID`, `LastManifestID` FROM `Depots` WHERE `DepotID` IN @Depots", new { Depots = requests.Select(x => x.DepotID) })
.ToDictionary(x => x.DepotID, x => x);
foreach (var request in requests)
{
Depot dbDepot;
if (dbDepots.ContainsKey(request.DepotID))
{
dbDepot = dbDepots[request.DepotID];
if (dbDepot.BuildID > request.BuildID)
{
// buildid went back in time? this either means a rollback, or a shared depot that isn't synced properly
Log.WriteDebug("Depot Processor", "Skipping depot {0} due to old buildid: {1} > {2}", request.DepotID, dbDepot.BuildID, request.BuildID);
continue;
}
if (dbDepot.LastManifestID == request.ManifestID && dbDepot.ManifestID == request.ManifestID && Settings.Current.FullRun <= FullRunState.Normal)
{
// Update depot name if changed
if (!request.DepotName.Equals(dbDepot.Name))
{
db.Execute("UPDATE `Depots` SET `Name` = @DepotName WHERE `DepotID` = @DepotID", new { request.DepotID, request.DepotName });
}
continue;
}
}
else
{
dbDepot = new Depot();
}
if (dbDepot.BuildID != request.BuildID || dbDepot.ManifestID != request.ManifestID || !request.DepotName.Equals(dbDepot.Name))
{
db.Execute(@"INSERT INTO `Depots` (`DepotID`, `Name`, `BuildID`, `ManifestID`) VALUES (@DepotID, @DepotName, @BuildID, @ManifestID)
ON DUPLICATE KEY UPDATE `LastUpdated` = unix_timestamp(), `Name` = VALUES(`Name`), `BuildID` = VALUES(`BuildID`), `ManifestID` = VALUES(`ManifestID`)",
new {
request.DepotID,
request.DepotName,
request.BuildID,
request.ManifestID
});
}
if (dbDepot.ManifestID != request.ManifestID)
{
MakeHistory(db, request, string.Empty, "manifest_change", dbDepot.ManifestID, request.ManifestID);
}
var owned = LicenseList.OwnedApps.ContainsKey(request.DepotID);
if (!owned)
{
request.Anonymous = owned = LicenseList.AnonymousApps.ContainsKey(request.DepotID);
if (owned)
{
Log.WriteWarn("Depot Processor", "Will download depot {0} using anonymous account", request.DepotID);
}
}
if (owned || Settings.Current.FullRun == FullRunState.WithForcedDepots || Settings.Current.FullRun == FullRunState.ImportantOnly)
{
lock (DepotLocks)
{
// This doesn't really save us from concurrency issues
if (DepotLocks.ContainsKey(request.DepotID))
{
Log.WriteWarn("Depot Processor", "Depot {0} was locked in another thread", request.DepotID);
continue;
}
DepotLocks.Add(request.DepotID, 1);
}
depotsToDownload.Add(request);
}
#if DEBUG
else
{
Log.WriteDebug("Depot Processor", "Skipping depot {0} from app {1} because we don't own it", request.DepotID, appID);
}
#endif
}
}
if (depotsToDownload.Any())
{
PICSProductInfo.ProcessorThreadPool.QueueWorkItem(async() =>
{
try
{
await DownloadDepots(appID, depotsToDownload);
}
catch (Exception e)
{
ErrorReporter.Notify(e);
}
foreach (var depot in depotsToDownload)
{
RemoveLock(depot.DepotID);
}
});
}
}
public async Task Process(uint appID, uint changeNumber, KeyValue depots)
{
var requests = new List <ManifestJob>();
// Get data in format we want first
foreach (var depot in depots.Children)
{
// Ignore these for now, parent app should be updated too anyway
if (depot["depotfromapp"].Value != null)
{
continue;
}
var request = new ManifestJob
{
ChangeNumber = changeNumber,
DepotName = depot["name"].AsString()
};
// Ignore keys that aren't integers, for example "branches"
if (!uint.TryParse(depot.Name, out request.DepotID))
{
continue;
}
// TODO: instead of locking we could wait for current process to finish
if (DepotLocks.ContainsKey(request.DepotID))
{
continue;
}
if (depot["manifests"]["public"].Value == null || !ulong.TryParse(depot["manifests"]["public"].Value, out request.ManifestID))
{
var branch = depot["manifests"].Children.FirstOrDefault(x => x.Name != "local");
if (branch == null || !ulong.TryParse(branch.Value, out request.ManifestID))
{
using (var db = Database.Get())
{
await db.ExecuteAsync("INSERT INTO `Depots` (`DepotID`, `Name`) VALUES (@DepotID, @DepotName) ON DUPLICATE KEY UPDATE `Name` = VALUES(`Name`)", new { request.DepotID, request.DepotName });
}
continue;
}
Log.WriteDebug("Depot Downloader", "Depot {0} (from {1}) has no public branch, but there is another one", request.DepotID, appID);
request.BuildID = depots["branches"][branch.Name]["buildid"].AsInteger();
}
else
{
request.BuildID = depots["branches"]["public"]["buildid"].AsInteger();
}
requests.Add(request);
}
if (!requests.Any())
{
return;
}
var depotsToDownload = new List <ManifestJob>();
using (var db = await Database.GetConnectionAsync())
{
var firstRequest = requests.First();
await db.ExecuteAsync("INSERT INTO `Builds` (`BuildID`, `ChangeID`, `AppID`) VALUES (@BuildID, @ChangeNumber, @AppID) ON DUPLICATE KEY UPDATE `AppID` = VALUES(`AppID`)",
new {
firstRequest.BuildID,
firstRequest.ChangeNumber,
appID
});
var dbDepots = (await db.QueryAsync <Depot>("SELECT `DepotID`, `Name`, `BuildID`, `ManifestID`, `LastManifestID` FROM `Depots` WHERE `DepotID` IN @Depots", new { Depots = requests.Select(x => x.DepotID) }))
.ToDictionary(x => x.DepotID, x => x);
foreach (var request in requests)
{
Depot dbDepot;
if (dbDepots.ContainsKey(request.DepotID))
{
dbDepot = dbDepots[request.DepotID];
if (dbDepot.BuildID > request.BuildID)
{
// buildid went back in time? this either means a rollback, or a shared depot that isn't synced properly
Log.WriteDebug("Depot Processor", "Skipping depot {0} due to old buildid: {1} > {2}", request.DepotID, dbDepot.BuildID, request.BuildID);
continue;
}
if (dbDepot.LastManifestID == request.ManifestID && dbDepot.ManifestID == request.ManifestID && Settings.Current.FullRun != FullRunState.WithForcedDepots)
{
// Update depot name if changed
if (!request.DepotName.Equals(dbDepot.Name))
{
await db.ExecuteAsync("UPDATE `Depots` SET `Name` = @DepotName WHERE `DepotID` = @DepotID", new { request.DepotID, request.DepotName });
}
continue;
}
}
else
{
dbDepot = new Depot();
}
if (dbDepot.BuildID != request.BuildID || dbDepot.ManifestID != request.ManifestID || !request.DepotName.Equals(dbDepot.Name))
{
await db.ExecuteAsync(@"INSERT INTO `Depots` (`DepotID`, `Name`, `BuildID`, `ManifestID`) VALUES (@DepotID, @DepotName, @BuildID, @ManifestID)
ON DUPLICATE KEY UPDATE `LastUpdated` = CURRENT_TIMESTAMP(), `Name` = VALUES(`Name`), `BuildID` = VALUES(`BuildID`), `ManifestID` = VALUES(`ManifestID`)",
new {
request.DepotID,
request.DepotName,
request.BuildID,
request.ManifestID
});
}
if (dbDepot.ManifestID != request.ManifestID)
{
await MakeHistory(db, null, request, string.Empty, "manifest_change", dbDepot.ManifestID, request.ManifestID);
}
var owned = LicenseList.OwnedApps.ContainsKey(request.DepotID);
if (!owned)
{
request.Anonymous = owned = LicenseList.AnonymousApps.ContainsKey(request.DepotID);
if (owned)
{
Log.WriteWarn("Depot Processor", "Will download depot {0} using anonymous account", request.DepotID);
}
}
if (owned)
{
lock (DepotLocks)
{
// This doesn't really save us from concurrency issues
if (DepotLocks.ContainsKey(request.DepotID))
{
Log.WriteWarn("Depot Processor", "Depot {0} was locked in another thread", request.DepotID);
continue;
}
DepotLocks.Add(request.DepotID, 1);
}
depotsToDownload.Add(request);
}
}
}
if (depotsToDownload.Any())
{
#pragma warning disable 4014
if (FileDownloader.IsImportantDepot(appID) && !Settings.IsFullRun && !string.IsNullOrEmpty(Settings.Current.PatchnotesNotifyURL))
{
TaskManager.Run(() => NotifyPatchnote(appID, depotsToDownload.First().BuildID));
}
TaskManager.Run(async() =>
{
try
{
await DownloadDepots(appID, depotsToDownload);
}
catch (Exception e)
{
ErrorReporter.Notify("Depot Processor", e);
}
foreach (var depot in depotsToDownload)
{
RemoveLock(depot.DepotID);
}
});
#pragma warning restore 4014
}
}
private static async Task <EResult> ProcessDepotAfterDownload(IDbConnection db, IDbTransaction transaction, ManifestJob request, DepotManifest depotManifest)
{
var filesOld = (await db.QueryAsync <DepotFile>("SELECT `ID`, `File`, `Hash`, `Size`, `Flags` FROM `DepotsFiles` WHERE `DepotID` = @DepotID", new { request.DepotID }, transaction: transaction)).ToDictionary(x => x.File, x => x);
var filesAdded = new List <DepotFile>();
var shouldHistorize = filesOld.Count > 0; // Don't historize file additions if we didn't have any data before
foreach (var file in depotManifest.Files.OrderByDescending(x => x.FileName))
{
var name = file.FileName.Replace('\\', '/');
byte[] hash = null;
// Store empty hashes as NULL (e.g. an empty file)
if (file.FileHash.Length > 0 && (file.Flags & EDepotFileFlag.Directory) == 0)
{
for (var i = 0; i < file.FileHash.Length; ++i)
{
if (file.FileHash[i] != 0)
{
hash = file.FileHash;
break;
}
}
}
// Limit path names to 260 characters (default windows max length)
// File column is varchar(260) and not higher to prevent reducing performance
// See https://stackoverflow.com/questions/1962310/importance-of-varchar-length-in-mysql-table/1962329#1962329
// Until 2019 there hasn't been a single file that went over this limit, so far there has been only one
// game with a big node_modules path, so we're safeguarding by limiting it.
if (name.Length > 260)
{
using var sha = new System.Security.Cryptography.SHA1Managed();
var nameHash = Utils.ByteArrayToString(sha.ComputeHash(System.Text.Encoding.UTF8.GetBytes(name)));
name = $"{{SteamDB file name is too long}}/{nameHash}/...{name.Substring(name.Length - 150)}";
}
if (filesOld.ContainsKey(name))
{
var oldFile = filesOld[name];
var updateFile = false;
if (oldFile.Size != file.TotalSize || !Utils.IsEqualSHA1(hash, oldFile.Hash))
{
await MakeHistory(db, transaction, request, name, "modified", oldFile.Size, file.TotalSize);
updateFile = true;
}
if (oldFile.Flags != file.Flags)
{
await MakeHistory(db, transaction, request, name, "modified_flags", (ulong)oldFile.Flags, (ulong)file.Flags);
updateFile = true;
}
if (updateFile)
{
await db.ExecuteAsync("UPDATE `DepotsFiles` SET `Hash` = @Hash, `Size` = @Size, `Flags` = @Flags WHERE `DepotID` = @DepotID AND `ID` = @ID", new DepotFile
{
ID = oldFile.ID,
DepotID = request.DepotID,
Hash = hash,
Size = file.TotalSize,
Flags = file.Flags
}, transaction : transaction);
}
filesOld.Remove(name);
}
else
{
// We want to historize modifications first, and only then deletions and additions
filesAdded.Add(new DepotFile
{
DepotID = request.DepotID,
Hash = hash,
File = name,
Size = file.TotalSize,
Flags = file.Flags
});
}
}
if (filesOld.Count > 0)
{
await db.ExecuteAsync("DELETE FROM `DepotsFiles` WHERE `DepotID` = @DepotID AND `ID` IN @Files", new { request.DepotID, Files = filesOld.Select(x => x.Value.ID) }, transaction : transaction);
await db.ExecuteAsync(HistoryQuery, filesOld.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ManifestID = request.ManifestID,
ChangeID = request.ChangeNumber,
Action = "removed",
File = x.Value.File,
OldValue = x.Value.Size
}), transaction : transaction);
}
if (filesAdded.Count > 0)
{
await db.ExecuteAsync("INSERT INTO `DepotsFiles` (`DepotID`, `File`, `Hash`, `Size`, `Flags`) VALUES (@DepotID, @File, @Hash, @Size, @Flags)", filesAdded, transaction : transaction);
if (shouldHistorize)
{
await db.ExecuteAsync(HistoryQuery, filesAdded.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ManifestID = request.ManifestID,
ChangeID = request.ChangeNumber,
Action = "added",
File = x.File,
NewValue = x.Size
}), transaction : transaction);
}
}
await db.ExecuteAsync("UPDATE `Depots` SET `LastManifestID` = @ManifestID, `LastUpdated` = CURRENT_TIMESTAMP() WHERE `DepotID` = @DepotID", new { request.DepotID, request.ManifestID }, transaction : transaction);
return(EResult.OK);
}
