public async Task Process(uint appID, uint changeNumber, KeyValue depots)
{
var requests = new List <ManifestJob>();
// Get data in format we want first
foreach (var depot in depots.Children)
{
// Ignore these for now, parent app should be updated too anyway
if (depot["depotfromapp"].Value != null)
{
continue;
}
var request = new ManifestJob
{
ChangeNumber = changeNumber,
};
// Ignore keys that aren't integers, for example "branches"
if (!uint.TryParse(depot.Name, out request.DepotID))
{
continue;
}
request.DepotName = depot["name"].AsString();
if (string.IsNullOrEmpty(request.DepotName))
{
request.DepotName = $"SteamDB Unnamed Depot {request.DepotID}";
}
// TODO: instead of locking we could wait for current process to finish
if (DepotLocks.ContainsKey(request.DepotID))
{
continue;
}
// SteamVR trickery
if (appID == 250820 &&
depot["manifests"]["beta"].Value != null &&
depots["branches"]["beta"]["buildid"].AsInteger() > depots["branches"]["public"]["buildid"].AsInteger())
{
request.BuildID = depots["branches"]["beta"]["buildid"].AsInteger();
request.ManifestID = ulong.Parse(depot["manifests"]["beta"].Value);
}
else
if (depot["manifests"]["public"].Value == null || !ulong.TryParse(depot["manifests"]["public"].Value, out request.ManifestID))
{
var branch = depot["manifests"].Children.Find(x => x.Name != "local");
if (branch == null || !ulong.TryParse(branch.Value, out request.ManifestID))
{
await using var db = await Database.GetConnectionAsync();
await db.ExecuteAsync("INSERT INTO `Depots` (`DepotID`, `Name`) VALUES (@DepotID, @DepotName) ON DUPLICATE KEY UPDATE `DepotID` = VALUES(`DepotID`)", new { request.DepotID, request.DepotName });
continue;
}
Log.WriteDebug(nameof(DepotProcessor), $"Depot {request.DepotID} (from {appID}) has no public branch, but there is another one");
request.BuildID = depots["branches"][branch.Name]["buildid"].AsInteger();
}
else
{
request.BuildID = depots["branches"]["public"]["buildid"].AsInteger();
}
requests.Add(request);
}
if (requests.Count == 0)
{
return;
}
var depotsToDownload = new List <ManifestJob>();
await using (var db = await Database.GetConnectionAsync())
{
await db.ExecuteAsync("INSERT INTO `Builds` (`BuildID`, `ChangeID`, `AppID`) VALUES (@BuildID, @ChangeNumber, @AppID) ON DUPLICATE KEY UPDATE `AppID` = VALUES(`AppID`)",
new
{
requests[0].BuildID,
requests[0].ChangeNumber,
appID
});
var depotIds = requests.Select(x => x.DepotID).ToList();
var dbDepots = (await db.QueryAsync <Depot>("SELECT `DepotID`, `Name`, `BuildID`, `ManifestID`, `LastManifestID`, `FilenamesEncrypted` FROM `Depots` WHERE `DepotID` IN @depotIds", new { depotIds }))
.ToDictionary(x => x.DepotID, x => x);
var decryptionKeys = (await db.QueryAsync <DepotKey>("SELECT `DepotID`, `Key` FROM `DepotsKeys` WHERE `DepotID` IN @depotIds", new { depotIds }))
.ToDictionary(x => x.DepotID, x => Utils.StringToByteArray(x.Key));
foreach (var request in requests)
{
Depot dbDepot;
decryptionKeys.TryGetValue(request.DepotID, out request.DepotKey);
if (dbDepots.ContainsKey(request.DepotID))
{
dbDepot = dbDepots[request.DepotID];
if (dbDepot.BuildID > request.BuildID)
{
// buildid went back in time? this either means a rollback, or a shared depot that isn't synced properly
Log.WriteDebug(nameof(DepotProcessor), $"Skipping depot {request.DepotID} due to old buildid: {dbDepot.BuildID} > {request.BuildID}");
continue;
}
if (dbDepot.LastManifestID == request.ManifestID &&
dbDepot.ManifestID == request.ManifestID &&
Settings.Current.FullRun != FullRunState.WithForcedDepots &&
!dbDepot.FilenamesEncrypted && request.DepotKey != null)
{
// Update depot name if changed
if (request.DepotName != dbDepot.Name)
{
await db.ExecuteAsync("UPDATE `Depots` SET `Name` = @DepotName WHERE `DepotID` = @DepotID", new { request.DepotID, request.DepotName });
}
continue;
}
request.StoredFilenamesEncrypted = dbDepot.FilenamesEncrypted;
request.LastManifestID = dbDepot.LastManifestID;
}
else
{
dbDepot = new Depot();
}
if (dbDepot.BuildID != request.BuildID || dbDepot.ManifestID != request.ManifestID || request.DepotName != dbDepot.Name)
{
await db.ExecuteAsync(@"INSERT INTO `Depots` (`DepotID`, `Name`, `BuildID`, `ManifestID`) VALUES (@DepotID, @DepotName, @BuildID, @ManifestID)
ON DUPLICATE KEY UPDATE `LastUpdated` = CURRENT_TIMESTAMP(), `Name` = VALUES(`Name`), `BuildID` = VALUES(`BuildID`), `ManifestID` = VALUES(`ManifestID`)",
new
{
request.DepotID,
request.DepotName,
request.BuildID,
request.ManifestID
});
}
if (dbDepot.ManifestID != request.ManifestID)
{
await MakeHistory(db, null, request, string.Empty, "manifest_change", dbDepot.ManifestID, request.ManifestID);
}
lock (DepotLocks)
{
// This doesn't really save us from concurrency issues
if (DepotLocks.ContainsKey(request.DepotID))
{
Log.WriteWarn(nameof(DepotProcessor), $"Depot {request.DepotID} was locked in another thread");
continue;
}
DepotLocks.Add(request.DepotID, 1);
}
depotsToDownload.Add(request);
}
}
if (depotsToDownload.Count > 0)
{
_ = TaskManager.Run(async() =>
{
try
{
await DownloadDepots(appID, depotsToDownload);
}
catch (Exception e)
{
ErrorReporter.Notify(nameof(DepotProcessor), e);
}
foreach (var depot in depotsToDownload)
{
RemoveLock(depot.DepotID);
}
});
}
}
private async Task DownloadDepots(uint appID, List <ManifestJob> depots)
{
Log.WriteDebug(nameof(DepotProcessor), $"Will process {depots.Count} depots ({DepotLocks.Count} depot locks left)");
var processTasks = new List <Task <EResult> >();
var anyFilesDownloaded = false;
var willDownloadFiles = false;
foreach (var depot in depots)
{
if (depot.DepotKey == null)
{
await GetDepotDecryptionKey(Steam.Instance.Apps, depot, appID);
if (depot.DepotKey == null && (depot.LastManifestID == depot.ManifestID || Settings.Current.OnlyOwnedDepots))
{
RemoveLock(depot.DepotID);
continue;
}
}
depot.Server = GetContentServer();
DepotManifest depotManifest = null;
var lastError = string.Empty;
for (var i = 0; i <= 5; i++)
{
try
{
await ManifestDownloadSemaphore.WaitAsync(TaskManager.TaskCancellationToken.Token).ConfigureAwait(false);
depotManifest = await CDNClient.DownloadManifestAsync(depot.DepotID, depot.ManifestID, depot.Server, string.Empty, depot.DepotKey);
break;
}
catch (Exception e)
{
lastError = e.Message;
Log.WriteError(nameof(DepotProcessor), $"Failed to download depot manifest for app {appID} depot {depot.DepotID} ({depot.Server}: {lastError}) (#{i})");
}
finally
{
ManifestDownloadSemaphore.Release();
}
if (depot.DepotKey != null)
{
RemoveErroredServer(depot.Server);
}
depot.Server = GetContentServer();
if (depotManifest == null && i < 5)
{
await Task.Delay(Utils.ExponentionalBackoff(i + 1));
}
}
if (depotManifest == null)
{
RemoveLock(depot.DepotID);
if (FileDownloader.IsImportantDepot(depot.DepotID))
{
IRC.Instance.SendOps($"{Colors.OLIVE}[{depot.DepotName}]{Colors.NORMAL} Failed to download manifest ({lastError})");
}
if (!Settings.IsFullRun && depot.DepotKey != null)
{
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
}
continue;
}
var task = ProcessDepotAfterDownload(depot, depotManifest);
processTasks.Add(task);
if (!FileDownloader.IsImportantDepot(depot.DepotID) || depot.DepotKey == null)
{
depot.Result = EResult.Ignored;
continue;
}
willDownloadFiles = true;
task = TaskManager.Run(async() =>
{
var result = EResult.Fail;
try
{
result = await FileDownloader.DownloadFilesFromDepot(depot, depotManifest);
if (result == EResult.OK)
{
anyFilesDownloaded = true;
}
}
catch (Exception e)
{
ErrorReporter.Notify($"Depot Processor {depot.DepotID}", e);
}
return(result);
});
processTasks.Add(task);
}
await Task.WhenAll(processTasks).ConfigureAwait(false);
Log.WriteDebug(nameof(DepotProcessor), $"{depots.Count} depot downloads finished for app {appID}");
// TODO: use ContinueWith on tasks
if (!anyFilesDownloaded && !willDownloadFiles)
{
foreach (var depot in depots)
{
RemoveLock(depot.DepotID);
}
return;
}
lock (UpdateScriptLock)
{
foreach (var depot in depots)
{
if (depot.Result == EResult.OK)
{
RunUpdateScript(UpdateScript, $"{depot.DepotID} no-git");
}
else if (depot.Result != EResult.Ignored)
{
Log.WriteWarn(nameof(DepotProcessor), $"Download failed for {depot.DepotID}: {depot.Result}");
// Mark this depot for redownload
using var db = Database.Get();
db.Execute("UPDATE `Depots` SET `LastManifestID` = 0 WHERE `DepotID` = @DepotID", new { depot.DepotID });
}
RemoveLock(depot.DepotID);
}
// Only commit changes if all depots downloaded
if (processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored))
{
if (!RunUpdateScriptForApp(appID, depots[0].BuildID))
{
RunUpdateScript(UpdateScript, "0");
}
}
else
{
Log.WriteDebug(nameof(DepotProcessor), $"Reprocessing the app {appID} because some files failed to download");
IRC.Instance.SendOps($"{Colors.OLIVE}[{Steam.GetAppName(appID)}]{Colors.NORMAL} Reprocessing the app due to download failures");
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
}
}
}
public static async Task <uint> TrySearchAppId(CommandArguments command)
{
uint appID = 0;
var name = command.Message;
Uri uri;
var query = new Dictionary <string, string>
{
{ "hitsPerPage", "1" },
{ "attributesToHighlight", "null" },
{ "attributesToSnippet", "null" },
{ "attributesToRetrieve", "[\"objectID\"]" },
{ "facetFilters", "[[\"appType:Game\",\"appType:Application\"]]" },
{ "advancedSyntax", "true" },
{ "query", name }
};
using (var content = new FormUrlEncodedContent(query))
{
uri = new UriBuilder("https://94he6yatei-dsn.algolia.net/1/indexes/steamdb/")
{
Query = await content.ReadAsStringAsync()
}.Uri;
}
using (var requestMessage = new HttpRequestMessage(HttpMethod.Get, uri))
{
requestMessage.Headers.Add("Referer", "https://github.com/SteamDatabase/SteamDatabaseBackend");
requestMessage.Headers.Add("X-Algolia-Application-Id", "94HE6YATEI");
requestMessage.Headers.Add("X-Algolia-API-Key", "2414d3366df67739fe6e73dad3f51a43");
try
{
var response = await Utils.HttpClient.SendAsync(requestMessage);
var data = await response.Content.ReadAsStringAsync();
var json = JsonConvert.DeserializeObject <AlgoliaSearchAppHits>(data);
if (json.Hits.Length > 0)
{
appID = json.Hits[0].AppID;
}
}
catch (Exception e)
{
ErrorReporter.Notify("Algolia search", e);
}
}
if (appID > 0)
{
return(appID);
}
await using (var db = await Database.GetConnectionAsync())
{
appID = await db.ExecuteScalarAsync <uint>($"SELECT `AppID` FROM `Apps` WHERE (`AppType` IN ({EAppType.Game:d},{EAppType.Application:d},{EAppType.Video:d}) AND (`Apps`.`StoreName` LIKE @Name OR `Apps`.`Name` LIKE @Name)) OR (`AppType` = {EAppType.Invalid:d} AND `Apps`.`LastKnownName` LIKE @Name) ORDER BY `LastUpdated` DESC LIMIT 1", new { Name = name });
}
if (appID == 0)
{
command.Reply("Nothing was found matching your request.");
}
return(appID);
}
private async Task DownloadDepots(uint appID, List <ManifestJob> depots)
{
Log.WriteDebug("Depot Downloader", "Will process {0} depots ({1} depot locks left)", depots.Count, DepotLocks.Count);
var processTasks = new List <Task <EResult> >();
var anyFilesDownloaded = false;
var willDownloadFiles = false;
foreach (var depot in depots)
{
if (depot.DepotKey == null)
{
await GetDepotDecryptionKey(Steam.Instance.Apps, depot, appID);
if (depot.DepotKey == null)
{
RemoveLock(depot.DepotID);
continue;
}
}
depot.Server = GetContentServer();
DepotManifest depotManifest = null;
var lastError = string.Empty;
for (var i = 0; i <= 5; i++)
{
try
{
depotManifest = await CDNClient.DownloadManifestAsync(depot.DepotID, depot.ManifestID, depot.Server, string.Empty, depot.DepotKey);
break;
}
catch (Exception e)
{
lastError = e.Message;
Log.WriteError("Depot Processor", "Failed to download depot manifest for app {0} depot {1} ({2}: {3}) (#{4})", appID, depot.DepotID, depot.Server, lastError, i);
}
depot.Server = GetContentServer();
if (depotManifest == null)
{
await Task.Delay(Utils.ExponentionalBackoff(i));
}
}
if (depotManifest == null)
{
RemoveLock(depot.DepotID);
if (FileDownloader.IsImportantDepot(depot.DepotID))
{
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download manifest ({3})",
Colors.OLIVE, depot.DepotName, Colors.NORMAL, lastError);
}
if (!Settings.IsFullRun)
{
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
}
continue;
}
var task = ProcessDepotAfterDownload(depot, depotManifest);
processTasks.Add(task);
if (!FileDownloader.IsImportantDepot(depot.DepotID))
{
continue;
}
willDownloadFiles = true;
task = TaskManager.Run(async() =>
{
var result = EResult.Fail;
try
{
result = await FileDownloader.DownloadFilesFromDepot(depot, depotManifest);
if (result == EResult.OK)
{
anyFilesDownloaded = true;
}
}
catch (Exception e)
{
ErrorReporter.Notify($"Depot Processor {depot.DepotID}", e);
}
return(result);
}).Unwrap();
processTasks.Add(task);
}
if (SaveLocalConfig)
{
SaveLocalConfig = false;
LocalConfig.Save();
}
await Task.WhenAll(processTasks).ConfigureAwait(false);
Log.WriteDebug("Depot Downloader", $"{depots.Count} depot downloads finished for app {appID}");
// TODO: use ContinueWith on tasks
if (!anyFilesDownloaded && !willDownloadFiles)
{
foreach (var depot in depots)
{
RemoveLock(depot.DepotID);
}
return;
}
lock (UpdateScriptLock)
{
foreach (var depot in depots)
{
if (depot.Result == EResult.OK)
{
RunUpdateScript(UpdateScript, string.Format("{0} no-git", depot.DepotID));
}
else if (depot.Result != EResult.Ignored)
{
Log.WriteWarn("Depot Processor", $"Download failed for {depot.DepotID}");
// Mark this depot for redownload
using var db = Database.Get();
db.Execute("UPDATE `Depots` SET `LastManifestID` = 0 WHERE `DepotID` = @DepotID", new { depot.DepotID });
}
RemoveLock(depot.DepotID);
}
// Only commit changes if all depots downloaded
if (processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored))
{
if (!RunUpdateScriptForApp(appID, depots[0].BuildID))
{
RunUpdateScript(UpdateScript, "0");
}
}
else
{
Log.WriteDebug("Depot Processor", "Reprocessing the app {0} because some files failed to download", appID);
IRC.Instance.SendOps("{0}[{1}]{2} Reprocessing the app due to download failures",
Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL
);
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
}
}
}
public static void Cleanup()
{
// If threads is null, app was not yet initialized and there is nothing to cleanup
if (Threads == null)
{
return;
}
Log.WriteInfo("Bootstrapper", "Exiting...");
ChangelistTimer.Stop();
Log.WriteInfo("Bootstrapper", "Disconnecting from Steam...");
try
{
Steam.Instance.IsRunning = false;
Steam.Instance.Client.Disconnect();
}
catch (Exception e)
{
ErrorReporter.Notify("Bootstrapper", e);
}
if (Settings.Current.IRC.Enabled)
{
Log.WriteInfo("Bootstrapper", "Closing IRC connection...");
RssReader.Timer.Stop();
IRC.Instance.Close();
}
Log.WriteInfo("Bootstrapper", "Cancelling {0} tasks...", TaskManager.TasksCount);
TaskManager.CancelAllTasks();
foreach (var thread in Threads.Where(thread => thread.ThreadState == ThreadState.Running))
{
Log.WriteInfo("Bootstrapper", "Joining thread {0}...", thread.Name);
thread.Join(TimeSpan.FromSeconds(5));
}
Log.WriteInfo("Bootstrapper", "Truncating GC table...");
using (var db = Database.Get())
{
db.Execute("DELETE FROM `GC`");
}
Log.WriteInfo("Bootstrapper", "Saving local config...");
LocalConfig.Save();
try
{
Steam.Anonymous.IsRunning = false;
Steam.Anonymous.Client.Disconnect();
}
catch (Exception e)
{
ErrorReporter.Notify("Bootstrapper", e);
}
}
public void Process(uint appID, uint changeNumber, KeyValue depots)
{
var requests = new List <ManifestJob>();
// Get data in format we want first
foreach (var depot in depots.Children)
{
// Ignore these for now, parent app should be updated too anyway
if (depot["depotfromapp"].Value != null)
{
continue;
}
var request = new ManifestJob
{
ChangeNumber = changeNumber,
DepotName = depot["name"].AsString()
};
// Ignore keys that aren't integers, for example "branches"
if (!uint.TryParse(depot.Name, out request.DepotID))
{
continue;
}
// TODO: instead of locking we could wait for current process to finish
if (DepotLocks.ContainsKey(request.DepotID))
{
continue;
}
// If there is no public manifest for this depot, it still could have some sort of open beta
if (depot["manifests"]["public"].Value == null || !ulong.TryParse(depot["manifests"]["public"].Value, out request.ManifestID))
{
var branch = depot["manifests"].Children.FirstOrDefault(x => x.Name != "local");
if (branch == null || !ulong.TryParse(branch.Value, out request.ManifestID))
{
using (var db = Database.GetConnection())
{
db.Execute("INSERT INTO `Depots` (`DepotID`, `Name`) VALUES (@DepotID, @DepotName) ON DUPLICATE KEY UPDATE `Name` = VALUES(`Name`)", new { request.DepotID, request.DepotName });
}
continue;
}
request.BuildID = branch["build"].AsInteger();
}
else
{
request.BuildID = depots["branches"]["public"]["buildid"].AsInteger();
}
requests.Add(request);
}
if (!requests.Any())
{
return;
}
var depotsToDownload = new List <ManifestJob>();
using (var db = Database.GetConnection())
{
var dbDepots = db.Query <Depot>("SELECT `DepotID`, `Name`, `BuildID`, `ManifestID`, `LastManifestID` FROM `Depots` WHERE `DepotID` IN @Depots", new { Depots = requests.Select(x => x.DepotID) })
.ToDictionary(x => x.DepotID, x => x);
foreach (var request in requests)
{
Depot dbDepot;
if (dbDepots.ContainsKey(request.DepotID))
{
dbDepot = dbDepots[request.DepotID];
if (dbDepot.BuildID > request.BuildID)
{
// buildid went back in time? this either means a rollback, or a shared depot that isn't synced properly
Log.WriteDebug("Depot Processor", "Skipping depot {0} due to old buildid: {1} > {2}", request.DepotID, dbDepot.BuildID, request.BuildID);
continue;
}
if (dbDepot.LastManifestID == request.ManifestID && dbDepot.ManifestID == request.ManifestID && Settings.Current.FullRun <= FullRunState.Normal)
{
// Update depot name if changed
if (!request.DepotName.Equals(dbDepot.Name))
{
db.Execute("UPDATE `Depots` SET `Name` = @DepotName WHERE `DepotID` = @DepotID", new { request.DepotID, request.DepotName });
}
continue;
}
}
else
{
dbDepot = new Depot();
}
if (dbDepot.BuildID != request.BuildID || dbDepot.ManifestID != request.ManifestID || !request.DepotName.Equals(dbDepot.Name))
{
db.Execute(@"INSERT INTO `Depots` (`DepotID`, `Name`, `BuildID`, `ManifestID`) VALUES (@DepotID, @DepotName, @BuildID, @ManifestID)
ON DUPLICATE KEY UPDATE `LastUpdated` = CURRENT_TIMESTAMP(), `Name` = VALUES(`Name`), `BuildID` = VALUES(`BuildID`), `ManifestID` = VALUES(`ManifestID`)",
new {
request.DepotID,
request.DepotName,
request.BuildID,
request.ManifestID
});
}
if (dbDepot.ManifestID != request.ManifestID)
{
MakeHistory(db, request, string.Empty, "manifest_change", dbDepot.ManifestID, request.ManifestID);
}
if (LicenseList.OwnedApps.ContainsKey(request.DepotID) || Settings.Current.FullRun >= FullRunState.WithForcedDepots)
{
lock (DepotLocks)
{
// This doesn't really save us from concurrency issues
if (DepotLocks.ContainsKey(request.DepotID))
{
Log.WriteWarn("Depot Processor", "Depot {0} was locked in another thread", request.DepotID);
continue;
}
DepotLocks.Add(request.DepotID, 1);
}
depotsToDownload.Add(request);
}
#if DEBUG
else
{
Log.WriteDebug("Depot Processor", "Skipping depot {0} from app {1} because we don't own it", request.DepotID, appID);
}
#endif
}
}
if (depotsToDownload.Any())
{
PICSProductInfo.ProcessorThreadPool.QueueWorkItem(async() =>
{
try
{
await DownloadDepots(appID, depotsToDownload);
}
catch (Exception e)
{
ErrorReporter.Notify(e);
}
foreach (var depot in depotsToDownload)
{
RemoveLock(depot.DepotID);
}
});
}
}
private EResult ProcessDepotAfterDownload(IDbConnection db, ManifestJob request, DepotManifest depotManifest)
{
var filesOld = db.Query <DepotFile>("SELECT `ID`, `File`, `Hash`, `Size`, `Flags` FROM `DepotsFiles` WHERE `DepotID` = @DepotID", new { request.DepotID }).ToDictionary(x => x.File, x => x);
var filesNew = new List <DepotFile>();
var filesAdded = new List <DepotFile>();
var shouldHistorize = filesOld.Any(); // Don't historize file additions if we didn't have any data before
foreach (var file in depotManifest.Files)
{
var name = file.FileName.Replace('\\', '/');
// safe guard
if (name.Length > 255)
{
ErrorReporter.Notify(new OverflowException(string.Format("File \"{0}\" in depot {1} is too long", name, request.DepotID)));
continue;
}
var depotFile = new DepotFile
{
DepotID = request.DepotID,
File = name,
Size = file.TotalSize,
Flags = file.Flags
};
if (file.FileHash.Length > 0 && !file.Flags.HasFlag(EDepotFileFlag.Directory))
{
depotFile.Hash = Utils.ByteArrayToString(file.FileHash);
}
else
{
depotFile.Hash = "0000000000000000000000000000000000000000";
}
filesNew.Add(depotFile);
}
foreach (var file in filesNew)
{
if (filesOld.ContainsKey(file.File))
{
var oldFile = filesOld[file.File];
var updateFile = false;
if (oldFile.Size != file.Size || !file.Hash.Equals(oldFile.Hash))
{
MakeHistory(db, request, file.File, "modified", oldFile.Size, file.Size);
updateFile = true;
}
if (oldFile.Flags != file.Flags)
{
MakeHistory(db, request, file.File, "modified_flags", (ulong)oldFile.Flags, (ulong)file.Flags);
updateFile = true;
}
if (updateFile)
{
file.ID = oldFile.ID;
db.Execute("UPDATE `DepotsFiles` SET `Hash` = @Hash, `Size` = @Size, `Flags` = @Flags WHERE `DepotID` = @DepotID AND `ID` = @ID", file);
}
filesOld.Remove(file.File);
}
else
{
// We want to historize modifications first, and only then deletions and additions
filesAdded.Add(file);
}
}
if (filesOld.Any())
{
db.Execute("DELETE FROM `DepotsFiles` WHERE `DepotID` = @DepotID AND `ID` IN @Files", new { request.DepotID, Files = filesOld.Select(x => x.Value.ID) });
db.Execute(GetHistoryQuery(), filesOld.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ChangeID = request.ChangeNumber,
Action = "removed",
File = x.Value.File
}));
}
if (filesAdded.Any())
{
db.Execute("INSERT INTO `DepotsFiles` (`DepotID`, `File`, `Hash`, `Size`, `Flags`) VALUES (@DepotID, @File, @Hash, @Size, @Flags)", filesAdded);
if (shouldHistorize)
{
db.Execute(GetHistoryQuery(), filesAdded.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ChangeID = request.ChangeNumber,
Action = "added",
File = x.File
}));
}
}
db.Execute("UPDATE `Depots` SET `LastManifestID` = @ManifestID WHERE `DepotID` = @DepotID", new { request.DepotID, request.ManifestID });
return(EResult.OK);
}
private static void OnPICSProductInfo(SteamApps.PICSProductInfoCallback callback)
{
JobManager.TryRemoveJob(callback.JobID);
var processors = new List <BaseProcessor>(
callback.Apps.Count +
callback.Packages.Count +
callback.UnknownApps.Count +
callback.UnknownPackages.Count
);
processors.AddRange(callback.Apps.Select(app => new AppProcessor(app.Key, app.Value)));
processors.AddRange(callback.Packages.Select(package => new SubProcessor(package.Key, package.Value)));
processors.AddRange(callback.UnknownApps.Select(app => new AppProcessor(app, null)));
processors.AddRange(callback.UnknownPackages.Select(package => new SubProcessor(package, null)));
foreach (var workaround in processors)
{
var processor = workaround;
Task mostRecentItem;
lock (CurrentlyProcessing)
{
CurrentlyProcessing.TryGetValue(processor.Id, out mostRecentItem);
}
var workerItem = TaskManager.Run(async() =>
{
try
{
await Semaphore.WaitAsync(TaskManager.TaskCancellationToken.Token).ConfigureAwait(false);
if (mostRecentItem != null && !mostRecentItem.IsCompleted)
{
Log.WriteDebug(processor.ToString(), $"Waiting for previous task to finish processing ({CurrentlyProcessing.Count})");
await mostRecentItem.ConfigureAwait(false);
#if DEBUG
Log.WriteDebug(processor.ToString(), "Previous task lock ended");
#endif
}
await processor.Process().ConfigureAwait(false);
}
catch (Exception e)
{
ErrorReporter.Notify(processor.ToString(), e);
}
finally
{
Semaphore.Release();
processor.Dispose();
}
return(processor);
}).Unwrap();
lock (CurrentlyProcessing)
{
CurrentlyProcessing[processor.Id] = workerItem;
}
// Register error handler on inner task and the continuation
TaskManager.RegisterErrorHandler(workerItem);
TaskManager.RegisterErrorHandler(workerItem.ContinueWith(RemoveProcessorLock, TaskManager.TaskCancellationToken.Token));
}
}
private async Task <EResult> ProcessDepotAfterDownload(IDbConnection db, IDbTransaction transaction, ManifestJob request, DepotManifest depotManifest)
{
var filesOld = (await db.QueryAsync <DepotFile>("SELECT `ID`, `File`, `Hash`, `Size`, `Flags` FROM `DepotsFiles` WHERE `DepotID` = @DepotID", new { request.DepotID }, transaction: transaction)).ToDictionary(x => x.File, x => x);
var filesAdded = new List <DepotFile>();
var shouldHistorize = filesOld.Count > 0; // Don't historize file additions if we didn't have any data before
foreach (var file in depotManifest.Files)
{
var name = file.FileName.Replace('\\', '/');
byte[] hash = null;
// Store empty hashes as NULL (e.g. an empty file)
if (file.FileHash.Length > 0 && (file.Flags & EDepotFileFlag.Directory) == 0)
{
for (var i = 0; i < file.FileHash.Length; ++i)
{
if (file.FileHash[i] != 0)
{
hash = file.FileHash;
break;
}
}
}
// safe guard
if (name.Length > 255)
{
ErrorReporter.Notify("Depot Processor", new OverflowException(string.Format("File \"{0}\" in depot {1} is too long", name, request.DepotID)));
continue;
}
if (filesOld.ContainsKey(name))
{
var oldFile = filesOld[name];
var updateFile = false;
if (oldFile.Size != file.TotalSize || !Utils.IsEqualSHA1(hash, oldFile.Hash))
{
await MakeHistory(db, transaction, request, name, "modified", oldFile.Size, file.TotalSize);
updateFile = true;
}
if (oldFile.Flags != file.Flags)
{
await MakeHistory(db, transaction, request, name, "modified_flags", (ulong)oldFile.Flags, (ulong)file.Flags);
updateFile = true;
}
if (updateFile)
{
await db.ExecuteAsync("UPDATE `DepotsFiles` SET `Hash` = @Hash, `Size` = @Size, `Flags` = @Flags WHERE `DepotID` = @DepotID AND `ID` = @ID", new DepotFile
{
ID = oldFile.ID,
DepotID = request.DepotID,
Hash = hash,
Size = file.TotalSize,
Flags = file.Flags
}, transaction : transaction);
}
filesOld.Remove(name);
}
else
{
// We want to historize modifications first, and only then deletions and additions
filesAdded.Add(new DepotFile
{
DepotID = request.DepotID,
Hash = hash,
File = name,
Size = file.TotalSize,
Flags = file.Flags
});
}
}
if (filesOld.Count > 0)
{
await db.ExecuteAsync("DELETE FROM `DepotsFiles` WHERE `DepotID` = @DepotID AND `ID` IN @Files", new { request.DepotID, Files = filesOld.Select(x => x.Value.ID) }, transaction : transaction);
await db.ExecuteAsync(HistoryQuery, filesOld.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ChangeID = request.ChangeNumber,
Action = "removed",
File = x.Value.File,
OldValue = x.Value.Size
}), transaction : transaction);
}
if (filesAdded.Count > 0)
{
await db.ExecuteAsync("INSERT INTO `DepotsFiles` (`DepotID`, `File`, `Hash`, `Size`, `Flags`) VALUES (@DepotID, @File, @Hash, @Size, @Flags)", filesAdded, transaction : transaction);
if (shouldHistorize)
{
await db.ExecuteAsync(HistoryQuery, filesAdded.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ChangeID = request.ChangeNumber,
Action = "added",
File = x.File,
NewValue = x.Size
}), transaction : transaction);
}
}
await db.ExecuteAsync("UPDATE `Depots` SET `LastManifestID` = @ManifestID, `LastUpdated` = CURRENT_TIMESTAMP() WHERE `DepotID` = @DepotID", new { request.DepotID, request.ManifestID }, transaction : transaction);
return(EResult.OK);
}
