public static void Cleanup()
{
// If threads is null, app was not yet initialized and there is nothing to cleanup
if (Threads == null)
{
return;
}
Log.WriteInfo("Bootstrapper", "Exiting...");
ChangelistTimer.Stop();
Log.WriteInfo("Bootstrapper", "Disconnecting from Steam...");
try
{
Steam.Instance.IsRunning = false;
Steam.Instance.Client.Disconnect();
}
catch (Exception e)
{
ErrorReporter.Notify("Bootstrapper", e);
}
if (Settings.Current.IRC.Enabled)
{
Log.WriteInfo("Bootstrapper", "Closing IRC connection...");
RssReader.Timer.Stop();
IRC.Instance.Close();
}
Log.WriteInfo("Bootstrapper", "Cancelling {0} tasks...", TaskManager.TasksCount);
TaskManager.CancelAllTasks();
foreach (var thread in Threads.Where(thread => thread.ThreadState == ThreadState.Running))
{
Log.WriteInfo("Bootstrapper", "Joining thread {0}...", thread.Name);
thread.Join(TimeSpan.FromSeconds(5));
}
Log.WriteInfo("Bootstrapper", "Saving local config...");
LocalConfig.Save();
try
{
Steam.Anonymous.IsRunning = false;
Steam.Anonymous.Client.Disconnect();
}
catch (Exception e)
{
ErrorReporter.Notify("Bootstrapper", e);
}
}
private void OnMachineAuth(SteamUser.UpdateMachineAuthCallback callback)
{
Log.WriteInfo("Steam", "Updating sentry file... {0}", callback.FileName);
if (callback.Data.Length != callback.BytesToWrite)
{
ErrorReporter.Notify(new InvalidDataException(string.Format("Data.Length ({0}) != BytesToWrite ({1}) in OnMachineAuth", callback.Data.Length, callback.BytesToWrite)));
}
int fileSize;
byte[] sentryHash;
using (var stream = new MemoryStream(LocalConfig.Sentry ?? new byte[callback.BytesToWrite]))
{
stream.Seek(callback.Offset, SeekOrigin.Begin);
stream.Write(callback.Data, 0, callback.BytesToWrite);
stream.Seek(0, SeekOrigin.Begin);
fileSize = (int)stream.Length;
using (var sha = new SHA1CryptoServiceProvider())
{
sentryHash = sha.ComputeHash(stream);
}
LocalConfig.Sentry = stream.ToArray();
}
LocalConfig.SentryFileName = callback.FileName;
LocalConfig.Save();
Steam.Instance.User.SendMachineAuthResponse(new SteamUser.MachineAuthDetails
{
JobID = callback.JobID,
FileName = callback.FileName,
BytesWritten = callback.BytesToWrite,
FileSize = fileSize,
Offset = callback.Offset,
Result = EResult.OK,
LastError = 0,
OneTimePassword = callback.OneTimePassword,
SentryFileHash = sentryHash
});
}
private void OnFreeLicenseCallback(SteamApps.FreeLicenseCallback callback)
{
JobManager.TryRemoveJob(callback.JobID);
var packageIDs = callback.GrantedPackages;
var appIDs = callback.GrantedApps;
Log.WriteDebug(nameof(FreeLicense), "Received free license: {0} ({1} apps: {2}, {3} packages: {4})",
callback.Result, appIDs.Count, string.Join(", ", appIDs), packageIDs.Count, string.Join(", ", packageIDs));
if (appIDs.Count > 0)
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(appIDs, Enumerable.Empty <uint>()),
new PICSTokens.RequestedTokens
{
Apps = appIDs.ToList()
});
foreach (var appid in appIDs)
{
LocalConfig.Current.FreeLicensesToRequest.Remove(appid);
}
LocalConfig.Save();
}
if (packageIDs.Count > 0)
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), packageIDs),
new PICSTokens.RequestedTokens
{
Packages = packageIDs.ToList()
});
TaskManager.RunAsync(async() =>
{
await RefreshPackageNames();
foreach (var subID in packageIDs)
{
IRC.Instance.SendAnnounce("New free license granted: {0}{1}{2} -{3} {4}",
Colors.BLUE, Steam.GetPackageName(subID), Colors.NORMAL,
Colors.DARKBLUE, SteamDB.GetPackageUrl(subID)
);
}
});
}
}
private static void AddToQueue(uint subId, uint appId)
{
if (!Settings.IsFullRun && !FreeLicenseTimer.Enabled)
{
FreeLicenseTimer.Start();
}
if (LocalConfig.Current.FreeLicensesToRequest.ContainsKey(subId))
{
return;
}
LocalConfig.Current.FreeLicensesToRequest.Add(subId, appId);
LocalConfig.Save();
}
private static void OnTimer(object sender, ElapsedEventArgs e)
{
var list = LocalConfig.Current.FreeLicensesToRequest.Take(REQUEST_RATE_LIMIT).ToList();
var now = DateUtils.DateTimeToUnixTime(DateTime.UtcNow) - 60;
Dictionary <uint, ulong> startTimes;
using (var db = Database.Get())
{
startTimes = db.Query(
"SELECT `SubID`, `Value` FROM `SubsInfo` WHERE `Key` = @Key AND `SubID` IN @Ids",
new
{
Key = KeyNameCache.GetSubKeyID("extended_starttime"),
Ids = list.Select(x => x.Key)
}
).ToDictionary(x => (uint)x.SubID, x => Convert.ToUInt64((string)x.Value));
}
foreach (var(subId, _) in list)
{
if (startTimes.TryGetValue(subId, out var startTime) && startTime > now)
{
// If start time has not been reached yet, don't remove this app from the list and keep trying to activate it
continue;
}
LocalConfig.Current.FreeLicensesToRequest.Remove(subId);
}
LocalConfig.Save();
var appids = list.Select(x => x.Value).Distinct();
AppsRequestedInHour = appids.Count();
Log.WriteDebug(nameof(FreeLicense), $"Requesting {AppsRequestedInHour} free apps as the rate limit timer ran: {string.Join(", ", appids)}");
JobManager.AddJob(() => Steam.Instance.Apps.RequestFreeLicense(appids));
if (LocalConfig.Current.FreeLicensesToRequest.Count > 0)
{
FreeLicenseTimer.Start();
}
}
private static async void Tick(object sender, ElapsedEventArgs e)
{
DateTime lastPostDate;
await using (var db = await Database.GetConnectionAsync())
{
lastPostDate = db.ExecuteScalar <DateTime>("SELECT `Value` FROM `LocalConfig` WHERE `ConfigKey` = @Key", new { Key = "backend.lastrsspost" });
}
var tasks = Settings.Current.RssFeeds.Select(uri => ProcessFeed(uri, lastPostDate));
var dates = await Task.WhenAll(tasks);
var maxDate = dates.Max();
if (maxDate > lastPostDate)
{
await LocalConfig.Update("backend.lastrsspost", maxDate.ToString(CultureInfo.InvariantCulture));
}
}
public static void Main()
{
Console.Title = "Steam Database";
var version = FileVersionInfo.GetVersionInfo(typeof(Steam).Assembly.Location);
ProductVersion = version.ProductVersion;
Log.WriteInfo("Bootstrapper", "Steam Database, built from commit: {0}", ProductVersion);
Log.WriteInfo("Bootstrapper", "Copyright (c) 2013-2015, SteamDB. See LICENSE file for more information.");
try
{
// Just create deepest folder we will use in the app
string filesDir = Path.Combine(Application.Path, "files", ".support", "chunks");
Directory.CreateDirectory(filesDir);
Settings.Load();
LocalConfig.Load();
}
catch (Exception e)
{
Log.WriteError("Settings", "{0}", e.Message);
return;
}
ErrorReporter.Init(Settings.Current.BugsnagApiKey);
if (Settings.Current.SteamKitDebug)
{
DebugLog.AddListener(new Log.SteamKitLogger());
DebugLog.Enabled = true;
}
AppDomain.CurrentDomain.UnhandledException += OnSillyCrashHandler;
Console.CancelKeyPress += OnCancelKey;
Application.Init();
}
private static async Task RequestUpdateForList(List <uint> appIDs, List <uint> packageIDs)
{
Log.WriteInfo(nameof(FullUpdateProcessor), $"Requesting info for {appIDs.Count} apps and {packageIDs.Count} packages");
foreach (var list in appIDs.Split(200))
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(list, Enumerable.Empty <uint>()),
new PICSTokens.RequestedTokens
{
Apps = list.ToList()
});
do
{
await Task.Delay(100);
}while (IsBusy());
}
if (Settings.Current.FullRun == FullRunState.WithForcedDepots)
{
return;
}
foreach (var list in packageIDs.Split(1000))
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), list),
new PICSTokens.RequestedTokens
{
Packages = list.ToList()
});
do
{
await Task.Delay(100);
}while (IsBusy());
}
LocalConfig.Save();
}
public static void Cleanup()
{
Log.WriteInfo(nameof(Application), "Exiting...");
try
{
Steam.Instance.IsRunning = false;
Steam.Instance.Client.Disconnect();
Steam.Instance.Dispose();
}
catch (Exception e)
{
ErrorReporter.Notify(nameof(Application), e);
}
if (Settings.Current.BuiltInHttpServerPort > 0)
{
HttpServer.Dispose();
}
if (Settings.Current.IRC.Enabled)
{
Log.WriteInfo(nameof(Application), "Closing IRC connection...");
RssReader.Timer.Stop();
RssReader.Dispose();
IRC.Instance.Close();
IrcThread.Join(TimeSpan.FromSeconds(5));
}
TaskManager.CancelAllTasks();
if (!Settings.IsFullRun)
{
LocalConfig.Update("backend.changenumber", Steam.Instance.PICSChanges.PreviousChangeNumber.ToString())
.GetAwaiter().GetResult();
}
}
private static void QueueRequest(uint appid)
{
if (Settings.IsFullRun || AppsRequestedInHour++ >= REQUEST_RATE_LIMIT)
{
if (LocalConfig.FreeLicensesToRequest.Contains(appid))
{
return;
}
Log.WriteDebug("Free Packages", $"Adding app {appid} to queue as rate limit is reached");
LocalConfig.FreeLicensesToRequest.Add(appid);
LocalConfig.Save();
return;
}
FreeLicenseTimer.Stop();
FreeLicenseTimer.Start();
JobManager.AddJob(() => Steam.Instance.Apps.RequestFreeLicense(appid));
}
private static void OnTimer(object sender, ElapsedEventArgs e)
{
var list = LocalConfig.FreeLicensesToRequest.Take(REQUEST_RATE_LIMIT).ToList();
foreach (var appid in list)
{
LocalConfig.FreeLicensesToRequest.Remove(appid);
}
LocalConfig.Save();
AppsRequestedInHour = list.Count;
Log.WriteDebug("Free Packages", $"Requesting {AppsRequestedInHour} free apps as the rate limit timer ran");
JobManager.AddJob(() => Steam.Instance.Apps.RequestFreeLicense(list));
if (LocalConfig.FreeLicensesToRequest.Count > 0)
{
FreeLicenseTimer.Start();
}
}
private async Task DownloadDepots(uint appID, List <ManifestJob> depots)
{
Log.WriteDebug("Depot Downloader", "Will process {0} depots ({1} depot locks left)", depots.Count(), DepotLocks.Count);
var processTasks = new List <Task <EResult> >();
bool anyFilesDownloaded = false;
foreach (var depot in depots)
{
var instance = depot.Anonymous ? Steam.Anonymous.Apps : Steam.Instance.Apps;
depot.DepotKey = await GetDepotDecryptionKey(instance, depot.DepotID, appID);
if (depot.DepotKey == null)
{
RemoveLock(depot.DepotID);
continue;
}
var cdnToken = await GetCDNAuthToken(instance, appID, depot.DepotID);
if (cdnToken == null)
{
RemoveLock(depot.DepotID);
Log.WriteDebug("Depot Downloader", "Got a depot key for depot {0} but no cdn auth token", depot.DepotID);
continue;
}
depot.CDNToken = cdnToken.Token;
depot.Server = GetContentServer();
DepotManifest depotManifest = null;
string lastError = string.Empty;
for (var i = 0; i <= 5; i++)
{
try
{
depotManifest = await CDNClient.DownloadManifestAsync(depot.DepotID, depot.ManifestID, depot.Server, depot.CDNToken, depot.DepotKey);
break;
}
catch (Exception e)
{
lastError = e.Message;
Log.WriteError("Depot Processor", "Failed to download depot manifest for app {0} depot {1} ({2}: {3}) (#{4})", appID, depot.DepotID, depot.Server, lastError, i);
}
// TODO: get new auth key if auth fails
depot.Server = GetContentServer();
if (depotManifest == null)
{
await Task.Delay(Utils.ExponentionalBackoff(i));
}
}
if (depotManifest == null)
{
LocalConfig.CDNAuthTokens.TryRemove(depot.DepotID, out _);
RemoveLock(depot.DepotID);
if (FileDownloader.IsImportantDepot(depot.DepotID))
{
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download manifest ({3})",
Colors.OLIVE, depot.DepotName, Colors.NORMAL, lastError);
}
if (!Settings.IsFullRun)
{
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
}
continue;
}
var task = ProcessDepotAfterDownload(depot, depotManifest);
processTasks.Add(task);
if (!FileDownloader.IsImportantDepot(depot.DepotID))
{
continue;
}
task = TaskManager.Run(async() =>
{
var result = EResult.Fail;
try
{
result = await FileDownloader.DownloadFilesFromDepot(depot, depotManifest);
if (result == EResult.OK)
{
anyFilesDownloaded = true;
}
}
catch (Exception e)
{
ErrorReporter.Notify($"Depot Processor {depot.DepotID}", e);
}
return(result);
}).Unwrap();
processTasks.Add(task);
}
if (SaveLocalConfig)
{
SaveLocalConfig = false;
LocalConfig.Save();
}
await Task.WhenAll(processTasks).ConfigureAwait(false);
Log.WriteDebug("Depot Downloader", "{0} depot downloads finished", depots.Count());
// TODO: use ContinueWith on tasks
if (!anyFilesDownloaded)
{
foreach (var depot in depots)
{
RemoveLock(depot.DepotID);
}
return;
}
if (!File.Exists(UpdateScript))
{
return;
}
lock (UpdateScriptLock)
{
foreach (var depot in depots)
{
if (depot.Result == EResult.OK)
{
RunUpdateScript(string.Format("{0} no-git", depot.DepotID));
}
else if (depot.Result != EResult.Ignored)
{
Log.WriteWarn("Depot Processor", "Dropping stored token for {0} due to download failures", depot.DepotID);
LocalConfig.CDNAuthTokens.TryRemove(depot.DepotID, out _);
using (var db = Database.Get())
{
// Mark this depot for redownload
db.Execute("UPDATE `Depots` SET `LastManifestID` = 0 WHERE `DepotID` = @DepotID", new { depot.DepotID });
}
}
RemoveLock(depot.DepotID);
}
// Only commit changes if all depots downloaded
if (processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored))
{
if (!RunUpdateScript(appID, depots.First().BuildID))
{
RunUpdateScript("0");
}
}
else
{
Log.WriteDebug("Depot Processor", "Reprocessing the app {0} because some files failed to download", appID);
IRC.Instance.SendOps("{0}[{1}]{2} Reprocessing the app due to download failures",
Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL
);
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
}
}
}
private async Task DownloadDepots(uint appID, List <ManifestJob> depots)
{
Log.WriteDebug("Depot Downloader", "Will process {0} depots ({1} depot locks left)", depots.Count, DepotLocks.Count);
var processTasks = new List <Task <EResult> >();
var anyFilesDownloaded = false;
var willDownloadFiles = false;
foreach (var depot in depots)
{
if (depot.DepotKey == null)
{
await GetDepotDecryptionKey(Steam.Instance.Apps, depot, appID);
if (depot.DepotKey == null && depot.LastManifestID == depot.ManifestID)
{
RemoveLock(depot.DepotID);
continue;
}
}
depot.Server = GetContentServer();
DepotManifest depotManifest = null;
var lastError = string.Empty;
for (var i = 0; i <= 5; i++)
{
try
{
depotManifest = await CDNClient.DownloadManifestAsync(depot.DepotID, depot.ManifestID, depot.Server, string.Empty, depot.DepotKey);
break;
}
catch (Exception e)
{
lastError = e.Message;
Log.WriteError("Depot Processor", "Failed to download depot manifest for app {0} depot {1} ({2}: {3}) (#{4})", appID, depot.DepotID, depot.Server, lastError, i);
}
if (depot.DepotKey != null)
{
RemoveErroredServer(depot.Server);
}
depot.Server = GetContentServer();
if (depotManifest == null)
{
await Task.Delay(Utils.ExponentionalBackoff(i));
}
}
if (depotManifest == null)
{
RemoveLock(depot.DepotID);
if (FileDownloader.IsImportantDepot(depot.DepotID))
{
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download manifest ({3})",
Colors.OLIVE, depot.DepotName, Colors.NORMAL, lastError);
}
if (!Settings.IsFullRun && depot.DepotKey != null)
{
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
}
continue;
}
var task = ProcessDepotAfterDownload(depot, depotManifest);
processTasks.Add(task);
if (!FileDownloader.IsImportantDepot(depot.DepotID) || depot.DepotKey == null)
{
continue;
}
willDownloadFiles = true;
task = TaskManager.Run(async() =>
{
var result = EResult.Fail;
try
{
result = await FileDownloader.DownloadFilesFromDepot(depot, depotManifest);
if (result == EResult.OK)
{
anyFilesDownloaded = true;
}
}
catch (Exception e)
{
ErrorReporter.Notify($"Depot Processor {depot.DepotID}", e);
}
return(result);
}).Unwrap();
processTasks.Add(task);
}
if (SaveLocalConfig)
{
SaveLocalConfig = false;
LocalConfig.Save();
}
await Task.WhenAll(processTasks).ConfigureAwait(false);
Log.WriteDebug("Depot Downloader", $"{depots.Count} depot downloads finished for app {appID}");
// TODO: use ContinueWith on tasks
if (!anyFilesDownloaded && !willDownloadFiles)
{
foreach (var depot in depots)
{
RemoveLock(depot.DepotID);
}
return;
}
lock (UpdateScriptLock)
{
foreach (var depot in depots)
{
if (depot.Result == EResult.OK)
{
RunUpdateScript(UpdateScript, string.Format("{0} no-git", depot.DepotID));
}
else if (depot.Result != EResult.Ignored)
{
Log.WriteWarn("Depot Processor", $"Download failed for {depot.DepotID}");
// Mark this depot for redownload
var db = Database.Get();
db.Execute("UPDATE `Depots` SET `LastManifestID` = 0 WHERE `DepotID` = @DepotID", new { depot.DepotID });
}
RemoveLock(depot.DepotID);
}
// Only commit changes if all depots downloaded
if (processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored))
{
if (!RunUpdateScriptForApp(appID, depots[0].BuildID))
{
RunUpdateScript(UpdateScript, "0");
}
}
else
{
Log.WriteDebug("Depot Processor", "Reprocessing the app {0} because some files failed to download", appID);
IRC.Instance.SendOps("{0}[{1}]{2} Reprocessing the app due to download failures",
Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL
);
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
}
}
}
/*
* Here be dragons.
*/
public static async Task <EResult> DownloadFilesFromDepot(DepotProcessor.ManifestJob job, DepotManifest depotManifest)
{
var filesRegex = Files[job.DepotID];
var files = depotManifest.Files.Where(x => filesRegex.IsMatch(x.FileName.Replace('\\', '/'))).ToList();
var downloadState = EResult.Fail;
ConcurrentDictionary <string, ExistingFileData> existingFileData;
await using (var db = await Database.GetConnectionAsync())
{
var data = db.ExecuteScalar <string>("SELECT `Value` FROM `LocalConfig` WHERE `ConfigKey` = @Key", new { Key = $"depot.{job.DepotID}" });
if (data != null)
{
existingFileData = JsonConvert.DeserializeObject <ConcurrentDictionary <string, ExistingFileData> >(data);
}
else
{
existingFileData = new ConcurrentDictionary <string, ExistingFileData>();
}
}
foreach (var file in existingFileData.Keys.Except(files.Select(x => x.FileName)))
{
Log.WriteWarn(nameof(FileDownloader), $"\"{file}\" no longer exists in manifest");
}
Log.WriteInfo($"FileDownloader {job.DepotID}", $"Will download {files.Count} files");
var downloadedFiles = 0;
var fileTasks = new Task[files.Count];
for (var i = 0; i < fileTasks.Length; i++)
{
var file = files[i];
fileTasks[i] = TaskManager.Run(async() =>
{
var existingFile = existingFileData.GetOrAdd(file.FileName, _ => new ExistingFileData());
EResult fileState;
try
{
await ChunkDownloadingSemaphore.WaitAsync().ConfigureAwait(false);
fileState = await DownloadFile(job, file, existingFile);
}
finally
{
ChunkDownloadingSemaphore.Release();
}
if (fileState == EResult.OK || fileState == EResult.SameAsPreviousValue)
{
existingFile.FileHash = file.FileHash;
downloadedFiles++;
}
if (fileState != EResult.SameAsPreviousValue)
{
// Do not write progress info to log file
Console.WriteLine($"{job.DepotName} [{downloadedFiles / (float) files.Count * 100.0f,6:#00.00}%] {files.Count - downloadedFiles} files left to download");
}
if (downloadState == EResult.DataCorruption)
{
return;
}
if (fileState == EResult.OK || fileState == EResult.DataCorruption)
{
downloadState = fileState;
}
});
}
await Task.WhenAll(fileTasks).ConfigureAwait(false);
await LocalConfig.Update($"depot.{job.DepotID}", JsonConvert.SerializeObject(existingFileData));
job.Result = downloadState switch
{
EResult.OK => EResult.OK,
EResult.DataCorruption => EResult.DataCorruption,
_ => EResult.Ignored
};
return(job.Result);
}
private Task SaveBetas()
{
return(LocalConfig.Update("backend.beta.requests", JsonConvert.SerializeObject(BetasToRequest)));
}
private void OnLoggedOn(SteamUser.LoggedOnCallback callback)
{
GameCoordinator.UpdateStatus(0, callback.Result.ToString());
if (callback.Result == EResult.AccountLogonDenied)
{
Console.Write("STEAM GUARD! Please enter the auth code sent to the email at {0}: ", callback.EmailDomain);
AuthCode = Console.ReadLine();
if (AuthCode != null)
{
AuthCode = AuthCode.Trim();
}
return;
}
if (callback.Result != EResult.OK)
{
Log.WriteInfo("Steam", "Failed to login: {0}", callback.Result);
IRC.Instance.SendEmoteAnnounce("failed to log in: {0}", callback.Result);
return;
}
var cellId = callback.CellID;
if (LocalConfig.CellID != cellId)
{
Log.WriteDebug("Local Config", "CellID differs, {0} != {1}, forcing server refetch", LocalConfig.CellID, cellId);
LocalConfig.CellID = cellId;
// TODO: is this really needed?
LocalConfig.LoadServers();
LocalConfig.Save();
}
LastSuccessfulLogin = DateTime.Now;
Log.WriteInfo("Steam", "Logged in, current Valve time is {0}", callback.ServerTime.ToString("R"));
IRC.Instance.SendEmoteAnnounce("logged in. Valve time: {0}", callback.ServerTime.ToString("R"));
if (Settings.IsFullRun)
{
if (Settings.Current.FullRun == FullRunState.ImportantOnly)
{
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(Application.ImportantApps.Keys, Enumerable.Empty <uint>()));
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetProductInfo(Enumerable.Empty <SteamApps.PICSRequest>(), Application.ImportantSubs.Keys.Select(package => Utils.NewPICSRequest(package))));
}
else if (Steam.Instance.PICSChanges.PreviousChangeNumber == 1)
{
Steam.Instance.PICSChanges.PerformSync();
//JobManager.AddJob(() => Steam.Instance.Apps.PICSGetChangesSince(1, true, true));
}
}
else
{
JobManager.RestartJobsIfAny();
Application.ChangelistTimer.Start();
}
}
private Task Save()
{
return(LocalConfig.Update("backend.freelicense.requests", JsonConvert.SerializeObject(FreeLicensesToRequest)));
}
private void OnPICSChanges(SteamApps.PICSChangesCallback callback)
{
if (PreviousChangeNumber == callback.CurrentChangeNumber)
{
return;
}
Log.WriteInfo(nameof(PICSChanges), $"Changelist {callback.LastChangeNumber} -> {callback.CurrentChangeNumber} ({callback.AppChanges.Count} apps, {callback.PackageChanges.Count} packages)");
PreviousChangeNumber = callback.CurrentChangeNumber;
TaskManager.Run(async() => await HandleChangeNumbers(callback));
if (callback.RequiresFullAppUpdate || callback.RequiresFullPackageUpdate)
{
TaskManager.Run(async() =>
{
if (callback.RequiresFullAppUpdate)
{
IRC.Instance.SendOps($"Changelist {callback.CurrentChangeNumber} has forced a full app update");
// When full update flag is set, presumably Steam client start hammering the servers
// and the PICS service just does not return any data for a while until it clears up
await FullUpdateProcessor.FullUpdateAppsMetadata(true);
}
if (callback.RequiresFullPackageUpdate)
{
IRC.Instance.SendOps($"Changelist {callback.CurrentChangeNumber} has forced a full package update");
await FullUpdateProcessor.FullUpdatePackagesMetadata();
}
IRC.Instance.SendOps($"Changelist {callback.CurrentChangeNumber} full update has finished");
});
}
if (callback.AppChanges.Count == 0 && callback.PackageChanges.Count == 0)
{
return;
}
const int appsPerJob = 50;
if (callback.AppChanges.Count > appsPerJob)
{
foreach (var list in callback.AppChanges.Keys.Split(appsPerJob))
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(list, Enumerable.Empty <uint>()),
new PICSTokens.RequestedTokens
{
Apps = list.ToList()
});
}
}
else if (callback.AppChanges.Count > 0)
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(callback.AppChanges.Keys, Enumerable.Empty <uint>()),
new PICSTokens.RequestedTokens
{
Apps = callback.AppChanges.Keys.ToList()
});
}
if (callback.PackageChanges.Count > appsPerJob)
{
foreach (var list in callback.PackageChanges.Keys.Split(appsPerJob))
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), list),
new PICSTokens.RequestedTokens
{
Packages = list.ToList()
});
}
}
else if (callback.PackageChanges.Count > 0)
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), callback.PackageChanges.Keys),
new PICSTokens.RequestedTokens
{
Packages = callback.PackageChanges.Keys.ToList()
});
}
if (callback.AppChanges.Count > 0)
{
_ = TaskManager.Run(async() => await HandleApps(callback));
}
if (callback.PackageChanges.Count > 0)
{
_ = TaskManager.Run(async() => await HandlePackages(callback));
_ = TaskManager.Run(async() => await HandlePackagesChangelists(callback));
}
if (PreviousChangeNumber - LastStoredChangeNumber >= 1000)
{
LastStoredChangeNumber = PreviousChangeNumber;
_ = TaskManager.Run(async() => await LocalConfig.Update("backend.changenumber", LastStoredChangeNumber.ToString()));
}
PrintImportants(callback);
}
private async Task DownloadDepots(uint appID, List <ManifestJob> depots)
{
Log.WriteDebug("Depot Downloader", "Will process {0} depots ({1} depot locks left)", depots.Count(), DepotLocks.Count);
var processTasks = new List <Task <EResult> >();
bool anyFilesDownloaded = false;
foreach (var depot in depots)
{
var instance = depot.Anonymous ? Steam.Anonymous.Apps : Steam.Instance.Apps;
depot.DepotKey = await GetDepotDecryptionKey(instance, depot.DepotID, appID);
if (depot.DepotKey == null)
{
RemoveLock(depot.DepotID);
continue;
}
var cdnToken = await GetCDNAuthToken(instance, appID, depot.DepotID);
if (cdnToken == null)
{
RemoveLock(depot.DepotID);
Log.WriteDebug("Depot Downloader", "Got a depot key for depot {0} but no cdn auth token", depot.DepotID);
continue;
}
depot.CDNToken = cdnToken.Token;
depot.Server = cdnToken.Server;
DepotManifest depotManifest = null;
string lastError = string.Empty;
for (var i = 0; i <= 5; i++)
{
try
{
depotManifest = CDNClient.DownloadManifest(depot.DepotID, depot.ManifestID, depot.Server, depot.CDNToken, depot.DepotKey);
break;
}
catch (Exception e)
{
Log.WriteWarn("Depot Downloader", "[{0}] Manifest download failed: {1} - {2}", depot.DepotID, e.GetType(), e.Message);
lastError = e.Message;
}
// TODO: get new auth key if auth fails
if (depotManifest == null)
{
await Task.Delay(Utils.ExponentionalBackoff(i));
}
}
if (depotManifest == null)
{
RemoveLock(depot.DepotID);
Log.WriteError("Depot Processor", "Failed to download depot manifest for app {0} depot {1} ({2}: {3})", appID, depot.DepotID, depot.Server, lastError);
if (FileDownloader.IsImportantDepot(depot.DepotID))
{
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download app {3} depot {4} manifest ({5}: {6})",
Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL, appID, depot.DepotID, depot.Server, lastError);
}
continue;
}
var task = TaskManager.Run(() =>
{
using (var db = Database.GetConnection())
{
using (var transaction = db.BeginTransaction())
{
var result = ProcessDepotAfterDownload(db, depot, depotManifest);
transaction.Commit();
return(result);
}
}
});
processTasks.Add(task);
if (FileDownloader.IsImportantDepot(depot.DepotID))
{
task = TaskManager.Run(() =>
{
var result = FileDownloader.DownloadFilesFromDepot(appID, depot, depotManifest);
if (result == EResult.OK)
{
anyFilesDownloaded = true;
}
return(result);
}, TaskCreationOptions.LongRunning);
TaskManager.RegisterErrorHandler(task);
processTasks.Add(task);
}
}
if (SaveLocalConfig)
{
SaveLocalConfig = false;
LocalConfig.Save();
}
await Task.WhenAll(processTasks);
Log.WriteDebug("Depot Downloader", "{0} depot downloads finished", depots.Count());
// TODO: use ContinueWith on tasks
if (!anyFilesDownloaded)
{
foreach (var depot in depots)
{
RemoveLock(depot.DepotID);
}
return;
}
if (!File.Exists(UpdateScript))
{
return;
}
bool lockTaken = false;
try
{
UpdateScriptLock.Enter(ref lockTaken);
foreach (var depot in depots)
{
if (depot.Result == EResult.OK)
{
RunUpdateScript(string.Format("{0} no-git", depot.DepotID));
}
RemoveLock(depot.DepotID);
}
// Only commit changes if all depots downloaded
if (processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored))
{
if (!RunUpdateScript(appID))
{
RunUpdateScript("0");
}
}
else
{
Log.WriteDebug("Depot Processor", "Reprocessing the app {0} because some files failed to download", appID);
IRC.Instance.SendOps("{0}[{1}]{2} Reprocessing the app due to download failures",
Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL
);
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
}
}
finally
{
if (lockTaken)
{
UpdateScriptLock.Exit();
}
}
}
private async Task DownloadDepots(IEnumerable <ManifestJob> depots)
{
Log.WriteDebug("Depot Downloader", "Will process {0} depots ({1} depot locks left)", depots.Count(), DepotLocks.Count);
var processTasks = new List <Task <EResult> >();
bool anyFilesDownloaded = false;
foreach (var depot in depots)
{
depot.DepotKey = await GetDepotDecryptionKey(depot.DepotID, depot.ParentAppID);
if (depot.DepotKey == null)
{
RemoveLock(depot.DepotID);
continue;
}
var cdnToken = await GetCDNAuthToken(depot.DepotID);
if (cdnToken == null)
{
RemoveLock(depot.DepotID);
Log.WriteDebug("Depot Downloader", "Got a depot key for depot {0} but no cdn auth token", depot.DepotID);
continue;
}
depot.CDNToken = cdnToken.Token;
depot.Server = cdnToken.Server;
DepotManifest depotManifest = null;
string lastError = string.Empty;
for (var i = 0; i <= 5; i++)
{
try
{
depotManifest = CDNClient.DownloadManifest(depot.DepotID, depot.ManifestID, depot.Server, depot.CDNToken, depot.DepotKey);
break;
}
catch (Exception e)
{
Log.WriteWarn("Depot Downloader", "{0} Manifest download failed: {1} - {2}", depot.DepotID, e.GetType(), e.Message);
lastError = e.Message;
}
// TODO: get new auth key if auth fails
if (depotManifest == null)
{
await Task.Delay(Utils.ExponentionalBackoff(i));
}
}
if (depotManifest == null)
{
RemoveLock(depot.DepotID);
Log.WriteError("Depot Processor", "Failed to download depot manifest for depot {0} ({1}: {2})", depot.DepotID, depot.Server, lastError);
if (FileDownloader.IsImportantDepot(depot.DepotID))
{
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download depot {3} manifest ({4}: {5})",
Colors.OLIVE, Steam.GetAppName(depot.ParentAppID), Colors.NORMAL, depot.DepotID, depot.Server, lastError);
}
continue;
}
var task = TaskManager.Run(() =>
{
using (var db = Database.GetConnection())
{
return(ProcessDepotAfterDownload(db, depot, depotManifest));
}
});
processTasks.Add(task);
if (FileDownloader.IsImportantDepot(depot.DepotID))
{
task = TaskManager.Run(() =>
{
var result = FileDownloader.DownloadFilesFromDepot(depot, depotManifest);
if (result == EResult.OK)
{
anyFilesDownloaded = true;
}
return(result);
}, TaskCreationOptions.LongRunning);
TaskManager.RegisterErrorHandler(task);
processTasks.Add(task);
}
}
if (SaveLocalConfig)
{
SaveLocalConfig = false;
LocalConfig.Save();
}
await Task.WhenAll(processTasks);
// TODO: use ContinueWith on tasks
if (!anyFilesDownloaded)
{
Log.WriteDebug("Depot Downloader", "Tasks awaited for {0} depot downloads", depots.Count());
foreach (var depot in depots)
{
RemoveLock(depot.DepotID);
}
return;
}
var canUpdate = processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored) && File.Exists(UpdateScript);
#if true
Log.WriteDebug("Depot Downloader", "Tasks awaited for {0} depot downloads (will run script: {1})", depots.Count(), canUpdate);
#endif
bool lockTaken = false;
try
{
UpdateScriptLock.Enter(ref lockTaken);
foreach (var depot in depots)
{
// TODO: this only needs to run if any downloaded files changed
if (canUpdate && FileDownloader.IsImportantDepot(depot.DepotID))
{
RunUpdateScript(string.Format("{0} no-git", depot.DepotID));
}
RemoveLock(depot.DepotID);
}
if (canUpdate)
{
RunUpdateScript("0");
}
}
finally
{
if (lockTaken)
{
UpdateScriptLock.Exit();
}
}
}
