private async Task DownloadDepots(uint appID, List <ManifestJob> depots)
{
Log.WriteDebug("Depot Downloader", "Will process {0} depots ({1} depot locks left)", depots.Count(), DepotLocks.Count);
var processTasks = new List <Task <EResult> >();
bool anyFilesDownloaded = false;
foreach (var depot in depots)
{
depot.DepotKey = await GetDepotDecryptionKey(depot.DepotID, appID);
if (depot.DepotKey == null)
{
RemoveLock(depot.DepotID);
continue;
}
var cdnToken = await GetCDNAuthToken(depot.DepotID);
if (cdnToken == null)
{
RemoveLock(depot.DepotID);
Log.WriteDebug("Depot Downloader", "Got a depot key for depot {0} but no cdn auth token", depot.DepotID);
continue;
}
depot.CDNToken = cdnToken.Token;
depot.Server = cdnToken.Server;
DepotManifest depotManifest = null;
string lastError = string.Empty;
for (var i = 0; i <= 5; i++)
{
try
{
depotManifest = CDNClient.DownloadManifest(depot.DepotID, depot.ManifestID, depot.Server, depot.CDNToken, depot.DepotKey);
break;
}
catch (Exception e)
{
Log.WriteWarn("Depot Downloader", "{0} Manifest download failed: {1} - {2}", depot.DepotID, e.GetType(), e.Message);
lastError = e.Message;
}
// TODO: get new auth key if auth fails
if (depotManifest == null)
{
await Task.Delay(Utils.ExponentionalBackoff(i));
}
}
if (depotManifest == null)
{
RemoveLock(depot.DepotID);
Log.WriteError("Depot Processor", "Failed to download depot manifest for depot {0} ({1}: {2})", depot.DepotID, depot.Server, lastError);
if (FileDownloader.IsImportantDepot(depot.DepotID))
{
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download depot {3} manifest ({4}: {5})",
Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL, depot.DepotID, depot.Server, lastError);
}
continue;
}
var task = TaskManager.Run(() =>
{
using (var db = Database.GetConnection())
{
using (var transaction = db.BeginTransaction())
{
var result = ProcessDepotAfterDownload(db, depot, depotManifest);
transaction.Commit();
return(result);
}
}
});
processTasks.Add(task);
if (FileDownloader.IsImportantDepot(depot.DepotID))
{
task = TaskManager.Run(() =>
{
var result = FileDownloader.DownloadFilesFromDepot(appID, depot, depotManifest);
if (result == EResult.OK)
{
anyFilesDownloaded = true;
}
return(result);
}, TaskCreationOptions.LongRunning);
TaskManager.RegisterErrorHandler(task);
processTasks.Add(task);
}
}
if (SaveLocalConfig)
{
SaveLocalConfig = false;
LocalConfig.Save();
}
await Task.WhenAll(processTasks);
Log.WriteDebug("Depot Downloader", "{0} depot downloads finished", depots.Count());
// TODO: use ContinueWith on tasks
if (!anyFilesDownloaded)
{
foreach (var depot in depots)
{
RemoveLock(depot.DepotID);
}
return;
}
if (!File.Exists(UpdateScript))
{
return;
}
bool lockTaken = false;
try
{
UpdateScriptLock.Enter(ref lockTaken);
foreach (var depot in depots)
{
if (depot.Result == EResult.OK)
{
RunUpdateScript(string.Format("{0} no-git", depot.DepotID));
}
RemoveLock(depot.DepotID);
}
// Only commit changes if all depots downloaded
if (processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored))
{
RunUpdateScript("0");
}
else
{
Log.WriteDebug("Depot Processor", "Reprocessing the app {0} because some files failed to download", appID);
IRC.Instance.SendOps("{0}[{1}]{2} Reprocessing the app because some files in {3} depots failed",
Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL,
depots.Count(x => x.Result != EResult.OK && x.Result != EResult.Ignored)
);
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
}
}
finally
{
if (lockTaken)
{
UpdateScriptLock.Exit();
}
}
}
public override async Task OnCommand(CommandArguments command)
{
if (command.CommandType != ECommandType.IRC || !IRC.IsRecipientChannel(command.Recipient))
{
command.Reply("This command is only available in channels.");
return;
}
var channel = command.Recipient;
var s = command.Message.Split(' ');
var count = s.Length;
if (count > 0)
{
uint id;
switch (s[0])
{
case "reload":
Application.ReloadImportant(command);
PICSTokens.Reload(command);
FileDownloader.ReloadFileList();
return;
case "add":
if (count < 3)
{
break;
}
if (!uint.TryParse(s[2], out id))
{
break;
}
switch (s[1])
{
case "app":
var exists = Application.ImportantApps.TryGetValue(id, out var channels);
if (exists && channels.Contains(channel))
{
command.Reply("App {0}{1}{2} ({3}) is already important in {4}{5}{6}.", Colors.BLUE, id, Colors.NORMAL, Steam.GetAppName(id), Colors.BLUE, channel, Colors.NORMAL);
}
else
{
if (exists)
{
Application.ImportantApps[id].Add(channel);
}
else
{
Application.ImportantApps.Add(id, new List <string> {
channel
});
}
using (var db = Database.Get())
{
await db.ExecuteAsync("INSERT INTO `ImportantApps` (`AppID`, `Channel`) VALUES (@AppID, @Channel)", new { AppID = id, Channel = channel });
}
command.Reply("Marked app {0}{1}{2} ({3}) as important in {4}{5}{6}.", Colors.BLUE, id, Colors.NORMAL, Steam.GetAppName(id), Colors.BLUE, channel, Colors.NORMAL);
}
return;
case "sub":
if (Application.ImportantSubs.ContainsKey(id))
{
command.Reply("Package {0}{1}{2} ({3}) is already important.", Colors.BLUE, id, Colors.NORMAL, Steam.GetPackageName(id));
}
else
{
Application.ImportantSubs.Add(id, 1);
using (var db = Database.Get())
{
await db.ExecuteAsync("INSERT INTO `ImportantSubs` (`SubID`) VALUES (@SubID)", new { SubID = id });
}
command.Reply("Marked package {0}{1}{2} ({3}) as important.", Colors.BLUE, id, Colors.NORMAL, Steam.GetPackageName(id));
}
return;
}
break;
case "remove":
if (count < 3)
{
break;
}
if (!uint.TryParse(s[2], out id))
{
break;
}
switch (s[1])
{
case "app":
if (!Application.ImportantApps.TryGetValue(id, out var channels) || !channels.Contains(channel))
{
command.Reply("App {0}{1}{2} ({3}) is not important in {4}{5}{6}.", Colors.BLUE, id, Colors.NORMAL, Steam.GetAppName(id), Colors.BLUE, channel, Colors.NORMAL);
}
else
{
if (channels.Count > 1)
{
Application.ImportantApps[id].Remove(channel);
}
else
{
Application.ImportantApps.Remove(id);
}
using (var db = Database.Get())
{
db.Execute("DELETE FROM `ImportantApps` WHERE `AppID` = @AppID AND `Channel` = @Channel", new { AppID = id, Channel = channel });
}
command.Reply("Removed app {0}{1}{2} ({3}) from the important list in {4}{5}{6}.", Colors.BLUE, id, Colors.NORMAL, Steam.GetAppName(id), Colors.BLUE, channel, Colors.NORMAL);
}
return;
case "sub":
if (!Application.ImportantSubs.ContainsKey(id))
{
command.Reply("Package {0}{1}{2} ({3}) is not important.", Colors.BLUE, id, Colors.NORMAL, Steam.GetPackageName(id));
}
else
{
Application.ImportantSubs.Remove(id);
using (var db = Database.Get())
{
await db.ExecuteAsync("DELETE FROM `ImportantSubs` WHERE `SubID` = @SubID", new { SubID = id });
}
command.Reply("Removed package {0}{1}{2} ({3}) from the important list.", Colors.BLUE, id, Colors.NORMAL, Steam.GetPackageName(id));
}
return;
}
break;
}
}
command.Reply("Usage:{0} important reload {1}or{2} important <add/remove> <app/sub> <id>", Colors.OLIVE, Colors.NORMAL, Colors.OLIVE);
}
private async Task DownloadDepots(uint appID, List <ManifestJob> depots)
{
Log.WriteDebug("Depot Downloader", "Will process {0} depots ({1} depot locks left)", depots.Count, DepotLocks.Count);
var processTasks = new List <Task <EResult> >();
var anyFilesDownloaded = false;
var willDownloadFiles = false;
foreach (var depot in depots)
{
if (depot.DepotKey == null)
{
await GetDepotDecryptionKey(Steam.Instance.Apps, depot, appID);
if (depot.DepotKey == null)
{
RemoveLock(depot.DepotID);
continue;
}
}
depot.Server = GetContentServer();
DepotManifest depotManifest = null;
var lastError = string.Empty;
for (var i = 0; i <= 5; i++)
{
try
{
depotManifest = await CDNClient.DownloadManifestAsync(depot.DepotID, depot.ManifestID, depot.Server, string.Empty, depot.DepotKey);
break;
}
catch (Exception e)
{
lastError = e.Message;
Log.WriteError("Depot Processor", "Failed to download depot manifest for app {0} depot {1} ({2}: {3}) (#{4})", appID, depot.DepotID, depot.Server, lastError, i);
}
depot.Server = GetContentServer();
if (depotManifest == null)
{
await Task.Delay(Utils.ExponentionalBackoff(i));
}
}
if (depotManifest == null)
{
RemoveLock(depot.DepotID);
if (FileDownloader.IsImportantDepot(depot.DepotID))
{
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download manifest ({3})",
Colors.OLIVE, depot.DepotName, Colors.NORMAL, lastError);
}
if (!Settings.IsFullRun)
{
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
}
continue;
}
var task = ProcessDepotAfterDownload(depot, depotManifest);
processTasks.Add(task);
if (!FileDownloader.IsImportantDepot(depot.DepotID))
{
continue;
}
willDownloadFiles = true;
task = TaskManager.Run(async() =>
{
var result = EResult.Fail;
try
{
result = await FileDownloader.DownloadFilesFromDepot(depot, depotManifest);
if (result == EResult.OK)
{
anyFilesDownloaded = true;
}
}
catch (Exception e)
{
ErrorReporter.Notify($"Depot Processor {depot.DepotID}", e);
}
return(result);
}).Unwrap();
processTasks.Add(task);
}
if (SaveLocalConfig)
{
SaveLocalConfig = false;
LocalConfig.Save();
}
await Task.WhenAll(processTasks).ConfigureAwait(false);
Log.WriteDebug("Depot Downloader", "{0} depot downloads finished", depots.Count);
// TODO: use ContinueWith on tasks
if (!anyFilesDownloaded && !willDownloadFiles)
{
foreach (var depot in depots)
{
RemoveLock(depot.DepotID);
}
return;
}
lock (UpdateScriptLock)
{
foreach (var depot in depots)
{
if (depot.Result == EResult.OK)
{
RunUpdateScript(UpdateScript, string.Format("{0} no-git", depot.DepotID));
}
else if (depot.Result != EResult.Ignored)
{
Log.WriteWarn("Depot Processor", $"Download failed for {depot.DepotID}");
using (var db = Database.Get())
{
// Mark this depot for redownload
db.Execute("UPDATE `Depots` SET `LastManifestID` = 0 WHERE `DepotID` = @DepotID", new { depot.DepotID });
}
}
RemoveLock(depot.DepotID);
}
// Only commit changes if all depots downloaded
if (processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored))
{
if (!RunUpdateScriptForApp(appID, depots[0].BuildID))
{
RunUpdateScript(UpdateScript, "0");
}
}
else
{
Log.WriteDebug("Depot Processor", "Reprocessing the app {0} because some files failed to download", appID);
IRC.Instance.SendOps("{0}[{1}]{2} Reprocessing the app due to download failures",
Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL
);
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
}
}
}