private static void OnSillyCrashHandler(object sender, UnhandledExceptionEventArgs args)
{
var parentException = args.ExceptionObject as Exception;
if (parentException is AggregateException aggregateException)
{
aggregateException.Flatten().Handle(e =>
{
ErrorReporter.Notify("Bootstrapper", e);
return(false);
});
}
else
{
ErrorReporter.Notify("Bootstrapper", parentException);
}
if (args.IsTerminating)
{
AppDomain.CurrentDomain.UnhandledException -= OnSillyCrashHandler;
IRC.Instance.SendOps("🥀 Backend process has crashed.");
Application.Cleanup();
}
}
public async Task Process()
{
Log.WriteInfo(ToString(), "Process");
try
{
DbConnection = await Database.GetConnectionAsync().ConfigureAwait(false);
await LoadData().ConfigureAwait(false);
if (ProductInfo == null)
{
await ProcessUnknown().ConfigureAwait(false);
}
else
{
await ProcessData().ConfigureAwait(false);
}
}
catch (MySqlException e)
{
ErrorReporter.Notify(ToString(), e);
JobManager.AddJob(() => RefreshSteam());
}
#if DEBUG
Log.WriteDebug(ToString(), "Processed");
#endif
}
public void Connect()
{
try
{
IrcClientConnectionOptions options = null;
if (Settings.Current.IRC.Ssl)
{
options = new IrcClientConnectionOptions
{
Ssl = true,
SslHostname = Settings.Current.IRC.Server
};
if (Settings.Current.IRC.SslAcceptInvalid)
{
options.SslCertificateValidationCallback = (sender, certificate, chain, policyErrors) => true;
}
}
Client.Connect(Settings.Current.IRC.Server, Settings.Current.IRC.Port, options);
}
catch (Exception e)
{
ErrorReporter.Notify("IRC", e);
}
}
private async void OnServerList(SteamClient.ServerListCallback callback)
{
IList <CDNClient.Server> serverList;
try
{
serverList = await CDNClient.FetchServerListAsync(maxServers : 60);
}
catch (Exception e)
{
ErrorReporter.Notify("Depot Downloader", e);
return;
}
CDNServers = new List <string>();
foreach (var server in serverList)
{
if (server.Type == "CDN" && (server.Host.StartsWith("valve") || server.Host.StartsWith("cache")))
{
Log.WriteDebug("Depot Processor", "Adding {0} to CDN server list", server.Host);
CDNServers.Add(server.Host);
}
else
{
Log.WriteDebug("Depot Processor", "Skipping {0}, not added to CDN server list", server.Host);
}
}
Log.WriteDebug("Depot Processor", "Finished adding CDN servers, added {0} out of {1}", CDNServers.Count, serverList.Count);
}
public static void Cleanup()
{
Log.WriteInfo(nameof(Application), "Exiting...");
try
{
Steam.Instance.IsRunning = false;
Steam.Instance.Client.Disconnect();
Steam.Instance.Dispose();
}
catch (Exception e)
{
ErrorReporter.Notify(nameof(Application), e);
}
if (Settings.Current.IRC.Enabled)
{
Log.WriteInfo(nameof(Application), "Closing IRC connection...");
RssReader.Timer.Stop();
RssReader.Dispose();
IRC.Instance.Close();
IrcThread.Join(TimeSpan.FromSeconds(5));
}
TaskManager.CancelAllTasks();
if (!Settings.IsFullRun)
{
LocalConfig.Update("backend.changenumber", Steam.Instance.PICSChanges.PreviousChangeNumber.ToString())
.GetAwaiter().GetResult();
}
}
private async void OnServerList(SteamClient.ServerListCallback callback)
{
IList <CDNClient.Server> serverList;
try
{
serverList = await CDNClient.FetchServerListAsync(maxServers : 30);
}
catch (Exception e)
{
if (CDNServers.Count == 0)
{
ErrorReporter.Notify("Depot Downloader", e);
}
return;
}
CDNServers = new List <string>();
foreach (var server in serverList)
{
// akamai.cdn.steampipe.steamcontent.com returns 404 Not Found unnecessarily
if (server.Type == "CDN" && !server.Host.Contains("cdn."))
{
CDNServers.Add(server.Host);
}
}
}
public static void RegisterErrorHandler(Task t)
{
Tasks.TryAdd(t, 1);
#if DEBUG
Log.WriteDebug("Task Manager", "New task: {0}", t);
#endif
t.ContinueWith(task =>
{
if (Tasks.TryRemove(task, out _))
{
#if DEBUG
Log.WriteDebug("Task Manager", "Removed task: {0} ({1} jobs left)", t, TasksCount);
#endif
}
});
t.ContinueWith(task =>
{
task.Exception.Flatten().Handle(e =>
{
ErrorReporter.Notify("Task Manager", e);
return(false);
});
}, TaskContinuationOptions.OnlyOnFaulted);
}
private static void RemoveProcessorLock(Task <BaseProcessor> task)
{
var processor = task.Result;
lock (CurrentlyProcessing)
{
#if false
if (CurrentlyProcessing[processor.Id]?.Id == task.Id)
{
CurrentlyProcessing.Remove(processor.Id);
}
#endif
if (CurrentlyProcessing.TryGetValue(processor.Id, out var mostRecentItem))
{
if (mostRecentItem.IsCompleted)
{
CurrentlyProcessing.Remove(processor.Id);
Log.WriteDebug(processor.ToString(), $"Removed completed lock ({CurrentlyProcessing.Count})");
}
else if (mostRecentItem.Id == task.Id)
{
ErrorReporter.Notify(processor.ToString(), new System.Exception($"Matching worker task ids, but task is not completed? @xPaw"));
}
}
}
}
private async void OnServerList(SteamClient.ServerListCallback callback)
{
IList <CDNClient.Server> serverList;
try
{
serverList = await CDNClient.FetchServerListAsync(maxServers : 60);
}
catch (Exception e)
{
if (CDNServers.Count == 0)
{
ErrorReporter.Notify("Depot Downloader", e);
}
return;
}
CDNServers = new List <string>();
foreach (var server in serverList)
{
if (server.Type == "CDN" && server.Host.StartsWith("valve"))
{
CDNServers.Add(server.Host);
}
}
}
private static void TryCommand(Command command, CommandArguments commandData)
{
#if false
if (commandData.CommandType == ECommandType.IRC && IRC.IsRecipientChannel(commandData.Recipient))
{
if (DateTime.Now.Subtract(LastCommandUseTime).TotalSeconds < 60)
{
commandData.ReplyAsNotice = ++LastCommandUseCount > 3;
}
else
{
LastCommandUseCount = 1;
}
LastCommandUseTime = DateTime.Now;
}
#endif
try
{
command.OnCommand(commandData);
}
catch (Exception e)
{
Log.WriteError("CommandHandler", "Exception while executing a command: {0}\n{1}", e.Message, e.StackTrace);
ReplyToCommand(commandData, "Exception: {0}", e.Message);
ErrorReporter.Notify(e);
}
}
public static void Cleanup()
{
Log.WriteInfo(nameof(Application), "Exiting...");
try
{
Steam.Instance.IsRunning = false;
Steam.Instance.Client.Disconnect();
Steam.Instance.Dispose();
}
catch (Exception e)
{
ErrorReporter.Notify(nameof(Application), e);
}
if (Settings.Current.IRC.Enabled)
{
Log.WriteInfo(nameof(Application), "Closing IRC connection...");
RssReader.Timer.Stop();
RssReader.Dispose();
IRC.Instance.Close();
IrcThread.Join(TimeSpan.FromSeconds(5));
}
TaskManager.CancelAllTasks();
LocalConfig.Save();
}
private void OnMachineAuth(SteamUser.UpdateMachineAuthCallback callback)
{
Log.WriteInfo("Steam", "Updating sentry file...");
byte[] sentryHash = CryptoHelper.SHAHash(callback.Data);
if (callback.Data.Length != callback.BytesToWrite)
{
ErrorReporter.Notify(new InvalidDataException(string.Format("Data.Length ({0}) != BytesToWrite ({1}) in OnMachineAuth", callback.Data.Length, callback.BytesToWrite)));
}
using (var file = File.OpenWrite(SentryFile))
{
file.Seek(callback.Offset, SeekOrigin.Begin);
file.Write(callback.Data, 0, callback.BytesToWrite);
}
Steam.Instance.User.SendMachineAuthResponse(new SteamUser.MachineAuthDetails
{
JobID = callback.JobID,
FileName = callback.FileName,
BytesWritten = callback.BytesToWrite,
FileSize = callback.Data.Length,
Offset = callback.Offset,
Result = EResult.OK,
LastError = 0,
OneTimePassword = callback.OneTimePassword,
SentryFileHash = sentryHash
});
}
public static void Cleanup()
{
// If threads is null, app was not yet initialized and there is nothing to cleanup
if (Threads == null)
{
return;
}
Log.WriteInfo("Bootstrapper", "Exiting...");
ChangelistTimer.Stop();
Log.WriteInfo("Bootstrapper", "Disconnecting from Steam...");
try
{
Steam.Instance.IsRunning = false;
Steam.Instance.Client.Disconnect();
}
catch (Exception e)
{
ErrorReporter.Notify("Bootstrapper", e);
}
if (Settings.Current.IRC.Enabled)
{
Log.WriteInfo("Bootstrapper", "Closing IRC connection...");
RssReader.Timer.Stop();
IRC.Instance.Close();
}
Log.WriteInfo("Bootstrapper", "Cancelling {0} tasks...", TaskManager.TasksCount);
TaskManager.CancelAllTasks();
foreach (var thread in Threads.Where(thread => thread.ThreadState == ThreadState.Running))
{
Log.WriteInfo("Bootstrapper", "Joining thread {0}...", thread.Name);
thread.Join(TimeSpan.FromSeconds(5));
}
Log.WriteInfo("Bootstrapper", "Saving local config...");
LocalConfig.Save();
try
{
Steam.Anonymous.IsRunning = false;
Steam.Anonymous.Client.Disconnect();
}
catch (Exception e)
{
ErrorReporter.Notify("Bootstrapper", e);
}
}
private static void OnUnobservedTaskException(object sender, UnobservedTaskExceptionEventArgs args)
{
args.Exception?.Flatten().Handle(e =>
{
ErrorReporter.Notify(nameof(Bootstrapper), e);
return(true);
});
}
public static void RegisterErrorHandler(Task t)
{
t.ContinueWith(task =>
{
task.Exception.Flatten().Handle(e =>
{
ErrorReporter.Notify("Task Manager", e);
return(false);
});
}, TaskContinuationOptions.OnlyOnFaulted);
}
private void OnMachineAuth(SteamUser.UpdateMachineAuthCallback callback)
{
Log.WriteInfo("Steam", "Updating sentry file... {0}", callback.FileName);
if (callback.Data.Length != callback.BytesToWrite)
{
ErrorReporter.Notify(new InvalidDataException(string.Format("Data.Length ({0}) != BytesToWrite ({1}) in OnMachineAuth", callback.Data.Length, callback.BytesToWrite)));
}
int fileSize;
byte[] sentryHash;
using (var stream = new MemoryStream(LocalConfig.Sentry ?? new byte[callback.BytesToWrite]))
{
stream.Seek(callback.Offset, SeekOrigin.Begin);
stream.Write(callback.Data, 0, callback.BytesToWrite);
stream.Seek(0, SeekOrigin.Begin);
fileSize = (int)stream.Length;
using (var sha = new SHA1CryptoServiceProvider())
{
sentryHash = sha.ComputeHash(stream);
}
LocalConfig.Sentry = stream.ToArray();
}
LocalConfig.SentryFileName = callback.FileName;
LocalConfig.Save();
Steam.Instance.User.SendMachineAuthResponse(new SteamUser.MachineAuthDetails
{
JobID = callback.JobID,
FileName = callback.FileName,
BytesWritten = callback.BytesToWrite,
FileSize = fileSize,
Offset = callback.Offset,
Result = EResult.OK,
LastError = 0,
OneTimePassword = callback.OneTimePassword,
SentryFileHash = sentryHash
});
}
private static void RegisterErrorHandler(Task t)
{
t.ContinueWith(task =>
{
task.Exception.Flatten().Handle(e =>
{
Log.WriteError("Task Manager", "Exception: {0}\n{1}", e.Message, e.StackTrace);
ErrorReporter.Notify(e);
return(false);
});
}, TaskContinuationOptions.OnlyOnFaulted);
}
private void Tick(object sender, ElapsedEventArgs e)
{
Parallel.ForEach(Settings.Current.RssFeeds, feed =>
{
try
{
ProcessFeed(feed);
}
catch (Exception ex)
{
ErrorReporter.Notify("RSS", ex);
}
});
}
public override void HandleMsg(IPacketMsg packetMsg)
{
if (packetMsg.MsgType == EMsg.ClientServiceCall ||
packetMsg.MsgType == EMsg.ClientServiceModule ||
packetMsg.MsgType == EMsg.ClientReadMachineAuth ||
packetMsg.MsgType == EMsg.ClientRequestMachineAuth)
{
IRC.Instance.SendOps("Yo xPaw and Netshroud, got {0}", packetMsg.MsgType);
ErrorReporter.Notify(new NotImplementedException(string.Format("Got {0}", packetMsg.MsgType)));
string file = Path.Combine(Application.Path, "files", ".support", string.Format("dump_{0}_{1}.bin", packetMsg.MsgType, DateTime.UtcNow.ToString("yyyyMMddHHmmssfff")));
Console.WriteLine(file);
File.WriteAllBytes(file, packetMsg.GetData());
}
}
private static void AddTask(Task t)
{
Tasks.TryAdd(t, 1);
t.ContinueWith(task =>
{
task.Exception?.Flatten().Handle(e =>
{
ErrorReporter.Notify(nameof(TaskManager), e);
return(false);
});
}, TaskContinuationOptions.OnlyOnFaulted);
t.ContinueWith(task => Tasks.TryRemove(task, out _));
}
public async void UpdateContentServerList()
{
LastServerRefreshTime = DateTime.Now;
KeyValue response;
using (var steamDirectory = Steam.Configuration.GetAsyncWebAPIInterface("ISteamDirectory"))
{
steamDirectory.Timeout = TimeSpan.FromSeconds(30);
try
{
response = await steamDirectory.CallAsync(HttpMethod.Get, "GetCSList", 1,
new Dictionary <string, object>
{
{ "cellid", LocalConfig.Current.CellID },
{ "maxcount", "20" } // Use the first 20 servers as they're sorted by cellid and we want low latency
});
if ((EResult)response["result"].AsInteger() != EResult.OK)
{
throw new Exception($"GetCSList result is EResult.${response["result"]}");
}
}
catch (Exception e)
{
ErrorReporter.Notify("Depot Downloader", e);
return;
}
}
// Note: There are servers like `origin5-sea1.steamcontent.com`
// which are hosted in Seattle, and most likely are the source of truth for
// game content, perhaps we should filter to these. But the latency is poor from Europe.
var newServers = response["serverlist"].Children.Select(x => (CDNClient.Server) new DnsEndPoint(x.Value.ToString(), 80)).ToList();
if (newServers.Count > 0)
{
CDNServers = newServers;
}
}
private static void RegisterErrorHandler(Task t)
{
Tasks.TryAdd(t, 1);
t.ContinueWith(task =>
{
Tasks.TryRemove(task, out _);
});
t.ContinueWith(task =>
{
task.Exception.Flatten().Handle(e =>
{
ErrorReporter.Notify("Task Manager", e);
return(false);
});
}, TaskContinuationOptions.OnlyOnFaulted);
}
private static async void TryCommand(Command command, CommandArguments commandData)
{
try
{
await command.OnCommand(commandData);
}
catch (TaskCanceledException)
{
commandData.Reply("Your command timed out.");
}
catch (AsyncJobFailedException)
{
commandData.Reply("Steam says this job failed, unable to execute your command.");
}
catch (Exception e)
{
commandData.Reply("Exception: {0}", e.Message);
ErrorReporter.Notify(e);
}
}
private static async void TryCommand(Command command, CommandArguments commandData)
{
try
{
await command.OnCommand(commandData);
}
catch (TaskCanceledException)
{
commandData.Reply("Your {0}{1}{2} command timed out.", Colors.OLIVE, command.Trigger, Colors.NORMAL);
}
catch (AsyncJobFailedException)
{
commandData.Reply("Steam says this job failed. Unable to execute your {0}{1}{2} command.", Colors.OLIVE, command.Trigger, Colors.NORMAL);
}
catch (Exception e)
{
ErrorReporter.Notify("IRC", e);
commandData.Reply("Exception: {0}", e.Message);
}
}
public void Connect()
{
try
{
IrcClientConnectionOptions options = null;
if (Settings.Current.IRC.Ssl)
{
options = new IrcClientConnectionOptions
{
Ssl = true,
SslHostname = Settings.Current.IRC.Server
};
}
Client.Connect(Settings.Current.IRC.Server, Settings.Current.IRC.Port, options);
}
catch (Exception e)
{
ErrorReporter.Notify("IRC", e);
}
}
public static async Task SendWebhook(object payload)
{
if (Settings.Current.WebhookURL == null)
{
return;
}
var json = JsonConvert.SerializeObject(payload);
var content = new StringContent(json, Encoding.UTF8, "application/json");
try
{
var result = await HttpClient.PostAsync(Settings.Current.WebhookURL, content);
var output = await result.Content.ReadAsStringAsync();
Log.WriteDebug("Webhook", $"Result: {output}");
}
catch (Exception e)
{
ErrorReporter.Notify("Webhook", e);
}
}
public async Task Process(uint appID, uint changeNumber, KeyValue depots)
{
var requests = new List <ManifestJob>();
// Get data in format we want first
foreach (var depot in depots.Children)
{
// Ignore these for now, parent app should be updated too anyway
if (depot["depotfromapp"].Value != null)
{
continue;
}
var request = new ManifestJob
{
ChangeNumber = changeNumber,
DepotName = depot["name"].AsString()
};
// Ignore keys that aren't integers, for example "branches"
if (!uint.TryParse(depot.Name, out request.DepotID))
{
continue;
}
// TODO: instead of locking we could wait for current process to finish
if (DepotLocks.ContainsKey(request.DepotID))
{
continue;
}
if (depot["manifests"]["public"].Value == null || !ulong.TryParse(depot["manifests"]["public"].Value, out request.ManifestID))
{
var branch = depot["manifests"].Children.FirstOrDefault(x => x.Name != "local");
if (branch == null || !ulong.TryParse(branch.Value, out request.ManifestID))
{
using (var db = Database.Get())
{
await db.ExecuteAsync("INSERT INTO `Depots` (`DepotID`, `Name`) VALUES (@DepotID, @DepotName) ON DUPLICATE KEY UPDATE `Name` = VALUES(`Name`)", new { request.DepotID, request.DepotName });
}
continue;
}
Log.WriteDebug("Depot Downloader", "Depot {0} (from {1}) has no public branch, but there is another one", request.DepotID, appID);
request.BuildID = depots["branches"][branch.Name]["buildid"].AsInteger();
}
else
{
request.BuildID = depots["branches"]["public"]["buildid"].AsInteger();
}
requests.Add(request);
}
if (!requests.Any())
{
return;
}
var depotsToDownload = new List <ManifestJob>();
using (var db = await Database.GetConnectionAsync())
{
var firstRequest = requests.First();
await db.ExecuteAsync("INSERT INTO `Builds` (`BuildID`, `ChangeID`, `AppID`) VALUES (@BuildID, @ChangeNumber, @AppID) ON DUPLICATE KEY UPDATE `AppID` = VALUES(`AppID`)",
new {
firstRequest.BuildID,
firstRequest.ChangeNumber,
appID
});
var dbDepots = (await db.QueryAsync <Depot>("SELECT `DepotID`, `Name`, `BuildID`, `ManifestID`, `LastManifestID` FROM `Depots` WHERE `DepotID` IN @Depots", new { Depots = requests.Select(x => x.DepotID) }))
.ToDictionary(x => x.DepotID, x => x);
foreach (var request in requests)
{
Depot dbDepot;
if (dbDepots.ContainsKey(request.DepotID))
{
dbDepot = dbDepots[request.DepotID];
if (dbDepot.BuildID > request.BuildID)
{
// buildid went back in time? this either means a rollback, or a shared depot that isn't synced properly
Log.WriteDebug("Depot Processor", "Skipping depot {0} due to old buildid: {1} > {2}", request.DepotID, dbDepot.BuildID, request.BuildID);
continue;
}
if (dbDepot.LastManifestID == request.ManifestID && dbDepot.ManifestID == request.ManifestID && Settings.Current.FullRun != FullRunState.WithForcedDepots)
{
// Update depot name if changed
if (!request.DepotName.Equals(dbDepot.Name))
{
await db.ExecuteAsync("UPDATE `Depots` SET `Name` = @DepotName WHERE `DepotID` = @DepotID", new { request.DepotID, request.DepotName });
}
continue;
}
}
else
{
dbDepot = new Depot();
}
if (dbDepot.BuildID != request.BuildID || dbDepot.ManifestID != request.ManifestID || !request.DepotName.Equals(dbDepot.Name))
{
await db.ExecuteAsync(@"INSERT INTO `Depots` (`DepotID`, `Name`, `BuildID`, `ManifestID`) VALUES (@DepotID, @DepotName, @BuildID, @ManifestID)
ON DUPLICATE KEY UPDATE `LastUpdated` = CURRENT_TIMESTAMP(), `Name` = VALUES(`Name`), `BuildID` = VALUES(`BuildID`), `ManifestID` = VALUES(`ManifestID`)",
new {
request.DepotID,
request.DepotName,
request.BuildID,
request.ManifestID
});
}
if (dbDepot.ManifestID != request.ManifestID)
{
await MakeHistory(db, null, request, string.Empty, "manifest_change", dbDepot.ManifestID, request.ManifestID);
}
var owned = LicenseList.OwnedApps.ContainsKey(request.DepotID);
if (!owned)
{
request.Anonymous = owned = LicenseList.AnonymousApps.ContainsKey(request.DepotID);
if (owned)
{
Log.WriteWarn("Depot Processor", "Will download depot {0} using anonymous account", request.DepotID);
}
}
if (owned)
{
lock (DepotLocks)
{
// This doesn't really save us from concurrency issues
if (DepotLocks.ContainsKey(request.DepotID))
{
Log.WriteWarn("Depot Processor", "Depot {0} was locked in another thread", request.DepotID);
continue;
}
DepotLocks.Add(request.DepotID, 1);
}
depotsToDownload.Add(request);
}
}
}
if (depotsToDownload.Any())
{
#pragma warning disable 4014
if (FileDownloader.IsImportantDepot(appID) && !Settings.IsFullRun && !string.IsNullOrEmpty(Settings.Current.PatchnotesNotifyURL))
{
TaskManager.Run(() => NotifyPatchnote(appID, depotsToDownload.First().BuildID));
}
TaskManager.Run(async() =>
{
try
{
await DownloadDepots(appID, depotsToDownload);
}
catch (Exception e)
{
ErrorReporter.Notify("Depot Processor", e);
}
foreach (var depot in depotsToDownload)
{
RemoveLock(depot.DepotID);
}
});
#pragma warning restore 4014
}
}
private async Task <EResult> ProcessDepotAfterDownload(IDbConnection db, IDbTransaction transaction, ManifestJob request, DepotManifest depotManifest)
{
var filesOld = (await db.QueryAsync <DepotFile>("SELECT `ID`, `File`, `Hash`, `Size`, `Flags` FROM `DepotsFiles` WHERE `DepotID` = @DepotID", new { request.DepotID }, transaction: transaction)).ToDictionary(x => x.File, x => x);
var filesAdded = new List <DepotFile>();
var shouldHistorize = filesOld.Any(); // Don't historize file additions if we didn't have any data before
foreach (var file in depotManifest.Files)
{
var name = file.FileName.Replace('\\', '/');
byte[] hash = null;
// Store empty hashes as NULL (e.g. an empty file)
if (file.FileHash.Length > 0 && !file.Flags.HasFlag(EDepotFileFlag.Directory))
{
for (int i = 0; i < file.FileHash.Length; ++i)
{
if (file.FileHash[i] != 0)
{
hash = file.FileHash;
break;
}
}
}
// safe guard
if (name.Length > 255)
{
ErrorReporter.Notify("Depot Processor", new OverflowException(string.Format("File \"{0}\" in depot {1} is too long", name, request.DepotID)));
continue;
}
if (filesOld.ContainsKey(name))
{
var oldFile = filesOld[name];
var updateFile = false;
if (oldFile.Size != file.TotalSize || !Utils.IsEqualSHA1(hash, oldFile.Hash))
{
await MakeHistory(db, transaction, request, name, "modified", oldFile.Size, file.TotalSize);
updateFile = true;
}
if (oldFile.Flags != file.Flags)
{
await MakeHistory(db, transaction, request, name, "modified_flags", (ulong)oldFile.Flags, (ulong)file.Flags);
updateFile = true;
}
if (updateFile)
{
await db.ExecuteAsync("UPDATE `DepotsFiles` SET `Hash` = @Hash, `Size` = @Size, `Flags` = @Flags WHERE `DepotID` = @DepotID AND `ID` = @ID", new DepotFile
{
ID = oldFile.ID,
DepotID = request.DepotID,
Hash = hash,
Size = file.TotalSize,
Flags = file.Flags
}, transaction : transaction);
}
filesOld.Remove(name);
}
else
{
// We want to historize modifications first, and only then deletions and additions
filesAdded.Add(new DepotFile
{
DepotID = request.DepotID,
Hash = hash,
File = name,
Size = file.TotalSize,
Flags = file.Flags
});
}
}
if (filesOld.Any())
{
await db.ExecuteAsync("DELETE FROM `DepotsFiles` WHERE `DepotID` = @DepotID AND `ID` IN @Files", new { request.DepotID, Files = filesOld.Select(x => x.Value.ID) }, transaction : transaction);
await db.ExecuteAsync(HistoryQuery, filesOld.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ChangeID = request.ChangeNumber,
Action = "removed",
File = x.Value.File
}), transaction : transaction);
}
if (filesAdded.Any())
{
await db.ExecuteAsync("INSERT INTO `DepotsFiles` (`DepotID`, `File`, `Hash`, `Size`, `Flags`) VALUES (@DepotID, @File, @Hash, @Size, @Flags)", filesAdded, transaction : transaction);
if (shouldHistorize)
{
await db.ExecuteAsync(HistoryQuery, filesAdded.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ChangeID = request.ChangeNumber,
Action = "added",
File = x.File
}), transaction : transaction);
}
}
await db.ExecuteAsync("UPDATE `Depots` SET `LastManifestID` = @ManifestID, `LastUpdated` = CURRENT_TIMESTAMP() WHERE `DepotID` = @DepotID", new { request.DepotID, request.ManifestID }, transaction : transaction);
return(EResult.OK);
}
private async Task DownloadDepots(uint appID, List <ManifestJob> depots)
{
Log.WriteDebug("Depot Downloader", "Will process {0} depots ({1} depot locks left)", depots.Count(), DepotLocks.Count);
var processTasks = new List <Task <EResult> >();
bool anyFilesDownloaded = false;
foreach (var depot in depots)
{
var instance = depot.Anonymous ? Steam.Anonymous.Apps : Steam.Instance.Apps;
depot.DepotKey = await GetDepotDecryptionKey(instance, depot.DepotID, appID);
if (depot.DepotKey == null)
{
RemoveLock(depot.DepotID);
continue;
}
var cdnToken = await GetCDNAuthToken(instance, appID, depot.DepotID);
if (cdnToken == null)
{
RemoveLock(depot.DepotID);
Log.WriteDebug("Depot Downloader", "Got a depot key for depot {0} but no cdn auth token", depot.DepotID);
continue;
}
depot.CDNToken = cdnToken.Token;
depot.Server = GetContentServer();
DepotManifest depotManifest = null;
string lastError = string.Empty;
for (var i = 0; i <= 5; i++)
{
try
{
depotManifest = await CDNClient.DownloadManifestAsync(depot.DepotID, depot.ManifestID, depot.Server, depot.CDNToken, depot.DepotKey);
break;
}
catch (Exception e)
{
lastError = e.Message;
Log.WriteError("Depot Processor", "Failed to download depot manifest for app {0} depot {1} ({2}: {3}) (#{4})", appID, depot.DepotID, depot.Server, lastError, i);
}
// TODO: get new auth key if auth fails
depot.Server = GetContentServer();
if (depotManifest == null)
{
await Task.Delay(Utils.ExponentionalBackoff(i));
}
}
if (depotManifest == null)
{
LocalConfig.CDNAuthTokens.TryRemove(depot.DepotID, out _);
RemoveLock(depot.DepotID);
if (FileDownloader.IsImportantDepot(depot.DepotID))
{
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download manifest ({3})",
Colors.OLIVE, depot.DepotName, Colors.NORMAL, lastError);
}
if (!Settings.IsFullRun)
{
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
}
continue;
}
var task = ProcessDepotAfterDownload(depot, depotManifest);
processTasks.Add(task);
if (!FileDownloader.IsImportantDepot(depot.DepotID))
{
continue;
}
task = TaskManager.Run(async() =>
{
var result = EResult.Fail;
try
{
result = await FileDownloader.DownloadFilesFromDepot(depot, depotManifest);
if (result == EResult.OK)
{
anyFilesDownloaded = true;
}
}
catch (Exception e)
{
ErrorReporter.Notify($"Depot Processor {depot.DepotID}", e);
}
return(result);
}).Unwrap();
processTasks.Add(task);
}
if (SaveLocalConfig)
{
SaveLocalConfig = false;
LocalConfig.Save();
}
await Task.WhenAll(processTasks).ConfigureAwait(false);
Log.WriteDebug("Depot Downloader", "{0} depot downloads finished", depots.Count());
// TODO: use ContinueWith on tasks
if (!anyFilesDownloaded)
{
foreach (var depot in depots)
{
RemoveLock(depot.DepotID);
}
return;
}
if (!File.Exists(UpdateScript))
{
return;
}
lock (UpdateScriptLock)
{
foreach (var depot in depots)
{
if (depot.Result == EResult.OK)
{
RunUpdateScript(string.Format("{0} no-git", depot.DepotID));
}
else if (depot.Result != EResult.Ignored)
{
Log.WriteWarn("Depot Processor", "Dropping stored token for {0} due to download failures", depot.DepotID);
LocalConfig.CDNAuthTokens.TryRemove(depot.DepotID, out _);
using (var db = Database.Get())
{
// Mark this depot for redownload
db.Execute("UPDATE `Depots` SET `LastManifestID` = 0 WHERE `DepotID` = @DepotID", new { depot.DepotID });
}
}
RemoveLock(depot.DepotID);
}
// Only commit changes if all depots downloaded
if (processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored))
{
if (!RunUpdateScript(appID, depots.First().BuildID))
{
RunUpdateScript("0");
}
}
else
{
Log.WriteDebug("Depot Processor", "Reprocessing the app {0} because some files failed to download", appID);
IRC.Instance.SendOps("{0}[{1}]{2} Reprocessing the app due to download failures",
Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL
);
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
}
}
}
public static async Task <uint> TrySearchAppId(CommandArguments command)
{
uint appID = 0;
var name = command.Message;
Uri uri;
var query = new Dictionary <string, string>
{
{ "hitsPerPage", "1" },
{ "attributesToHighlight", "null" },
{ "attributesToSnippet", "null" },
{ "attributesToRetrieve", "[\"objectID\"]" },
{ "facetFilters", "[[\"appType:Game\",\"appType:Application\"]]" },
{ "advancedSyntax", "true" },
{ "query", name }
};
using (var content = new FormUrlEncodedContent(query))
{
uri = new UriBuilder("https://94he6yatei-dsn.algolia.net/1/indexes/steamdb/")
{
Query = await content.ReadAsStringAsync()
}.Uri;
}
using (var requestMessage = new HttpRequestMessage(HttpMethod.Get, uri))
{
requestMessage.Headers.Add("Referer", "https://github.com/SteamDatabase/SteamDatabaseBackend");
requestMessage.Headers.Add("X-Algolia-Application-Id", "94HE6YATEI");
requestMessage.Headers.Add("X-Algolia-API-Key", "2414d3366df67739fe6e73dad3f51a43");
try
{
var response = await Utils.HttpClient.SendAsync(requestMessage);
var data = await response.Content.ReadAsStringAsync();
var json = JsonConvert.DeserializeObject <AlgoliaSearchAppHits>(data);
if (json.Hits.Length > 0)
{
appID = json.Hits[0].AppID;
}
}
catch (Exception e)
{
ErrorReporter.Notify("Algolia search", e);
}
}
if (appID > 0)
{
return(appID);
}
await using (var db = await Database.GetConnectionAsync())
{
appID = await db.ExecuteScalarAsync <uint>("SELECT `AppID` FROM `Apps` LEFT JOIN `AppsTypes` ON `Apps`.`AppType` = `AppsTypes`.`AppType` WHERE (`AppsTypes`.`Name` IN ('game', 'application', 'video', 'hardware') AND (`Apps`.`StoreName` LIKE @Name OR `Apps`.`Name` LIKE @Name)) OR (`AppsTypes`.`Name` = 'unknown' AND `Apps`.`LastKnownName` LIKE @Name) ORDER BY `LastUpdated` DESC LIMIT 1", new { Name = name });
}
if (appID == 0)
{
command.Reply("Nothing was found matching your request.");
}
return(appID);
}