private static void HandleClientRequestFreeLicenseResponse(IPacketMsg packetMsg)
{
var resp = new ClientMsgProtobuf <CMsgClientRequestFreeLicenseResponse>(packetMsg);
JobAction job;
JobManager.TryRemoveJob(packetMsg.TargetJobID, out job);
var packageIDs = resp.Body.granted_packageids;
var appIDs = resp.Body.granted_appids;
Log.WriteDebug("FreeLicense", "Received free license: {0} ({1} apps, {2} packages)", (EResult)resp.Body.eresult, appIDs.Count, packageIDs.Count);
if (appIDs.Count > 0)
{
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appIDs, Enumerable.Empty <uint>()));
}
if (packageIDs.Count > 0)
{
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetProductInfo(Enumerable.Empty <uint>(), packageIDs));
// TODO: We could re-queue apps in these packages as well
// We don't want to block our main thread with web requests
TaskManager.Run(() =>
{
string data = null;
try
{
var response = WebAuth.PerformRequest("GET", "https://store.steampowered.com/account/");
using (var responseStream = response.GetResponseStream())
{
using (var reader = new StreamReader(responseStream))
{
data = reader.ReadToEnd();
}
}
}
catch (WebException e)
{
Log.WriteError("FreeLicense", "Failed to fetch account details page: {0}", e.Message);
}
foreach (var package in packageIDs)
{
Package packageData;
using (var db = Database.GetConnection())
{
packageData = db.Query <Package>("SELECT `SubID`, `Name`, `LastKnownName` FROM `Subs` WHERE `SubID` = @SubID", new { SubID = package }).FirstOrDefault();
}
if (!string.IsNullOrEmpty(data))
{
// Tell me all about using regex
var match = Regex.Match(data, string.Format("RemoveFreeLicense\\( ?{0}, ?'(.+)' ?\\)", package));
if (match.Success)
{
var grantedName = Encoding.UTF8.GetString(Convert.FromBase64String(match.Groups[1].Value));
// Update last known name if we can
if (packageData.SubID > 0 && (string.IsNullOrEmpty(packageData.LastKnownName) || packageData.LastKnownName.StartsWith("Steam Sub ", StringComparison.Ordinal)))
{
using (var db = Database.GetConnection())
{
db.Execute("UPDATE `Subs` SET `LastKnownName` = @Name WHERE `SubID` = @SubID", new { SubID = package, Name = grantedName });
db.Execute(SubProcessor.GetHistoryQuery(),
new PICSHistory
{
ID = package,
Key = SteamDB.DATABASE_NAME_TYPE,
OldValue = "free on demand; account page",
NewValue = grantedName,
Action = "created_info"
}
);
// Add a app comment on each app in this package
var comment = string.Format("This app is in a free on demand package called <b>{0}</b>", System.Security.SecurityElement.Escape(grantedName));
var apps = db.Query <PackageApp>("SELECT `AppID` FROM `SubsApps` WHERE `SubID` = @SubID", new { SubID = package }).ToList();
var types = db.Query <App>("SELECT `AppID` FROM `Apps` WHERE `AppType` > 0 AND `AppID` IN @Ids", new { Ids = apps.Select(x => x.AppID) }).ToDictionary(x => x.AppID, x => true);
var key = db.ExecuteScalar <uint>("SELECT `ID` FROM `KeyNames` WHERE `Name` = 'website_comment'");
foreach (var app in apps)
{
if (types.ContainsKey(app.AppID))
{
continue;
}
db.Execute("INSERT INTO `AppsInfo` VALUES (@AppID, @Key, @Value) ON DUPLICATE KEY UPDATE `Key` = `Key`", new { AppID = app.AppID, Key = key, value = comment });
}
}
}
packageData.LastKnownName = grantedName;
}
}
IRC.Instance.SendMain("New free license granted: {0}{1}{2} -{3} {4}",
Colors.BLUE, Steam.FormatPackageName(package, packageData), Colors.NORMAL,
Colors.DARKBLUE, SteamDB.GetPackageURL(package)
);
}
});
}
}
private void OnLoggedOn(SteamUser.LoggedOnCallback callback)
{
TaskManager.Run(AuthenticateUser);
}
public void StartTick()
{
TickerHash++;
TaskManager.Run(Tick);
}
private static async void OnClanState(SteamFriends.ClanStateCallback callback)
{
if (callback.Events.Count == 0 && callback.Announcements.Count == 0)
{
return;
}
var groupName = callback.ClanName;
if (string.IsNullOrEmpty(groupName))
{
groupName = Steam.Instance.Friends.GetClanName(callback.ClanID);
// Check once more, because that can fail too
if (string.IsNullOrEmpty(groupName))
{
groupName = "Group";
}
}
foreach (var announcement in callback.Announcements)
{
var url = $"https://steamcommunity.com/gid/{callback.ClanID.AccountID}/announcements/detail/{announcement.ID}";
IRC.Instance.SendAnnounce($"{Colors.BLUE}{groupName}{Colors.NORMAL} announcement: {Colors.OLIVE}{announcement.Headline}{Colors.NORMAL} -{Colors.DARKBLUE} {url}");
_ = TaskManager.Run(async() => await Utils.SendWebhook(new
{
Type = "GroupAnnouncement",
Title = announcement.Headline,
Group = groupName,
Url = url,
}));
Log.WriteInfo(nameof(ClanState), $"{groupName} \"{announcement.Headline}\"");
}
await using var db = await Database.GetConnectionAsync();
foreach (var groupEvent in callback.Events)
{
var link = $"https://steamcommunity.com/gid/{callback.ClanID.AccountID}/events/{groupEvent.ID}";
var id = await db.ExecuteScalarAsync <int>("SELECT `ID` FROM `RSS` WHERE `Link` = @Link", new { Link = link });
if (id > 0)
{
continue;
}
IRC.Instance.SendAnnounce(
$"{Colors.BLUE}{groupName}{Colors.NORMAL} event: {Colors.OLIVE}{groupEvent.Headline}{Colors.NORMAL} -{Colors.DARKBLUE} {link} {Colors.DARKGRAY}({groupEvent.EventTime.ToString("s", CultureInfo.InvariantCulture).Replace("T", " ")})"
);
Log.WriteInfo(nameof(ClanState), $"{groupName} Event \"{groupEvent.Headline}\" {link}");
await db.ExecuteAsync("INSERT INTO `RSS` (`Link`, `Title`) VALUES(@Link, @Title)", new { Link = link, Title = groupEvent.Headline });
_ = TaskManager.Run(async() => await Utils.SendWebhook(new
{
Type = "GroupAnnouncement",
Title = groupEvent.Headline,
Group = groupName,
Url = link,
}));
}
}
private static void OnPICSProductInfo(SteamApps.PICSProductInfoCallback callback)
{
JobManager.TryRemoveJob(callback.JobID);
var apps = callback.Apps.Concat(callback.UnknownApps.ToDictionary(x => x, x => (SteamApps.PICSProductInfoCallback.PICSProductInfo)null));
var packages = callback.Packages.Concat(callback.UnknownPackages.ToDictionary(x => x, x => (SteamApps.PICSProductInfoCallback.PICSProductInfo)null));
foreach (var workaround in apps)
{
var app = workaround;
Log.WriteInfo("PICSProductInfo", "{0}AppID: {1}", app.Value == null ? "Unknown " : "", app.Key);
Task mostRecentItem;
lock (ProcessedApps)
{
ProcessedApps.TryGetValue(app.Key, out mostRecentItem);
}
var workerItem = TaskManager.Run(async delegate
{
if (mostRecentItem != null && !mostRecentItem.IsCompleted)
{
Log.WriteDebug("PICSProductInfo", "Waiting for app {0} to finish processing", app.Key);
await mostRecentItem;
}
using (var processor = new AppProcessor(app.Key))
{
if (app.Value == null)
{
processor.ProcessUnknown();
}
else
{
processor.Process(app.Value);
}
}
});
if (Settings.IsFullRun)
{
continue;
}
lock (ProcessedApps)
{
ProcessedApps[app.Key] = workerItem;
}
workerItem.ContinueWith(task =>
{
lock (ProcessedApps)
{
if (ProcessedApps.TryGetValue(app.Key, out mostRecentItem) && mostRecentItem.IsCompleted)
{
ProcessedApps.Remove(app.Key);
}
}
});
}
foreach (var workaround in packages)
{
var package = workaround;
Log.WriteInfo("PICSProductInfo", "{0}SubID: {1}", package.Value == null ? "Unknown " : "", package.Key);
Task mostRecentItem;
lock (ProcessedSubs)
{
ProcessedSubs.TryGetValue(package.Key, out mostRecentItem);
}
var workerItem = TaskManager.Run(async delegate
{
if (mostRecentItem != null && !mostRecentItem.IsCompleted)
{
Log.WriteDebug("PICSProductInfo", "Waiting for package {0} to finish processing", package.Key);
await mostRecentItem;
}
using (var processor = new SubProcessor(package.Key))
{
if (package.Value == null)
{
processor.ProcessUnknown();
}
else
{
processor.Process(package.Value);
}
}
});
if (Settings.IsFullRun)
{
continue;
}
lock (ProcessedSubs)
{
ProcessedSubs[package.Key] = workerItem;
}
workerItem.ContinueWith(task =>
{
lock (ProcessedSubs)
{
if (ProcessedSubs.TryGetValue(package.Key, out mostRecentItem) && mostRecentItem.IsCompleted)
{
ProcessedSubs.Remove(package.Key);
}
}
});
}
}
private async Task DownloadDepots(uint appID, List <ManifestJob> depots)
{
Log.WriteDebug(nameof(DepotProcessor), $"Will process {depots.Count} depots ({DepotLocks.Count} depot locks left)");
var processTasks = new List <Task <EResult> >();
var anyFilesDownloaded = false;
var willDownloadFiles = false;
foreach (var depot in depots)
{
if (depot.DepotKey == null)
{
await GetDepotDecryptionKey(Steam.Instance.Apps, depot, appID);
if (depot.DepotKey == null && (depot.LastManifestID == depot.ManifestID || Settings.Current.OnlyOwnedDepots))
{
RemoveLock(depot.DepotID);
continue;
}
}
depot.Server = GetContentServer();
DepotManifest depotManifest = null;
var lastError = string.Empty;
for (var i = 0; i <= 5; i++)
{
try
{
await ManifestDownloadSemaphore.WaitAsync(TaskManager.TaskCancellationToken.Token).ConfigureAwait(false);
depotManifest = await CDNClient.DownloadManifestAsync(depot.DepotID, depot.ManifestID, depot.Server, string.Empty, depot.DepotKey);
break;
}
catch (Exception e)
{
lastError = e.Message;
Log.WriteError(nameof(DepotProcessor), $"Failed to download depot manifest for app {appID} depot {depot.DepotID} ({depot.Server}: {lastError}) (#{i})");
}
finally
{
ManifestDownloadSemaphore.Release();
}
if (depot.DepotKey != null)
{
RemoveErroredServer(depot.Server);
}
depot.Server = GetContentServer();
if (depotManifest == null && i < 5)
{
await Task.Delay(Utils.ExponentionalBackoff(i + 1));
}
}
if (depotManifest == null)
{
RemoveLock(depot.DepotID);
if (FileDownloader.IsImportantDepot(depot.DepotID))
{
IRC.Instance.SendOps($"{Colors.OLIVE}[{depot.DepotName}]{Colors.NORMAL} Failed to download manifest ({lastError})");
}
if (!Settings.IsFullRun && depot.DepotKey != null)
{
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
}
continue;
}
var task = ProcessDepotAfterDownload(depot, depotManifest);
processTasks.Add(task);
if (!FileDownloader.IsImportantDepot(depot.DepotID) || depot.DepotKey == null)
{
depot.Result = EResult.Ignored;
continue;
}
willDownloadFiles = true;
task = TaskManager.Run(async() =>
{
var result = EResult.Fail;
try
{
result = await FileDownloader.DownloadFilesFromDepot(depot, depotManifest);
if (result == EResult.OK)
{
anyFilesDownloaded = true;
}
}
catch (Exception e)
{
ErrorReporter.Notify($"Depot Processor {depot.DepotID}", e);
}
return(result);
});
processTasks.Add(task);
}
await Task.WhenAll(processTasks).ConfigureAwait(false);
Log.WriteDebug(nameof(DepotProcessor), $"{depots.Count} depot downloads finished for app {appID}");
// TODO: use ContinueWith on tasks
if (!anyFilesDownloaded && !willDownloadFiles)
{
foreach (var depot in depots)
{
RemoveLock(depot.DepotID);
}
return;
}
lock (UpdateScriptLock)
{
foreach (var depot in depots)
{
if (depot.Result == EResult.OK)
{
RunUpdateScript(UpdateScript, $"{depot.DepotID} no-git");
}
else if (depot.Result != EResult.Ignored)
{
Log.WriteWarn(nameof(DepotProcessor), $"Download failed for {depot.DepotID}: {depot.Result}");
// Mark this depot for redownload
using var db = Database.Get();
db.Execute("UPDATE `Depots` SET `LastManifestID` = 0 WHERE `DepotID` = @DepotID", new { depot.DepotID });
}
RemoveLock(depot.DepotID);
}
// Only commit changes if all depots downloaded
if (processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored))
{
if (!RunUpdateScriptForApp(appID, depots[0].BuildID))
{
RunUpdateScript(UpdateScript, "0");
}
}
else
{
Log.WriteDebug(nameof(DepotProcessor), $"Reprocessing the app {appID} because some files failed to download");
IRC.Instance.SendOps($"{Colors.OLIVE}[{Steam.GetAppName(appID)}]{Colors.NORMAL} Reprocessing the app due to download failures");
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
}
}
}
private async Task DownloadDepots(uint appID, List <ManifestJob> depots)
{
Log.WriteDebug("Depot Downloader", "Will process {0} depots ({1} depot locks left)", depots.Count, DepotLocks.Count);
var processTasks = new List <Task <EResult> >();
bool anyFilesDownloaded = false;
foreach (var depot in depots)
{
var instance = depot.Anonymous ? Steam.Anonymous.Apps : Steam.Instance.Apps;
depot.DepotKey = await GetDepotDecryptionKey(instance, depot.DepotID, appID);
if (depot.DepotKey == null)
{
RemoveLock(depot.DepotID);
continue;
}
var cdnToken = await GetCDNAuthToken(instance, appID, depot.DepotID);
if (cdnToken == null)
{
RemoveLock(depot.DepotID);
Log.WriteDebug("Depot Downloader", "Got a depot key for depot {0} but no cdn auth token", depot.DepotID);
continue;
}
depot.CDNToken = cdnToken.Token;
depot.Server = GetContentServer();
DepotManifest depotManifest = null;
string lastError = string.Empty;
for (var i = 0; i <= 5; i++)
{
try
{
depotManifest = await CDNClient.DownloadManifestAsync(depot.DepotID, depot.ManifestID, depot.Server, depot.CDNToken, depot.DepotKey);
break;
}
catch (Exception e)
{
lastError = e.Message;
Log.WriteError("Depot Processor", "Failed to download depot manifest for app {0} depot {1} ({2}: {3}) (#{4})", appID, depot.DepotID, depot.Server, lastError, i);
}
// TODO: get new auth key if auth fails
depot.Server = GetContentServer();
if (depotManifest == null)
{
await Task.Delay(Utils.ExponentionalBackoff(i));
}
}
if (depotManifest == null)
{
LocalConfig.CDNAuthTokens.TryRemove(depot.DepotID, out _);
RemoveLock(depot.DepotID);
if (FileDownloader.IsImportantDepot(depot.DepotID))
{
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download manifest ({3})",
Colors.OLIVE, depot.DepotName, Colors.NORMAL, lastError);
}
if (!Settings.IsFullRun)
{
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
}
continue;
}
var task = ProcessDepotAfterDownload(depot, depotManifest);
processTasks.Add(task);
if (!FileDownloader.IsImportantDepot(depot.DepotID))
{
continue;
}
task = TaskManager.Run(async() =>
{
var result = EResult.Fail;
try
{
result = await FileDownloader.DownloadFilesFromDepot(depot, depotManifest);
if (result == EResult.OK)
{
anyFilesDownloaded = true;
}
}
catch (Exception e)
{
ErrorReporter.Notify($"Depot Processor {depot.DepotID}", e);
}
return(result);
}).Unwrap();
processTasks.Add(task);
}
if (SaveLocalConfig)
{
SaveLocalConfig = false;
LocalConfig.Save();
}
await Task.WhenAll(processTasks).ConfigureAwait(false);
Log.WriteDebug("Depot Downloader", "{0} depot downloads finished", depots.Count);
// TODO: use ContinueWith on tasks
if (!anyFilesDownloaded)
{
foreach (var depot in depots)
{
RemoveLock(depot.DepotID);
}
return;
}
if (!File.Exists(UpdateScript))
{
return;
}
lock (UpdateScriptLock)
{
foreach (var depot in depots)
{
if (depot.Result == EResult.OK)
{
RunUpdateScript(UpdateScript, string.Format("{0} no-git", depot.DepotID));
}
else if (depot.Result != EResult.Ignored)
{
Log.WriteWarn("Depot Processor", "Dropping stored token for {0} due to download failures", depot.DepotID);
LocalConfig.CDNAuthTokens.TryRemove(depot.DepotID, out _);
using (var db = Database.Get())
{
// Mark this depot for redownload
db.Execute("UPDATE `Depots` SET `LastManifestID` = 0 WHERE `DepotID` = @DepotID", new { depot.DepotID });
}
}
RemoveLock(depot.DepotID);
}
// Only commit changes if all depots downloaded
if (processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored))
{
if (!RunUpdateScript(appID, depots[0].BuildID))
{
RunUpdateScript(UpdateScript, "0");
}
}
else
{
Log.WriteDebug("Depot Processor", "Reprocessing the app {0} because some files failed to download", appID);
IRC.Instance.SendOps("{0}[{1}]{2} Reprocessing the app due to download failures",
Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL
);
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
}
}
}
private static async Task <EResult> DownloadFile(DepotProcessor.ManifestJob job, DepotManifest.FileData file, byte[] hash)
{
var directory = Path.Combine(Application.Path, "files", DownloadFolders[job.DepotID], Path.GetDirectoryName(file.FileName));
var finalPath = new FileInfo(Path.Combine(directory, Path.GetFileName(file.FileName)));
var downloadPath = new FileInfo(Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".steamdb_tmp")));
if (!Directory.Exists(directory))
{
Directory.CreateDirectory(directory);
}
else if (file.TotalSize == 0)
{
if (!finalPath.Exists)
{
using (var _ = finalPath.Create())
{
// FileInfo.Create returns a stream but we don't need it
}
Log.WriteInfo("FileDownloader", "{0} created an empty file", file.FileName);
return(EResult.SameAsPreviousValue);
}
else if (finalPath.Length == 0)
{
#if DEBUG
Log.WriteDebug("FileDownloader", "{0} is already empty", file.FileName);
#endif
return(EResult.SameAsPreviousValue);
}
}
else if (hash != null && file.FileHash.SequenceEqual(hash))
{
#if DEBUG
Log.WriteDebug("FileDownloader", "{0} already matches the file we have", file.FileName);
#endif
return(EResult.SameAsPreviousValue);
}
byte[] checksum;
using (var sha = SHA1.Create())
{
checksum = sha.ComputeHash(Encoding.UTF8.GetBytes(file.FileName));
}
var neededChunks = new List <DepotManifest.ChunkData>();
var chunks = file.Chunks.OrderBy(x => x.Offset).ToList();
var oldChunksFile = Path.Combine(Application.Path, "files", ".support", "chunks", string.Format("{0}-{1}.json", job.DepotID, BitConverter.ToString(checksum)));
using (var fs = downloadPath.Open(FileMode.OpenOrCreate, FileAccess.ReadWrite))
{
fs.SetLength((long)file.TotalSize);
if (finalPath.Exists && File.Exists(oldChunksFile))
{
var oldChunks = JsonConvert.DeserializeObject <List <DepotManifest.ChunkData> >(File.ReadAllText(oldChunksFile), JsonHandleAllReferences);
using (var fsOld = finalPath.Open(FileMode.Open, FileAccess.Read))
{
foreach (var chunk in chunks)
{
var oldChunk = oldChunks.Find(c => c.ChunkID.SequenceEqual(chunk.ChunkID));
if (oldChunk != null)
{
var oldData = new byte[oldChunk.UncompressedLength];
fsOld.Seek((long)oldChunk.Offset, SeekOrigin.Begin);
fsOld.Read(oldData, 0, oldData.Length);
var existingChecksum = Utils.AdlerHash(oldData);
if (existingChecksum.SequenceEqual(chunk.Checksum))
{
fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
fs.Write(oldData, 0, oldData.Length);
#if DEBUG
Log.WriteDebug("FileDownloader", "{0} Found chunk ({1}), not downloading", file.FileName, chunk.Offset);
#endif
}
else
{
neededChunks.Add(chunk);
#if DEBUG
Log.WriteDebug("FileDownloader", "{0} Found chunk ({1}), but checksum differs", file.FileName, chunk.Offset);
#endif
}
}
else
{
neededChunks.Add(chunk);
}
}
}
}
else
{
neededChunks = chunks;
}
}
var downloadedSize = file.TotalSize - (ulong)neededChunks.Sum(x => x.UncompressedLength);
var chunkCancellation = new CancellationTokenSource();
var chunkTasks = new Task[neededChunks.Count];
Log.WriteInfo("FileDownloader", "Downloading {0} ({1} bytes, {2} out of {3} chunks)", file.FileName, downloadedSize, neededChunks.Count, chunks.Count);
for (var i = 0; i < chunkTasks.Length; i++)
{
var chunk = neededChunks[i];
chunkTasks[i] = TaskManager.Run(async() =>
{
try
{
chunkCancellation.Token.ThrowIfCancellationRequested();
await ChunkDownloadingSemaphore.WaitAsync(chunkCancellation.Token).ConfigureAwait(false);
var result = await DownloadChunk(job, chunk, downloadPath);
if (!result)
{
Log.WriteWarn("FileDownloader", "Failed to download chunk for {0}", file.FileName);
chunkCancellation.Cancel();
}
else
{
downloadedSize += chunk.UncompressedLength;
// Do not write progress info to log file
Console.WriteLine("{2} [{0,6:#00.00}%] {1}", downloadedSize / (float)file.TotalSize * 100.0f, file.FileName, job.DepotName);
}
}
finally
{
ChunkDownloadingSemaphore.Release();
}
}).Unwrap();
// Register error handler on inner task
TaskManager.RegisterErrorHandler(chunkTasks[i]);
}
await Task.WhenAll(chunkTasks).ConfigureAwait(false);
using (var fs = downloadPath.Open(FileMode.Open, FileAccess.ReadWrite))
{
fs.Seek(0, SeekOrigin.Begin);
using (var sha = SHA1.Create())
{
checksum = sha.ComputeHash(fs);
}
}
if (!file.FileHash.SequenceEqual(checksum))
{
IRC.Instance.SendOps("{0}[{1}]{2} Failed to correctly download {3}{4}",
Colors.OLIVE, job.DepotName, Colors.NORMAL, Colors.BLUE, file.FileName);
Log.WriteWarn("FileDownloader", "Failed to download file {0} ({1})", file.FileName, job.Server);
downloadPath.Delete();
return(EResult.DataCorruption);
}
Log.WriteInfo("FileDownloader", "Downloaded {0} from {1}", file.FileName, job.DepotName);
finalPath.Delete();
downloadPath.MoveTo(finalPath.FullName);
if (chunks.Count > 1)
{
File.WriteAllText(oldChunksFile, JsonConvert.SerializeObject(chunks, Formatting.None, JsonHandleAllReferences));
}
else if (File.Exists(oldChunksFile))
{
File.Delete(oldChunksFile);
}
return(EResult.OK);
}
private async Task DownloadDepots(uint appID, List <ManifestJob> depots)
{
Log.WriteDebug("Depot Downloader", "Will process {0} depots ({1} depot locks left)", depots.Count(), DepotLocks.Count);
var processTasks = new List <Task <EResult> >();
bool anyFilesDownloaded = false;
foreach (var depot in depots)
{
var instance = depot.Anonymous ? Steam.Anonymous.Apps : Steam.Instance.Apps;
depot.DepotKey = await GetDepotDecryptionKey(instance, depot.DepotID, appID);
if (depot.DepotKey == null)
{
RemoveLock(depot.DepotID);
continue;
}
var cdnToken = await GetCDNAuthToken(instance, appID, depot.DepotID);
if (cdnToken == null)
{
RemoveLock(depot.DepotID);
Log.WriteDebug("Depot Downloader", "Got a depot key for depot {0} but no cdn auth token", depot.DepotID);
continue;
}
depot.CDNToken = cdnToken.Token;
depot.Server = cdnToken.Server;
DepotManifest depotManifest = null;
string lastError = string.Empty;
for (var i = 0; i <= 5; i++)
{
try
{
depotManifest = CDNClient.DownloadManifest(depot.DepotID, depot.ManifestID, depot.Server, depot.CDNToken, depot.DepotKey);
break;
}
catch (Exception e)
{
Log.WriteWarn("Depot Downloader", "[{0}] Manifest download failed: {1} - {2}", depot.DepotID, e.GetType(), e.Message);
lastError = e.Message;
}
// TODO: get new auth key if auth fails
if (depotManifest == null)
{
await Task.Delay(Utils.ExponentionalBackoff(i));
}
}
if (depotManifest == null)
{
RemoveLock(depot.DepotID);
Log.WriteError("Depot Processor", "Failed to download depot manifest for app {0} depot {1} ({2}: {3})", appID, depot.DepotID, depot.Server, lastError);
if (FileDownloader.IsImportantDepot(depot.DepotID))
{
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download app {3} depot {4} manifest ({5}: {6})",
Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL, appID, depot.DepotID, depot.Server, lastError);
}
continue;
}
var task = TaskManager.Run(() =>
{
using (var db = Database.GetConnection())
{
using (var transaction = db.BeginTransaction())
{
var result = ProcessDepotAfterDownload(db, depot, depotManifest);
transaction.Commit();
return(result);
}
}
});
processTasks.Add(task);
if (FileDownloader.IsImportantDepot(depot.DepotID))
{
task = TaskManager.Run(() =>
{
var result = FileDownloader.DownloadFilesFromDepot(appID, depot, depotManifest);
if (result == EResult.OK)
{
anyFilesDownloaded = true;
}
return(result);
}, TaskCreationOptions.LongRunning);
TaskManager.RegisterErrorHandler(task);
processTasks.Add(task);
}
}
if (SaveLocalConfig)
{
SaveLocalConfig = false;
LocalConfig.Save();
}
await Task.WhenAll(processTasks);
Log.WriteDebug("Depot Downloader", "{0} depot downloads finished", depots.Count());
// TODO: use ContinueWith on tasks
if (!anyFilesDownloaded)
{
foreach (var depot in depots)
{
RemoveLock(depot.DepotID);
}
return;
}
if (!File.Exists(UpdateScript))
{
return;
}
bool lockTaken = false;
try
{
UpdateScriptLock.Enter(ref lockTaken);
foreach (var depot in depots)
{
if (depot.Result == EResult.OK)
{
RunUpdateScript(string.Format("{0} no-git", depot.DepotID));
}
RemoveLock(depot.DepotID);
}
// Only commit changes if all depots downloaded
if (processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored))
{
if (!RunUpdateScript(appID))
{
RunUpdateScript("0");
}
}
else
{
Log.WriteDebug("Depot Processor", "Reprocessing the app {0} because some files failed to download", appID);
IRC.Instance.SendOps("{0}[{1}]{2} Reprocessing the app due to download failures",
Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL
);
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
}
}
finally
{
if (lockTaken)
{
UpdateScriptLock.Exit();
}
}
}
private void OnPICSChanges(SteamApps.PICSChangesCallback callback)
{
if (PreviousChangeNumber == callback.CurrentChangeNumber)
{
return;
}
Log.WriteInfo(nameof(PICSChanges), $"Changelist {callback.LastChangeNumber} -> {callback.CurrentChangeNumber} ({callback.AppChanges.Count} apps, {callback.PackageChanges.Count} packages)");
PreviousChangeNumber = callback.CurrentChangeNumber;
TaskManager.Run(async() => await HandleChangeNumbers(callback));
if (callback.RequiresFullAppUpdate || callback.RequiresFullPackageUpdate)
{
TaskManager.Run(async() =>
{
if (callback.RequiresFullAppUpdate)
{
IRC.Instance.SendOps($"Changelist {callback.CurrentChangeNumber} has forced a full app update");
// When full update flag is set, presumably Steam client start hammering the servers
// and the PICS service just does not return any data for a while until it clears up
await FullUpdateProcessor.FullUpdateAppsMetadata(true);
}
if (callback.RequiresFullPackageUpdate)
{
IRC.Instance.SendOps($"Changelist {callback.CurrentChangeNumber} has forced a full package update");
await FullUpdateProcessor.FullUpdatePackagesMetadata();
}
IRC.Instance.SendOps($"Changelist {callback.CurrentChangeNumber} full update has finished");
});
}
if (callback.AppChanges.Count == 0 && callback.PackageChanges.Count == 0)
{
return;
}
const int appsPerJob = 50;
if (callback.AppChanges.Count > appsPerJob)
{
foreach (var list in callback.AppChanges.Keys.Split(appsPerJob))
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(list, Enumerable.Empty <uint>()),
new PICSTokens.RequestedTokens
{
Apps = list.ToList()
});
}
}
else if (callback.AppChanges.Count > 0)
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(callback.AppChanges.Keys, Enumerable.Empty <uint>()),
new PICSTokens.RequestedTokens
{
Apps = callback.AppChanges.Keys.ToList()
});
}
if (callback.PackageChanges.Count > appsPerJob)
{
foreach (var list in callback.PackageChanges.Keys.Split(appsPerJob))
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), list),
new PICSTokens.RequestedTokens
{
Packages = list.ToList()
});
}
}
else if (callback.PackageChanges.Count > 0)
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), callback.PackageChanges.Keys),
new PICSTokens.RequestedTokens
{
Packages = callback.PackageChanges.Keys.ToList()
});
}
if (callback.AppChanges.Count > 0)
{
_ = TaskManager.Run(async() => await HandleApps(callback));
}
if (callback.PackageChanges.Count > 0)
{
_ = TaskManager.Run(async() => await HandlePackages(callback));
_ = TaskManager.Run(async() => await HandlePackagesChangelists(callback));
}
if (PreviousChangeNumber - LastStoredChangeNumber >= 1000)
{
LastStoredChangeNumber = PreviousChangeNumber;
_ = TaskManager.Run(async() => await LocalConfig.Update("backend.changenumber", LastStoredChangeNumber.ToString()));
}
PrintImportants(callback);
}
private void OnTimer(object sender, ElapsedEventArgs e)
{
if (!Steam.Instance.IsLoggedOn)
{
lock (FreeLicenseTimer)
{
FreeLicenseTimer.Start();
}
return;
}
if (FreeLicensesToRequest.IsEmpty)
{
TaskManager.Run(RequestBetas);
return;
}
var list = FreeLicensesToRequest.Take(REQUEST_RATE_LIMIT).ToList();
var now = DateUtils.DateTimeToUnixTime(DateTime.UtcNow) - 60;
Dictionary <uint, ulong> startTimes;
using (var db = Database.Get())
{
startTimes = db.Query(
"SELECT `SubID`, `Value` FROM `SubsInfo` WHERE `Key` = @Key AND `SubID` IN @Ids",
new
{
Key = KeyNameCache.GetSubKeyID("extended_starttime"),
Ids = list.Select(x => x.Key)
}
).ToDictionary(x => (uint)x.SubID, x => Convert.ToUInt64((string)x.Value));
}
foreach (var(subId, _) in list)
{
if (startTimes.TryGetValue(subId, out var startTime) && startTime > now)
{
// If start time has not been reached yet, don't remove this app from the list and keep trying to activate it
continue;
}
FreeLicensesToRequest.TryRemove(subId, out _);
}
TaskManager.Run(Save);
TaskManager.Run(RequestBetas);
var appids = list.Select(x => x.Value).Distinct();
AppsRequestedInHour = appids.Count();
Log.WriteDebug(nameof(FreeLicense), $"Requesting {AppsRequestedInHour} free apps as the rate limit timer ran: {string.Join(", ", appids)}");
JobManager.AddJob(() => Steam.Instance.Apps.RequestFreeLicense(appids));
if (!FreeLicensesToRequest.IsEmpty)
{
lock (FreeLicenseTimer)
{
FreeLicenseTimer.Start();
}
}
}
public override async Task OnCommand(CommandArguments command)
{
var s = command.Message.Split(' ');
var count = s.Length;
if (count > 0)
{
uint id;
switch (s[0])
{
case "reload":
await Application.ReloadImportant();
await PICSTokens.Reload();
command.Notice("Reloaded important apps and pics tokens");
return;
case "fullrun":
_ = TaskManager.Run(async() =>
{
command.Reply("Started full metadata scan, this will take a while…");
await FullUpdateProcessor.FullUpdateAppsMetadata(true);
command.Reply("App full scan finished, starting packages, this will take even longer…");
await FullUpdateProcessor.FullUpdatePackagesMetadata();
command.Reply("Full metadata scan finished.");
});
return;
case "add":
if (count < 3)
{
break;
}
if (!uint.TryParse(s[2], out id))
{
break;
}
switch (s[1])
{
case "app":
if (Application.ImportantApps.Contains(id))
{
command.Reply($"App {Colors.BLUE}{id}{Colors.NORMAL} ({Steam.GetAppName(id)}) is already important.");
}
else
{
Application.ImportantApps.Add(id);
await using (var db = await Database.GetConnectionAsync())
{
await db.ExecuteAsync("INSERT INTO `ImportantApps` (`AppID`) VALUES (@AppID)", new { AppID = id });
}
command.Reply($"Marked app {Colors.BLUE}{id}{Colors.NORMAL} ({Steam.GetAppName(id)}) as important.");
}
return;
case "sub":
if (Application.ImportantSubs.Contains(id))
{
command.Reply($"Package {Colors.BLUE}{id}{Colors.NORMAL} ({Steam.GetPackageName(id)}) is already important.");
}
else
{
Application.ImportantSubs.Add(id);
await using (var db = await Database.GetConnectionAsync())
{
await db.ExecuteAsync("INSERT INTO `ImportantSubs` (`SubID`) VALUES (@SubID)", new { SubID = id });
}
command.Reply($"Marked package {Colors.BLUE}{id}{Colors.NORMAL} ({Steam.GetPackageName(id)}) as important.");
}
return;
}
break;
case "remove":
if (count < 3)
{
break;
}
if (!uint.TryParse(s[2], out id))
{
break;
}
switch (s[1])
{
case "app":
if (!Application.ImportantApps.Contains(id))
{
command.Reply($"App {Colors.BLUE}{id}{Colors.NORMAL} ({Steam.GetAppName(id)}) is not important.");
}
else
{
Application.ImportantApps.Remove(id);
await using (var db = await Database.GetConnectionAsync())
{
await db.ExecuteAsync("DELETE FROM `ImportantApps` WHERE `AppID` = @AppID", new { AppID = id });
}
command.Reply($"Removed app {Colors.BLUE}{id}{Colors.NORMAL} ({Steam.GetAppName(id)}) from the important list.");
}
return;
case "sub":
if (!Application.ImportantSubs.Contains(id))
{
command.Reply($"Package {Colors.BLUE}{id}{Colors.NORMAL} ({Steam.GetPackageName(id)}) is not important.");
}
else
{
Application.ImportantSubs.Remove(id);
await using (var db = await Database.GetConnectionAsync())
{
await db.ExecuteAsync("DELETE FROM `ImportantSubs` WHERE `SubID` = @SubID", new { SubID = id });
}
command.Reply($"Removed package {Colors.BLUE}{id}{Colors.NORMAL} ({Steam.GetPackageName(id)}) from the important list.");
}
return;
}
break;
}
}
command.Reply($"Usage:{Colors.OLIVE} important reload {Colors.NORMAL}or{Colors.OLIVE} important <add/remove> <app/sub> <id> {Colors.NORMAL}or{Colors.OLIVE} important fullrun");
}
/*
* Here be dragons.
*/
public static async Task <EResult> DownloadFilesFromDepot(DepotProcessor.ManifestJob job, DepotManifest depotManifest)
{
var filesRegex = Files[job.DepotID];
var files = depotManifest.Files.Where(x => filesRegex.IsMatch(x.FileName.Replace('\\', '/'))).ToList();
var downloadState = EResult.Fail;
ConcurrentDictionary <string, ExistingFileData> existingFileData;
await using (var db = await Database.GetConnectionAsync())
{
var data = db.ExecuteScalar <string>("SELECT `Value` FROM `LocalConfig` WHERE `ConfigKey` = @Key", new { Key = $"depot.{job.DepotID}" });
if (data != null)
{
existingFileData = JsonConvert.DeserializeObject <ConcurrentDictionary <string, ExistingFileData> >(data);
}
else
{
existingFileData = new ConcurrentDictionary <string, ExistingFileData>();
}
}
foreach (var file in existingFileData.Keys.Except(files.Select(x => x.FileName)))
{
Log.WriteWarn(nameof(FileDownloader), $"\"{file}\" no longer exists in manifest");
}
Log.WriteInfo($"FileDownloader {job.DepotID}", $"Will download {files.Count} files");
var downloadedFiles = 0;
var fileTasks = new Task[files.Count];
for (var i = 0; i < fileTasks.Length; i++)
{
var file = files[i];
fileTasks[i] = TaskManager.Run(async() =>
{
var existingFile = existingFileData.GetOrAdd(file.FileName, _ => new ExistingFileData());
EResult fileState;
try
{
await ChunkDownloadingSemaphore.WaitAsync().ConfigureAwait(false);
fileState = await DownloadFile(job, file, existingFile);
}
finally
{
ChunkDownloadingSemaphore.Release();
}
if (fileState == EResult.OK || fileState == EResult.SameAsPreviousValue)
{
existingFile.FileHash = file.FileHash;
downloadedFiles++;
}
if (fileState != EResult.SameAsPreviousValue)
{
// Do not write progress info to log file
Console.WriteLine($"{job.DepotName} [{downloadedFiles / (float) files.Count * 100.0f,6:#00.00}%] {files.Count - downloadedFiles} files left to download");
}
if (downloadState == EResult.DataCorruption)
{
return;
}
if (fileState == EResult.OK || fileState == EResult.DataCorruption)
{
downloadState = fileState;
}
});
}
await Task.WhenAll(fileTasks).ConfigureAwait(false);
await LocalConfig.Update($"depot.{job.DepotID}", JsonConvert.SerializeObject(existingFileData));
job.Result = downloadState switch
{
EResult.OK => EResult.OK,
EResult.DataCorruption => EResult.DataCorruption,
_ => EResult.Ignored
};
return(job.Result);
}
private static async Task <EResult> DownloadFile(DepotProcessor.ManifestJob job, DepotManifest.FileData file, ExistingFileData existingFile)
{
var directory = Path.Combine(Application.Path, "files", DownloadFolders[job.DepotID], Path.GetDirectoryName(file.FileName));
var finalPath = new FileInfo(Path.Combine(directory, Path.GetFileName(file.FileName)));
var downloadPath = new FileInfo(Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".steamdb_tmp")));
if (!Directory.Exists(directory))
{
Directory.CreateDirectory(directory);
}
else if (file.TotalSize == 0)
{
if (!finalPath.Exists)
{
await using (var _ = finalPath.Create())
{
// FileInfo.Create returns a stream but we don't need it
}
Log.WriteInfo($"FileDownloader {job.DepotID}", $"{file.FileName} created an empty file");
return(EResult.SameAsPreviousValue);
}
else if (finalPath.Length == 0)
{
#if DEBUG
Log.WriteDebug($"FileDownloader {job.DepotID}", $"{file.FileName} is already empty");
#endif
return(EResult.SameAsPreviousValue);
}
}
else if (existingFile.FileHash != null && file.FileHash.SequenceEqual(existingFile.FileHash))
{
#if DEBUG
Log.WriteDebug($"FileDownloader {job.DepotID}", $"{file.FileName} already matches the file we have");
#endif
return(EResult.SameAsPreviousValue);
}
using var sha = SHA1.Create();
var neededChunks = new List <DepotManifest.ChunkData>();
var chunks = file.Chunks.OrderBy(x => x.Offset).ToList();
await using (var fs = downloadPath.Open(FileMode.OpenOrCreate, FileAccess.ReadWrite))
{
fs.SetLength((long)file.TotalSize);
if (finalPath.Exists)
{
await using var fsOld = finalPath.Open(FileMode.Open, FileAccess.Read);
foreach (var chunk in chunks)
{
var oldChunk = existingFile.Chunks.FirstOrDefault(c => c.Value.SequenceEqual(chunk.ChunkID));
if (oldChunk.Value != null)
{
var oldData = new byte[chunk.UncompressedLength];
fsOld.Seek((long)oldChunk.Key, SeekOrigin.Begin);
fsOld.Read(oldData, 0, oldData.Length);
var existingChecksum = sha.ComputeHash(oldData);
if (existingChecksum.SequenceEqual(chunk.ChunkID))
{
fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
fs.Write(oldData, 0, oldData.Length);
#if DEBUG
Log.WriteDebug($"FileDownloader {job.DepotID}", $"{file.FileName} Found chunk ({chunk.Offset}), not downloading");
#endif
}
else
{
neededChunks.Add(chunk);
#if DEBUG
Log.WriteDebug($"FileDownloader {job.DepotID}", $"{file.FileName} Found chunk ({chunk.Offset}), but checksum differs");
#endif
}
}
else
{
neededChunks.Add(chunk);
}
}
}
else
{
neededChunks = chunks;
}
}
using var chunkCancellation = new CancellationTokenSource();
var downloadedSize = file.TotalSize - (ulong)neededChunks.Sum(x => x.UncompressedLength);
var chunkTasks = new Task[neededChunks.Count];
Log.WriteInfo($"FileDownloader {job.DepotID}", $"Downloading {file.FileName} ({neededChunks.Count} out of {chunks.Count} chunks to download)");
for (var i = 0; i < chunkTasks.Length; i++)
{
var chunk = neededChunks[i];
chunkTasks[i] = TaskManager.Run(async() =>
{
var result = await DownloadChunk(job, chunk, downloadPath, chunkCancellation);
if (!result)
{
Log.WriteWarn($"FileDownloader {job.DepotID}", $"Failed to download chunk for {file.FileName} ({chunk.Offset})");
chunkCancellation.Cancel();
}
else
{
downloadedSize += chunk.UncompressedLength;
// Do not write progress info to log file
Console.WriteLine($"{job.DepotName} [{downloadedSize / (float) file.TotalSize * 100.0f,6:#00.00}%] {file.FileName}");
}
});
}
await Task.WhenAll(chunkTasks).ConfigureAwait(false);
byte[] checksum;
await using (var fs = downloadPath.Open(FileMode.Open, FileAccess.ReadWrite))
{
checksum = sha.ComputeHash(fs);
}
if (!file.FileHash.SequenceEqual(checksum))
{
if (!job.DownloadCorrupted)
{
job.DownloadCorrupted = true;
IRC.Instance.SendOps($"{Colors.OLIVE}[{job.DepotName}]{Colors.NORMAL} Failed to correctly download {Colors.BLUE}{file.FileName}");
}
Log.WriteWarn($"FileDownloader {job.DepotID}", $"Hash check failed for {file.FileName} ({job.Server})");
downloadPath.Delete();
existingFile.FileHash = null;
existingFile.Chunks.Clear();
return(EResult.DataCorruption);
}
Log.WriteInfo($"FileDownloader {job.DepotID}", $"Downloaded {file.FileName}");
finalPath.Delete();
downloadPath.MoveTo(finalPath.FullName);
if (chunks.Count > 0)
{
existingFile.Chunks = chunks.ToDictionary(chunk => chunk.Offset, chunk => chunk.ChunkID);
}
else
{
existingFile.Chunks.Clear();
}
return(EResult.OK);
}
public void StartTick()
{
TaskManager.Run(Tick);
}
private void OnLoggedOn(SteamUser.LoggedOnCallback callback)
{
TaskManager.Run(UpdateContentServerList);
}
public async Task Process(uint appID, uint changeNumber, KeyValue depots)
{
var requests = new List <ManifestJob>();
// Get data in format we want first
foreach (var depot in depots.Children)
{
// Ignore these for now, parent app should be updated too anyway
if (depot["depotfromapp"].Value != null)
{
continue;
}
var request = new ManifestJob
{
ChangeNumber = changeNumber,
};
// Ignore keys that aren't integers, for example "branches"
if (!uint.TryParse(depot.Name, out request.DepotID))
{
continue;
}
request.DepotName = depot["name"].AsString();
if (string.IsNullOrEmpty(request.DepotName))
{
request.DepotName = $"SteamDB Unnamed Depot {request.DepotID}";
}
// TODO: instead of locking we could wait for current process to finish
if (DepotLocks.ContainsKey(request.DepotID))
{
continue;
}
// SteamVR trickery
if (appID == 250820 &&
depot["manifests"]["beta"].Value != null &&
depots["branches"]["beta"]["buildid"].AsInteger() > depots["branches"]["public"]["buildid"].AsInteger())
{
request.BuildID = depots["branches"]["beta"]["buildid"].AsInteger();
request.ManifestID = ulong.Parse(depot["manifests"]["beta"].Value);
}
else
if (depot["manifests"]["public"].Value == null || !ulong.TryParse(depot["manifests"]["public"].Value, out request.ManifestID))
{
var branch = depot["manifests"].Children.Find(x => x.Name != "local");
if (branch == null || !ulong.TryParse(branch.Value, out request.ManifestID))
{
await using var db = await Database.GetConnectionAsync();
await db.ExecuteAsync("INSERT INTO `Depots` (`DepotID`, `Name`) VALUES (@DepotID, @DepotName) ON DUPLICATE KEY UPDATE `DepotID` = VALUES(`DepotID`)", new { request.DepotID, request.DepotName });
continue;
}
Log.WriteDebug(nameof(DepotProcessor), $"Depot {request.DepotID} (from {appID}) has no public branch, but there is another one");
request.BuildID = depots["branches"][branch.Name]["buildid"].AsInteger();
}
else
{
request.BuildID = depots["branches"]["public"]["buildid"].AsInteger();
}
requests.Add(request);
}
if (requests.Count == 0)
{
return;
}
var depotsToDownload = new List <ManifestJob>();
await using (var db = await Database.GetConnectionAsync())
{
await db.ExecuteAsync("INSERT INTO `Builds` (`BuildID`, `ChangeID`, `AppID`) VALUES (@BuildID, @ChangeNumber, @AppID) ON DUPLICATE KEY UPDATE `AppID` = VALUES(`AppID`)",
new
{
requests[0].BuildID,
requests[0].ChangeNumber,
appID
});
var depotIds = requests.Select(x => x.DepotID).ToList();
var dbDepots = (await db.QueryAsync <Depot>("SELECT `DepotID`, `Name`, `BuildID`, `ManifestID`, `LastManifestID`, `FilenamesEncrypted` FROM `Depots` WHERE `DepotID` IN @depotIds", new { depotIds }))
.ToDictionary(x => x.DepotID, x => x);
var decryptionKeys = (await db.QueryAsync <DepotKey>("SELECT `DepotID`, `Key` FROM `DepotsKeys` WHERE `DepotID` IN @depotIds", new { depotIds }))
.ToDictionary(x => x.DepotID, x => Utils.StringToByteArray(x.Key));
foreach (var request in requests)
{
Depot dbDepot;
decryptionKeys.TryGetValue(request.DepotID, out request.DepotKey);
if (dbDepots.ContainsKey(request.DepotID))
{
dbDepot = dbDepots[request.DepotID];
if (dbDepot.BuildID > request.BuildID)
{
// buildid went back in time? this either means a rollback, or a shared depot that isn't synced properly
Log.WriteDebug(nameof(DepotProcessor), $"Skipping depot {request.DepotID} due to old buildid: {dbDepot.BuildID} > {request.BuildID}");
continue;
}
if (dbDepot.LastManifestID == request.ManifestID &&
dbDepot.ManifestID == request.ManifestID &&
Settings.Current.FullRun != FullRunState.WithForcedDepots &&
!dbDepot.FilenamesEncrypted && request.DepotKey != null)
{
// Update depot name if changed
if (request.DepotName != dbDepot.Name)
{
await db.ExecuteAsync("UPDATE `Depots` SET `Name` = @DepotName WHERE `DepotID` = @DepotID", new { request.DepotID, request.DepotName });
}
continue;
}
request.StoredFilenamesEncrypted = dbDepot.FilenamesEncrypted;
request.LastManifestID = dbDepot.LastManifestID;
}
else
{
dbDepot = new Depot();
}
if (dbDepot.BuildID != request.BuildID || dbDepot.ManifestID != request.ManifestID || request.DepotName != dbDepot.Name)
{
await db.ExecuteAsync(@"INSERT INTO `Depots` (`DepotID`, `Name`, `BuildID`, `ManifestID`) VALUES (@DepotID, @DepotName, @BuildID, @ManifestID)
ON DUPLICATE KEY UPDATE `LastUpdated` = CURRENT_TIMESTAMP(), `Name` = VALUES(`Name`), `BuildID` = VALUES(`BuildID`), `ManifestID` = VALUES(`ManifestID`)",
new
{
request.DepotID,
request.DepotName,
request.BuildID,
request.ManifestID
});
}
if (dbDepot.ManifestID != request.ManifestID)
{
await MakeHistory(db, null, request, string.Empty, "manifest_change", dbDepot.ManifestID, request.ManifestID);
}
lock (DepotLocks)
{
// This doesn't really save us from concurrency issues
if (DepotLocks.ContainsKey(request.DepotID))
{
Log.WriteWarn(nameof(DepotProcessor), $"Depot {request.DepotID} was locked in another thread");
continue;
}
DepotLocks.Add(request.DepotID, 1);
}
depotsToDownload.Add(request);
}
}
if (depotsToDownload.Count > 0)
{
_ = TaskManager.Run(async() =>
{
try
{
await DownloadDepots(appID, depotsToDownload);
}
catch (Exception e)
{
ErrorReporter.Notify(nameof(DepotProcessor), e);
}
foreach (var depot in depotsToDownload)
{
RemoveLock(depot.DepotID);
}
});
}
}
private static void OnPICSProductInfo(SteamApps.PICSProductInfoCallback callback)
{
JobManager.TryRemoveJob(callback.JobID);
var apps = callback.Apps.Concat(callback.UnknownApps.ToDictionary(x => x, x => (SteamApps.PICSProductInfoCallback.PICSProductInfo)null));
var packages = callback.Packages.Concat(callback.UnknownPackages.ToDictionary(x => x, x => (SteamApps.PICSProductInfoCallback.PICSProductInfo)null));
foreach (var workaround in apps)
{
var app = workaround;
Log.WriteInfo("PICSProductInfo", "{0}AppID: {1}", app.Value == null ? "Unknown " : "", app.Key);
Task mostRecentItem;
lock (ProcessedApps)
{
ProcessedApps.TryGetValue(app.Key, out mostRecentItem);
}
var workerItem = TaskManager.Run(async() =>
{
try
{
await ProcessorSemaphore.WaitAsync().ConfigureAwait(false);
if (mostRecentItem != null && !mostRecentItem.IsCompleted)
{
Log.WriteDebug("PICSProductInfo", "Waiting for app {0} to finish processing", app.Key);
await mostRecentItem.ConfigureAwait(false);
}
using (var processor = new AppProcessor(app.Key))
{
if (app.Value == null)
{
processor.ProcessUnknown();
}
else
{
processor.Process(app.Value);
}
}
}
catch (MySqlException e)
{
ErrorReporter.Notify($"App {app.Key}", e);
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(app.Key, null));
}
catch (Exception e)
{
ErrorReporter.Notify($"App {app.Key}", e);
}
finally
{
lock (ProcessedApps)
{
if (ProcessedApps.TryGetValue(app.Key, out mostRecentItem) && mostRecentItem.IsCompleted)
{
ProcessedApps.Remove(app.Key);
}
}
ProcessorSemaphore.Release();
}
});
if (Settings.IsFullRun)
{
continue;
}
lock (ProcessedApps)
{
ProcessedApps[app.Key] = workerItem;
}
}
foreach (var workaround in packages)
{
var package = workaround;
Log.WriteInfo("PICSProductInfo", "{0}SubID: {1}", package.Value == null ? "Unknown " : "", package.Key);
Task mostRecentItem;
lock (ProcessedSubs)
{
ProcessedSubs.TryGetValue(package.Key, out mostRecentItem);
}
var workerItem = TaskManager.Run(async() =>
{
try
{
await ProcessorSemaphore.WaitAsync().ConfigureAwait(false);
if (mostRecentItem != null && !mostRecentItem.IsCompleted)
{
Log.WriteDebug("PICSProductInfo", "Waiting for package {0} to finish processing", package.Key);
await mostRecentItem.ConfigureAwait(false);
}
using (var processor = new SubProcessor(package.Key))
{
if (package.Value == null)
{
processor.ProcessUnknown();
}
else
{
processor.Process(package.Value);
}
}
}
catch (MySqlException e)
{
ErrorReporter.Notify($"Package {package.Key}", e);
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetProductInfo(null, package.Key, false, false));
}
catch (Exception e)
{
ErrorReporter.Notify($"Package {package.Key}", e);
}
finally
{
lock (ProcessedSubs)
{
if (ProcessedSubs.TryGetValue(package.Key, out mostRecentItem) && mostRecentItem.IsCompleted)
{
ProcessedSubs.Remove(package.Key);
}
}
ProcessorSemaphore.Release();
}
});
if (Settings.IsFullRun)
{
continue;
}
lock (ProcessedSubs)
{
ProcessedSubs[package.Key] = workerItem;
}
}
}
private void DownloadManifest(ManifestJob request)
{
Log.WriteInfo("Depot Processor", "DepotID: {0}", request.DepotID);
DepotManifest depotManifest = null;
string lastError = string.Empty;
// CDN is very random, just keep trying
for (var i = 0; i <= 5; i++)
{
try
{
depotManifest = CDNClient.DownloadManifest(request.DepotID, request.ManifestID, request.Server, request.CDNToken, request.DepotKey);
break;
}
catch (Exception e)
{
lastError = e.Message;
}
}
if (depotManifest == null)
{
Log.WriteError("Depot Processor", "Failed to download depot manifest for depot {0} ({1}: {2}) (#{3})", request.DepotID, request.Server, lastError, request.Tries);
if (--request.Tries >= 0)
{
request.Server = GetContentServer(request.Tries);
JobManager.AddJob(() => Steam.Instance.Apps.GetCDNAuthToken(request.DepotID, request.Server), request);
return;
}
RemoveLock(request.DepotID); // TODO: Remove this once task in OnCDNAuthTokenCallback is used
if (FileDownloader.IsImportantDepot(request.DepotID))
{
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download depot {3} manifest ({4}: {5})",
Colors.OLIVE, Steam.GetAppName(request.ParentAppID), Colors.NORMAL, request.DepotID, request.Server, lastError);
}
return;
}
if (FileDownloader.IsImportantDepot(request.DepotID))
{
TaskManager.Run(() => FileDownloader.DownloadFilesFromDepot(request, depotManifest));
}
// TODO: Task here instead of in OnCDNAuthTokenCallback due to mono's silly threadpool
TaskManager.Run(() =>
{
using (var db = Database.GetConnection())
{
ProcessDepotAfterDownload(db, request, depotManifest);
}
}).ContinueWith(task =>
{
RemoveLock(request.DepotID);
Log.WriteDebug("Depot Processor", "Processed depot {0} ({1} depot locks left)", request.DepotID, DepotLocks.Count);
});
}
private static void OnPICSProductInfo(SteamApps.PICSProductInfoCallback callback)
{
JobManager.TryRemoveJob(callback.JobID);
var processors = new List <BaseProcessor>(
callback.Apps.Count +
callback.Packages.Count +
callback.UnknownApps.Count +
callback.UnknownPackages.Count
);
processors.AddRange(callback.Apps.Select(app => new AppProcessor(app.Key, app.Value)));
processors.AddRange(callback.Packages.Select(package => new SubProcessor(package.Key, package.Value)));
processors.AddRange(callback.UnknownApps.Select(app => new AppProcessor(app, null)));
processors.AddRange(callback.UnknownPackages.Select(package => new SubProcessor(package, null)));
foreach (var workaround in processors)
{
var processor = workaround;
Task mostRecentItem;
lock (CurrentlyProcessing)
{
CurrentlyProcessing.TryGetValue(processor.Id, out mostRecentItem);
}
var workerItem = TaskManager.Run(async() =>
{
try
{
await Semaphore.WaitAsync(TaskManager.TaskCancellationToken.Token).ConfigureAwait(false);
if (mostRecentItem?.IsCompleted == false)
{
Log.WriteDebug(processor.ToString(), $"Waiting for previous task to finish processing ({CurrentlyProcessing.Count})");
await mostRecentItem.ConfigureAwait(false);
#if DEBUG
Log.WriteDebug(processor.ToString(), "Previous task lock ended");
#endif
}
await processor.Process().ConfigureAwait(false);
}
catch (Exception e)
{
ErrorReporter.Notify(processor.ToString(), e);
}
finally
{
Semaphore.Release();
processor.Dispose();
}
return(processor);
}).Unwrap();
lock (CurrentlyProcessing)
{
CurrentlyProcessing[processor.Id] = workerItem;
}
// Register error handler on inner task and the continuation
TaskManager.RegisterErrorHandler(workerItem);
TaskManager.RegisterErrorHandler(workerItem.ContinueWith(RemoveProcessorLock, TaskManager.TaskCancellationToken.Token));
}
}
public async Task Process(uint appID, uint changeNumber, KeyValue depots)
{
var requests = new List <ManifestJob>();
// Get data in format we want first
foreach (var depot in depots.Children)
{
// Ignore these for now, parent app should be updated too anyway
if (depot["depotfromapp"].Value != null)
{
continue;
}
var request = new ManifestJob
{
ChangeNumber = changeNumber,
DepotName = depot["name"].AsString()
};
// Ignore keys that aren't integers, for example "branches"
if (!uint.TryParse(depot.Name, out request.DepotID))
{
continue;
}
// TODO: instead of locking we could wait for current process to finish
if (DepotLocks.ContainsKey(request.DepotID))
{
continue;
}
if (depot["manifests"]["public"].Value == null || !ulong.TryParse(depot["manifests"]["public"].Value, out request.ManifestID))
{
var branch = depot["manifests"].Children.Find(x => x.Name != "local");
if (branch == null || !ulong.TryParse(branch.Value, out request.ManifestID))
{
using (var db = Database.Get())
{
await db.ExecuteAsync("INSERT INTO `Depots` (`DepotID`, `Name`) VALUES (@DepotID, @DepotName) ON DUPLICATE KEY UPDATE `DepotID` = VALUES(`DepotID`)", new { request.DepotID, request.DepotName });
}
continue;
}
Log.WriteDebug("Depot Downloader", "Depot {0} (from {1}) has no public branch, but there is another one", request.DepotID, appID);
request.BuildID = depots["branches"][branch.Name]["buildid"].AsInteger();
}
else
{
request.BuildID = depots["branches"]["public"]["buildid"].AsInteger();
}
requests.Add(request);
}
if (requests.Count == 0)
{
return;
}
var depotsToDownload = new List <ManifestJob>();
using (var db = await Database.GetConnectionAsync())
{
await db.ExecuteAsync("INSERT INTO `Builds` (`BuildID`, `ChangeID`, `AppID`) VALUES (@BuildID, @ChangeNumber, @AppID) ON DUPLICATE KEY UPDATE `AppID` = VALUES(`AppID`)",
new {
requests[0].BuildID,
requests[0].ChangeNumber,
appID
});
var dbDepots = (await db.QueryAsync <Depot>("SELECT `DepotID`, `Name`, `BuildID`, `ManifestID`, `LastManifestID` FROM `Depots` WHERE `DepotID` IN @Depots", new { Depots = requests.Select(x => x.DepotID) }))
.ToDictionary(x => x.DepotID, x => x);
foreach (var request in requests)
{
Depot dbDepot;
if (dbDepots.ContainsKey(request.DepotID))
{
dbDepot = dbDepots[request.DepotID];
if (dbDepot.BuildID > request.BuildID)
{
// buildid went back in time? this either means a rollback, or a shared depot that isn't synced properly
Log.WriteDebug("Depot Processor", "Skipping depot {0} due to old buildid: {1} > {2}", request.DepotID, dbDepot.BuildID, request.BuildID);
continue;
}
if (dbDepot.LastManifestID == request.ManifestID && dbDepot.ManifestID == request.ManifestID && Settings.Current.FullRun != FullRunState.WithForcedDepots)
{
// Update depot name if changed
if (!request.DepotName.Equals(dbDepot.Name))
{
await db.ExecuteAsync("UPDATE `Depots` SET `Name` = @DepotName WHERE `DepotID` = @DepotID", new { request.DepotID, request.DepotName });
}
continue;
}
}
else
{
dbDepot = new Depot();
}
if (dbDepot.BuildID != request.BuildID || dbDepot.ManifestID != request.ManifestID || !request.DepotName.Equals(dbDepot.Name))
{
await db.ExecuteAsync(@"INSERT INTO `Depots` (`DepotID`, `Name`, `BuildID`, `ManifestID`) VALUES (@DepotID, @DepotName, @BuildID, @ManifestID)
ON DUPLICATE KEY UPDATE `LastUpdated` = CURRENT_TIMESTAMP(), `Name` = VALUES(`Name`), `BuildID` = VALUES(`BuildID`), `ManifestID` = VALUES(`ManifestID`)",
new {
request.DepotID,
request.DepotName,
request.BuildID,
request.ManifestID
});
}
if (dbDepot.ManifestID != request.ManifestID)
{
await MakeHistory(db, null, request, string.Empty, "manifest_change", dbDepot.ManifestID, request.ManifestID);
}
var owned = LicenseList.OwnedApps.ContainsKey(request.DepotID);
if (!owned)
{
request.Anonymous = owned = LicenseList.AnonymousApps.ContainsKey(request.DepotID);
if (owned)
{
Log.WriteWarn("Depot Processor", "Will download depot {0} using anonymous account", request.DepotID);
}
}
if (owned)
{
lock (DepotLocks)
{
// This doesn't really save us from concurrency issues
if (DepotLocks.ContainsKey(request.DepotID))
{
Log.WriteWarn("Depot Processor", "Depot {0} was locked in another thread", request.DepotID);
continue;
}
DepotLocks.Add(request.DepotID, 1);
}
depotsToDownload.Add(request);
}
}
}
if (depotsToDownload.Count > 0)
{
_ = TaskManager.Run(async() =>
{
try
{
await DownloadDepots(appID, depotsToDownload);
}
catch (Exception e)
{
ErrorReporter.Notify("Depot Processor", e);
}
foreach (var depot in depotsToDownload)
{
RemoveLock(depot.DepotID);
}
});
}
}
private static async Task <DateTime> ProcessFeed(Uri feedUrl, DateTime lastPostDate)
{
var feed = await LoadRSS(feedUrl);
if (feed == null)
{
return(DateTime.MinValue);
}
if (feed.Items.Count == 0)
{
Log.WriteError(nameof(RSS), $"Did not find any items in {feedUrl}");
return(DateTime.MinValue);
}
await using var db = await Database.GetConnectionAsync();
var items = (await db.QueryAsync <GenericFeedItem>("SELECT `Link` FROM `RSS` WHERE `Link` IN @Ids", new { Ids = feed.Items.Select(x => x.Link) })).ToDictionary(x => x.Link, _ => (byte)1);
var newItems = feed.Items.Where(item => item.PubDate > lastPostDate && !items.ContainsKey(item.Link));
var maxDate = DateTime.MinValue;
foreach (var item in newItems)
{
if (maxDate < item.PubDate)
{
maxDate = item.PubDate;
}
Log.WriteInfo(nameof(RSS), $"[{feed.Title}] {item.Title}: {item.Link} ({item.PubDate})");
IRC.Instance.SendAnnounce($"{Colors.BLUE}{feed.Title}{Colors.NORMAL}: {item.Title} -{Colors.DARKBLUE} {item.Link}");
await db.ExecuteAsync("INSERT INTO `RSS` (`Link`, `Title`, `Date`) VALUES(@Link, @Title, @PubDate)", new
{
item.Link,
item.Title,
item.PubDate,
});
_ = TaskManager.Run(async() => await Utils.SendWebhook(new
{
Type = "RSS",
item.Title,
Url = item.Link,
}));
if (!Settings.IsMillhaven)
{
continue;
}
uint appID = 0;
if (feed.Title == "Steam RSS News Feed")
{
if (item.Title.StartsWith("Dota 2 Update", StringComparison.Ordinal))
{
appID = 570;
}
else if (item.Title == "Team Fortress 2 Update Released")
{
appID = 440;
// tf2 changelog cleanup
item.Content = item.Content.Replace("<br/>", "\n");
item.Content = item.Content.Replace("<ul style=\"padding-bottom: 0px; margin-bottom: 0px;\">", "\n");
item.Content = item.Content.Replace("<ul style=\"padding-bottom: 0px; margin-bottom: 0px;\" >", "\n");
item.Content = item.Content.Replace("</ul>", "\n");
item.Content = item.Content.Replace("<li>", "* ");
}
else if (item.Title == "Left 4 Dead 2 - Update")
{
appID = 550;
}
else if (item.Title == "Left 4 Dead - Update")
{
appID = 500;
}
else if (item.Title == "Portal 2 - Update")
{
appID = 620;
}
}
else if (feed.Title.Contains("Counter-Strike: Global Offensive") && item.Title.StartsWith("Release Notes", StringComparison.Ordinal))
{
appID = 730;
// csgo changelog cleanup
item.Content = item.Content.Replace("</p>", "\n");
item.Content = new Regex("<p>\\[\\s*(.+)\\s*\\]", RegexOptions.Multiline | RegexOptions.CultureInvariant).Replace(item.Content, "## $1");
item.Content = item.Content.Replace("<p>", "");
}
if (appID > 0)
{
var build = (await db.QueryAsync <Build>(
"SELECT `Builds`.`BuildID`, `Builds`.`ChangeID`, `Builds`.`AppID`, `Changelists`.`Date`, LENGTH(`Official`) as `Official` FROM `Builds` " +
"LEFT JOIN `Patchnotes` ON `Patchnotes`.`BuildID` = `Builds`.`BuildID` " +
"JOIN `Apps` ON `Apps`.`AppID` = `Builds`.`AppID` " +
"JOIN `Changelists` ON `Builds`.`ChangeID` = `Changelists`.`ChangeID` " +
"WHERE `Builds`.`AppID` = @AppID ORDER BY `Builds`.`BuildID` DESC LIMIT 1",
new { appID }
)).SingleOrDefault();
if (build == null)
{
continue;
}
if (DateTime.UtcNow > build.Date.AddMinutes(60))
{
Log.WriteDebug(nameof(RSS), $"Got {appID} update patch notes, but there is no build within last 10 minutes. {item.Link}");
IRC.Instance.SendOps($"{Colors.GREEN}[Patch notes]{Colors.NORMAL} Got {appID} update patch notes, but there is no build within last 10 minutes. {item.Link}");
continue;
}
if (build.Official > 0)
{
Log.WriteDebug(nameof(RSS), $"Got {appID} update patch notes, but official patch notes is already filled. {item.Link}");
IRC.Instance.SendOps($"{Colors.GREEN}[Patch notes]{Colors.NORMAL} Got {appID} update patch notes, but official patch notes is already filled. {item.Link}");
continue;
}
// breaks
item.Content = new Regex(@"<br( \/)?>\r?\n?", RegexOptions.Multiline | RegexOptions.CultureInvariant).Replace(item.Content, "\n");
// dashes (CS:GO mainly)
item.Content = new Regex("^(?:-|&#(?:8208|8209|8210|8211|8212|8213);|–|—) ?", RegexOptions.Multiline | RegexOptions.CultureInvariant).Replace(item.Content, "* ");
item.Content = WebUtility.HtmlDecode(item.Content);
Log.WriteDebug(nameof(RSS), $"Inserting {build.AppID} patchnotes for build {build.BuildID}:\n{item.Content}");
var accountId = Steam.Instance.Client?.SteamID?.AccountID ?? 0;
await db.ExecuteAsync(
"INSERT INTO `Patchnotes` (`BuildID`, `AppID`, `ChangeID`, `Date`, `Official`, `OfficialURL`) " +
"VALUES (@BuildID, @AppID, @ChangeID, @Date, @Content, @Link) ON DUPLICATE KEY UPDATE `Official` = VALUES(`Official`), `OfficialURL` = VALUES(`OfficialURL`), `LastEditor` = @AccountID",
new
{
build.BuildID,
build.AppID,
build.ChangeID,
Date = build.Date.AddSeconds(1).ToString("yyyy-MM-dd HH:mm:ss"),
item.Content,
item.Link,
accountId
}
);
IRC.Instance.SendAnnounce($"\u2699 Official patch notes:{Colors.BLUE} {Steam.GetAppName(build.AppID)}{Colors.NORMAL} -{Colors.DARKBLUE} {SteamDB.GetPatchnotesUrl(build.BuildID)}");
}
}
return(maxDate);
}