private static void HandleClientRequestFreeLicenseResponse(IPacketMsg packetMsg) { var resp = new ClientMsgProtobuf <CMsgClientRequestFreeLicenseResponse>(packetMsg); JobAction job; JobManager.TryRemoveJob(packetMsg.TargetJobID, out job); var packageIDs = resp.Body.granted_packageids; var appIDs = resp.Body.granted_appids; Log.WriteDebug("FreeLicense", "Received free license: {0} ({1} apps, {2} packages)", (EResult)resp.Body.eresult, appIDs.Count, packageIDs.Count); if (appIDs.Count > 0) { JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appIDs, Enumerable.Empty <uint>())); } if (packageIDs.Count > 0) { JobManager.AddJob(() => Steam.Instance.Apps.PICSGetProductInfo(Enumerable.Empty <uint>(), packageIDs)); // TODO: We could re-queue apps in these packages as well // We don't want to block our main thread with web requests TaskManager.Run(() => { string data = null; try { var response = WebAuth.PerformRequest("GET", "https://store.steampowered.com/account/"); using (var responseStream = response.GetResponseStream()) { using (var reader = new StreamReader(responseStream)) { data = reader.ReadToEnd(); } } } catch (WebException e) { Log.WriteError("FreeLicense", "Failed to fetch account details page: {0}", e.Message); } foreach (var package in packageIDs) { Package packageData; using (var db = Database.GetConnection()) { packageData = db.Query <Package>("SELECT `SubID`, `Name`, `LastKnownName` FROM `Subs` WHERE `SubID` = @SubID", new { SubID = package }).FirstOrDefault(); } if (!string.IsNullOrEmpty(data)) { // Tell me all about using regex var match = Regex.Match(data, string.Format("RemoveFreeLicense\\( ?{0}, ?'(.+)' ?\\)", package)); if (match.Success) { var grantedName = Encoding.UTF8.GetString(Convert.FromBase64String(match.Groups[1].Value)); // Update last known name if we can if (packageData.SubID > 0 && (string.IsNullOrEmpty(packageData.LastKnownName) || packageData.LastKnownName.StartsWith("Steam Sub ", StringComparison.Ordinal))) { using (var db = Database.GetConnection()) { db.Execute("UPDATE `Subs` SET `LastKnownName` = @Name WHERE `SubID` = @SubID", new { SubID = package, Name = grantedName }); db.Execute(SubProcessor.GetHistoryQuery(), new PICSHistory { ID = package, Key = SteamDB.DATABASE_NAME_TYPE, OldValue = "free on demand; account page", NewValue = grantedName, Action = "created_info" } ); // Add a app comment on each app in this package var comment = string.Format("This app is in a free on demand package called <b>{0}</b>", System.Security.SecurityElement.Escape(grantedName)); var apps = db.Query <PackageApp>("SELECT `AppID` FROM `SubsApps` WHERE `SubID` = @SubID", new { SubID = package }).ToList(); var types = db.Query <App>("SELECT `AppID` FROM `Apps` WHERE `AppType` > 0 AND `AppID` IN @Ids", new { Ids = apps.Select(x => x.AppID) }).ToDictionary(x => x.AppID, x => true); var key = db.ExecuteScalar <uint>("SELECT `ID` FROM `KeyNames` WHERE `Name` = 'website_comment'"); foreach (var app in apps) { if (types.ContainsKey(app.AppID)) { continue; } db.Execute("INSERT INTO `AppsInfo` VALUES (@AppID, @Key, @Value) ON DUPLICATE KEY UPDATE `Key` = `Key`", new { AppID = app.AppID, Key = key, value = comment }); } } } packageData.LastKnownName = grantedName; } } IRC.Instance.SendMain("New free license granted: {0}{1}{2} -{3} {4}", Colors.BLUE, Steam.FormatPackageName(package, packageData), Colors.NORMAL, Colors.DARKBLUE, SteamDB.GetPackageURL(package) ); } }); } }
private void OnLoggedOn(SteamUser.LoggedOnCallback callback) { TaskManager.Run(AuthenticateUser); }
public void StartTick() { TickerHash++; TaskManager.Run(Tick); }
private static async void OnClanState(SteamFriends.ClanStateCallback callback) { if (callback.Events.Count == 0 && callback.Announcements.Count == 0) { return; } var groupName = callback.ClanName; if (string.IsNullOrEmpty(groupName)) { groupName = Steam.Instance.Friends.GetClanName(callback.ClanID); // Check once more, because that can fail too if (string.IsNullOrEmpty(groupName)) { groupName = "Group"; } } foreach (var announcement in callback.Announcements) { var url = $"https://steamcommunity.com/gid/{callback.ClanID.AccountID}/announcements/detail/{announcement.ID}"; IRC.Instance.SendAnnounce($"{Colors.BLUE}{groupName}{Colors.NORMAL} announcement: {Colors.OLIVE}{announcement.Headline}{Colors.NORMAL} -{Colors.DARKBLUE} {url}"); _ = TaskManager.Run(async() => await Utils.SendWebhook(new { Type = "GroupAnnouncement", Title = announcement.Headline, Group = groupName, Url = url, })); Log.WriteInfo(nameof(ClanState), $"{groupName} \"{announcement.Headline}\""); } await using var db = await Database.GetConnectionAsync(); foreach (var groupEvent in callback.Events) { var link = $"https://steamcommunity.com/gid/{callback.ClanID.AccountID}/events/{groupEvent.ID}"; var id = await db.ExecuteScalarAsync <int>("SELECT `ID` FROM `RSS` WHERE `Link` = @Link", new { Link = link }); if (id > 0) { continue; } IRC.Instance.SendAnnounce( $"{Colors.BLUE}{groupName}{Colors.NORMAL} event: {Colors.OLIVE}{groupEvent.Headline}{Colors.NORMAL} -{Colors.DARKBLUE} {link} {Colors.DARKGRAY}({groupEvent.EventTime.ToString("s", CultureInfo.InvariantCulture).Replace("T", " ")})" ); Log.WriteInfo(nameof(ClanState), $"{groupName} Event \"{groupEvent.Headline}\" {link}"); await db.ExecuteAsync("INSERT INTO `RSS` (`Link`, `Title`) VALUES(@Link, @Title)", new { Link = link, Title = groupEvent.Headline }); _ = TaskManager.Run(async() => await Utils.SendWebhook(new { Type = "GroupAnnouncement", Title = groupEvent.Headline, Group = groupName, Url = link, })); } }
private static void OnPICSProductInfo(SteamApps.PICSProductInfoCallback callback) { JobManager.TryRemoveJob(callback.JobID); var apps = callback.Apps.Concat(callback.UnknownApps.ToDictionary(x => x, x => (SteamApps.PICSProductInfoCallback.PICSProductInfo)null)); var packages = callback.Packages.Concat(callback.UnknownPackages.ToDictionary(x => x, x => (SteamApps.PICSProductInfoCallback.PICSProductInfo)null)); foreach (var workaround in apps) { var app = workaround; Log.WriteInfo("PICSProductInfo", "{0}AppID: {1}", app.Value == null ? "Unknown " : "", app.Key); Task mostRecentItem; lock (ProcessedApps) { ProcessedApps.TryGetValue(app.Key, out mostRecentItem); } var workerItem = TaskManager.Run(async delegate { if (mostRecentItem != null && !mostRecentItem.IsCompleted) { Log.WriteDebug("PICSProductInfo", "Waiting for app {0} to finish processing", app.Key); await mostRecentItem; } using (var processor = new AppProcessor(app.Key)) { if (app.Value == null) { processor.ProcessUnknown(); } else { processor.Process(app.Value); } } }); if (Settings.IsFullRun) { continue; } lock (ProcessedApps) { ProcessedApps[app.Key] = workerItem; } workerItem.ContinueWith(task => { lock (ProcessedApps) { if (ProcessedApps.TryGetValue(app.Key, out mostRecentItem) && mostRecentItem.IsCompleted) { ProcessedApps.Remove(app.Key); } } }); } foreach (var workaround in packages) { var package = workaround; Log.WriteInfo("PICSProductInfo", "{0}SubID: {1}", package.Value == null ? "Unknown " : "", package.Key); Task mostRecentItem; lock (ProcessedSubs) { ProcessedSubs.TryGetValue(package.Key, out mostRecentItem); } var workerItem = TaskManager.Run(async delegate { if (mostRecentItem != null && !mostRecentItem.IsCompleted) { Log.WriteDebug("PICSProductInfo", "Waiting for package {0} to finish processing", package.Key); await mostRecentItem; } using (var processor = new SubProcessor(package.Key)) { if (package.Value == null) { processor.ProcessUnknown(); } else { processor.Process(package.Value); } } }); if (Settings.IsFullRun) { continue; } lock (ProcessedSubs) { ProcessedSubs[package.Key] = workerItem; } workerItem.ContinueWith(task => { lock (ProcessedSubs) { if (ProcessedSubs.TryGetValue(package.Key, out mostRecentItem) && mostRecentItem.IsCompleted) { ProcessedSubs.Remove(package.Key); } } }); } }
private async Task DownloadDepots(uint appID, List <ManifestJob> depots) { Log.WriteDebug(nameof(DepotProcessor), $"Will process {depots.Count} depots ({DepotLocks.Count} depot locks left)"); var processTasks = new List <Task <EResult> >(); var anyFilesDownloaded = false; var willDownloadFiles = false; foreach (var depot in depots) { if (depot.DepotKey == null) { await GetDepotDecryptionKey(Steam.Instance.Apps, depot, appID); if (depot.DepotKey == null && (depot.LastManifestID == depot.ManifestID || Settings.Current.OnlyOwnedDepots)) { RemoveLock(depot.DepotID); continue; } } depot.Server = GetContentServer(); DepotManifest depotManifest = null; var lastError = string.Empty; for (var i = 0; i <= 5; i++) { try { await ManifestDownloadSemaphore.WaitAsync(TaskManager.TaskCancellationToken.Token).ConfigureAwait(false); depotManifest = await CDNClient.DownloadManifestAsync(depot.DepotID, depot.ManifestID, depot.Server, string.Empty, depot.DepotKey); break; } catch (Exception e) { lastError = e.Message; Log.WriteError(nameof(DepotProcessor), $"Failed to download depot manifest for app {appID} depot {depot.DepotID} ({depot.Server}: {lastError}) (#{i})"); } finally { ManifestDownloadSemaphore.Release(); } if (depot.DepotKey != null) { RemoveErroredServer(depot.Server); } depot.Server = GetContentServer(); if (depotManifest == null && i < 5) { await Task.Delay(Utils.ExponentionalBackoff(i + 1)); } } if (depotManifest == null) { RemoveLock(depot.DepotID); if (FileDownloader.IsImportantDepot(depot.DepotID)) { IRC.Instance.SendOps($"{Colors.OLIVE}[{depot.DepotName}]{Colors.NORMAL} Failed to download manifest ({lastError})"); } if (!Settings.IsFullRun && depot.DepotKey != null) { JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null)); } continue; } var task = ProcessDepotAfterDownload(depot, depotManifest); processTasks.Add(task); if (!FileDownloader.IsImportantDepot(depot.DepotID) || depot.DepotKey == null) { depot.Result = EResult.Ignored; continue; } willDownloadFiles = true; task = TaskManager.Run(async() => { var result = EResult.Fail; try { result = await FileDownloader.DownloadFilesFromDepot(depot, depotManifest); if (result == EResult.OK) { anyFilesDownloaded = true; } } catch (Exception e) { ErrorReporter.Notify($"Depot Processor {depot.DepotID}", e); } return(result); }); processTasks.Add(task); } await Task.WhenAll(processTasks).ConfigureAwait(false); Log.WriteDebug(nameof(DepotProcessor), $"{depots.Count} depot downloads finished for app {appID}"); // TODO: use ContinueWith on tasks if (!anyFilesDownloaded && !willDownloadFiles) { foreach (var depot in depots) { RemoveLock(depot.DepotID); } return; } lock (UpdateScriptLock) { foreach (var depot in depots) { if (depot.Result == EResult.OK) { RunUpdateScript(UpdateScript, $"{depot.DepotID} no-git"); } else if (depot.Result != EResult.Ignored) { Log.WriteWarn(nameof(DepotProcessor), $"Download failed for {depot.DepotID}: {depot.Result}"); // Mark this depot for redownload using var db = Database.Get(); db.Execute("UPDATE `Depots` SET `LastManifestID` = 0 WHERE `DepotID` = @DepotID", new { depot.DepotID }); } RemoveLock(depot.DepotID); } // Only commit changes if all depots downloaded if (processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored)) { if (!RunUpdateScriptForApp(appID, depots[0].BuildID)) { RunUpdateScript(UpdateScript, "0"); } } else { Log.WriteDebug(nameof(DepotProcessor), $"Reprocessing the app {appID} because some files failed to download"); IRC.Instance.SendOps($"{Colors.OLIVE}[{Steam.GetAppName(appID)}]{Colors.NORMAL} Reprocessing the app due to download failures"); JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null)); } } }
private async Task DownloadDepots(uint appID, List <ManifestJob> depots) { Log.WriteDebug("Depot Downloader", "Will process {0} depots ({1} depot locks left)", depots.Count, DepotLocks.Count); var processTasks = new List <Task <EResult> >(); bool anyFilesDownloaded = false; foreach (var depot in depots) { var instance = depot.Anonymous ? Steam.Anonymous.Apps : Steam.Instance.Apps; depot.DepotKey = await GetDepotDecryptionKey(instance, depot.DepotID, appID); if (depot.DepotKey == null) { RemoveLock(depot.DepotID); continue; } var cdnToken = await GetCDNAuthToken(instance, appID, depot.DepotID); if (cdnToken == null) { RemoveLock(depot.DepotID); Log.WriteDebug("Depot Downloader", "Got a depot key for depot {0} but no cdn auth token", depot.DepotID); continue; } depot.CDNToken = cdnToken.Token; depot.Server = GetContentServer(); DepotManifest depotManifest = null; string lastError = string.Empty; for (var i = 0; i <= 5; i++) { try { depotManifest = await CDNClient.DownloadManifestAsync(depot.DepotID, depot.ManifestID, depot.Server, depot.CDNToken, depot.DepotKey); break; } catch (Exception e) { lastError = e.Message; Log.WriteError("Depot Processor", "Failed to download depot manifest for app {0} depot {1} ({2}: {3}) (#{4})", appID, depot.DepotID, depot.Server, lastError, i); } // TODO: get new auth key if auth fails depot.Server = GetContentServer(); if (depotManifest == null) { await Task.Delay(Utils.ExponentionalBackoff(i)); } } if (depotManifest == null) { LocalConfig.CDNAuthTokens.TryRemove(depot.DepotID, out _); RemoveLock(depot.DepotID); if (FileDownloader.IsImportantDepot(depot.DepotID)) { IRC.Instance.SendOps("{0}[{1}]{2} Failed to download manifest ({3})", Colors.OLIVE, depot.DepotName, Colors.NORMAL, lastError); } if (!Settings.IsFullRun) { JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null)); } continue; } var task = ProcessDepotAfterDownload(depot, depotManifest); processTasks.Add(task); if (!FileDownloader.IsImportantDepot(depot.DepotID)) { continue; } task = TaskManager.Run(async() => { var result = EResult.Fail; try { result = await FileDownloader.DownloadFilesFromDepot(depot, depotManifest); if (result == EResult.OK) { anyFilesDownloaded = true; } } catch (Exception e) { ErrorReporter.Notify($"Depot Processor {depot.DepotID}", e); } return(result); }).Unwrap(); processTasks.Add(task); } if (SaveLocalConfig) { SaveLocalConfig = false; LocalConfig.Save(); } await Task.WhenAll(processTasks).ConfigureAwait(false); Log.WriteDebug("Depot Downloader", "{0} depot downloads finished", depots.Count); // TODO: use ContinueWith on tasks if (!anyFilesDownloaded) { foreach (var depot in depots) { RemoveLock(depot.DepotID); } return; } if (!File.Exists(UpdateScript)) { return; } lock (UpdateScriptLock) { foreach (var depot in depots) { if (depot.Result == EResult.OK) { RunUpdateScript(UpdateScript, string.Format("{0} no-git", depot.DepotID)); } else if (depot.Result != EResult.Ignored) { Log.WriteWarn("Depot Processor", "Dropping stored token for {0} due to download failures", depot.DepotID); LocalConfig.CDNAuthTokens.TryRemove(depot.DepotID, out _); using (var db = Database.Get()) { // Mark this depot for redownload db.Execute("UPDATE `Depots` SET `LastManifestID` = 0 WHERE `DepotID` = @DepotID", new { depot.DepotID }); } } RemoveLock(depot.DepotID); } // Only commit changes if all depots downloaded if (processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored)) { if (!RunUpdateScript(appID, depots[0].BuildID)) { RunUpdateScript(UpdateScript, "0"); } } else { Log.WriteDebug("Depot Processor", "Reprocessing the app {0} because some files failed to download", appID); IRC.Instance.SendOps("{0}[{1}]{2} Reprocessing the app due to download failures", Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL ); JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null)); } } }
private static async Task <EResult> DownloadFile(DepotProcessor.ManifestJob job, DepotManifest.FileData file, byte[] hash) { var directory = Path.Combine(Application.Path, "files", DownloadFolders[job.DepotID], Path.GetDirectoryName(file.FileName)); var finalPath = new FileInfo(Path.Combine(directory, Path.GetFileName(file.FileName))); var downloadPath = new FileInfo(Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".steamdb_tmp"))); if (!Directory.Exists(directory)) { Directory.CreateDirectory(directory); } else if (file.TotalSize == 0) { if (!finalPath.Exists) { using (var _ = finalPath.Create()) { // FileInfo.Create returns a stream but we don't need it } Log.WriteInfo("FileDownloader", "{0} created an empty file", file.FileName); return(EResult.SameAsPreviousValue); } else if (finalPath.Length == 0) { #if DEBUG Log.WriteDebug("FileDownloader", "{0} is already empty", file.FileName); #endif return(EResult.SameAsPreviousValue); } } else if (hash != null && file.FileHash.SequenceEqual(hash)) { #if DEBUG Log.WriteDebug("FileDownloader", "{0} already matches the file we have", file.FileName); #endif return(EResult.SameAsPreviousValue); } byte[] checksum; using (var sha = SHA1.Create()) { checksum = sha.ComputeHash(Encoding.UTF8.GetBytes(file.FileName)); } var neededChunks = new List <DepotManifest.ChunkData>(); var chunks = file.Chunks.OrderBy(x => x.Offset).ToList(); var oldChunksFile = Path.Combine(Application.Path, "files", ".support", "chunks", string.Format("{0}-{1}.json", job.DepotID, BitConverter.ToString(checksum))); using (var fs = downloadPath.Open(FileMode.OpenOrCreate, FileAccess.ReadWrite)) { fs.SetLength((long)file.TotalSize); if (finalPath.Exists && File.Exists(oldChunksFile)) { var oldChunks = JsonConvert.DeserializeObject <List <DepotManifest.ChunkData> >(File.ReadAllText(oldChunksFile), JsonHandleAllReferences); using (var fsOld = finalPath.Open(FileMode.Open, FileAccess.Read)) { foreach (var chunk in chunks) { var oldChunk = oldChunks.Find(c => c.ChunkID.SequenceEqual(chunk.ChunkID)); if (oldChunk != null) { var oldData = new byte[oldChunk.UncompressedLength]; fsOld.Seek((long)oldChunk.Offset, SeekOrigin.Begin); fsOld.Read(oldData, 0, oldData.Length); var existingChecksum = Utils.AdlerHash(oldData); if (existingChecksum.SequenceEqual(chunk.Checksum)) { fs.Seek((long)chunk.Offset, SeekOrigin.Begin); fs.Write(oldData, 0, oldData.Length); #if DEBUG Log.WriteDebug("FileDownloader", "{0} Found chunk ({1}), not downloading", file.FileName, chunk.Offset); #endif } else { neededChunks.Add(chunk); #if DEBUG Log.WriteDebug("FileDownloader", "{0} Found chunk ({1}), but checksum differs", file.FileName, chunk.Offset); #endif } } else { neededChunks.Add(chunk); } } } } else { neededChunks = chunks; } } var downloadedSize = file.TotalSize - (ulong)neededChunks.Sum(x => x.UncompressedLength); var chunkCancellation = new CancellationTokenSource(); var chunkTasks = new Task[neededChunks.Count]; Log.WriteInfo("FileDownloader", "Downloading {0} ({1} bytes, {2} out of {3} chunks)", file.FileName, downloadedSize, neededChunks.Count, chunks.Count); for (var i = 0; i < chunkTasks.Length; i++) { var chunk = neededChunks[i]; chunkTasks[i] = TaskManager.Run(async() => { try { chunkCancellation.Token.ThrowIfCancellationRequested(); await ChunkDownloadingSemaphore.WaitAsync(chunkCancellation.Token).ConfigureAwait(false); var result = await DownloadChunk(job, chunk, downloadPath); if (!result) { Log.WriteWarn("FileDownloader", "Failed to download chunk for {0}", file.FileName); chunkCancellation.Cancel(); } else { downloadedSize += chunk.UncompressedLength; // Do not write progress info to log file Console.WriteLine("{2} [{0,6:#00.00}%] {1}", downloadedSize / (float)file.TotalSize * 100.0f, file.FileName, job.DepotName); } } finally { ChunkDownloadingSemaphore.Release(); } }).Unwrap(); // Register error handler on inner task TaskManager.RegisterErrorHandler(chunkTasks[i]); } await Task.WhenAll(chunkTasks).ConfigureAwait(false); using (var fs = downloadPath.Open(FileMode.Open, FileAccess.ReadWrite)) { fs.Seek(0, SeekOrigin.Begin); using (var sha = SHA1.Create()) { checksum = sha.ComputeHash(fs); } } if (!file.FileHash.SequenceEqual(checksum)) { IRC.Instance.SendOps("{0}[{1}]{2} Failed to correctly download {3}{4}", Colors.OLIVE, job.DepotName, Colors.NORMAL, Colors.BLUE, file.FileName); Log.WriteWarn("FileDownloader", "Failed to download file {0} ({1})", file.FileName, job.Server); downloadPath.Delete(); return(EResult.DataCorruption); } Log.WriteInfo("FileDownloader", "Downloaded {0} from {1}", file.FileName, job.DepotName); finalPath.Delete(); downloadPath.MoveTo(finalPath.FullName); if (chunks.Count > 1) { File.WriteAllText(oldChunksFile, JsonConvert.SerializeObject(chunks, Formatting.None, JsonHandleAllReferences)); } else if (File.Exists(oldChunksFile)) { File.Delete(oldChunksFile); } return(EResult.OK); }
private async Task DownloadDepots(uint appID, List <ManifestJob> depots) { Log.WriteDebug("Depot Downloader", "Will process {0} depots ({1} depot locks left)", depots.Count(), DepotLocks.Count); var processTasks = new List <Task <EResult> >(); bool anyFilesDownloaded = false; foreach (var depot in depots) { var instance = depot.Anonymous ? Steam.Anonymous.Apps : Steam.Instance.Apps; depot.DepotKey = await GetDepotDecryptionKey(instance, depot.DepotID, appID); if (depot.DepotKey == null) { RemoveLock(depot.DepotID); continue; } var cdnToken = await GetCDNAuthToken(instance, appID, depot.DepotID); if (cdnToken == null) { RemoveLock(depot.DepotID); Log.WriteDebug("Depot Downloader", "Got a depot key for depot {0} but no cdn auth token", depot.DepotID); continue; } depot.CDNToken = cdnToken.Token; depot.Server = cdnToken.Server; DepotManifest depotManifest = null; string lastError = string.Empty; for (var i = 0; i <= 5; i++) { try { depotManifest = CDNClient.DownloadManifest(depot.DepotID, depot.ManifestID, depot.Server, depot.CDNToken, depot.DepotKey); break; } catch (Exception e) { Log.WriteWarn("Depot Downloader", "[{0}] Manifest download failed: {1} - {2}", depot.DepotID, e.GetType(), e.Message); lastError = e.Message; } // TODO: get new auth key if auth fails if (depotManifest == null) { await Task.Delay(Utils.ExponentionalBackoff(i)); } } if (depotManifest == null) { RemoveLock(depot.DepotID); Log.WriteError("Depot Processor", "Failed to download depot manifest for app {0} depot {1} ({2}: {3})", appID, depot.DepotID, depot.Server, lastError); if (FileDownloader.IsImportantDepot(depot.DepotID)) { IRC.Instance.SendOps("{0}[{1}]{2} Failed to download app {3} depot {4} manifest ({5}: {6})", Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL, appID, depot.DepotID, depot.Server, lastError); } continue; } var task = TaskManager.Run(() => { using (var db = Database.GetConnection()) { using (var transaction = db.BeginTransaction()) { var result = ProcessDepotAfterDownload(db, depot, depotManifest); transaction.Commit(); return(result); } } }); processTasks.Add(task); if (FileDownloader.IsImportantDepot(depot.DepotID)) { task = TaskManager.Run(() => { var result = FileDownloader.DownloadFilesFromDepot(appID, depot, depotManifest); if (result == EResult.OK) { anyFilesDownloaded = true; } return(result); }, TaskCreationOptions.LongRunning); TaskManager.RegisterErrorHandler(task); processTasks.Add(task); } } if (SaveLocalConfig) { SaveLocalConfig = false; LocalConfig.Save(); } await Task.WhenAll(processTasks); Log.WriteDebug("Depot Downloader", "{0} depot downloads finished", depots.Count()); // TODO: use ContinueWith on tasks if (!anyFilesDownloaded) { foreach (var depot in depots) { RemoveLock(depot.DepotID); } return; } if (!File.Exists(UpdateScript)) { return; } bool lockTaken = false; try { UpdateScriptLock.Enter(ref lockTaken); foreach (var depot in depots) { if (depot.Result == EResult.OK) { RunUpdateScript(string.Format("{0} no-git", depot.DepotID)); } RemoveLock(depot.DepotID); } // Only commit changes if all depots downloaded if (processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored)) { if (!RunUpdateScript(appID)) { RunUpdateScript("0"); } } else { Log.WriteDebug("Depot Processor", "Reprocessing the app {0} because some files failed to download", appID); IRC.Instance.SendOps("{0}[{1}]{2} Reprocessing the app due to download failures", Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL ); JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null)); } } finally { if (lockTaken) { UpdateScriptLock.Exit(); } } }
private void OnPICSChanges(SteamApps.PICSChangesCallback callback) { if (PreviousChangeNumber == callback.CurrentChangeNumber) { return; } Log.WriteInfo(nameof(PICSChanges), $"Changelist {callback.LastChangeNumber} -> {callback.CurrentChangeNumber} ({callback.AppChanges.Count} apps, {callback.PackageChanges.Count} packages)"); PreviousChangeNumber = callback.CurrentChangeNumber; TaskManager.Run(async() => await HandleChangeNumbers(callback)); if (callback.RequiresFullAppUpdate || callback.RequiresFullPackageUpdate) { TaskManager.Run(async() => { if (callback.RequiresFullAppUpdate) { IRC.Instance.SendOps($"Changelist {callback.CurrentChangeNumber} has forced a full app update"); // When full update flag is set, presumably Steam client start hammering the servers // and the PICS service just does not return any data for a while until it clears up await FullUpdateProcessor.FullUpdateAppsMetadata(true); } if (callback.RequiresFullPackageUpdate) { IRC.Instance.SendOps($"Changelist {callback.CurrentChangeNumber} has forced a full package update"); await FullUpdateProcessor.FullUpdatePackagesMetadata(); } IRC.Instance.SendOps($"Changelist {callback.CurrentChangeNumber} full update has finished"); }); } if (callback.AppChanges.Count == 0 && callback.PackageChanges.Count == 0) { return; } const int appsPerJob = 50; if (callback.AppChanges.Count > appsPerJob) { foreach (var list in callback.AppChanges.Keys.Split(appsPerJob)) { JobManager.AddJob( () => Steam.Instance.Apps.PICSGetAccessTokens(list, Enumerable.Empty <uint>()), new PICSTokens.RequestedTokens { Apps = list.ToList() }); } } else if (callback.AppChanges.Count > 0) { JobManager.AddJob( () => Steam.Instance.Apps.PICSGetAccessTokens(callback.AppChanges.Keys, Enumerable.Empty <uint>()), new PICSTokens.RequestedTokens { Apps = callback.AppChanges.Keys.ToList() }); } if (callback.PackageChanges.Count > appsPerJob) { foreach (var list in callback.PackageChanges.Keys.Split(appsPerJob)) { JobManager.AddJob( () => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), list), new PICSTokens.RequestedTokens { Packages = list.ToList() }); } } else if (callback.PackageChanges.Count > 0) { JobManager.AddJob( () => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), callback.PackageChanges.Keys), new PICSTokens.RequestedTokens { Packages = callback.PackageChanges.Keys.ToList() }); } if (callback.AppChanges.Count > 0) { _ = TaskManager.Run(async() => await HandleApps(callback)); } if (callback.PackageChanges.Count > 0) { _ = TaskManager.Run(async() => await HandlePackages(callback)); _ = TaskManager.Run(async() => await HandlePackagesChangelists(callback)); } if (PreviousChangeNumber - LastStoredChangeNumber >= 1000) { LastStoredChangeNumber = PreviousChangeNumber; _ = TaskManager.Run(async() => await LocalConfig.Update("backend.changenumber", LastStoredChangeNumber.ToString())); } PrintImportants(callback); }
private void OnTimer(object sender, ElapsedEventArgs e) { if (!Steam.Instance.IsLoggedOn) { lock (FreeLicenseTimer) { FreeLicenseTimer.Start(); } return; } if (FreeLicensesToRequest.IsEmpty) { TaskManager.Run(RequestBetas); return; } var list = FreeLicensesToRequest.Take(REQUEST_RATE_LIMIT).ToList(); var now = DateUtils.DateTimeToUnixTime(DateTime.UtcNow) - 60; Dictionary <uint, ulong> startTimes; using (var db = Database.Get()) { startTimes = db.Query( "SELECT `SubID`, `Value` FROM `SubsInfo` WHERE `Key` = @Key AND `SubID` IN @Ids", new { Key = KeyNameCache.GetSubKeyID("extended_starttime"), Ids = list.Select(x => x.Key) } ).ToDictionary(x => (uint)x.SubID, x => Convert.ToUInt64((string)x.Value)); } foreach (var(subId, _) in list) { if (startTimes.TryGetValue(subId, out var startTime) && startTime > now) { // If start time has not been reached yet, don't remove this app from the list and keep trying to activate it continue; } FreeLicensesToRequest.TryRemove(subId, out _); } TaskManager.Run(Save); TaskManager.Run(RequestBetas); var appids = list.Select(x => x.Value).Distinct(); AppsRequestedInHour = appids.Count(); Log.WriteDebug(nameof(FreeLicense), $"Requesting {AppsRequestedInHour} free apps as the rate limit timer ran: {string.Join(", ", appids)}"); JobManager.AddJob(() => Steam.Instance.Apps.RequestFreeLicense(appids)); if (!FreeLicensesToRequest.IsEmpty) { lock (FreeLicenseTimer) { FreeLicenseTimer.Start(); } } }
public override async Task OnCommand(CommandArguments command) { var s = command.Message.Split(' '); var count = s.Length; if (count > 0) { uint id; switch (s[0]) { case "reload": await Application.ReloadImportant(); await PICSTokens.Reload(); command.Notice("Reloaded important apps and pics tokens"); return; case "fullrun": _ = TaskManager.Run(async() => { command.Reply("Started full metadata scan, this will take a while…"); await FullUpdateProcessor.FullUpdateAppsMetadata(true); command.Reply("App full scan finished, starting packages, this will take even longer…"); await FullUpdateProcessor.FullUpdatePackagesMetadata(); command.Reply("Full metadata scan finished."); }); return; case "add": if (count < 3) { break; } if (!uint.TryParse(s[2], out id)) { break; } switch (s[1]) { case "app": if (Application.ImportantApps.Contains(id)) { command.Reply($"App {Colors.BLUE}{id}{Colors.NORMAL} ({Steam.GetAppName(id)}) is already important."); } else { Application.ImportantApps.Add(id); await using (var db = await Database.GetConnectionAsync()) { await db.ExecuteAsync("INSERT INTO `ImportantApps` (`AppID`) VALUES (@AppID)", new { AppID = id }); } command.Reply($"Marked app {Colors.BLUE}{id}{Colors.NORMAL} ({Steam.GetAppName(id)}) as important."); } return; case "sub": if (Application.ImportantSubs.Contains(id)) { command.Reply($"Package {Colors.BLUE}{id}{Colors.NORMAL} ({Steam.GetPackageName(id)}) is already important."); } else { Application.ImportantSubs.Add(id); await using (var db = await Database.GetConnectionAsync()) { await db.ExecuteAsync("INSERT INTO `ImportantSubs` (`SubID`) VALUES (@SubID)", new { SubID = id }); } command.Reply($"Marked package {Colors.BLUE}{id}{Colors.NORMAL} ({Steam.GetPackageName(id)}) as important."); } return; } break; case "remove": if (count < 3) { break; } if (!uint.TryParse(s[2], out id)) { break; } switch (s[1]) { case "app": if (!Application.ImportantApps.Contains(id)) { command.Reply($"App {Colors.BLUE}{id}{Colors.NORMAL} ({Steam.GetAppName(id)}) is not important."); } else { Application.ImportantApps.Remove(id); await using (var db = await Database.GetConnectionAsync()) { await db.ExecuteAsync("DELETE FROM `ImportantApps` WHERE `AppID` = @AppID", new { AppID = id }); } command.Reply($"Removed app {Colors.BLUE}{id}{Colors.NORMAL} ({Steam.GetAppName(id)}) from the important list."); } return; case "sub": if (!Application.ImportantSubs.Contains(id)) { command.Reply($"Package {Colors.BLUE}{id}{Colors.NORMAL} ({Steam.GetPackageName(id)}) is not important."); } else { Application.ImportantSubs.Remove(id); await using (var db = await Database.GetConnectionAsync()) { await db.ExecuteAsync("DELETE FROM `ImportantSubs` WHERE `SubID` = @SubID", new { SubID = id }); } command.Reply($"Removed package {Colors.BLUE}{id}{Colors.NORMAL} ({Steam.GetPackageName(id)}) from the important list."); } return; } break; } } command.Reply($"Usage:{Colors.OLIVE} important reload {Colors.NORMAL}or{Colors.OLIVE} important <add/remove> <app/sub> <id> {Colors.NORMAL}or{Colors.OLIVE} important fullrun"); }
/* * Here be dragons. */ public static async Task <EResult> DownloadFilesFromDepot(DepotProcessor.ManifestJob job, DepotManifest depotManifest) { var filesRegex = Files[job.DepotID]; var files = depotManifest.Files.Where(x => filesRegex.IsMatch(x.FileName.Replace('\\', '/'))).ToList(); var downloadState = EResult.Fail; ConcurrentDictionary <string, ExistingFileData> existingFileData; await using (var db = await Database.GetConnectionAsync()) { var data = db.ExecuteScalar <string>("SELECT `Value` FROM `LocalConfig` WHERE `ConfigKey` = @Key", new { Key = $"depot.{job.DepotID}" }); if (data != null) { existingFileData = JsonConvert.DeserializeObject <ConcurrentDictionary <string, ExistingFileData> >(data); } else { existingFileData = new ConcurrentDictionary <string, ExistingFileData>(); } } foreach (var file in existingFileData.Keys.Except(files.Select(x => x.FileName))) { Log.WriteWarn(nameof(FileDownloader), $"\"{file}\" no longer exists in manifest"); } Log.WriteInfo($"FileDownloader {job.DepotID}", $"Will download {files.Count} files"); var downloadedFiles = 0; var fileTasks = new Task[files.Count]; for (var i = 0; i < fileTasks.Length; i++) { var file = files[i]; fileTasks[i] = TaskManager.Run(async() => { var existingFile = existingFileData.GetOrAdd(file.FileName, _ => new ExistingFileData()); EResult fileState; try { await ChunkDownloadingSemaphore.WaitAsync().ConfigureAwait(false); fileState = await DownloadFile(job, file, existingFile); } finally { ChunkDownloadingSemaphore.Release(); } if (fileState == EResult.OK || fileState == EResult.SameAsPreviousValue) { existingFile.FileHash = file.FileHash; downloadedFiles++; } if (fileState != EResult.SameAsPreviousValue) { // Do not write progress info to log file Console.WriteLine($"{job.DepotName} [{downloadedFiles / (float) files.Count * 100.0f,6:#00.00}%] {files.Count - downloadedFiles} files left to download"); } if (downloadState == EResult.DataCorruption) { return; } if (fileState == EResult.OK || fileState == EResult.DataCorruption) { downloadState = fileState; } }); } await Task.WhenAll(fileTasks).ConfigureAwait(false); await LocalConfig.Update($"depot.{job.DepotID}", JsonConvert.SerializeObject(existingFileData)); job.Result = downloadState switch { EResult.OK => EResult.OK, EResult.DataCorruption => EResult.DataCorruption, _ => EResult.Ignored }; return(job.Result); }
private static async Task <EResult> DownloadFile(DepotProcessor.ManifestJob job, DepotManifest.FileData file, ExistingFileData existingFile) { var directory = Path.Combine(Application.Path, "files", DownloadFolders[job.DepotID], Path.GetDirectoryName(file.FileName)); var finalPath = new FileInfo(Path.Combine(directory, Path.GetFileName(file.FileName))); var downloadPath = new FileInfo(Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".steamdb_tmp"))); if (!Directory.Exists(directory)) { Directory.CreateDirectory(directory); } else if (file.TotalSize == 0) { if (!finalPath.Exists) { await using (var _ = finalPath.Create()) { // FileInfo.Create returns a stream but we don't need it } Log.WriteInfo($"FileDownloader {job.DepotID}", $"{file.FileName} created an empty file"); return(EResult.SameAsPreviousValue); } else if (finalPath.Length == 0) { #if DEBUG Log.WriteDebug($"FileDownloader {job.DepotID}", $"{file.FileName} is already empty"); #endif return(EResult.SameAsPreviousValue); } } else if (existingFile.FileHash != null && file.FileHash.SequenceEqual(existingFile.FileHash)) { #if DEBUG Log.WriteDebug($"FileDownloader {job.DepotID}", $"{file.FileName} already matches the file we have"); #endif return(EResult.SameAsPreviousValue); } using var sha = SHA1.Create(); var neededChunks = new List <DepotManifest.ChunkData>(); var chunks = file.Chunks.OrderBy(x => x.Offset).ToList(); await using (var fs = downloadPath.Open(FileMode.OpenOrCreate, FileAccess.ReadWrite)) { fs.SetLength((long)file.TotalSize); if (finalPath.Exists) { await using var fsOld = finalPath.Open(FileMode.Open, FileAccess.Read); foreach (var chunk in chunks) { var oldChunk = existingFile.Chunks.FirstOrDefault(c => c.Value.SequenceEqual(chunk.ChunkID)); if (oldChunk.Value != null) { var oldData = new byte[chunk.UncompressedLength]; fsOld.Seek((long)oldChunk.Key, SeekOrigin.Begin); fsOld.Read(oldData, 0, oldData.Length); var existingChecksum = sha.ComputeHash(oldData); if (existingChecksum.SequenceEqual(chunk.ChunkID)) { fs.Seek((long)chunk.Offset, SeekOrigin.Begin); fs.Write(oldData, 0, oldData.Length); #if DEBUG Log.WriteDebug($"FileDownloader {job.DepotID}", $"{file.FileName} Found chunk ({chunk.Offset}), not downloading"); #endif } else { neededChunks.Add(chunk); #if DEBUG Log.WriteDebug($"FileDownloader {job.DepotID}", $"{file.FileName} Found chunk ({chunk.Offset}), but checksum differs"); #endif } } else { neededChunks.Add(chunk); } } } else { neededChunks = chunks; } } using var chunkCancellation = new CancellationTokenSource(); var downloadedSize = file.TotalSize - (ulong)neededChunks.Sum(x => x.UncompressedLength); var chunkTasks = new Task[neededChunks.Count]; Log.WriteInfo($"FileDownloader {job.DepotID}", $"Downloading {file.FileName} ({neededChunks.Count} out of {chunks.Count} chunks to download)"); for (var i = 0; i < chunkTasks.Length; i++) { var chunk = neededChunks[i]; chunkTasks[i] = TaskManager.Run(async() => { var result = await DownloadChunk(job, chunk, downloadPath, chunkCancellation); if (!result) { Log.WriteWarn($"FileDownloader {job.DepotID}", $"Failed to download chunk for {file.FileName} ({chunk.Offset})"); chunkCancellation.Cancel(); } else { downloadedSize += chunk.UncompressedLength; // Do not write progress info to log file Console.WriteLine($"{job.DepotName} [{downloadedSize / (float) file.TotalSize * 100.0f,6:#00.00}%] {file.FileName}"); } }); } await Task.WhenAll(chunkTasks).ConfigureAwait(false); byte[] checksum; await using (var fs = downloadPath.Open(FileMode.Open, FileAccess.ReadWrite)) { checksum = sha.ComputeHash(fs); } if (!file.FileHash.SequenceEqual(checksum)) { if (!job.DownloadCorrupted) { job.DownloadCorrupted = true; IRC.Instance.SendOps($"{Colors.OLIVE}[{job.DepotName}]{Colors.NORMAL} Failed to correctly download {Colors.BLUE}{file.FileName}"); } Log.WriteWarn($"FileDownloader {job.DepotID}", $"Hash check failed for {file.FileName} ({job.Server})"); downloadPath.Delete(); existingFile.FileHash = null; existingFile.Chunks.Clear(); return(EResult.DataCorruption); } Log.WriteInfo($"FileDownloader {job.DepotID}", $"Downloaded {file.FileName}"); finalPath.Delete(); downloadPath.MoveTo(finalPath.FullName); if (chunks.Count > 0) { existingFile.Chunks = chunks.ToDictionary(chunk => chunk.Offset, chunk => chunk.ChunkID); } else { existingFile.Chunks.Clear(); } return(EResult.OK); }
public void StartTick() { TaskManager.Run(Tick); }
private void OnLoggedOn(SteamUser.LoggedOnCallback callback) { TaskManager.Run(UpdateContentServerList); }
public async Task Process(uint appID, uint changeNumber, KeyValue depots) { var requests = new List <ManifestJob>(); // Get data in format we want first foreach (var depot in depots.Children) { // Ignore these for now, parent app should be updated too anyway if (depot["depotfromapp"].Value != null) { continue; } var request = new ManifestJob { ChangeNumber = changeNumber, }; // Ignore keys that aren't integers, for example "branches" if (!uint.TryParse(depot.Name, out request.DepotID)) { continue; } request.DepotName = depot["name"].AsString(); if (string.IsNullOrEmpty(request.DepotName)) { request.DepotName = $"SteamDB Unnamed Depot {request.DepotID}"; } // TODO: instead of locking we could wait for current process to finish if (DepotLocks.ContainsKey(request.DepotID)) { continue; } // SteamVR trickery if (appID == 250820 && depot["manifests"]["beta"].Value != null && depots["branches"]["beta"]["buildid"].AsInteger() > depots["branches"]["public"]["buildid"].AsInteger()) { request.BuildID = depots["branches"]["beta"]["buildid"].AsInteger(); request.ManifestID = ulong.Parse(depot["manifests"]["beta"].Value); } else if (depot["manifests"]["public"].Value == null || !ulong.TryParse(depot["manifests"]["public"].Value, out request.ManifestID)) { var branch = depot["manifests"].Children.Find(x => x.Name != "local"); if (branch == null || !ulong.TryParse(branch.Value, out request.ManifestID)) { await using var db = await Database.GetConnectionAsync(); await db.ExecuteAsync("INSERT INTO `Depots` (`DepotID`, `Name`) VALUES (@DepotID, @DepotName) ON DUPLICATE KEY UPDATE `DepotID` = VALUES(`DepotID`)", new { request.DepotID, request.DepotName }); continue; } Log.WriteDebug(nameof(DepotProcessor), $"Depot {request.DepotID} (from {appID}) has no public branch, but there is another one"); request.BuildID = depots["branches"][branch.Name]["buildid"].AsInteger(); } else { request.BuildID = depots["branches"]["public"]["buildid"].AsInteger(); } requests.Add(request); } if (requests.Count == 0) { return; } var depotsToDownload = new List <ManifestJob>(); await using (var db = await Database.GetConnectionAsync()) { await db.ExecuteAsync("INSERT INTO `Builds` (`BuildID`, `ChangeID`, `AppID`) VALUES (@BuildID, @ChangeNumber, @AppID) ON DUPLICATE KEY UPDATE `AppID` = VALUES(`AppID`)", new { requests[0].BuildID, requests[0].ChangeNumber, appID }); var depotIds = requests.Select(x => x.DepotID).ToList(); var dbDepots = (await db.QueryAsync <Depot>("SELECT `DepotID`, `Name`, `BuildID`, `ManifestID`, `LastManifestID`, `FilenamesEncrypted` FROM `Depots` WHERE `DepotID` IN @depotIds", new { depotIds })) .ToDictionary(x => x.DepotID, x => x); var decryptionKeys = (await db.QueryAsync <DepotKey>("SELECT `DepotID`, `Key` FROM `DepotsKeys` WHERE `DepotID` IN @depotIds", new { depotIds })) .ToDictionary(x => x.DepotID, x => Utils.StringToByteArray(x.Key)); foreach (var request in requests) { Depot dbDepot; decryptionKeys.TryGetValue(request.DepotID, out request.DepotKey); if (dbDepots.ContainsKey(request.DepotID)) { dbDepot = dbDepots[request.DepotID]; if (dbDepot.BuildID > request.BuildID) { // buildid went back in time? this either means a rollback, or a shared depot that isn't synced properly Log.WriteDebug(nameof(DepotProcessor), $"Skipping depot {request.DepotID} due to old buildid: {dbDepot.BuildID} > {request.BuildID}"); continue; } if (dbDepot.LastManifestID == request.ManifestID && dbDepot.ManifestID == request.ManifestID && Settings.Current.FullRun != FullRunState.WithForcedDepots && !dbDepot.FilenamesEncrypted && request.DepotKey != null) { // Update depot name if changed if (request.DepotName != dbDepot.Name) { await db.ExecuteAsync("UPDATE `Depots` SET `Name` = @DepotName WHERE `DepotID` = @DepotID", new { request.DepotID, request.DepotName }); } continue; } request.StoredFilenamesEncrypted = dbDepot.FilenamesEncrypted; request.LastManifestID = dbDepot.LastManifestID; } else { dbDepot = new Depot(); } if (dbDepot.BuildID != request.BuildID || dbDepot.ManifestID != request.ManifestID || request.DepotName != dbDepot.Name) { await db.ExecuteAsync(@"INSERT INTO `Depots` (`DepotID`, `Name`, `BuildID`, `ManifestID`) VALUES (@DepotID, @DepotName, @BuildID, @ManifestID) ON DUPLICATE KEY UPDATE `LastUpdated` = CURRENT_TIMESTAMP(), `Name` = VALUES(`Name`), `BuildID` = VALUES(`BuildID`), `ManifestID` = VALUES(`ManifestID`)", new { request.DepotID, request.DepotName, request.BuildID, request.ManifestID }); } if (dbDepot.ManifestID != request.ManifestID) { await MakeHistory(db, null, request, string.Empty, "manifest_change", dbDepot.ManifestID, request.ManifestID); } lock (DepotLocks) { // This doesn't really save us from concurrency issues if (DepotLocks.ContainsKey(request.DepotID)) { Log.WriteWarn(nameof(DepotProcessor), $"Depot {request.DepotID} was locked in another thread"); continue; } DepotLocks.Add(request.DepotID, 1); } depotsToDownload.Add(request); } } if (depotsToDownload.Count > 0) { _ = TaskManager.Run(async() => { try { await DownloadDepots(appID, depotsToDownload); } catch (Exception e) { ErrorReporter.Notify(nameof(DepotProcessor), e); } foreach (var depot in depotsToDownload) { RemoveLock(depot.DepotID); } }); } }
private static void OnPICSProductInfo(SteamApps.PICSProductInfoCallback callback) { JobManager.TryRemoveJob(callback.JobID); var apps = callback.Apps.Concat(callback.UnknownApps.ToDictionary(x => x, x => (SteamApps.PICSProductInfoCallback.PICSProductInfo)null)); var packages = callback.Packages.Concat(callback.UnknownPackages.ToDictionary(x => x, x => (SteamApps.PICSProductInfoCallback.PICSProductInfo)null)); foreach (var workaround in apps) { var app = workaround; Log.WriteInfo("PICSProductInfo", "{0}AppID: {1}", app.Value == null ? "Unknown " : "", app.Key); Task mostRecentItem; lock (ProcessedApps) { ProcessedApps.TryGetValue(app.Key, out mostRecentItem); } var workerItem = TaskManager.Run(async() => { try { await ProcessorSemaphore.WaitAsync().ConfigureAwait(false); if (mostRecentItem != null && !mostRecentItem.IsCompleted) { Log.WriteDebug("PICSProductInfo", "Waiting for app {0} to finish processing", app.Key); await mostRecentItem.ConfigureAwait(false); } using (var processor = new AppProcessor(app.Key)) { if (app.Value == null) { processor.ProcessUnknown(); } else { processor.Process(app.Value); } } } catch (MySqlException e) { ErrorReporter.Notify($"App {app.Key}", e); JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(app.Key, null)); } catch (Exception e) { ErrorReporter.Notify($"App {app.Key}", e); } finally { lock (ProcessedApps) { if (ProcessedApps.TryGetValue(app.Key, out mostRecentItem) && mostRecentItem.IsCompleted) { ProcessedApps.Remove(app.Key); } } ProcessorSemaphore.Release(); } }); if (Settings.IsFullRun) { continue; } lock (ProcessedApps) { ProcessedApps[app.Key] = workerItem; } } foreach (var workaround in packages) { var package = workaround; Log.WriteInfo("PICSProductInfo", "{0}SubID: {1}", package.Value == null ? "Unknown " : "", package.Key); Task mostRecentItem; lock (ProcessedSubs) { ProcessedSubs.TryGetValue(package.Key, out mostRecentItem); } var workerItem = TaskManager.Run(async() => { try { await ProcessorSemaphore.WaitAsync().ConfigureAwait(false); if (mostRecentItem != null && !mostRecentItem.IsCompleted) { Log.WriteDebug("PICSProductInfo", "Waiting for package {0} to finish processing", package.Key); await mostRecentItem.ConfigureAwait(false); } using (var processor = new SubProcessor(package.Key)) { if (package.Value == null) { processor.ProcessUnknown(); } else { processor.Process(package.Value); } } } catch (MySqlException e) { ErrorReporter.Notify($"Package {package.Key}", e); JobManager.AddJob(() => Steam.Instance.Apps.PICSGetProductInfo(null, package.Key, false, false)); } catch (Exception e) { ErrorReporter.Notify($"Package {package.Key}", e); } finally { lock (ProcessedSubs) { if (ProcessedSubs.TryGetValue(package.Key, out mostRecentItem) && mostRecentItem.IsCompleted) { ProcessedSubs.Remove(package.Key); } } ProcessorSemaphore.Release(); } }); if (Settings.IsFullRun) { continue; } lock (ProcessedSubs) { ProcessedSubs[package.Key] = workerItem; } } }
private void DownloadManifest(ManifestJob request) { Log.WriteInfo("Depot Processor", "DepotID: {0}", request.DepotID); DepotManifest depotManifest = null; string lastError = string.Empty; // CDN is very random, just keep trying for (var i = 0; i <= 5; i++) { try { depotManifest = CDNClient.DownloadManifest(request.DepotID, request.ManifestID, request.Server, request.CDNToken, request.DepotKey); break; } catch (Exception e) { lastError = e.Message; } } if (depotManifest == null) { Log.WriteError("Depot Processor", "Failed to download depot manifest for depot {0} ({1}: {2}) (#{3})", request.DepotID, request.Server, lastError, request.Tries); if (--request.Tries >= 0) { request.Server = GetContentServer(request.Tries); JobManager.AddJob(() => Steam.Instance.Apps.GetCDNAuthToken(request.DepotID, request.Server), request); return; } RemoveLock(request.DepotID); // TODO: Remove this once task in OnCDNAuthTokenCallback is used if (FileDownloader.IsImportantDepot(request.DepotID)) { IRC.Instance.SendOps("{0}[{1}]{2} Failed to download depot {3} manifest ({4}: {5})", Colors.OLIVE, Steam.GetAppName(request.ParentAppID), Colors.NORMAL, request.DepotID, request.Server, lastError); } return; } if (FileDownloader.IsImportantDepot(request.DepotID)) { TaskManager.Run(() => FileDownloader.DownloadFilesFromDepot(request, depotManifest)); } // TODO: Task here instead of in OnCDNAuthTokenCallback due to mono's silly threadpool TaskManager.Run(() => { using (var db = Database.GetConnection()) { ProcessDepotAfterDownload(db, request, depotManifest); } }).ContinueWith(task => { RemoveLock(request.DepotID); Log.WriteDebug("Depot Processor", "Processed depot {0} ({1} depot locks left)", request.DepotID, DepotLocks.Count); }); }
private static void OnPICSProductInfo(SteamApps.PICSProductInfoCallback callback) { JobManager.TryRemoveJob(callback.JobID); var processors = new List <BaseProcessor>( callback.Apps.Count + callback.Packages.Count + callback.UnknownApps.Count + callback.UnknownPackages.Count ); processors.AddRange(callback.Apps.Select(app => new AppProcessor(app.Key, app.Value))); processors.AddRange(callback.Packages.Select(package => new SubProcessor(package.Key, package.Value))); processors.AddRange(callback.UnknownApps.Select(app => new AppProcessor(app, null))); processors.AddRange(callback.UnknownPackages.Select(package => new SubProcessor(package, null))); foreach (var workaround in processors) { var processor = workaround; Task mostRecentItem; lock (CurrentlyProcessing) { CurrentlyProcessing.TryGetValue(processor.Id, out mostRecentItem); } var workerItem = TaskManager.Run(async() => { try { await Semaphore.WaitAsync(TaskManager.TaskCancellationToken.Token).ConfigureAwait(false); if (mostRecentItem?.IsCompleted == false) { Log.WriteDebug(processor.ToString(), $"Waiting for previous task to finish processing ({CurrentlyProcessing.Count})"); await mostRecentItem.ConfigureAwait(false); #if DEBUG Log.WriteDebug(processor.ToString(), "Previous task lock ended"); #endif } await processor.Process().ConfigureAwait(false); } catch (Exception e) { ErrorReporter.Notify(processor.ToString(), e); } finally { Semaphore.Release(); processor.Dispose(); } return(processor); }).Unwrap(); lock (CurrentlyProcessing) { CurrentlyProcessing[processor.Id] = workerItem; } // Register error handler on inner task and the continuation TaskManager.RegisterErrorHandler(workerItem); TaskManager.RegisterErrorHandler(workerItem.ContinueWith(RemoveProcessorLock, TaskManager.TaskCancellationToken.Token)); } }
public async Task Process(uint appID, uint changeNumber, KeyValue depots) { var requests = new List <ManifestJob>(); // Get data in format we want first foreach (var depot in depots.Children) { // Ignore these for now, parent app should be updated too anyway if (depot["depotfromapp"].Value != null) { continue; } var request = new ManifestJob { ChangeNumber = changeNumber, DepotName = depot["name"].AsString() }; // Ignore keys that aren't integers, for example "branches" if (!uint.TryParse(depot.Name, out request.DepotID)) { continue; } // TODO: instead of locking we could wait for current process to finish if (DepotLocks.ContainsKey(request.DepotID)) { continue; } if (depot["manifests"]["public"].Value == null || !ulong.TryParse(depot["manifests"]["public"].Value, out request.ManifestID)) { var branch = depot["manifests"].Children.Find(x => x.Name != "local"); if (branch == null || !ulong.TryParse(branch.Value, out request.ManifestID)) { using (var db = Database.Get()) { await db.ExecuteAsync("INSERT INTO `Depots` (`DepotID`, `Name`) VALUES (@DepotID, @DepotName) ON DUPLICATE KEY UPDATE `DepotID` = VALUES(`DepotID`)", new { request.DepotID, request.DepotName }); } continue; } Log.WriteDebug("Depot Downloader", "Depot {0} (from {1}) has no public branch, but there is another one", request.DepotID, appID); request.BuildID = depots["branches"][branch.Name]["buildid"].AsInteger(); } else { request.BuildID = depots["branches"]["public"]["buildid"].AsInteger(); } requests.Add(request); } if (requests.Count == 0) { return; } var depotsToDownload = new List <ManifestJob>(); using (var db = await Database.GetConnectionAsync()) { await db.ExecuteAsync("INSERT INTO `Builds` (`BuildID`, `ChangeID`, `AppID`) VALUES (@BuildID, @ChangeNumber, @AppID) ON DUPLICATE KEY UPDATE `AppID` = VALUES(`AppID`)", new { requests[0].BuildID, requests[0].ChangeNumber, appID }); var dbDepots = (await db.QueryAsync <Depot>("SELECT `DepotID`, `Name`, `BuildID`, `ManifestID`, `LastManifestID` FROM `Depots` WHERE `DepotID` IN @Depots", new { Depots = requests.Select(x => x.DepotID) })) .ToDictionary(x => x.DepotID, x => x); foreach (var request in requests) { Depot dbDepot; if (dbDepots.ContainsKey(request.DepotID)) { dbDepot = dbDepots[request.DepotID]; if (dbDepot.BuildID > request.BuildID) { // buildid went back in time? this either means a rollback, or a shared depot that isn't synced properly Log.WriteDebug("Depot Processor", "Skipping depot {0} due to old buildid: {1} > {2}", request.DepotID, dbDepot.BuildID, request.BuildID); continue; } if (dbDepot.LastManifestID == request.ManifestID && dbDepot.ManifestID == request.ManifestID && Settings.Current.FullRun != FullRunState.WithForcedDepots) { // Update depot name if changed if (!request.DepotName.Equals(dbDepot.Name)) { await db.ExecuteAsync("UPDATE `Depots` SET `Name` = @DepotName WHERE `DepotID` = @DepotID", new { request.DepotID, request.DepotName }); } continue; } } else { dbDepot = new Depot(); } if (dbDepot.BuildID != request.BuildID || dbDepot.ManifestID != request.ManifestID || !request.DepotName.Equals(dbDepot.Name)) { await db.ExecuteAsync(@"INSERT INTO `Depots` (`DepotID`, `Name`, `BuildID`, `ManifestID`) VALUES (@DepotID, @DepotName, @BuildID, @ManifestID) ON DUPLICATE KEY UPDATE `LastUpdated` = CURRENT_TIMESTAMP(), `Name` = VALUES(`Name`), `BuildID` = VALUES(`BuildID`), `ManifestID` = VALUES(`ManifestID`)", new { request.DepotID, request.DepotName, request.BuildID, request.ManifestID }); } if (dbDepot.ManifestID != request.ManifestID) { await MakeHistory(db, null, request, string.Empty, "manifest_change", dbDepot.ManifestID, request.ManifestID); } var owned = LicenseList.OwnedApps.ContainsKey(request.DepotID); if (!owned) { request.Anonymous = owned = LicenseList.AnonymousApps.ContainsKey(request.DepotID); if (owned) { Log.WriteWarn("Depot Processor", "Will download depot {0} using anonymous account", request.DepotID); } } if (owned) { lock (DepotLocks) { // This doesn't really save us from concurrency issues if (DepotLocks.ContainsKey(request.DepotID)) { Log.WriteWarn("Depot Processor", "Depot {0} was locked in another thread", request.DepotID); continue; } DepotLocks.Add(request.DepotID, 1); } depotsToDownload.Add(request); } } } if (depotsToDownload.Count > 0) { _ = TaskManager.Run(async() => { try { await DownloadDepots(appID, depotsToDownload); } catch (Exception e) { ErrorReporter.Notify("Depot Processor", e); } foreach (var depot in depotsToDownload) { RemoveLock(depot.DepotID); } }); } }
private static async Task <DateTime> ProcessFeed(Uri feedUrl, DateTime lastPostDate) { var feed = await LoadRSS(feedUrl); if (feed == null) { return(DateTime.MinValue); } if (feed.Items.Count == 0) { Log.WriteError(nameof(RSS), $"Did not find any items in {feedUrl}"); return(DateTime.MinValue); } await using var db = await Database.GetConnectionAsync(); var items = (await db.QueryAsync <GenericFeedItem>("SELECT `Link` FROM `RSS` WHERE `Link` IN @Ids", new { Ids = feed.Items.Select(x => x.Link) })).ToDictionary(x => x.Link, _ => (byte)1); var newItems = feed.Items.Where(item => item.PubDate > lastPostDate && !items.ContainsKey(item.Link)); var maxDate = DateTime.MinValue; foreach (var item in newItems) { if (maxDate < item.PubDate) { maxDate = item.PubDate; } Log.WriteInfo(nameof(RSS), $"[{feed.Title}] {item.Title}: {item.Link} ({item.PubDate})"); IRC.Instance.SendAnnounce($"{Colors.BLUE}{feed.Title}{Colors.NORMAL}: {item.Title} -{Colors.DARKBLUE} {item.Link}"); await db.ExecuteAsync("INSERT INTO `RSS` (`Link`, `Title`, `Date`) VALUES(@Link, @Title, @PubDate)", new { item.Link, item.Title, item.PubDate, }); _ = TaskManager.Run(async() => await Utils.SendWebhook(new { Type = "RSS", item.Title, Url = item.Link, })); if (!Settings.IsMillhaven) { continue; } uint appID = 0; if (feed.Title == "Steam RSS News Feed") { if (item.Title.StartsWith("Dota 2 Update", StringComparison.Ordinal)) { appID = 570; } else if (item.Title == "Team Fortress 2 Update Released") { appID = 440; // tf2 changelog cleanup item.Content = item.Content.Replace("<br/>", "\n"); item.Content = item.Content.Replace("<ul style=\"padding-bottom: 0px; margin-bottom: 0px;\">", "\n"); item.Content = item.Content.Replace("<ul style=\"padding-bottom: 0px; margin-bottom: 0px;\" >", "\n"); item.Content = item.Content.Replace("</ul>", "\n"); item.Content = item.Content.Replace("<li>", "* "); } else if (item.Title == "Left 4 Dead 2 - Update") { appID = 550; } else if (item.Title == "Left 4 Dead - Update") { appID = 500; } else if (item.Title == "Portal 2 - Update") { appID = 620; } } else if (feed.Title.Contains("Counter-Strike: Global Offensive") && item.Title.StartsWith("Release Notes", StringComparison.Ordinal)) { appID = 730; // csgo changelog cleanup item.Content = item.Content.Replace("</p>", "\n"); item.Content = new Regex("<p>\\[\\s*(.+)\\s*\\]", RegexOptions.Multiline | RegexOptions.CultureInvariant).Replace(item.Content, "## $1"); item.Content = item.Content.Replace("<p>", ""); } if (appID > 0) { var build = (await db.QueryAsync <Build>( "SELECT `Builds`.`BuildID`, `Builds`.`ChangeID`, `Builds`.`AppID`, `Changelists`.`Date`, LENGTH(`Official`) as `Official` FROM `Builds` " + "LEFT JOIN `Patchnotes` ON `Patchnotes`.`BuildID` = `Builds`.`BuildID` " + "JOIN `Apps` ON `Apps`.`AppID` = `Builds`.`AppID` " + "JOIN `Changelists` ON `Builds`.`ChangeID` = `Changelists`.`ChangeID` " + "WHERE `Builds`.`AppID` = @AppID ORDER BY `Builds`.`BuildID` DESC LIMIT 1", new { appID } )).SingleOrDefault(); if (build == null) { continue; } if (DateTime.UtcNow > build.Date.AddMinutes(60)) { Log.WriteDebug(nameof(RSS), $"Got {appID} update patch notes, but there is no build within last 10 minutes. {item.Link}"); IRC.Instance.SendOps($"{Colors.GREEN}[Patch notes]{Colors.NORMAL} Got {appID} update patch notes, but there is no build within last 10 minutes. {item.Link}"); continue; } if (build.Official > 0) { Log.WriteDebug(nameof(RSS), $"Got {appID} update patch notes, but official patch notes is already filled. {item.Link}"); IRC.Instance.SendOps($"{Colors.GREEN}[Patch notes]{Colors.NORMAL} Got {appID} update patch notes, but official patch notes is already filled. {item.Link}"); continue; } // breaks item.Content = new Regex(@"<br( \/)?>\r?\n?", RegexOptions.Multiline | RegexOptions.CultureInvariant).Replace(item.Content, "\n"); // dashes (CS:GO mainly) item.Content = new Regex("^(?:-|&#(?:8208|8209|8210|8211|8212|8213);|–|—) ?", RegexOptions.Multiline | RegexOptions.CultureInvariant).Replace(item.Content, "* "); item.Content = WebUtility.HtmlDecode(item.Content); Log.WriteDebug(nameof(RSS), $"Inserting {build.AppID} patchnotes for build {build.BuildID}:\n{item.Content}"); var accountId = Steam.Instance.Client?.SteamID?.AccountID ?? 0; await db.ExecuteAsync( "INSERT INTO `Patchnotes` (`BuildID`, `AppID`, `ChangeID`, `Date`, `Official`, `OfficialURL`) " + "VALUES (@BuildID, @AppID, @ChangeID, @Date, @Content, @Link) ON DUPLICATE KEY UPDATE `Official` = VALUES(`Official`), `OfficialURL` = VALUES(`OfficialURL`), `LastEditor` = @AccountID", new { build.BuildID, build.AppID, build.ChangeID, Date = build.Date.AddSeconds(1).ToString("yyyy-MM-dd HH:mm:ss"), item.Content, item.Link, accountId } ); IRC.Instance.SendAnnounce($"\u2699 Official patch notes:{Colors.BLUE} {Steam.GetAppName(build.AppID)}{Colors.NORMAL} -{Colors.DARKBLUE} {SteamDB.GetPatchnotesUrl(build.BuildID)}"); } } return(maxDate); }