private void PrintImportants(SteamApps.PICSChangesCallback callback) { // Apps var important = callback.AppChanges.Keys.Intersect(Application.ImportantApps.Keys); foreach (var app in important) { var appName = Steam.GetAppName(app, out var appType); IRC.Instance.AnnounceImportantAppUpdate(app, "{0} update: {1}{2}{3} -{4} {5}", appType, Colors.BLUE, appName, Colors.NORMAL, Colors.DARKBLUE, SteamDB.GetAppURL(app, "history")); if (Settings.Current.CanQueryStore) { Steam.Instance.UnifiedMessages.SendMessage("ChatRoom.SendChatMessage#1", new CChatRoom_SendChatMessage_Request { chat_group_id = 1147, chat_id = 10208600, message = $"[Changelist {callback.CurrentChangeNumber}] {appType} update: {appName}\n{SteamDB.GetAppURL(app, "history")}?utm_source=Steam&utm_medium=Steam&utm_campaign=SteamDB%20Group%20Chat" }); } } // Packages important = callback.PackageChanges.Keys.Intersect(Application.ImportantSubs.Keys); foreach (var package in important) { IRC.Instance.AnnounceImportantPackageUpdate(package, "Package update: {0}{1}{2} -{3} {4}", Colors.BLUE, Steam.GetPackageName(package), Colors.NORMAL, Colors.DARKBLUE, SteamDB.GetPackageURL(package, "history")); } }
private void PrintImportants(SteamApps.PICSChangesCallback callback) { // Apps var important = callback.AppChanges.Keys.Intersect(Application.ImportantApps.Keys); string appType; string appName; foreach (var app in important) { appName = Steam.GetAppName(app, out appType); IRC.Instance.AnnounceImportantAppUpdate(app, "{0} update: {1}{2}{3} -{4} {5}", appType, Colors.BLUE, appName, Colors.NORMAL, Colors.DARKBLUE, SteamDB.GetAppURL(app, "history")); } // Packages important = callback.PackageChanges.Keys.Intersect(Application.ImportantSubs.Keys); foreach (var package in important) { IRC.Instance.AnnounceImportantPackageUpdate(package, "Package update: {0}{1}{2} -{3} {4}", Colors.BLUE, Steam.GetPackageName(package), Colors.NORMAL, Colors.DARKBLUE, SteamDB.GetPackageURL(package, "history")); } }
private void OnPICSChangesFullRun(SteamApps.PICSChangesCallback callback) { PreviousChangeNumber = 2; Log.WriteInfo("PICSChanges", "Requesting info for {0} apps and {1} packages", callback.AppChanges.Count, callback.PackageChanges.Count); var apps = callback.AppChanges.Keys.ToList(); // Horribly unoptimized mess, but it's a full run so whatever while (apps.Any()) { JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(apps.Take(500), Enumerable.Empty <uint>())); apps = apps.Skip(500).ToList(); } var packages = callback.PackageChanges.Keys.Select(package => Utils.NewPICSRequest(package)).ToList(); while (packages.Any()) { JobManager.AddJob(() => Steam.Instance.Apps.PICSGetProductInfo(Enumerable.Empty <SteamApps.PICSRequest>(), packages.Take(500))); packages = packages.Skip(500).ToList(); } }
private static void PrintImportants(SteamApps.PICSChangesCallback callback) { // Apps var important = callback.AppChanges.Keys.Intersect(Application.ImportantApps.Keys); foreach (var app in important) { var appName = Steam.GetAppName(app, out var appType); IRC.Instance.AnnounceImportantAppUpdate(app, $"{appType} update: {Colors.BLUE}{appName}{Colors.NORMAL} -{Colors.DARKBLUE} {SteamDB.GetAppUrl(app, "history")}"); if (Settings.Current.CanQueryStore) { Steam.Instance.UnifiedMessages.SendMessage("ChatRoom.SendChatMessage#1", new CChatRoom_SendChatMessage_Request { chat_group_id = 1147, chat_id = 10208600, message = $"{appType} update: {appName}\n<{SteamDB.GetAppUrl(app, "history")}?changeid={callback.CurrentChangeNumber}>" }); } } // Packages important = callback.PackageChanges.Keys.Intersect(Application.ImportantSubs.Keys); foreach (var package in important) { IRC.Instance.SendMain($"Package update: {Colors.BLUE}{Steam.GetPackageName(package)}{Colors.NORMAL} -{Colors.DARKBLUE} {SteamDB.GetPackageUrl(package, "history")}"); } }
private void OnPICSChanges(SteamApps.PICSChangesCallback callback) { if (PreviousChangeNumber == callback.CurrentChangeNumber) { return; } Bootstrap.Log("Changelist {0} -> {1} ({2} apps, {3} packages)", PreviousChangeNumber, callback.CurrentChangeNumber, callback.AppChanges.Count, callback.PackageChanges.Count); PreviousChangeNumber = callback.CurrentChangeNumber; // Group apps and package changes by changelist, this will seperate into individual changelists var appGrouping = callback.AppChanges.Values.GroupBy(a => a.ChangeNumber); var packageGrouping = callback.PackageChanges.Values.GroupBy(p => p.ChangeNumber); // Join apps and packages back together based on changelist number var changeLists = Utils.FullOuterJoin(appGrouping, packageGrouping, a => a.Key, p => p.Key, (a, p, key) => new SteamChangelist { ChangeNumber = key, Apps = a.Select(x => x.ID), Packages = p.Select(x => x.ID), }, new EmptyGrouping <uint, SteamApps.PICSChangesCallback.PICSChangeData>(), new EmptyGrouping <uint, SteamApps.PICSChangesCallback.PICSChangeData>()) .OrderBy(c => c.ChangeNumber); foreach (var changeList in changeLists) { Bootstrap.Broadcast(new ChangelistEvent(changeList)); } Bootstrap.SendAppsToSubscribers(callback.AppChanges); }
private static async Task HandlePackagesChangelists(SteamApps.PICSChangesCallback callback) { await using var db = await Database.GetConnectionAsync(); await db.ExecuteAsync("INSERT INTO `ChangelistsSubs` (`ChangeID`, `SubID`) VALUES (@ChangeNumber, @ID) ON DUPLICATE KEY UPDATE `SubID` = `SubID`", callback.PackageChanges.Values); await db.ExecuteAsync("UPDATE `Subs` SET `LastUpdated` = CURRENT_TIMESTAMP() WHERE `SubID` IN @Ids", new { Ids = callback.PackageChanges.Values.Select(x => x.ID) }); }
private static async Task RefreshChanges() { if (!await RefreshSemaphore.WaitAsync(0).ConfigureAwait(false)) { return; } try { Bot refreshBot = null; SteamApps.PICSChangesCallback picsChanges = null; for (byte i = 0; (i < WebBrowser.MaxTries) && (picsChanges == null); i++) { refreshBot = Bot.Bots.Values.FirstOrDefault(bot => bot.IsConnectedAndLoggedOn); if (refreshBot == null) { return; } try { picsChanges = await refreshBot.SteamApps.PICSGetChangesSince(LastChangeNumber, true, true).ToLongRunningTask().ConfigureAwait(false); } catch (Exception e) { refreshBot.ArchiLogger.LogGenericWarningException(e); } } if ((refreshBot == null) || (picsChanges == null)) { ASF.ArchiLogger.LogGenericWarning(Strings.WarningFailed); return; } if (picsChanges.CurrentChangeNumber == picsChanges.LastChangeNumber) { return; } LastChangeNumber = picsChanges.CurrentChangeNumber; if (picsChanges.RequiresFullAppUpdate || picsChanges.RequiresFullPackageUpdate || ((picsChanges.AppChanges.Count == 0) && (picsChanges.PackageChanges.Count == 0))) { await PluginsCore.OnPICSChangesRestart(picsChanges.CurrentChangeNumber).ConfigureAwait(false); return; } if (picsChanges.PackageChanges.Count > 0) { await ASF.GlobalDatabase.RefreshPackages(refreshBot, picsChanges.PackageChanges.ToDictionary(package => package.Key, package => package.Value.ChangeNumber)).ConfigureAwait(false); } await PluginsCore.OnPICSChanges(picsChanges.CurrentChangeNumber, picsChanges.AppChanges, picsChanges.PackageChanges).ConfigureAwait(false); } finally { RefreshSemaphore.Release(); } }
private void OnPICSChangesFullRun(SteamApps.PICSChangesCallback callback) { PreviousChangeNumber = 2; var apps = callback.AppChanges.Keys; var packages = callback.PackageChanges.Keys; TaskManager.Run(() => RequestUpdateForList(apps, packages)); }
private void HandlePackagesChangelists(SteamApps.PICSChangesCallback callback) { using (var db = Database.GetConnection()) { db.Execute("INSERT INTO `ChangelistsSubs` (`ChangeID`, `SubID`) VALUES (@ChangeNumber, @ID) ON DUPLICATE KEY UPDATE `SubID` = `SubID`", callback.PackageChanges.Values); db.Execute("UPDATE `Subs` SET `LastUpdated` = CURRENT_TIMESTAMP() WHERE `SubID` IN @Ids", new { Ids = callback.PackageChanges.Values.Select(x => x.ID) }); } }
private void HandleApps(SteamApps.PICSChangesCallback callback) { StoreQueue.AddAppToQueue(callback.AppChanges.Values.Select(x => x.ID)); using (var db = Database.GetConnection()) { db.Execute("INSERT INTO `ChangelistsApps` (`ChangeID`, `AppID`) VALUES (@ChangeNumber, @ID) ON DUPLICATE KEY UPDATE `AppID` = `AppID`", callback.AppChanges.Values); db.Execute("UPDATE `Apps` SET `LastUpdated` = CURRENT_TIMESTAMP() WHERE `AppID` IN @Ids", new { Ids = callback.AppChanges.Values.Select(x => x.ID) }); } }
private async Task HandlePackages(SteamApps.PICSChangesCallback callback) { Dictionary <uint, byte> ignoredPackages; await using (var db = await Database.GetConnectionAsync()) { ignoredPackages = (await db.QueryAsync("SELECT `SubID`, `SubID` FROM `SubsInfo` WHERE `SubID` IN @Subs AND `Key` = @Key AND `Value` IN @Types", new { Key = BillingTypeKey, Subs = callback.PackageChanges.Values.Select(x => x.ID), Types = IgnorableBillingTypes } )).ToDictionary(x => (uint)x.SubID, _ => (byte)1); } // Steam comp if (!ignoredPackages.ContainsKey(0)) { ignoredPackages.Add(0, 1); } // Anon dedi comp if (!ignoredPackages.ContainsKey(17906)) { ignoredPackages.Add(17906, 1); } var subids = callback.PackageChanges.Values .Select(x => x.ID).Where(x => !ignoredPackages.ContainsKey(x)) .ToList(); if (subids.Count == 0) { return; } List <uint> appids; // Queue all the apps in the package as well await using (var db = await Database.GetConnectionAsync()) { appids = (await db.QueryAsync <uint>("SELECT `AppID` FROM `SubsApps` WHERE `SubID` IN @Ids AND `Type` = 'app'", new { Ids = subids })).ToList(); } if (appids.Count > 0) { await StoreQueue.AddAppToQueue(appids); } await StoreQueue.AddPackageToQueue(subids); }
private void OnPICSChangesFullRun(SteamApps.PICSChangesCallback callback, JobID job) { // Hackiness to prevent processing legit changelists after our request if that ever happens if (PreviousChange != 1) { Log.WriteWarn("Steam", "Got changelist {0}, but ignoring it because we're in a full run", callback.CurrentChangeNumber); return; } PreviousChange = 2; Log.WriteInfo("Steam", "Requesting info for {0} apps and {1} packages", callback.AppChanges.Count, callback.PackageChanges.Count); Apps.PICSGetProductInfo(Enumerable.Empty <SteamApps.PICSRequest>(), callback.PackageChanges.Keys.Select(package => NewPICSRequest(package))); Apps.PICSGetAccessTokens(callback.AppChanges.Keys, Enumerable.Empty <uint>()); }
private async void OnPICSChanges(SteamApps.PICSChangesCallback callback) { if (PreviousChangeNumber == callback.CurrentChangeNumber) { return; } var packageChangesCount = callback.PackageChanges.Count; var appChangesCount = callback.AppChanges.Count; Log.WriteInfo("PICSChanges", "Changelist {0} -> {1} ({2} apps, {3} packages)", PreviousChangeNumber, callback.CurrentChangeNumber, appChangesCount, packageChangesCount); LocalConfig.Current.ChangeNumber = PreviousChangeNumber = callback.CurrentChangeNumber; await HandleChangeNumbers(callback); if (appChangesCount == 0 && packageChangesCount == 0) { IRC.Instance.SendAnnounce("{0}»{1} Changelist {2}{3}{4} (empty)", Colors.RED, Colors.NORMAL, Colors.BLUE, PreviousChangeNumber, Colors.DARKGRAY); return; } // We don't care about waiting for separate database queries to finish, so don't await them #pragma warning disable 4014 if (appChangesCount > 0) { JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(callback.AppChanges.Keys, Enumerable.Empty <uint>())); TaskManager.RunAsync(async() => await HandleApps(callback)); } if (packageChangesCount > 0) { JobManager.AddJob(() => Steam.Instance.Apps.PICSGetProductInfo(Enumerable.Empty <SteamApps.PICSRequest>(), callback.PackageChanges.Keys.Select(Utils.NewPICSRequest))); TaskManager.RunAsync(async() => await HandlePackages(callback)); TaskManager.RunAsync(async() => await HandlePackagesChangelists(callback)); } TaskManager.RunAsync(async() => await SendChangelistsToIRC(callback)); #pragma warning restore 4014 PrintImportants(callback); }
private static async Task HandleChangeNumbers(SteamApps.PICSChangesCallback callback) { var changeNumbers = callback.AppChanges.Values .Select(x => x.ChangeNumber) .Union(callback.PackageChanges.Values.Select(x => x.ChangeNumber)) .Distinct() .Where(x => x != callback.CurrentChangeNumber) .ToList(); changeNumbers.Add(callback.CurrentChangeNumber); // Silly thing var changeLists = changeNumbers.Select(x => new Changelist { ChangeID = x }); using var db = Database.Get(); await db.ExecuteAsync("INSERT INTO `Changelists` (`ChangeID`) VALUES (@ChangeID) ON DUPLICATE KEY UPDATE `Date` = `Date`", changeLists); }
private void OnPICSChanges(SteamApps.PICSChangesCallback callback) { if (PreviousChangeNumber == callback.CurrentChangeNumber) { return; } var packageChangesCount = callback.PackageChanges.Count; var appChangesCount = callback.AppChanges.Count; Log.WriteInfo("PICSChanges", "Changelist {0} -> {1} ({2} apps, {3} packages)", PreviousChangeNumber, callback.CurrentChangeNumber, appChangesCount, packageChangesCount); PreviousChangeNumber = callback.CurrentChangeNumber; HandleChangeNumbers(callback); if (appChangesCount == 0 && packageChangesCount == 0) { IRC.Instance.SendAnnounce("{0}»{1} Changelist {2}{3}{4} (empty)", Colors.RED, Colors.NORMAL, Colors.BLUE, PreviousChangeNumber, Colors.DARKGRAY); return; } if (appChangesCount > 0) { JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(callback.AppChanges.Keys, Enumerable.Empty <uint>())); TaskManager.Run(() => HandleApps(callback)); } if (packageChangesCount > 0) { JobManager.AddJob(() => Steam.Instance.Apps.PICSGetProductInfo(Enumerable.Empty <SteamApps.PICSRequest>(), callback.PackageChanges.Keys.Select(package => Utils.NewPICSRequest(package)))); TaskManager.Run(() => HandlePackages(callback)); TaskManager.Run(() => HandlePackagesChangelists(callback)); } TaskManager.Run(() => SendChangelistsToIRC(callback)); PrintImportants(callback); }
private static void PrintImportants(SteamApps.PICSChangesCallback callback) { // Apps var important = callback.AppChanges.Keys.Intersect(Application.ImportantApps); foreach (var app in important) { var changeNumber = callback.AppChanges[app].ChangeNumber; TaskManager.Run(async() => await Utils.SendWebhook(new { Type = "ImportantAppUpdate", AppID = app, ChangeNumber = changeNumber, Url = $"{SteamDB.GetAppUrl(app, "history")}?changeid={changeNumber}", })); var appName = Steam.GetAppName(app, out var appType); IRC.Instance.SendAnnounce($"{appType} update: {Colors.BLUE}{appName}{Colors.NORMAL} -{Colors.DARKBLUE} {SteamDB.GetAppUrl(app, "history")}"); } // Packages important = callback.PackageChanges.Keys.Intersect(Application.ImportantSubs); foreach (var package in important) { var changeNumber = callback.PackageChanges[package].ChangeNumber; TaskManager.Run(async() => await Utils.SendWebhook(new { Type = "ImportantSubUpdate", SubID = package, ChangeNumber = changeNumber, Url = $"{SteamDB.GetPackageUrl(package, "history")}?changeid={changeNumber}", })); var subName = Steam.GetPackageName(package); IRC.Instance.SendAnnounce($"Package update: {Colors.BLUE}{subName}{Colors.NORMAL} -{Colors.DARKBLUE} {SteamDB.GetPackageUrl(package, "history")}"); } }
private void OnPICSChanges(SteamApps.PICSChangesCallback callback, JobID job) { if (PreviousChange == callback.CurrentChangeNumber) { return; } if (ProcessorPool.IsIdle) { Log.WriteDebug("Steam", "Cleaning processed apps and subs"); ProcessedApps.Clear(); ProcessedSubs.Clear(); } var packageChangesCount = callback.PackageChanges.Count; var appChangesCount = callback.AppChanges.Count; Log.WriteInfo("Steam", "Changelist {0} -> {1} ({2} apps, {3} packages)", PreviousChange, callback.CurrentChangeNumber, appChangesCount, packageChangesCount); PreviousChange = callback.CurrentChangeNumber; DbWorker.ExecuteNonQuery("INSERT INTO `Changelists` (`ChangeID`) VALUES (@ChangeID) ON DUPLICATE KEY UPDATE `Date` = CURRENT_TIMESTAMP()", new MySqlParameter("@ChangeID", callback.CurrentChangeNumber)); if (appChangesCount == 0 && packageChangesCount == 0) { IRC.SendAnnounce("{0}»{1} Changelist {2}{3}{4} (empty)", Colors.RED, Colors.NORMAL, Colors.OLIVE, PreviousChange, Colors.DARK_GRAY); return; } SecondaryPool.QueueWorkItem(SteamProxy.Instance.OnPICSChanges, callback); // Packages have no tokens so we request info for them right away if (packageChangesCount > 0) { Apps.PICSGetProductInfo(Enumerable.Empty <SteamApps.PICSRequest>(), callback.PackageChanges.Keys.Select(package => NewPICSRequest(package))); } if (appChangesCount > 0) { // Get all app tokens Apps.PICSGetAccessTokens(callback.AppChanges.Keys, Enumerable.Empty <uint>()); SecondaryPool.QueueWorkItem(delegate { string changes = string.Empty; foreach (var app in callback.AppChanges.Values) { if (callback.CurrentChangeNumber != app.ChangeNumber) { DbWorker.ExecuteNonQuery("INSERT INTO `Changelists` (`ChangeID`) VALUES (@ChangeID) ON DUPLICATE KEY UPDATE `Date` = `Date`", new MySqlParameter("@ChangeID", app.ChangeNumber)); } DbWorker.ExecuteNonQuery("UPDATE `Apps` SET `LastUpdated` = CURRENT_TIMESTAMP() WHERE `AppID` = @AppID", new MySqlParameter("@AppID", app.ID)); changes += string.Format("({0}, {1}),", app.ChangeNumber, app.ID); } if (!changes.Equals(string.Empty)) { changes = string.Format("INSERT INTO `ChangelistsApps` (`ChangeID`, `AppID`) VALUES {0} ON DUPLICATE KEY UPDATE `AppID` = `AppID`", changes.Remove(changes.Length - 1)); DbWorker.ExecuteNonQuery(changes); } }); } if (packageChangesCount > 0) { SecondaryPool.QueueWorkItem(delegate { string changes = string.Empty; foreach (var package in callback.PackageChanges.Values) { if (callback.CurrentChangeNumber != package.ChangeNumber) { DbWorker.ExecuteNonQuery("INSERT INTO `Changelists` (`ChangeID`) VALUES (@ChangeID) ON DUPLICATE KEY UPDATE `Date` = `Date`", new MySqlParameter("@ChangeID", package.ChangeNumber)); } DbWorker.ExecuteNonQuery("UPDATE `Subs` SET `LastUpdated` = CURRENT_TIMESTAMP() WHERE `SubID` = @SubID", new MySqlParameter("@SubID", package.ID)); changes += string.Format("({0}, {1}),", package.ChangeNumber, package.ID); } if (!changes.Equals(string.Empty)) { changes = string.Format("INSERT INTO `ChangelistsSubs` (`ChangeID`, `SubID`) VALUES {0} ON DUPLICATE KEY UPDATE `SubID` = `SubID`", changes.Remove(changes.Length - 1)); DbWorker.ExecuteNonQuery(changes); } }); } }
private void OnPICSChanges(SteamApps.PICSChangesCallback callback) { if (PreviousChangeNumber == callback.CurrentChangeNumber) { return; } Log.WriteInfo(nameof(PICSChanges), $"Changelist {PreviousChangeNumber} -> {callback.CurrentChangeNumber} ({callback.AppChanges.Count} apps, {callback.PackageChanges.Count} packages)"); LocalConfig.Current.ChangeNumber = PreviousChangeNumber = callback.CurrentChangeNumber; TaskManager.Run(async() => await HandleChangeNumbers(callback)); if (callback.RequiresFullAppUpdate || callback.RequiresFullPackageUpdate) { TaskManager.Run(async() => { if (callback.RequiresFullAppUpdate) { IRC.Instance.SendOps($"Changelist {callback.CurrentChangeNumber} has forced a full app update"); await FullUpdateProcessor.FullUpdateAppsMetadata(); } if (callback.RequiresFullPackageUpdate) { IRC.Instance.SendOps($"Changelist {callback.CurrentChangeNumber} has forced a full package update"); await FullUpdateProcessor.FullUpdatePackagesMetadata(); } }); } if (callback.AppChanges.Count == 0 && callback.PackageChanges.Count == 0) { IRC.Instance.SendAnnounce($"{Colors.RED}»{Colors.NORMAL} Changelist {Colors.BLUE}{PreviousChangeNumber}{Colors.DARKGRAY} (empty)"); return; } const int appsPerJob = 50; if (callback.AppChanges.Count > appsPerJob) { foreach (var list in callback.AppChanges.Keys.Split(appsPerJob)) { JobManager.AddJob( () => Steam.Instance.Apps.PICSGetAccessTokens(list, Enumerable.Empty <uint>()), new PICSTokens.RequestedTokens { Apps = list.ToList() }); } } else if (callback.AppChanges.Count > 0) { JobManager.AddJob( () => Steam.Instance.Apps.PICSGetAccessTokens(callback.AppChanges.Keys, Enumerable.Empty <uint>()), new PICSTokens.RequestedTokens { Apps = callback.AppChanges.Keys.ToList() }); } if (callback.PackageChanges.Count > appsPerJob) { foreach (var list in callback.PackageChanges.Keys.Split(appsPerJob)) { JobManager.AddJob( () => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), list), new PICSTokens.RequestedTokens { Packages = list.ToList() }); } } else if (callback.PackageChanges.Count > 0) { JobManager.AddJob( () => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), callback.PackageChanges.Keys), new PICSTokens.RequestedTokens { Packages = callback.PackageChanges.Keys.ToList() }); } if (callback.AppChanges.Count > 0) { _ = TaskManager.Run(async() => await HandleApps(callback)); } if (callback.PackageChanges.Count > 0) { _ = TaskManager.Run(async() => await HandlePackages(callback)); _ = TaskManager.Run(async() => await HandlePackagesChangelists(callback)); } _ = TaskManager.Run(async() => await SendChangelistsToIRC(callback)); PrintImportants(callback); }
private void SendChangelistsToIRC(SteamApps.PICSChangesCallback callback) { // Group apps and package changes by changelist, this will seperate into individual changelists var appGrouping = callback.AppChanges.Values.GroupBy(a => a.ChangeNumber); var packageGrouping = callback.PackageChanges.Values.GroupBy(p => p.ChangeNumber); // Join apps and packages back together based on changelist number var changeLists = Utils.FullOuterJoin(appGrouping, packageGrouping, a => a.Key, p => p.Key, (a, p, key) => new { ChangeNumber = key, Apps = a.ToList(), Packages = p.ToList(), }, new EmptyGrouping <uint, SteamApps.PICSChangesCallback.PICSChangeData>(), new EmptyGrouping <uint, SteamApps.PICSChangesCallback.PICSChangeData>()) .OrderBy(c => c.ChangeNumber); foreach (var changeList in changeLists) { var appCount = changeList.Apps.Count; var packageCount = changeList.Packages.Count; string Message = string.Format("Changelist {0}{1}{2} {3}({4:N0} apps and {5:N0} packages){6} -{7} {8}", Colors.BLUE, changeList.ChangeNumber, Colors.NORMAL, Colors.DARKGRAY, appCount, packageCount, Colors.NORMAL, Colors.DARKBLUE, SteamDB.GetChangelistURL(changeList.ChangeNumber) ); var changesCount = appCount + packageCount; if (changesCount >= 50) { IRC.Instance.SendMain(Message); } IRC.Instance.SendAnnounce("{0}»{1} {2}", Colors.RED, Colors.NORMAL, Message); // If this changelist is very big, freenode will hate us forever if we decide to print all that stuff if (changesCount > 300) { IRC.Instance.SendAnnounce("{0} This changelist is too big to be printed in IRC, please view it online", Colors.RED); continue; } if (appCount > 0) { Dictionary <uint, App> apps; App data; using (var db = Database.GetConnection()) { apps = db.Query <App>("SELECT `AppID`, `Name`, `LastKnownName` FROM `Apps` WHERE `AppID` IN @Ids", new { Ids = changeList.Apps.Select(x => x.ID) }).ToDictionary(x => x.AppID, x => x); } foreach (var app in changeList.Apps) { apps.TryGetValue(app.ID, out data); IRC.Instance.SendAnnounce(" App: {0}{1}{2} - {3}{4}", Colors.BLUE, app.ID, Colors.NORMAL, Steam.FormatAppName(app.ID, data), app.NeedsToken ? SteamDB.StringNeedToken : string.Empty ); } } if (packageCount > 0) { Dictionary <uint, Package> packages; Package data; using (var db = Database.GetConnection()) { packages = db.Query <Package>("SELECT `SubID`, `Name`, `LastKnownName` FROM `Subs` WHERE `SubID` IN @Ids", new { Ids = changeList.Packages.Select(x => x.ID) }).ToDictionary(x => x.SubID, x => x); } foreach (var package in changeList.Packages) { packages.TryGetValue(package.ID, out data); IRC.Instance.SendAnnounce(" Package: {0}{1}{2} - {3}{4}", Colors.BLUE, package.ID, Colors.NORMAL, Steam.FormatPackageName(package.ID, data), package.NeedsToken ? SteamDB.StringNeedToken : string.Empty ); } } } }
void OnPICSChanges(SteamApps.PICSChangesCallback callback) { // group apps and package changes by changelist, this will seperate into individual changelists var appGrouping = callback.AppChanges .Select(a => a.Value) .GroupBy(a => a.ChangeNumber); var packageGrouping = callback.PackageChanges .Select(p => p.Value) .GroupBy(p => p.ChangeNumber); // join apps and packages back together based on changelist number var changeLists = appGrouping .FullOuterJoin(packageGrouping, a => a.Key, p => p.Key, (a, p, key) => new { ChangeNumber = key, Apps = a.ToList(), Packages = p.ToList(), }, new EmptyGrouping <uint, SteamApps.PICSChangesCallback.PICSChangeData>(), new EmptyGrouping <uint, SteamApps.PICSChangesCallback.PICSChangeData>()) .OrderBy(c => c.ChangeNumber); // the number of changes required in a changelist in order to be important enough to display // to all broadcast channels const int ChangesReqToBeImportant = 50; if (changeLists.Count() == 0) { // this will happen on the first changes request lastChangeNumber = callback.CurrentChangeNumber; return; } foreach (var changeList in changeLists) { if (changeList.ChangeNumber <= lastChangeNumber) { return; // old changelist } lastChangeNumber = changeList.ChangeNumber; int numAppChanges = changeList.Apps.Count; int numPackageChanges = changeList.Packages.Count; string message = string.Format("Got PICS changelist {0} for {1} apps and {2} packages - {3}", changeList.ChangeNumber, changeList.Apps.Count, changeList.Packages.Count, GetChangelistURL(changeList.ChangeNumber)); if (numPackageChanges >= ChangesReqToBeImportant || numAppChanges >= ChangesReqToBeImportant) { // if this changelist contains a number of changes over a specific threshold, we'll consider it "important" and send to all channels IRC.Instance.SendToTag("pics", message); } if (numAppChanges > 0) { var importantApps = changeList.Apps.Select(a => a.ID) .Intersect(Settings.Current.ImportantApps.Select(a => a.AppID)); foreach (var app in importantApps) { SettingsXml.ImportantApp importantApp = Settings.Current.ImportantApps .FirstOrDefault(a => a.AppID == app); string tag = "pics"; if (!string.IsNullOrEmpty(importantApp.Tag)) { tag = string.Format("{0}-{1}", tag, importantApp.Tag); } // get the channels interested in this pics update var picsChannels = Settings.Current.GetChannelsForTag(tag); if (picsChannels.Count() == 0) { Log.WriteWarn("PICSJob", "No channels setup for tag: {0}", tag); } string targetChans = string.Join(",", picsChannels.Select(c => c.Channel)); IRC.Instance.Send(targetChans, "Important App Update: {0} - {1}", Steam.Instance.GetAppName(app), GetAppHistoryUrl(app)); } } if (numPackageChanges > 0) { // todo: important packages } } }
public void OnPICSChanges(SteamApps.PICSChangesCallback callback) { // Print any apps importants first var important = callback.AppChanges.Keys.Intersect(ImportantApps); if (important.Count() > 5) { IRC.SendMain("{0}{1}{2} important apps updated -{3} {4}", Colors.OLIVE, important.Count(), Colors.NORMAL, Colors.DARK_BLUE, SteamDB.GetChangelistURL(callback.CurrentChangeNumber)); } else { foreach (var app in important) { IRC.SendMain("Important app update: {0}{1}{2} -{3} {4}", Colors.OLIVE, GetAppName(app), Colors.NORMAL, Colors.DARK_BLUE, SteamDB.GetAppURL(app, "history")); } } // And then important packages important = callback.PackageChanges.Keys.Intersect(ImportantSubs); if (important.Count() > 5) { IRC.SendMain("{0}{1}{2} important packages updated -{3} {4}", Colors.OLIVE, important.Count(), Colors.NORMAL, Colors.DARK_BLUE, SteamDB.GetChangelistURL(callback.CurrentChangeNumber)); } else { foreach (var package in important) { IRC.SendMain("Important package update: {0}{1}{2} -{3} {4}", Colors.OLIVE, GetPackageName(package), Colors.NORMAL, Colors.DARK_BLUE, SteamDB.GetPackageURL(package, "history")); } } // Group apps and package changes by changelist, this will seperate into individual changelists var appGrouping = callback.AppChanges.Values.GroupBy(a => a.ChangeNumber); var packageGrouping = callback.PackageChanges.Values.GroupBy(p => p.ChangeNumber); // Join apps and packages back together based on changelist number var changeLists = Utils.FullOuterJoin(appGrouping, packageGrouping, a => a.Key, p => p.Key, (a, p, key) => new { ChangeNumber = key, Apps = a.ToList(), Packages = p.ToList(), }, new EmptyGrouping <uint, SteamApps.PICSChangesCallback.PICSChangeData>(), new EmptyGrouping <uint, SteamApps.PICSChangesCallback.PICSChangeData>()) .OrderBy(c => c.ChangeNumber); foreach (var changeList in changeLists) { var appCount = changeList.Apps.Count; var packageCount = changeList.Packages.Count; string Message = string.Format("Changelist {0}{1}{2} {3}({4:N0} apps and {5:N0} packages){6} -{7} {8}", Colors.OLIVE, changeList.ChangeNumber, Colors.NORMAL, Colors.DARK_GRAY, appCount, packageCount, Colors.NORMAL, Colors.DARK_BLUE, SteamDB.GetChangelistURL(changeList.ChangeNumber) ); var changesCount = appCount + packageCount; if (changesCount >= 50) { IRC.SendMain(Message); } IRC.SendAnnounce("{0}»{1} {2}", Colors.RED, Colors.NORMAL, Message); // If this changelist is very big, freenode will hate us forever if we decide to print all that stuff if (changesCount > 500) { IRC.SendAnnounce("{0} This changelist is too big to be printed in IRC, please view it on our website", Colors.RED); continue; } string name; if (appCount > 0) { foreach (var app in changeList.Apps) { name = GetAppName(app.ID, true); if (string.IsNullOrEmpty(name)) { name = string.Format("{0}{1}{2}", Colors.GREEN, app.ID, Colors.NORMAL); } else { name = string.Format("{0}{1}{2} - {3}", Colors.LIGHT_GRAY, app.ID, Colors.NORMAL, name); } IRC.SendAnnounce(" App: {0}{1}", name, app.NeedsToken ? StringNeedToken : string.Empty); } } if (packageCount > 0) { foreach (var package in changeList.Packages) { name = GetPackageName(package.ID, true); if (string.IsNullOrEmpty(name)) { name = string.Format("{0}{1}{2}", Colors.GREEN, package.ID, Colors.NORMAL); } else { name = string.Format("{0}{1}{2} - {3}", Colors.LIGHT_GRAY, package.ID, Colors.NORMAL, name); } IRC.SendAnnounce(" Package: {0}{1}{2}", name, package.NeedsToken ? StringNeedToken : string.Empty, Steam.Instance.OwnedPackages.Contains(package.ID) ? StringCheckmark : string.Empty ); } } } }
private void OnPICSChanges(SteamApps.PICSChangesCallback callback) { if (PreviousChangeNumber == callback.CurrentChangeNumber) { return; } Log.WriteInfo(nameof(PICSChanges), $"Changelist {callback.LastChangeNumber} -> {callback.CurrentChangeNumber} ({callback.AppChanges.Count} apps, {callback.PackageChanges.Count} packages)"); PreviousChangeNumber = callback.CurrentChangeNumber; TaskManager.Run(async() => await HandleChangeNumbers(callback)); if (callback.RequiresFullAppUpdate || callback.RequiresFullPackageUpdate) { TaskManager.Run(async() => { if (callback.RequiresFullAppUpdate) { IRC.Instance.SendOps($"Changelist {callback.CurrentChangeNumber} has forced a full app update"); // When full update flag is set, presumably Steam client start hammering the servers // and the PICS service just does not return any data for a while until it clears up await FullUpdateProcessor.FullUpdateAppsMetadata(true); } if (callback.RequiresFullPackageUpdate) { IRC.Instance.SendOps($"Changelist {callback.CurrentChangeNumber} has forced a full package update"); await FullUpdateProcessor.FullUpdatePackagesMetadata(); } IRC.Instance.SendOps($"Changelist {callback.CurrentChangeNumber} full update has finished"); }); } if (callback.AppChanges.Count == 0 && callback.PackageChanges.Count == 0) { return; } const int appsPerJob = 50; if (callback.AppChanges.Count > appsPerJob) { foreach (var list in callback.AppChanges.Keys.Split(appsPerJob)) { JobManager.AddJob( () => Steam.Instance.Apps.PICSGetAccessTokens(list, Enumerable.Empty <uint>()), new PICSTokens.RequestedTokens { Apps = list.ToList() }); } } else if (callback.AppChanges.Count > 0) { JobManager.AddJob( () => Steam.Instance.Apps.PICSGetAccessTokens(callback.AppChanges.Keys, Enumerable.Empty <uint>()), new PICSTokens.RequestedTokens { Apps = callback.AppChanges.Keys.ToList() }); } if (callback.PackageChanges.Count > appsPerJob) { foreach (var list in callback.PackageChanges.Keys.Split(appsPerJob)) { JobManager.AddJob( () => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), list), new PICSTokens.RequestedTokens { Packages = list.ToList() }); } } else if (callback.PackageChanges.Count > 0) { JobManager.AddJob( () => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), callback.PackageChanges.Keys), new PICSTokens.RequestedTokens { Packages = callback.PackageChanges.Keys.ToList() }); } if (callback.AppChanges.Count > 0) { _ = TaskManager.Run(async() => await HandleApps(callback)); } if (callback.PackageChanges.Count > 0) { _ = TaskManager.Run(async() => await HandlePackages(callback)); _ = TaskManager.Run(async() => await HandlePackagesChangelists(callback)); } if (PreviousChangeNumber - LastStoredChangeNumber >= 1000) { LastStoredChangeNumber = PreviousChangeNumber; _ = TaskManager.Run(async() => await LocalConfig.Update("backend.changenumber", LastStoredChangeNumber.ToString())); } PrintImportants(callback); }
private async void OnPICSChanges(SteamApps.PICSChangesCallback callback) { if (PreviousChangeNumber == callback.CurrentChangeNumber) { return; } Log.WriteInfo("PICSChanges", $"Changelist {PreviousChangeNumber} -> {callback.CurrentChangeNumber} ({callback.AppChanges.Count} apps, {callback.PackageChanges.Count} packages)"); LocalConfig.Current.ChangeNumber = PreviousChangeNumber = callback.CurrentChangeNumber; await HandleChangeNumbers(callback); if (callback.AppChanges.Count == 0 && callback.PackageChanges.Count == 0) { IRC.Instance.SendAnnounce("{0}»{1} Changelist {2}{3}{4} (empty)", Colors.RED, Colors.NORMAL, Colors.BLUE, PreviousChangeNumber, Colors.DARKGRAY); return; } const int appsPerJob = 50; if (callback.AppChanges.Count > appsPerJob) { foreach (var list in callback.AppChanges.Keys.Split(appsPerJob)) { JobManager.AddJob( () => Steam.Instance.Apps.PICSGetAccessTokens(list, Enumerable.Empty <uint>()), new PICSTokens.RequestedTokens { Apps = list.ToList() }); } } else if (callback.AppChanges.Count > 0) { JobManager.AddJob( () => Steam.Instance.Apps.PICSGetAccessTokens(callback.AppChanges.Keys, Enumerable.Empty <uint>()), new PICSTokens.RequestedTokens { Apps = callback.AppChanges.Keys.ToList() }); } if (callback.PackageChanges.Count > appsPerJob) { foreach (var list in callback.PackageChanges.Keys.Split(appsPerJob)) { JobManager.AddJob( () => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), list), new PICSTokens.RequestedTokens { Packages = list.ToList() }); } } else if (callback.PackageChanges.Count > 0) { JobManager.AddJob( () => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), callback.PackageChanges.Keys), new PICSTokens.RequestedTokens { Packages = callback.PackageChanges.Keys.ToList() }); } if (callback.AppChanges.Count > 0) { _ = TaskManager.RunAsync(async() => await HandleApps(callback)); } if (callback.PackageChanges.Count > 0) { _ = TaskManager.RunAsync(async() => await HandlePackages(callback)); _ = TaskManager.RunAsync(async() => await HandlePackagesChangelists(callback)); } _ = TaskManager.RunAsync(async() => await SendChangelistsToIRC(callback)); PrintImportants(callback); }
private async Task SendChangelistsToIRC(SteamApps.PICSChangesCallback callback) { if (DateTime.Now > ChangelistBurstTime) { ChangelistBurstTime = DateTime.Now.AddMinutes(5); ChangelistBurstCount = 0; } // Group apps and package changes by changelist number var changelists = new Dictionary <uint, IrcChangelistGroup>(); foreach (var app in callback.AppChanges.Values) { if (!changelists.ContainsKey(app.ChangeNumber)) { changelists[app.ChangeNumber] = new IrcChangelistGroup(); } changelists[app.ChangeNumber].Apps.Add(app.ID); } foreach (var package in callback.PackageChanges.Values) { if (!changelists.ContainsKey(package.ChangeNumber)) { changelists[package.ChangeNumber] = new IrcChangelistGroup(); } changelists[package.ChangeNumber].Packages.Add(package.ID); } foreach (var(changeNumber, changeList) in changelists.OrderBy(x => x.Key)) { var appCount = changeList.Apps.Count; var packageCount = changeList.Packages.Count; var message = $"Changelist {Colors.BLUE}{changeNumber}{Colors.NORMAL} {Colors.DARKGRAY}({appCount:N0} apps and {packageCount:N0} packages)"; var changesCount = appCount + packageCount; if (changesCount >= 50) { IRC.Instance.SendMain($"Big {message}{Colors.DARKBLUE} {SteamDB.GetChangelistUrl(changeNumber)}"); } if (ChangelistBurstCount++ >= CHANGELIST_BURST_MIN || changesCount > 300) { if (appCount > 0) { message += $" (Apps: {string.Join(", ", changeList.Apps)})"; } if (packageCount > 0) { message += $" (Packages: {string.Join(", ", changeList.Packages)})"; } IRC.Instance.SendAnnounce($"{Colors.RED}»{Colors.NORMAL} {message}"); continue; } IRC.Instance.SendAnnounce($"{Colors.RED}»{Colors.NORMAL} {message}"); if (appCount > 0) { Dictionary <uint, App> apps; await using (var db = await Database.GetConnectionAsync()) { apps = (await db.QueryAsync <App>("SELECT `AppID`, `Name`, `LastKnownName` FROM `Apps` WHERE `AppID` IN @Ids", new { Ids = changeList.Apps })).ToDictionary(x => x.AppID, x => x); } foreach (var appId in changeList.Apps) { apps.TryGetValue(appId, out var data); IRC.Instance.SendAnnounce($" App: {Colors.BLUE}{appId}{Colors.NORMAL} - {Steam.FormatAppName(appId, data)}"); } } if (packageCount > 0) { Dictionary <uint, Package> packages; await using (var db = await Database.GetConnectionAsync()) { packages = (await db.QueryAsync <Package>("SELECT `SubID`, `Name`, `LastKnownName` FROM `Subs` WHERE `SubID` IN @Ids", new { Ids = changeList.Packages })).ToDictionary(x => x.SubID, x => x); } foreach (var packageId in changeList.Packages) { packages.TryGetValue(packageId, out var data); IRC.Instance.SendAnnounce($" Package: {Colors.BLUE}{packageId}{Colors.NORMAL} - {Steam.FormatPackageName(packageId, data)}"); } } } }
private async Task SendChangelistsToIRC(SteamApps.PICSChangesCallback callback) { if (DateTime.Now > ChangelistBurstTime) { ChangelistBurstTime = DateTime.Now.AddMinutes(5); ChangelistBurstCount = 0; } // Group apps and package changes by changelist, this will seperate into individual changelists var appGrouping = callback.AppChanges.Values.GroupBy(a => a.ChangeNumber); var packageGrouping = callback.PackageChanges.Values.GroupBy(p => p.ChangeNumber); // Join apps and packages back together based on changelist number var changeLists = Utils.FullOuterJoin(appGrouping, packageGrouping, a => a.Key, p => p.Key, (a, p, key) => new { ChangeNumber = key, Apps = a.ToList(), Packages = p.ToList(), }, new EmptyGrouping <uint, SteamApps.PICSChangesCallback.PICSChangeData>(), new EmptyGrouping <uint, SteamApps.PICSChangesCallback.PICSChangeData>()) .OrderBy(c => c.ChangeNumber); foreach (var changeList in changeLists) { var appCount = changeList.Apps.Count; var packageCount = changeList.Packages.Count; var message = $"Changelist {Colors.BLUE}{changeList.ChangeNumber}{Colors.NORMAL} {Colors.DARKGRAY}({appCount:N0} apps and {packageCount:N0} packages)"; var changesCount = appCount + packageCount; if (changesCount >= 50) { IRC.Instance.SendMain($"Big {message}{Colors.DARKBLUE} {SteamDB.GetChangelistURL(changeList.ChangeNumber)}"); } if (ChangelistBurstCount++ >= CHANGELIST_BURST_MIN || changesCount > 300) { if (appCount > 0) { message += $" (Apps: {string.Join(", ", changeList.Apps.Select(x => x.ID))})"; } if (packageCount > 0) { message += $" (Packages: {string.Join(", ", changeList.Packages.Select(x => x.ID))})"; } IRC.Instance.SendAnnounce($"{Colors.RED}»{Colors.NORMAL} {message}"); continue; } IRC.Instance.SendAnnounce("{0}»{1} {2}", Colors.RED, Colors.NORMAL, message); if (appCount > 0) { Dictionary <uint, App> apps; using (var db = Database.Get()) { apps = (await db.QueryAsync <App>("SELECT `AppID`, `Name`, `LastKnownName` FROM `Apps` WHERE `AppID` IN @Ids", new { Ids = changeList.Apps.Select(x => x.ID) })).ToDictionary(x => x.AppID, x => x); } foreach (var app in changeList.Apps) { apps.TryGetValue(app.ID, out var data); IRC.Instance.SendAnnounce(" App: {0}{1}{2} - {3}{4}", Colors.BLUE, app.ID, Colors.NORMAL, Steam.FormatAppName(app.ID, data), app.NeedsToken ? SteamDB.StringNeedToken : string.Empty ); } } if (packageCount > 0) { Dictionary <uint, Package> packages; using (var db = Database.Get()) { packages = (await db.QueryAsync <Package>("SELECT `SubID`, `Name`, `LastKnownName` FROM `Subs` WHERE `SubID` IN @Ids", new { Ids = changeList.Packages.Select(x => x.ID) })).ToDictionary(x => x.SubID, x => x); } foreach (var package in changeList.Packages) { packages.TryGetValue(package.ID, out var data); IRC.Instance.SendAnnounce(" Package: {0}{1}{2} - {3}{4}", Colors.BLUE, package.ID, Colors.NORMAL, Steam.FormatPackageName(package.ID, data), package.NeedsToken ? SteamDB.StringNeedToken : string.Empty ); } } } }