private void PrintImportants(SteamApps.PICSChangesCallback callback)
{
// Apps
var important = callback.AppChanges.Keys.Intersect(Application.ImportantApps.Keys);
foreach (var app in important)
{
var appName = Steam.GetAppName(app, out var appType);
IRC.Instance.AnnounceImportantAppUpdate(app, "{0} update: {1}{2}{3} -{4} {5}",
appType,
Colors.BLUE, appName, Colors.NORMAL,
Colors.DARKBLUE, SteamDB.GetAppURL(app, "history"));
if (Settings.Current.CanQueryStore)
{
Steam.Instance.UnifiedMessages.SendMessage("ChatRoom.SendChatMessage#1", new CChatRoom_SendChatMessage_Request
{
chat_group_id = 1147,
chat_id = 10208600,
message = $"[Changelist {callback.CurrentChangeNumber}] {appType} update: {appName}\n{SteamDB.GetAppURL(app, "history")}?utm_source=Steam&utm_medium=Steam&utm_campaign=SteamDB%20Group%20Chat"
});
}
}
// Packages
important = callback.PackageChanges.Keys.Intersect(Application.ImportantSubs.Keys);
foreach (var package in important)
{
IRC.Instance.AnnounceImportantPackageUpdate(package, "Package update: {0}{1}{2} -{3} {4}", Colors.BLUE, Steam.GetPackageName(package), Colors.NORMAL, Colors.DARKBLUE, SteamDB.GetPackageURL(package, "history"));
}
}
private void PrintImportants(SteamApps.PICSChangesCallback callback)
{
// Apps
var important = callback.AppChanges.Keys.Intersect(Application.ImportantApps.Keys);
string appType;
string appName;
foreach (var app in important)
{
appName = Steam.GetAppName(app, out appType);
IRC.Instance.AnnounceImportantAppUpdate(app, "{0} update: {1}{2}{3} -{4} {5}",
appType,
Colors.BLUE, appName, Colors.NORMAL,
Colors.DARKBLUE, SteamDB.GetAppURL(app, "history"));
}
// Packages
important = callback.PackageChanges.Keys.Intersect(Application.ImportantSubs.Keys);
foreach (var package in important)
{
IRC.Instance.AnnounceImportantPackageUpdate(package, "Package update: {0}{1}{2} -{3} {4}", Colors.BLUE, Steam.GetPackageName(package), Colors.NORMAL, Colors.DARKBLUE, SteamDB.GetPackageURL(package, "history"));
}
}
private void OnPICSChangesFullRun(SteamApps.PICSChangesCallback callback)
{
PreviousChangeNumber = 2;
Log.WriteInfo("PICSChanges", "Requesting info for {0} apps and {1} packages", callback.AppChanges.Count, callback.PackageChanges.Count);
var apps = callback.AppChanges.Keys.ToList();
// Horribly unoptimized mess, but it's a full run so whatever
while (apps.Any())
{
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(apps.Take(500), Enumerable.Empty <uint>()));
apps = apps.Skip(500).ToList();
}
var packages = callback.PackageChanges.Keys.Select(package => Utils.NewPICSRequest(package)).ToList();
while (packages.Any())
{
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetProductInfo(Enumerable.Empty <SteamApps.PICSRequest>(), packages.Take(500)));
packages = packages.Skip(500).ToList();
}
}
private static void PrintImportants(SteamApps.PICSChangesCallback callback)
{
// Apps
var important = callback.AppChanges.Keys.Intersect(Application.ImportantApps.Keys);
foreach (var app in important)
{
var appName = Steam.GetAppName(app, out var appType);
IRC.Instance.AnnounceImportantAppUpdate(app, $"{appType} update: {Colors.BLUE}{appName}{Colors.NORMAL} -{Colors.DARKBLUE} {SteamDB.GetAppUrl(app, "history")}");
if (Settings.Current.CanQueryStore)
{
Steam.Instance.UnifiedMessages.SendMessage("ChatRoom.SendChatMessage#1", new CChatRoom_SendChatMessage_Request
{
chat_group_id = 1147,
chat_id = 10208600,
message = $"{appType} update: {appName}\n<{SteamDB.GetAppUrl(app, "history")}?changeid={callback.CurrentChangeNumber}>"
});
}
}
// Packages
important = callback.PackageChanges.Keys.Intersect(Application.ImportantSubs.Keys);
foreach (var package in important)
{
IRC.Instance.SendMain($"Package update: {Colors.BLUE}{Steam.GetPackageName(package)}{Colors.NORMAL} -{Colors.DARKBLUE} {SteamDB.GetPackageUrl(package, "history")}");
}
}
private void OnPICSChanges(SteamApps.PICSChangesCallback callback)
{
if (PreviousChangeNumber == callback.CurrentChangeNumber)
{
return;
}
Bootstrap.Log("Changelist {0} -> {1} ({2} apps, {3} packages)", PreviousChangeNumber, callback.CurrentChangeNumber, callback.AppChanges.Count, callback.PackageChanges.Count);
PreviousChangeNumber = callback.CurrentChangeNumber;
// Group apps and package changes by changelist, this will seperate into individual changelists
var appGrouping = callback.AppChanges.Values.GroupBy(a => a.ChangeNumber);
var packageGrouping = callback.PackageChanges.Values.GroupBy(p => p.ChangeNumber);
// Join apps and packages back together based on changelist number
var changeLists = Utils.FullOuterJoin(appGrouping, packageGrouping, a => a.Key, p => p.Key, (a, p, key) => new SteamChangelist
{
ChangeNumber = key,
Apps = a.Select(x => x.ID),
Packages = p.Select(x => x.ID),
},
new EmptyGrouping <uint, SteamApps.PICSChangesCallback.PICSChangeData>(),
new EmptyGrouping <uint, SteamApps.PICSChangesCallback.PICSChangeData>())
.OrderBy(c => c.ChangeNumber);
foreach (var changeList in changeLists)
{
Bootstrap.Broadcast(new ChangelistEvent(changeList));
}
Bootstrap.SendAppsToSubscribers(callback.AppChanges);
}
private static async Task HandlePackagesChangelists(SteamApps.PICSChangesCallback callback)
{
await using var db = await Database.GetConnectionAsync();
await db.ExecuteAsync("INSERT INTO `ChangelistsSubs` (`ChangeID`, `SubID`) VALUES (@ChangeNumber, @ID) ON DUPLICATE KEY UPDATE `SubID` = `SubID`", callback.PackageChanges.Values);
await db.ExecuteAsync("UPDATE `Subs` SET `LastUpdated` = CURRENT_TIMESTAMP() WHERE `SubID` IN @Ids", new { Ids = callback.PackageChanges.Values.Select(x => x.ID) });
}
private static async Task RefreshChanges()
{
if (!await RefreshSemaphore.WaitAsync(0).ConfigureAwait(false))
{
return;
}
try {
Bot refreshBot = null;
SteamApps.PICSChangesCallback picsChanges = null;
for (byte i = 0; (i < WebBrowser.MaxTries) && (picsChanges == null); i++)
{
refreshBot = Bot.Bots.Values.FirstOrDefault(bot => bot.IsConnectedAndLoggedOn);
if (refreshBot == null)
{
return;
}
try {
picsChanges = await refreshBot.SteamApps.PICSGetChangesSince(LastChangeNumber, true, true).ToLongRunningTask().ConfigureAwait(false);
} catch (Exception e) {
refreshBot.ArchiLogger.LogGenericWarningException(e);
}
}
if ((refreshBot == null) || (picsChanges == null))
{
ASF.ArchiLogger.LogGenericWarning(Strings.WarningFailed);
return;
}
if (picsChanges.CurrentChangeNumber == picsChanges.LastChangeNumber)
{
return;
}
LastChangeNumber = picsChanges.CurrentChangeNumber;
if (picsChanges.RequiresFullAppUpdate || picsChanges.RequiresFullPackageUpdate || ((picsChanges.AppChanges.Count == 0) && (picsChanges.PackageChanges.Count == 0)))
{
await PluginsCore.OnPICSChangesRestart(picsChanges.CurrentChangeNumber).ConfigureAwait(false);
return;
}
if (picsChanges.PackageChanges.Count > 0)
{
await ASF.GlobalDatabase.RefreshPackages(refreshBot, picsChanges.PackageChanges.ToDictionary(package => package.Key, package => package.Value.ChangeNumber)).ConfigureAwait(false);
}
await PluginsCore.OnPICSChanges(picsChanges.CurrentChangeNumber, picsChanges.AppChanges, picsChanges.PackageChanges).ConfigureAwait(false);
} finally {
RefreshSemaphore.Release();
}
}
private void OnPICSChangesFullRun(SteamApps.PICSChangesCallback callback)
{
PreviousChangeNumber = 2;
var apps = callback.AppChanges.Keys;
var packages = callback.PackageChanges.Keys;
TaskManager.Run(() => RequestUpdateForList(apps, packages));
}
private void HandlePackagesChangelists(SteamApps.PICSChangesCallback callback)
{
using (var db = Database.GetConnection())
{
db.Execute("INSERT INTO `ChangelistsSubs` (`ChangeID`, `SubID`) VALUES (@ChangeNumber, @ID) ON DUPLICATE KEY UPDATE `SubID` = `SubID`", callback.PackageChanges.Values);
db.Execute("UPDATE `Subs` SET `LastUpdated` = CURRENT_TIMESTAMP() WHERE `SubID` IN @Ids", new { Ids = callback.PackageChanges.Values.Select(x => x.ID) });
}
}
private void HandleApps(SteamApps.PICSChangesCallback callback)
{
StoreQueue.AddAppToQueue(callback.AppChanges.Values.Select(x => x.ID));
using (var db = Database.GetConnection())
{
db.Execute("INSERT INTO `ChangelistsApps` (`ChangeID`, `AppID`) VALUES (@ChangeNumber, @ID) ON DUPLICATE KEY UPDATE `AppID` = `AppID`", callback.AppChanges.Values);
db.Execute("UPDATE `Apps` SET `LastUpdated` = CURRENT_TIMESTAMP() WHERE `AppID` IN @Ids", new { Ids = callback.AppChanges.Values.Select(x => x.ID) });
}
}
private async Task HandlePackages(SteamApps.PICSChangesCallback callback)
{
Dictionary <uint, byte> ignoredPackages;
await using (var db = await Database.GetConnectionAsync())
{
ignoredPackages = (await db.QueryAsync("SELECT `SubID`, `SubID` FROM `SubsInfo` WHERE `SubID` IN @Subs AND `Key` = @Key AND `Value` IN @Types",
new
{
Key = BillingTypeKey,
Subs = callback.PackageChanges.Values.Select(x => x.ID),
Types = IgnorableBillingTypes
}
)).ToDictionary(x => (uint)x.SubID, _ => (byte)1);
}
// Steam comp
if (!ignoredPackages.ContainsKey(0))
{
ignoredPackages.Add(0, 1);
}
// Anon dedi comp
if (!ignoredPackages.ContainsKey(17906))
{
ignoredPackages.Add(17906, 1);
}
var subids = callback.PackageChanges.Values
.Select(x => x.ID).Where(x => !ignoredPackages.ContainsKey(x))
.ToList();
if (subids.Count == 0)
{
return;
}
List <uint> appids;
// Queue all the apps in the package as well
await using (var db = await Database.GetConnectionAsync())
{
appids = (await db.QueryAsync <uint>("SELECT `AppID` FROM `SubsApps` WHERE `SubID` IN @Ids AND `Type` = 'app'", new { Ids = subids })).ToList();
}
if (appids.Count > 0)
{
await StoreQueue.AddAppToQueue(appids);
}
await StoreQueue.AddPackageToQueue(subids);
}
private void OnPICSChangesFullRun(SteamApps.PICSChangesCallback callback, JobID job)
{
// Hackiness to prevent processing legit changelists after our request if that ever happens
if (PreviousChange != 1)
{
Log.WriteWarn("Steam", "Got changelist {0}, but ignoring it because we're in a full run", callback.CurrentChangeNumber);
return;
}
PreviousChange = 2;
Log.WriteInfo("Steam", "Requesting info for {0} apps and {1} packages", callback.AppChanges.Count, callback.PackageChanges.Count);
Apps.PICSGetProductInfo(Enumerable.Empty <SteamApps.PICSRequest>(), callback.PackageChanges.Keys.Select(package => NewPICSRequest(package)));
Apps.PICSGetAccessTokens(callback.AppChanges.Keys, Enumerable.Empty <uint>());
}
private async void OnPICSChanges(SteamApps.PICSChangesCallback callback)
{
if (PreviousChangeNumber == callback.CurrentChangeNumber)
{
return;
}
var packageChangesCount = callback.PackageChanges.Count;
var appChangesCount = callback.AppChanges.Count;
Log.WriteInfo("PICSChanges", "Changelist {0} -> {1} ({2} apps, {3} packages)", PreviousChangeNumber, callback.CurrentChangeNumber, appChangesCount, packageChangesCount);
LocalConfig.Current.ChangeNumber = PreviousChangeNumber = callback.CurrentChangeNumber;
await HandleChangeNumbers(callback);
if (appChangesCount == 0 && packageChangesCount == 0)
{
IRC.Instance.SendAnnounce("{0}»{1} Changelist {2}{3}{4} (empty)", Colors.RED, Colors.NORMAL, Colors.BLUE, PreviousChangeNumber, Colors.DARKGRAY);
return;
}
// We don't care about waiting for separate database queries to finish, so don't await them
#pragma warning disable 4014
if (appChangesCount > 0)
{
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(callback.AppChanges.Keys, Enumerable.Empty <uint>()));
TaskManager.RunAsync(async() => await HandleApps(callback));
}
if (packageChangesCount > 0)
{
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetProductInfo(Enumerable.Empty <SteamApps.PICSRequest>(), callback.PackageChanges.Keys.Select(Utils.NewPICSRequest)));
TaskManager.RunAsync(async() => await HandlePackages(callback));
TaskManager.RunAsync(async() => await HandlePackagesChangelists(callback));
}
TaskManager.RunAsync(async() => await SendChangelistsToIRC(callback));
#pragma warning restore 4014
PrintImportants(callback);
}
private static async Task HandleChangeNumbers(SteamApps.PICSChangesCallback callback)
{
var changeNumbers = callback.AppChanges.Values
.Select(x => x.ChangeNumber)
.Union(callback.PackageChanges.Values.Select(x => x.ChangeNumber))
.Distinct()
.Where(x => x != callback.CurrentChangeNumber)
.ToList();
changeNumbers.Add(callback.CurrentChangeNumber);
// Silly thing
var changeLists = changeNumbers.Select(x => new Changelist {
ChangeID = x
});
using var db = Database.Get();
await db.ExecuteAsync("INSERT INTO `Changelists` (`ChangeID`) VALUES (@ChangeID) ON DUPLICATE KEY UPDATE `Date` = `Date`", changeLists);
}
private void OnPICSChanges(SteamApps.PICSChangesCallback callback)
{
if (PreviousChangeNumber == callback.CurrentChangeNumber)
{
return;
}
var packageChangesCount = callback.PackageChanges.Count;
var appChangesCount = callback.AppChanges.Count;
Log.WriteInfo("PICSChanges", "Changelist {0} -> {1} ({2} apps, {3} packages)", PreviousChangeNumber, callback.CurrentChangeNumber, appChangesCount, packageChangesCount);
PreviousChangeNumber = callback.CurrentChangeNumber;
HandleChangeNumbers(callback);
if (appChangesCount == 0 && packageChangesCount == 0)
{
IRC.Instance.SendAnnounce("{0}»{1} Changelist {2}{3}{4} (empty)", Colors.RED, Colors.NORMAL, Colors.BLUE, PreviousChangeNumber, Colors.DARKGRAY);
return;
}
if (appChangesCount > 0)
{
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(callback.AppChanges.Keys, Enumerable.Empty <uint>()));
TaskManager.Run(() => HandleApps(callback));
}
if (packageChangesCount > 0)
{
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetProductInfo(Enumerable.Empty <SteamApps.PICSRequest>(), callback.PackageChanges.Keys.Select(package => Utils.NewPICSRequest(package))));
TaskManager.Run(() => HandlePackages(callback));
TaskManager.Run(() => HandlePackagesChangelists(callback));
}
TaskManager.Run(() => SendChangelistsToIRC(callback));
PrintImportants(callback);
}
private static void PrintImportants(SteamApps.PICSChangesCallback callback)
{
// Apps
var important = callback.AppChanges.Keys.Intersect(Application.ImportantApps);
foreach (var app in important)
{
var changeNumber = callback.AppChanges[app].ChangeNumber;
TaskManager.Run(async() => await Utils.SendWebhook(new
{
Type = "ImportantAppUpdate",
AppID = app,
ChangeNumber = changeNumber,
Url = $"{SteamDB.GetAppUrl(app, "history")}?changeid={changeNumber}",
}));
var appName = Steam.GetAppName(app, out var appType);
IRC.Instance.SendAnnounce($"{appType} update: {Colors.BLUE}{appName}{Colors.NORMAL} -{Colors.DARKBLUE} {SteamDB.GetAppUrl(app, "history")}");
}
// Packages
important = callback.PackageChanges.Keys.Intersect(Application.ImportantSubs);
foreach (var package in important)
{
var changeNumber = callback.PackageChanges[package].ChangeNumber;
TaskManager.Run(async() => await Utils.SendWebhook(new
{
Type = "ImportantSubUpdate",
SubID = package,
ChangeNumber = changeNumber,
Url = $"{SteamDB.GetPackageUrl(package, "history")}?changeid={changeNumber}",
}));
var subName = Steam.GetPackageName(package);
IRC.Instance.SendAnnounce($"Package update: {Colors.BLUE}{subName}{Colors.NORMAL} -{Colors.DARKBLUE} {SteamDB.GetPackageUrl(package, "history")}");
}
}
private void OnPICSChanges(SteamApps.PICSChangesCallback callback, JobID job)
{
if (PreviousChange == callback.CurrentChangeNumber)
{
return;
}
if (ProcessorPool.IsIdle)
{
Log.WriteDebug("Steam", "Cleaning processed apps and subs");
ProcessedApps.Clear();
ProcessedSubs.Clear();
}
var packageChangesCount = callback.PackageChanges.Count;
var appChangesCount = callback.AppChanges.Count;
Log.WriteInfo("Steam", "Changelist {0} -> {1} ({2} apps, {3} packages)", PreviousChange, callback.CurrentChangeNumber, appChangesCount, packageChangesCount);
PreviousChange = callback.CurrentChangeNumber;
DbWorker.ExecuteNonQuery("INSERT INTO `Changelists` (`ChangeID`) VALUES (@ChangeID) ON DUPLICATE KEY UPDATE `Date` = CURRENT_TIMESTAMP()", new MySqlParameter("@ChangeID", callback.CurrentChangeNumber));
if (appChangesCount == 0 && packageChangesCount == 0)
{
IRC.SendAnnounce("{0}»{1} Changelist {2}{3}{4} (empty)", Colors.RED, Colors.NORMAL, Colors.OLIVE, PreviousChange, Colors.DARK_GRAY);
return;
}
SecondaryPool.QueueWorkItem(SteamProxy.Instance.OnPICSChanges, callback);
// Packages have no tokens so we request info for them right away
if (packageChangesCount > 0)
{
Apps.PICSGetProductInfo(Enumerable.Empty <SteamApps.PICSRequest>(), callback.PackageChanges.Keys.Select(package => NewPICSRequest(package)));
}
if (appChangesCount > 0)
{
// Get all app tokens
Apps.PICSGetAccessTokens(callback.AppChanges.Keys, Enumerable.Empty <uint>());
SecondaryPool.QueueWorkItem(delegate
{
string changes = string.Empty;
foreach (var app in callback.AppChanges.Values)
{
if (callback.CurrentChangeNumber != app.ChangeNumber)
{
DbWorker.ExecuteNonQuery("INSERT INTO `Changelists` (`ChangeID`) VALUES (@ChangeID) ON DUPLICATE KEY UPDATE `Date` = `Date`", new MySqlParameter("@ChangeID", app.ChangeNumber));
}
DbWorker.ExecuteNonQuery("UPDATE `Apps` SET `LastUpdated` = CURRENT_TIMESTAMP() WHERE `AppID` = @AppID", new MySqlParameter("@AppID", app.ID));
changes += string.Format("({0}, {1}),", app.ChangeNumber, app.ID);
}
if (!changes.Equals(string.Empty))
{
changes = string.Format("INSERT INTO `ChangelistsApps` (`ChangeID`, `AppID`) VALUES {0} ON DUPLICATE KEY UPDATE `AppID` = `AppID`", changes.Remove(changes.Length - 1));
DbWorker.ExecuteNonQuery(changes);
}
});
}
if (packageChangesCount > 0)
{
SecondaryPool.QueueWorkItem(delegate
{
string changes = string.Empty;
foreach (var package in callback.PackageChanges.Values)
{
if (callback.CurrentChangeNumber != package.ChangeNumber)
{
DbWorker.ExecuteNonQuery("INSERT INTO `Changelists` (`ChangeID`) VALUES (@ChangeID) ON DUPLICATE KEY UPDATE `Date` = `Date`", new MySqlParameter("@ChangeID", package.ChangeNumber));
}
DbWorker.ExecuteNonQuery("UPDATE `Subs` SET `LastUpdated` = CURRENT_TIMESTAMP() WHERE `SubID` = @SubID", new MySqlParameter("@SubID", package.ID));
changes += string.Format("({0}, {1}),", package.ChangeNumber, package.ID);
}
if (!changes.Equals(string.Empty))
{
changes = string.Format("INSERT INTO `ChangelistsSubs` (`ChangeID`, `SubID`) VALUES {0} ON DUPLICATE KEY UPDATE `SubID` = `SubID`", changes.Remove(changes.Length - 1));
DbWorker.ExecuteNonQuery(changes);
}
});
}
}
private void OnPICSChanges(SteamApps.PICSChangesCallback callback)
{
if (PreviousChangeNumber == callback.CurrentChangeNumber)
{
return;
}
Log.WriteInfo(nameof(PICSChanges), $"Changelist {PreviousChangeNumber} -> {callback.CurrentChangeNumber} ({callback.AppChanges.Count} apps, {callback.PackageChanges.Count} packages)");
LocalConfig.Current.ChangeNumber = PreviousChangeNumber = callback.CurrentChangeNumber;
TaskManager.Run(async() => await HandleChangeNumbers(callback));
if (callback.RequiresFullAppUpdate || callback.RequiresFullPackageUpdate)
{
TaskManager.Run(async() =>
{
if (callback.RequiresFullAppUpdate)
{
IRC.Instance.SendOps($"Changelist {callback.CurrentChangeNumber} has forced a full app update");
await FullUpdateProcessor.FullUpdateAppsMetadata();
}
if (callback.RequiresFullPackageUpdate)
{
IRC.Instance.SendOps($"Changelist {callback.CurrentChangeNumber} has forced a full package update");
await FullUpdateProcessor.FullUpdatePackagesMetadata();
}
});
}
if (callback.AppChanges.Count == 0 && callback.PackageChanges.Count == 0)
{
IRC.Instance.SendAnnounce($"{Colors.RED}»{Colors.NORMAL} Changelist {Colors.BLUE}{PreviousChangeNumber}{Colors.DARKGRAY} (empty)");
return;
}
const int appsPerJob = 50;
if (callback.AppChanges.Count > appsPerJob)
{
foreach (var list in callback.AppChanges.Keys.Split(appsPerJob))
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(list, Enumerable.Empty <uint>()),
new PICSTokens.RequestedTokens
{
Apps = list.ToList()
});
}
}
else if (callback.AppChanges.Count > 0)
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(callback.AppChanges.Keys, Enumerable.Empty <uint>()),
new PICSTokens.RequestedTokens
{
Apps = callback.AppChanges.Keys.ToList()
});
}
if (callback.PackageChanges.Count > appsPerJob)
{
foreach (var list in callback.PackageChanges.Keys.Split(appsPerJob))
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), list),
new PICSTokens.RequestedTokens
{
Packages = list.ToList()
});
}
}
else if (callback.PackageChanges.Count > 0)
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), callback.PackageChanges.Keys),
new PICSTokens.RequestedTokens
{
Packages = callback.PackageChanges.Keys.ToList()
});
}
if (callback.AppChanges.Count > 0)
{
_ = TaskManager.Run(async() => await HandleApps(callback));
}
if (callback.PackageChanges.Count > 0)
{
_ = TaskManager.Run(async() => await HandlePackages(callback));
_ = TaskManager.Run(async() => await HandlePackagesChangelists(callback));
}
_ = TaskManager.Run(async() => await SendChangelistsToIRC(callback));
PrintImportants(callback);
}
private void SendChangelistsToIRC(SteamApps.PICSChangesCallback callback)
{
// Group apps and package changes by changelist, this will seperate into individual changelists
var appGrouping = callback.AppChanges.Values.GroupBy(a => a.ChangeNumber);
var packageGrouping = callback.PackageChanges.Values.GroupBy(p => p.ChangeNumber);
// Join apps and packages back together based on changelist number
var changeLists = Utils.FullOuterJoin(appGrouping, packageGrouping, a => a.Key, p => p.Key, (a, p, key) => new
{
ChangeNumber = key,
Apps = a.ToList(),
Packages = p.ToList(),
},
new EmptyGrouping <uint, SteamApps.PICSChangesCallback.PICSChangeData>(),
new EmptyGrouping <uint, SteamApps.PICSChangesCallback.PICSChangeData>())
.OrderBy(c => c.ChangeNumber);
foreach (var changeList in changeLists)
{
var appCount = changeList.Apps.Count;
var packageCount = changeList.Packages.Count;
string Message = string.Format("Changelist {0}{1}{2} {3}({4:N0} apps and {5:N0} packages){6} -{7} {8}",
Colors.BLUE, changeList.ChangeNumber, Colors.NORMAL,
Colors.DARKGRAY, appCount, packageCount, Colors.NORMAL,
Colors.DARKBLUE, SteamDB.GetChangelistURL(changeList.ChangeNumber)
);
var changesCount = appCount + packageCount;
if (changesCount >= 50)
{
IRC.Instance.SendMain(Message);
}
IRC.Instance.SendAnnounce("{0}»{1} {2}", Colors.RED, Colors.NORMAL, Message);
// If this changelist is very big, freenode will hate us forever if we decide to print all that stuff
if (changesCount > 300)
{
IRC.Instance.SendAnnounce("{0} This changelist is too big to be printed in IRC, please view it online", Colors.RED);
continue;
}
if (appCount > 0)
{
Dictionary <uint, App> apps;
App data;
using (var db = Database.GetConnection())
{
apps = db.Query <App>("SELECT `AppID`, `Name`, `LastKnownName` FROM `Apps` WHERE `AppID` IN @Ids", new { Ids = changeList.Apps.Select(x => x.ID) }).ToDictionary(x => x.AppID, x => x);
}
foreach (var app in changeList.Apps)
{
apps.TryGetValue(app.ID, out data);
IRC.Instance.SendAnnounce(" App: {0}{1}{2} - {3}{4}",
Colors.BLUE, app.ID, Colors.NORMAL,
Steam.FormatAppName(app.ID, data),
app.NeedsToken ? SteamDB.StringNeedToken : string.Empty
);
}
}
if (packageCount > 0)
{
Dictionary <uint, Package> packages;
Package data;
using (var db = Database.GetConnection())
{
packages = db.Query <Package>("SELECT `SubID`, `Name`, `LastKnownName` FROM `Subs` WHERE `SubID` IN @Ids", new { Ids = changeList.Packages.Select(x => x.ID) }).ToDictionary(x => x.SubID, x => x);
}
foreach (var package in changeList.Packages)
{
packages.TryGetValue(package.ID, out data);
IRC.Instance.SendAnnounce(" Package: {0}{1}{2} - {3}{4}",
Colors.BLUE, package.ID, Colors.NORMAL,
Steam.FormatPackageName(package.ID, data),
package.NeedsToken ? SteamDB.StringNeedToken : string.Empty
);
}
}
}
}
void OnPICSChanges(SteamApps.PICSChangesCallback callback)
{
// group apps and package changes by changelist, this will seperate into individual changelists
var appGrouping = callback.AppChanges
.Select(a => a.Value)
.GroupBy(a => a.ChangeNumber);
var packageGrouping = callback.PackageChanges
.Select(p => p.Value)
.GroupBy(p => p.ChangeNumber);
// join apps and packages back together based on changelist number
var changeLists = appGrouping
.FullOuterJoin(packageGrouping, a => a.Key, p => p.Key, (a, p, key) => new
{
ChangeNumber = key,
Apps = a.ToList(),
Packages = p.ToList(),
},
new EmptyGrouping <uint, SteamApps.PICSChangesCallback.PICSChangeData>(),
new EmptyGrouping <uint, SteamApps.PICSChangesCallback.PICSChangeData>())
.OrderBy(c => c.ChangeNumber);
// the number of changes required in a changelist in order to be important enough to display
// to all broadcast channels
const int ChangesReqToBeImportant = 50;
if (changeLists.Count() == 0)
{
// this will happen on the first changes request
lastChangeNumber = callback.CurrentChangeNumber;
return;
}
foreach (var changeList in changeLists)
{
if (changeList.ChangeNumber <= lastChangeNumber)
{
return; // old changelist
}
lastChangeNumber = changeList.ChangeNumber;
int numAppChanges = changeList.Apps.Count;
int numPackageChanges = changeList.Packages.Count;
string message = string.Format("Got PICS changelist {0} for {1} apps and {2} packages - {3}",
changeList.ChangeNumber, changeList.Apps.Count, changeList.Packages.Count, GetChangelistURL(changeList.ChangeNumber));
if (numPackageChanges >= ChangesReqToBeImportant || numAppChanges >= ChangesReqToBeImportant)
{
// if this changelist contains a number of changes over a specific threshold, we'll consider it "important" and send to all channels
IRC.Instance.SendToTag("pics", message);
}
if (numAppChanges > 0)
{
var importantApps = changeList.Apps.Select(a => a.ID)
.Intersect(Settings.Current.ImportantApps.Select(a => a.AppID));
foreach (var app in importantApps)
{
SettingsXml.ImportantApp importantApp = Settings.Current.ImportantApps
.FirstOrDefault(a => a.AppID == app);
string tag = "pics";
if (!string.IsNullOrEmpty(importantApp.Tag))
{
tag = string.Format("{0}-{1}", tag, importantApp.Tag);
}
// get the channels interested in this pics update
var picsChannels = Settings.Current.GetChannelsForTag(tag);
if (picsChannels.Count() == 0)
{
Log.WriteWarn("PICSJob", "No channels setup for tag: {0}", tag);
}
string targetChans = string.Join(",", picsChannels.Select(c => c.Channel));
IRC.Instance.Send(targetChans, "Important App Update: {0} - {1}", Steam.Instance.GetAppName(app), GetAppHistoryUrl(app));
}
}
if (numPackageChanges > 0)
{
// todo: important packages
}
}
}
public void OnPICSChanges(SteamApps.PICSChangesCallback callback)
{
// Print any apps importants first
var important = callback.AppChanges.Keys.Intersect(ImportantApps);
if (important.Count() > 5)
{
IRC.SendMain("{0}{1}{2} important apps updated -{3} {4}", Colors.OLIVE, important.Count(), Colors.NORMAL, Colors.DARK_BLUE, SteamDB.GetChangelistURL(callback.CurrentChangeNumber));
}
else
{
foreach (var app in important)
{
IRC.SendMain("Important app update: {0}{1}{2} -{3} {4}", Colors.OLIVE, GetAppName(app), Colors.NORMAL, Colors.DARK_BLUE, SteamDB.GetAppURL(app, "history"));
}
}
// And then important packages
important = callback.PackageChanges.Keys.Intersect(ImportantSubs);
if (important.Count() > 5)
{
IRC.SendMain("{0}{1}{2} important packages updated -{3} {4}", Colors.OLIVE, important.Count(), Colors.NORMAL, Colors.DARK_BLUE, SteamDB.GetChangelistURL(callback.CurrentChangeNumber));
}
else
{
foreach (var package in important)
{
IRC.SendMain("Important package update: {0}{1}{2} -{3} {4}", Colors.OLIVE, GetPackageName(package), Colors.NORMAL, Colors.DARK_BLUE, SteamDB.GetPackageURL(package, "history"));
}
}
// Group apps and package changes by changelist, this will seperate into individual changelists
var appGrouping = callback.AppChanges.Values.GroupBy(a => a.ChangeNumber);
var packageGrouping = callback.PackageChanges.Values.GroupBy(p => p.ChangeNumber);
// Join apps and packages back together based on changelist number
var changeLists = Utils.FullOuterJoin(appGrouping, packageGrouping, a => a.Key, p => p.Key, (a, p, key) => new
{
ChangeNumber = key,
Apps = a.ToList(),
Packages = p.ToList(),
},
new EmptyGrouping <uint, SteamApps.PICSChangesCallback.PICSChangeData>(),
new EmptyGrouping <uint, SteamApps.PICSChangesCallback.PICSChangeData>())
.OrderBy(c => c.ChangeNumber);
foreach (var changeList in changeLists)
{
var appCount = changeList.Apps.Count;
var packageCount = changeList.Packages.Count;
string Message = string.Format("Changelist {0}{1}{2} {3}({4:N0} apps and {5:N0} packages){6} -{7} {8}",
Colors.OLIVE, changeList.ChangeNumber, Colors.NORMAL,
Colors.DARK_GRAY, appCount, packageCount, Colors.NORMAL,
Colors.DARK_BLUE, SteamDB.GetChangelistURL(changeList.ChangeNumber)
);
var changesCount = appCount + packageCount;
if (changesCount >= 50)
{
IRC.SendMain(Message);
}
IRC.SendAnnounce("{0}»{1} {2}", Colors.RED, Colors.NORMAL, Message);
// If this changelist is very big, freenode will hate us forever if we decide to print all that stuff
if (changesCount > 500)
{
IRC.SendAnnounce("{0} This changelist is too big to be printed in IRC, please view it on our website", Colors.RED);
continue;
}
string name;
if (appCount > 0)
{
foreach (var app in changeList.Apps)
{
name = GetAppName(app.ID, true);
if (string.IsNullOrEmpty(name))
{
name = string.Format("{0}{1}{2}", Colors.GREEN, app.ID, Colors.NORMAL);
}
else
{
name = string.Format("{0}{1}{2} - {3}", Colors.LIGHT_GRAY, app.ID, Colors.NORMAL, name);
}
IRC.SendAnnounce(" App: {0}{1}", name, app.NeedsToken ? StringNeedToken : string.Empty);
}
}
if (packageCount > 0)
{
foreach (var package in changeList.Packages)
{
name = GetPackageName(package.ID, true);
if (string.IsNullOrEmpty(name))
{
name = string.Format("{0}{1}{2}", Colors.GREEN, package.ID, Colors.NORMAL);
}
else
{
name = string.Format("{0}{1}{2} - {3}", Colors.LIGHT_GRAY, package.ID, Colors.NORMAL, name);
}
IRC.SendAnnounce(" Package: {0}{1}{2}",
name,
package.NeedsToken ? StringNeedToken : string.Empty,
Steam.Instance.OwnedPackages.Contains(package.ID) ? StringCheckmark : string.Empty
);
}
}
}
}
private void OnPICSChanges(SteamApps.PICSChangesCallback callback)
{
if (PreviousChangeNumber == callback.CurrentChangeNumber)
{
return;
}
Log.WriteInfo(nameof(PICSChanges), $"Changelist {callback.LastChangeNumber} -> {callback.CurrentChangeNumber} ({callback.AppChanges.Count} apps, {callback.PackageChanges.Count} packages)");
PreviousChangeNumber = callback.CurrentChangeNumber;
TaskManager.Run(async() => await HandleChangeNumbers(callback));
if (callback.RequiresFullAppUpdate || callback.RequiresFullPackageUpdate)
{
TaskManager.Run(async() =>
{
if (callback.RequiresFullAppUpdate)
{
IRC.Instance.SendOps($"Changelist {callback.CurrentChangeNumber} has forced a full app update");
// When full update flag is set, presumably Steam client start hammering the servers
// and the PICS service just does not return any data for a while until it clears up
await FullUpdateProcessor.FullUpdateAppsMetadata(true);
}
if (callback.RequiresFullPackageUpdate)
{
IRC.Instance.SendOps($"Changelist {callback.CurrentChangeNumber} has forced a full package update");
await FullUpdateProcessor.FullUpdatePackagesMetadata();
}
IRC.Instance.SendOps($"Changelist {callback.CurrentChangeNumber} full update has finished");
});
}
if (callback.AppChanges.Count == 0 && callback.PackageChanges.Count == 0)
{
return;
}
const int appsPerJob = 50;
if (callback.AppChanges.Count > appsPerJob)
{
foreach (var list in callback.AppChanges.Keys.Split(appsPerJob))
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(list, Enumerable.Empty <uint>()),
new PICSTokens.RequestedTokens
{
Apps = list.ToList()
});
}
}
else if (callback.AppChanges.Count > 0)
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(callback.AppChanges.Keys, Enumerable.Empty <uint>()),
new PICSTokens.RequestedTokens
{
Apps = callback.AppChanges.Keys.ToList()
});
}
if (callback.PackageChanges.Count > appsPerJob)
{
foreach (var list in callback.PackageChanges.Keys.Split(appsPerJob))
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), list),
new PICSTokens.RequestedTokens
{
Packages = list.ToList()
});
}
}
else if (callback.PackageChanges.Count > 0)
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), callback.PackageChanges.Keys),
new PICSTokens.RequestedTokens
{
Packages = callback.PackageChanges.Keys.ToList()
});
}
if (callback.AppChanges.Count > 0)
{
_ = TaskManager.Run(async() => await HandleApps(callback));
}
if (callback.PackageChanges.Count > 0)
{
_ = TaskManager.Run(async() => await HandlePackages(callback));
_ = TaskManager.Run(async() => await HandlePackagesChangelists(callback));
}
if (PreviousChangeNumber - LastStoredChangeNumber >= 1000)
{
LastStoredChangeNumber = PreviousChangeNumber;
_ = TaskManager.Run(async() => await LocalConfig.Update("backend.changenumber", LastStoredChangeNumber.ToString()));
}
PrintImportants(callback);
}
private async void OnPICSChanges(SteamApps.PICSChangesCallback callback)
{
if (PreviousChangeNumber == callback.CurrentChangeNumber)
{
return;
}
Log.WriteInfo("PICSChanges", $"Changelist {PreviousChangeNumber} -> {callback.CurrentChangeNumber} ({callback.AppChanges.Count} apps, {callback.PackageChanges.Count} packages)");
LocalConfig.Current.ChangeNumber = PreviousChangeNumber = callback.CurrentChangeNumber;
await HandleChangeNumbers(callback);
if (callback.AppChanges.Count == 0 && callback.PackageChanges.Count == 0)
{
IRC.Instance.SendAnnounce("{0}»{1} Changelist {2}{3}{4} (empty)", Colors.RED, Colors.NORMAL, Colors.BLUE, PreviousChangeNumber, Colors.DARKGRAY);
return;
}
const int appsPerJob = 50;
if (callback.AppChanges.Count > appsPerJob)
{
foreach (var list in callback.AppChanges.Keys.Split(appsPerJob))
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(list, Enumerable.Empty <uint>()),
new PICSTokens.RequestedTokens
{
Apps = list.ToList()
});
}
}
else if (callback.AppChanges.Count > 0)
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(callback.AppChanges.Keys, Enumerable.Empty <uint>()),
new PICSTokens.RequestedTokens
{
Apps = callback.AppChanges.Keys.ToList()
});
}
if (callback.PackageChanges.Count > appsPerJob)
{
foreach (var list in callback.PackageChanges.Keys.Split(appsPerJob))
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), list),
new PICSTokens.RequestedTokens
{
Packages = list.ToList()
});
}
}
else if (callback.PackageChanges.Count > 0)
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), callback.PackageChanges.Keys),
new PICSTokens.RequestedTokens
{
Packages = callback.PackageChanges.Keys.ToList()
});
}
if (callback.AppChanges.Count > 0)
{
_ = TaskManager.RunAsync(async() => await HandleApps(callback));
}
if (callback.PackageChanges.Count > 0)
{
_ = TaskManager.RunAsync(async() => await HandlePackages(callback));
_ = TaskManager.RunAsync(async() => await HandlePackagesChangelists(callback));
}
_ = TaskManager.RunAsync(async() => await SendChangelistsToIRC(callback));
PrintImportants(callback);
}
private async Task SendChangelistsToIRC(SteamApps.PICSChangesCallback callback)
{
if (DateTime.Now > ChangelistBurstTime)
{
ChangelistBurstTime = DateTime.Now.AddMinutes(5);
ChangelistBurstCount = 0;
}
// Group apps and package changes by changelist number
var changelists = new Dictionary <uint, IrcChangelistGroup>();
foreach (var app in callback.AppChanges.Values)
{
if (!changelists.ContainsKey(app.ChangeNumber))
{
changelists[app.ChangeNumber] = new IrcChangelistGroup();
}
changelists[app.ChangeNumber].Apps.Add(app.ID);
}
foreach (var package in callback.PackageChanges.Values)
{
if (!changelists.ContainsKey(package.ChangeNumber))
{
changelists[package.ChangeNumber] = new IrcChangelistGroup();
}
changelists[package.ChangeNumber].Packages.Add(package.ID);
}
foreach (var(changeNumber, changeList) in changelists.OrderBy(x => x.Key))
{
var appCount = changeList.Apps.Count;
var packageCount = changeList.Packages.Count;
var message = $"Changelist {Colors.BLUE}{changeNumber}{Colors.NORMAL} {Colors.DARKGRAY}({appCount:N0} apps and {packageCount:N0} packages)";
var changesCount = appCount + packageCount;
if (changesCount >= 50)
{
IRC.Instance.SendMain($"Big {message}{Colors.DARKBLUE} {SteamDB.GetChangelistUrl(changeNumber)}");
}
if (ChangelistBurstCount++ >= CHANGELIST_BURST_MIN || changesCount > 300)
{
if (appCount > 0)
{
message += $" (Apps: {string.Join(", ", changeList.Apps)})";
}
if (packageCount > 0)
{
message += $" (Packages: {string.Join(", ", changeList.Packages)})";
}
IRC.Instance.SendAnnounce($"{Colors.RED}»{Colors.NORMAL} {message}");
continue;
}
IRC.Instance.SendAnnounce($"{Colors.RED}»{Colors.NORMAL} {message}");
if (appCount > 0)
{
Dictionary <uint, App> apps;
await using (var db = await Database.GetConnectionAsync())
{
apps = (await db.QueryAsync <App>("SELECT `AppID`, `Name`, `LastKnownName` FROM `Apps` WHERE `AppID` IN @Ids", new { Ids = changeList.Apps })).ToDictionary(x => x.AppID, x => x);
}
foreach (var appId in changeList.Apps)
{
apps.TryGetValue(appId, out var data);
IRC.Instance.SendAnnounce($" App: {Colors.BLUE}{appId}{Colors.NORMAL} - {Steam.FormatAppName(appId, data)}");
}
}
if (packageCount > 0)
{
Dictionary <uint, Package> packages;
await using (var db = await Database.GetConnectionAsync())
{
packages = (await db.QueryAsync <Package>("SELECT `SubID`, `Name`, `LastKnownName` FROM `Subs` WHERE `SubID` IN @Ids", new { Ids = changeList.Packages })).ToDictionary(x => x.SubID, x => x);
}
foreach (var packageId in changeList.Packages)
{
packages.TryGetValue(packageId, out var data);
IRC.Instance.SendAnnounce($" Package: {Colors.BLUE}{packageId}{Colors.NORMAL} - {Steam.FormatPackageName(packageId, data)}");
}
}
}
}
private async Task SendChangelistsToIRC(SteamApps.PICSChangesCallback callback)
{
if (DateTime.Now > ChangelistBurstTime)
{
ChangelistBurstTime = DateTime.Now.AddMinutes(5);
ChangelistBurstCount = 0;
}
// Group apps and package changes by changelist, this will seperate into individual changelists
var appGrouping = callback.AppChanges.Values.GroupBy(a => a.ChangeNumber);
var packageGrouping = callback.PackageChanges.Values.GroupBy(p => p.ChangeNumber);
// Join apps and packages back together based on changelist number
var changeLists = Utils.FullOuterJoin(appGrouping, packageGrouping, a => a.Key, p => p.Key, (a, p, key) => new
{
ChangeNumber = key,
Apps = a.ToList(),
Packages = p.ToList(),
},
new EmptyGrouping <uint, SteamApps.PICSChangesCallback.PICSChangeData>(),
new EmptyGrouping <uint, SteamApps.PICSChangesCallback.PICSChangeData>())
.OrderBy(c => c.ChangeNumber);
foreach (var changeList in changeLists)
{
var appCount = changeList.Apps.Count;
var packageCount = changeList.Packages.Count;
var message = $"Changelist {Colors.BLUE}{changeList.ChangeNumber}{Colors.NORMAL} {Colors.DARKGRAY}({appCount:N0} apps and {packageCount:N0} packages)";
var changesCount = appCount + packageCount;
if (changesCount >= 50)
{
IRC.Instance.SendMain($"Big {message}{Colors.DARKBLUE} {SteamDB.GetChangelistURL(changeList.ChangeNumber)}");
}
if (ChangelistBurstCount++ >= CHANGELIST_BURST_MIN || changesCount > 300)
{
if (appCount > 0)
{
message += $" (Apps: {string.Join(", ", changeList.Apps.Select(x => x.ID))})";
}
if (packageCount > 0)
{
message += $" (Packages: {string.Join(", ", changeList.Packages.Select(x => x.ID))})";
}
IRC.Instance.SendAnnounce($"{Colors.RED}»{Colors.NORMAL} {message}");
continue;
}
IRC.Instance.SendAnnounce("{0}»{1} {2}", Colors.RED, Colors.NORMAL, message);
if (appCount > 0)
{
Dictionary <uint, App> apps;
using (var db = Database.Get())
{
apps = (await db.QueryAsync <App>("SELECT `AppID`, `Name`, `LastKnownName` FROM `Apps` WHERE `AppID` IN @Ids", new { Ids = changeList.Apps.Select(x => x.ID) })).ToDictionary(x => x.AppID, x => x);
}
foreach (var app in changeList.Apps)
{
apps.TryGetValue(app.ID, out var data);
IRC.Instance.SendAnnounce(" App: {0}{1}{2} - {3}{4}",
Colors.BLUE, app.ID, Colors.NORMAL,
Steam.FormatAppName(app.ID, data),
app.NeedsToken ? SteamDB.StringNeedToken : string.Empty
);
}
}
if (packageCount > 0)
{
Dictionary <uint, Package> packages;
using (var db = Database.Get())
{
packages = (await db.QueryAsync <Package>("SELECT `SubID`, `Name`, `LastKnownName` FROM `Subs` WHERE `SubID` IN @Ids", new { Ids = changeList.Packages.Select(x => x.ID) })).ToDictionary(x => x.SubID, x => x);
}
foreach (var package in changeList.Packages)
{
packages.TryGetValue(package.ID, out var data);
IRC.Instance.SendAnnounce(" Package: {0}{1}{2} - {3}{4}",
Colors.BLUE, package.ID, Colors.NORMAL,
Steam.FormatPackageName(package.ID, data),
package.NeedsToken ? SteamDB.StringNeedToken : string.Empty
);
}
}
}
}
