protected override async Task ProcessData() { await LoadData(); ChangeNumber = ProductInfo.ChangeNumber; if (Settings.IsFullRun) { await DbConnection.ExecuteAsync("INSERT INTO `Changelists` (`ChangeID`) VALUES (@ChangeNumber) ON DUPLICATE KEY UPDATE `Date` = `Date`", new { ProductInfo.ChangeNumber }); await DbConnection.ExecuteAsync("INSERT INTO `ChangelistsSubs` (`ChangeID`, `SubID`) VALUES (@ChangeNumber, @SubID) ON DUPLICATE KEY UPDATE `SubID` = `SubID`", new { SubID, ProductInfo.ChangeNumber }); } await ProcessKey("root_changenumber", "changenumber", ChangeNumber.ToString()); var appAddedToThisPackage = false; var packageOwned = LicenseList.OwnedSubs.ContainsKey(SubID); var newPackageName = ProductInfo.KeyValues["name"].AsString(); var apps = (await DbConnection.QueryAsync <PackageApp>("SELECT `AppID`, `Type` FROM `SubsApps` WHERE `SubID` = @SubID", new { SubID })).ToDictionary(x => x.AppID, x => x.Type); // TODO: Ideally this should be SteamDB Unknown Package and proper checks like app processor does if (newPackageName == null) { newPackageName = string.Concat("Steam Sub ", SubID); } if (newPackageName != null) { if (string.IsNullOrEmpty(PackageName)) { await DbConnection.ExecuteAsync("INSERT INTO `Subs` (`SubID`, `Name`, `LastKnownName`) VALUES (@SubID, @Name, @Name)", new { SubID, Name = newPackageName }); await MakeHistory("created_sub"); await MakeHistory("created_info", SteamDB.DATABASE_NAME_TYPE, string.Empty, newPackageName); } else if (!PackageName.Equals(newPackageName)) { if (newPackageName.StartsWith("Steam Sub ", StringComparison.Ordinal)) { await DbConnection.ExecuteAsync("UPDATE `Subs` SET `Name` = @Name WHERE `SubID` = @SubID", new { SubID, Name = newPackageName }); } else { await DbConnection.ExecuteAsync("UPDATE `Subs` SET `Name` = @Name, `LastKnownName` = @Name WHERE `SubID` = @SubID", new { SubID, Name = newPackageName }); } await MakeHistory("modified_info", SteamDB.DATABASE_NAME_TYPE, PackageName, newPackageName); } } foreach (var section in ProductInfo.KeyValues.Children) { var sectionName = section.Name.ToLower(); if (string.IsNullOrEmpty(sectionName) || sectionName.Equals("packageid") || sectionName.Equals("changenumber") || sectionName.Equals("name")) { // Ignore common keys continue; } if (sectionName.Equals("appids") || sectionName.Equals("depotids")) { // Remove "ids", so we get "app" from appids and "depot" from depotids var type = sectionName.Replace("ids", string.Empty); var isAppSection = type.Equals("app"); var typeID = (uint)(isAppSection ? 0 : 1); // 0 = app, 1 = depot; can't store as string because it's in the `key` field foreach (var childrenApp in section.Children) { var appID = uint.Parse(childrenApp.Value); // Is this appid already in this package? if (apps.ContainsKey(appID)) { // Is this appid's type the same? if (apps[appID] != type) { await DbConnection.ExecuteAsync("UPDATE `SubsApps` SET `Type` = @Type WHERE `SubID` = @SubID AND `AppID` = @AppID", new { SubID, AppID = appID, Type = type }); await MakeHistory("added_to_sub", typeID, apps[appID].Equals("app")? "0" : "1", childrenApp.Value); appAddedToThisPackage = true; // TODO: Log relevant add/remove history for depot/app? } apps.Remove(appID); } else { await DbConnection.ExecuteAsync("INSERT INTO `SubsApps` (`SubID`, `AppID`, `Type`) VALUES(@SubID, @AppID, @Type)", new { SubID, AppID = appID, Type = type }); await MakeHistory("added_to_sub", typeID, string.Empty, childrenApp.Value); if (isAppSection) { await DbConnection.ExecuteAsync(AppProcessor.HistoryQuery, new PICSHistory { ID = appID, ChangeID = ChangeNumber, NewValue = SubID.ToString(), Action = "added_to_sub" } ); } else { await DbConnection.ExecuteAsync(DepotProcessor.HistoryQuery, new DepotHistory { DepotID = appID, ChangeID = ChangeNumber, NewValue = SubID, Action = "added_to_sub" } ); } appAddedToThisPackage = true; if (packageOwned && !LicenseList.OwnedApps.ContainsKey(appID)) { LicenseList.OwnedApps.Add(appID, 1); } } } } else if (sectionName.Equals("extended")) { foreach (var children in section.Children) { var keyName = string.Format("{0}_{1}", sectionName, children.Name); if (children.Children.Count > 0) { await ProcessKey(keyName, children.Name, Utils.JsonifyKeyValue(children), true); } else { await ProcessKey(keyName, children.Name, children.Value); } } } else if (section.Children.Any()) { sectionName = string.Format("root_{0}", sectionName); await ProcessKey(sectionName, sectionName, Utils.JsonifyKeyValue(section), true); } else if (!string.IsNullOrEmpty(section.Value)) { var keyName = string.Format("root_{0}", sectionName); await ProcessKey(keyName, sectionName, section.Value); } } foreach (var data in CurrentData.Values.Where(data => !data.Processed && !data.KeyName.StartsWith("website", StringComparison.Ordinal))) { await DbConnection.ExecuteAsync("DELETE FROM `SubsInfo` WHERE `SubID` = @SubID AND `Key` = @Key", new { SubID, data.Key }); await MakeHistory("removed_key", data.Key, data.Value); } var appsRemoved = apps.Any(); foreach (var app in apps) { await DbConnection.ExecuteAsync("DELETE FROM `SubsApps` WHERE `SubID` = @SubID AND `AppID` = @AppID AND `Type` = @Type", new { SubID, AppID = app.Key, Type = app.Value }); var isAppSection = app.Value.Equals("app"); var typeID = (uint)(isAppSection ? 0 : 1); // 0 = app, 1 = depot; can't store as string because it's in the `key` field await MakeHistory("removed_from_sub", typeID, app.Key.ToString()); if (isAppSection) { await DbConnection.ExecuteAsync(AppProcessor.HistoryQuery, new PICSHistory { ID = app.Key, ChangeID = ChangeNumber, OldValue = SubID.ToString(), Action = "removed_from_sub" } ); } else { await DbConnection.ExecuteAsync(DepotProcessor.HistoryQuery, new DepotHistory { DepotID = app.Key, ChangeID = ChangeNumber, OldValue = SubID, Action = "removed_from_sub" } ); } } if (appsRemoved) { LicenseList.RefreshApps(); } if (!packageOwned && SubID != 17906) { FreeLicense.RequestFromPackage(SubID, ProductInfo.KeyValues); } // Re-queue apps in this package so we can update depots and whatnot if (appAddedToThisPackage && !Settings.IsFullRun && !string.IsNullOrEmpty(PackageName)) { JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(ProductInfo.KeyValues["appids"].Children.Select(x => (uint)x.AsInteger()), Enumerable.Empty <uint>())); } // Maintain a list of anonymous content if (appAddedToThisPackage && SubID == 17906) { LicenseList.RefreshAnonymous(); } }
protected override async Task ProcessData() { await LoadData(); ChangeNumber = ProductInfo.ChangeNumber; if (Settings.IsFullRun) { await DbConnection.ExecuteAsync("INSERT INTO `Changelists` (`ChangeID`) VALUES (@ChangeNumber) ON DUPLICATE KEY UPDATE `Date` = `Date`", new { ProductInfo.ChangeNumber }); await DbConnection.ExecuteAsync("INSERT INTO `ChangelistsSubs` (`ChangeID`, `SubID`) VALUES (@ChangeNumber, @SubID) ON DUPLICATE KEY UPDATE `SubID` = `SubID`", new { SubID, ProductInfo.ChangeNumber }); } await ProcessKey("root_changenumber", "changenumber", ChangeNumber.ToString()); var appAddedToThisPackage = false; var hasPackageInfo = ProductInfo.KeyValues.Children.Count > 0; var packageOwned = LicenseList.OwnedSubs.ContainsKey(SubID); var newPackageName = ProductInfo.KeyValues["name"].AsString() ?? string.Concat("Steam Sub ", SubID); var apps = (await DbConnection.QueryAsync <PackageApp>("SELECT `AppID`, `Type` FROM `SubsApps` WHERE `SubID` = @SubID", new { SubID })).ToDictionary(x => x.AppID, x => x.Type); var alreadySeenAppIds = new HashSet <uint>(); if (!hasPackageInfo) { ProductInfo.KeyValues.Children.Add(new KeyValue("steamdb_requires_token", "1")); } if (string.IsNullOrEmpty(PackageName)) { await DbConnection.ExecuteAsync("INSERT INTO `Subs` (`SubID`, `Name`, `LastKnownName`) VALUES (@SubID, @Name, @Name)", new { SubID, Name = newPackageName }); await MakeHistory("created_sub"); await MakeHistory("created_info", SteamDB.DatabaseNameType, string.Empty, newPackageName); } else if (PackageName != newPackageName) { if (newPackageName.StartsWith("Steam Sub ", StringComparison.Ordinal)) { await DbConnection.ExecuteAsync("UPDATE `Subs` SET `Name` = @Name WHERE `SubID` = @SubID", new { SubID, Name = newPackageName }); } else { await DbConnection.ExecuteAsync("UPDATE `Subs` SET `Name` = @Name, `LastKnownName` = @Name WHERE `SubID` = @SubID", new { SubID, Name = newPackageName }); } await MakeHistory("modified_info", SteamDB.DatabaseNameType, PackageName, newPackageName); } foreach (var section in ProductInfo.KeyValues.Children) { var sectionName = section.Name.ToLowerInvariant(); if (string.IsNullOrEmpty(sectionName) || sectionName == "packageid" || sectionName == "changenumber" || sectionName == "name") { // Ignore common keys continue; } if (sectionName == "appids" || sectionName == "depotids") { // Remove "ids", so we get "app" from appids and "depot" from depotids var type = sectionName.Replace("ids", string.Empty); var isAppSection = type == "app"; var typeID = (uint)(isAppSection ? 0 : 1); // 0 = app, 1 = depot; can't store as string because it's in the `key` field foreach (var childrenApp in section.Children) { if (!uint.TryParse(childrenApp.Value, out var appID)) { Log.WriteWarn(nameof(SubProcessor), $"Package {SubID} has an invalid uint: {childrenApp.Value}"); continue; } if (alreadySeenAppIds.Contains(appID)) { Log.WriteWarn(nameof(SubProcessor), $"Package {SubID} has a duplicate app: {appID}"); continue; } alreadySeenAppIds.Add(appID); // Is this appid already in this package? if (apps.ContainsKey(appID)) { // Is this appid's type the same? if (apps[appID] != type) { await DbConnection.ExecuteAsync("UPDATE `SubsApps` SET `Type` = @Type WHERE `SubID` = @SubID AND `AppID` = @AppID", new { SubID, AppID = appID, Type = type }); await MakeHistory("added_to_sub", typeID, apps[appID] == "app"? "0" : "1", childrenApp.Value); appAddedToThisPackage = true; // Log relevant add/remove history events for depot and app var appHistory = new PICSHistory { ID = appID, ChangeID = ChangeNumber, }; if (isAppSection) { appHistory.NewValue = SubID.ToString(); appHistory.Action = "added_to_sub"; } else { appHistory.OldValue = SubID.ToString(); appHistory.Action = "removed_from_sub"; } await DbConnection.ExecuteAsync(AppProcessor.HistoryQuery, appHistory); var depotHistory = new DepotHistory { DepotID = appID, ManifestID = 0, ChangeID = ChangeNumber, OldValue = SubID, Action = isAppSection ? "removed_from_sub" : "added_to_sub" }; if (isAppSection) { depotHistory.OldValue = SubID; depotHistory.Action = "removed_from_sub"; } else { depotHistory.NewValue = SubID; depotHistory.Action = "added_to_sub"; } await DbConnection.ExecuteAsync(DepotProcessor.HistoryQuery, depotHistory); } apps.Remove(appID); } else { await DbConnection.ExecuteAsync("INSERT INTO `SubsApps` (`SubID`, `AppID`, `Type`) VALUES(@SubID, @AppID, @Type)", new { SubID, AppID = appID, Type = type }); await MakeHistory("added_to_sub", typeID, string.Empty, childrenApp.Value); if (isAppSection) { await DbConnection.ExecuteAsync(AppProcessor.HistoryQuery, new PICSHistory { ID = appID, ChangeID = ChangeNumber, NewValue = SubID.ToString(), Action = "added_to_sub" } ); } else { await DbConnection.ExecuteAsync(DepotProcessor.HistoryQuery, new DepotHistory { DepotID = appID, ManifestID = 0, ChangeID = ChangeNumber, NewValue = SubID, Action = "added_to_sub" } ); } appAddedToThisPackage = true; if (packageOwned && !LicenseList.OwnedApps.ContainsKey(appID)) { LicenseList.OwnedApps.Add(appID, 1); } } } } else if (sectionName == "extended") { foreach (var children in section.Children) { var keyName = $"{sectionName}_{children.Name}"; if (children.Children.Count > 0) { await ProcessKey(keyName, children.Name, Utils.JsonifyKeyValue(children), true); } else { await ProcessKey(keyName, children.Name, children.Value); } } } else if (sectionName == "appitems" && section.Children.Count > 1) { sectionName = $"root_{sectionName}"; var fixedAppItems = new KeyValue(section.Name); // Valve for some reason creates a new children for each item, // instead of actually making it an array. // This causes json_decode in php override the key, thus lose data. foreach (var item in section.Children) { var appItem = fixedAppItems.Children.Find(s => s.Name == item.Name); if (appItem == default) { appItem = new KeyValue(item.Name); fixedAppItems.Children.Add(appItem); } foreach (var itemId in item.Children) { appItem.Children.Add(new KeyValue(itemId.Name, itemId.Value)); } } await ProcessKey(sectionName, sectionName, Utils.JsonifyKeyValue(fixedAppItems), true); } else if (section.Children.Count > 0) { sectionName = $"root_{sectionName}"; await ProcessKey(sectionName, sectionName, Utils.JsonifyKeyValue(section), true); } else if (!string.IsNullOrEmpty(section.Value)) { var keyName = $"root_{sectionName}"; await ProcessKey(keyName, sectionName, section.Value); } } // If this package no longer returns any package info, keep the existing info we have if (hasPackageInfo) { foreach (var data in CurrentData.Values.Where(data => !data.Processed && !data.KeyName.StartsWith("website", StringComparison.Ordinal))) { await DbConnection.ExecuteAsync("DELETE FROM `SubsInfo` WHERE `SubID` = @SubID AND `Key` = @Key", new { SubID, data.Key }); await MakeHistory("removed_key", data.Key, data.Value); } var appsRemoved = apps.Count > 0; foreach (var(appid, type) in apps) { await DbConnection.ExecuteAsync("DELETE FROM `SubsApps` WHERE `SubID` = @SubID AND `AppID` = @AppID AND `Type` = @Type", new { SubID, AppID = appid, Type = type }); var isAppSection = type == "app"; var typeID = (uint)(isAppSection ? 0 : 1); // 0 = app, 1 = depot; can't store as string because it's in the `key` field await MakeHistory("removed_from_sub", typeID, appid.ToString()); if (isAppSection) { await DbConnection.ExecuteAsync(AppProcessor.HistoryQuery, new PICSHistory { ID = appid, ChangeID = ChangeNumber, OldValue = SubID.ToString(), Action = "removed_from_sub" } ); } else { await DbConnection.ExecuteAsync(DepotProcessor.HistoryQuery, new DepotHistory { DepotID = appid, ManifestID = 0, ChangeID = ChangeNumber, OldValue = SubID, Action = "removed_from_sub" } ); } } if (appsRemoved) { LicenseList.RefreshApps(); } if (!packageOwned && SubID != 17906 && Settings.IsMillhaven) { Steam.Instance.FreeLicense.RequestFromPackage(SubID, ProductInfo.KeyValues); } } // Re-queue apps in this package so we can update depots and whatnot if (appAddedToThisPackage && !Settings.IsFullRun && !string.IsNullOrEmpty(PackageName)) { var appsToRequest = ProductInfo.KeyValues["appids"].Children.Select(x => (uint)x.AsInteger()).ToList(); JobManager.AddJob( () => Steam.Instance.Apps.PICSGetAccessTokens(appsToRequest, Enumerable.Empty <uint>()), new PICSTokens.RequestedTokens { Apps = appsToRequest }); } if (ProductInfo.MissingToken && PICSTokens.HasPackageToken(SubID)) { Log.WriteError(nameof(PICSTokens), $"Overridden token for subid {SubID} is invalid?"); IRC.Instance.SendOps($"[Tokens] Looks like the overridden token for subid {SubID} ({newPackageName}) is invalid"); } }
public void Process(SteamApps.PICSProductInfoCallback.PICSProductInfo productInfo) { ChangeNumber = productInfo.ChangeNumber; #if !DEBUG if (Settings.IsFullRun) #endif { Log.WriteDebug("Sub Processor", "SubID: {0}", SubID); } var appAddedToThisPackage = false; var packageOwned = LicenseList.OwnedSubs.ContainsKey(SubID); var kv = productInfo.KeyValues.Children.FirstOrDefault(); var newPackageName = kv["name"].AsString(); var apps = DbConnection.Query <PackageApp>("SELECT `AppID`, `Type` FROM `SubsApps` WHERE `SubID` = @SubID", new { SubID }).ToDictionary(x => x.AppID, x => x.Type); // TODO: Ideally this should be SteamDB Unknown Package and proper checks like app processor does if (newPackageName == null) { newPackageName = string.Concat("Steam Sub ", SubID); } if (newPackageName != null) { if (string.IsNullOrEmpty(PackageName)) { DbConnection.Execute("INSERT INTO `Subs` (`SubID`, `Name`, `LastKnownName`) VALUES (@SubID, @Name, @Name) ON DUPLICATE KEY UPDATE `Name` = @Name", new { SubID, Name = newPackageName }); MakeHistory("created_sub"); MakeHistory("created_info", SteamDB.DATABASE_NAME_TYPE, string.Empty, newPackageName); } else if (!PackageName.Equals(newPackageName)) { if (newPackageName.StartsWith("Steam Sub ", StringComparison.Ordinal)) { DbConnection.Execute("UPDATE `Subs` SET `Name` = @Name WHERE `SubID` = @SubID", new { SubID, Name = newPackageName }); } else { DbConnection.Execute("UPDATE `Subs` SET `Name` = @Name, `LastKnownName` = @Name WHERE `SubID` = @SubID", new { SubID, Name = newPackageName }); } MakeHistory("modified_info", SteamDB.DATABASE_NAME_TYPE, PackageName, newPackageName); } } foreach (var section in kv.Children) { string sectionName = section.Name.ToLower(); if (string.IsNullOrEmpty(sectionName) || sectionName.Equals("packageid") || sectionName.Equals("name")) { // Ignore common keys continue; } if (sectionName.Equals("appids") || sectionName.Equals("depotids")) { // Remove "ids", so we get "app" from appids and "depot" from depotids string type = sectionName.Replace("ids", string.Empty); var isAppSection = type.Equals("app"); var typeID = (uint)(isAppSection ? 0 : 1); // 0 = app, 1 = depot; can't store as string because it's in the `key` field foreach (var childrenApp in section.Children) { uint appID = uint.Parse(childrenApp.Value); // Is this appid already in this package? if (apps.ContainsKey(appID)) { // Is this appid's type the same? if (apps[appID] != type) { DbConnection.Execute("UPDATE `SubsApps` SET `Type` = @Type WHERE `SubID` = @SubID AND `AppID` = @AppID", new { SubID, AppID = appID, Type = type }); MakeHistory("added_to_sub", typeID, apps[appID].Equals("app") ? "0" : "1", childrenApp.Value); appAddedToThisPackage = true; // TODO: Log relevant add/remove history for depot/app? } apps.Remove(appID); } else { DbConnection.Execute("INSERT INTO `SubsApps` (`SubID`, `AppID`, `Type`) VALUES(@SubID, @AppID, @Type) ON DUPLICATE KEY UPDATE `Type` = @Type", new { SubID, AppID = appID, Type = type }); MakeHistory("added_to_sub", typeID, string.Empty, childrenApp.Value); if (isAppSection) { DbConnection.Execute(AppProcessor.GetHistoryQuery(), new PICSHistory { ID = appID, ChangeID = ChangeNumber, NewValue = SubID.ToString(), Action = "added_to_sub" } ); } else { DbConnection.Execute(DepotProcessor.GetHistoryQuery(), new DepotHistory { DepotID = appID, ChangeID = ChangeNumber, NewValue = SubID, Action = "added_to_sub" } ); } appAddedToThisPackage = true; if (packageOwned && !LicenseList.OwnedApps.ContainsKey(appID)) { LicenseList.OwnedApps.Add(appID, (byte)1); } } } } else if (sectionName.Equals("extended")) { string keyName; foreach (var children in section.Children) { keyName = string.Format("{0}_{1}", sectionName, children.Name); if (children.Children.Count > 0) { ProcessKey(keyName, children.Name, Utils.JsonifyKeyValue(children), true); } else { ProcessKey(keyName, children.Name, children.Value); } } } else if (section.Children.Any()) { sectionName = string.Format("root_{0}", sectionName); ProcessKey(sectionName, sectionName, Utils.JsonifyKeyValue(section), true); } else if (!string.IsNullOrEmpty(section.Value)) { string keyName = string.Format("root_{0}", sectionName); ProcessKey(keyName, sectionName, section.Value); } } foreach (var data in CurrentData.Values) { if (!data.Processed && !data.KeyName.StartsWith("website", StringComparison.Ordinal)) { DbConnection.Execute("DELETE FROM `SubsInfo` WHERE `SubID` = @SubID AND `Key` = @Key", new { SubID, data.Key }); MakeHistory("removed_key", data.Key, data.Value); } } var appsRemoved = apps.Any(); foreach (var app in apps) { DbConnection.Execute("DELETE FROM `SubsApps` WHERE `SubID` = @SubID AND `AppID` = @AppID AND `Type` = @Type", new { SubID, AppID = app.Key, Type = app.Value }); var isAppSection = app.Value.Equals("app"); var typeID = (uint)(isAppSection ? 0 : 1); // 0 = app, 1 = depot; can't store as string because it's in the `key` field MakeHistory("removed_from_sub", typeID, app.Key.ToString()); if (isAppSection) { DbConnection.Execute(AppProcessor.GetHistoryQuery(), new PICSHistory { ID = app.Key, ChangeID = ChangeNumber, OldValue = SubID.ToString(), Action = "removed_from_sub" } ); } else { DbConnection.Execute(DepotProcessor.GetHistoryQuery(), new DepotHistory { DepotID = app.Key, ChangeID = ChangeNumber, OldValue = SubID, Action = "removed_from_sub" } ); } } if (appsRemoved) { LicenseList.RefreshApps(); } if (kv["billingtype"].AsInteger() == 12 && !packageOwned) // 12 == free on demand { Log.WriteDebug("Sub Processor", "Requesting apps in SubID {0} as a free license", SubID); JobManager.AddJob(() => SteamDB.RequestFreeLicense(kv["appids"].Children.Select(appid => (uint)appid.AsInteger()).ToList())); } // Re-queue apps in this package so we can update depots and whatnot if (appAddedToThisPackage && !Settings.IsFullRun && !string.IsNullOrEmpty(PackageName)) { JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(kv["appids"].Children.Select(x => (uint)x.AsInteger()), Enumerable.Empty <uint>())); } }