private static void OnPICSProductInfo(SteamApps.PICSProductInfoCallback callback) { var apps = callback.Apps.Concat(callback.UnknownApps.ToDictionary(x => x, x => (SteamApps.PICSProductInfoCallback.PICSProductInfo)null)); var packages = callback.Packages.Concat(callback.UnknownPackages.ToDictionary(x => x, x => (SteamApps.PICSProductInfoCallback.PICSProductInfo)null)); foreach (var workaround in apps) { var app = workaround; Log.WriteInfo("PICSProductInfo", "{0}AppID: {1}", app.Value == null ? "Unknown " : "", app.Key); Task mostRecentItem; lock (ProcessedApps) { ProcessedApps.TryGetValue(app.Key, out mostRecentItem); } var workerItem = TaskManager.Run(async delegate { if (mostRecentItem != null && !mostRecentItem.IsCompleted) { Log.WriteDebug("PICSProductInfo", "Waiting for app {0} to finish processing", app.Key); await mostRecentItem; } using (var processor = new AppProcessor(app.Key)) { if (app.Value == null) { processor.ProcessUnknown(); } else { processor.Process(app.Value); } } }); if (Settings.IsFullRun) { continue; } lock (ProcessedApps) { ProcessedApps[app.Key] = workerItem; } workerItem.ContinueWith(task => { lock (ProcessedApps) { if (ProcessedApps.TryGetValue(app.Key, out mostRecentItem) && mostRecentItem.IsCompleted) { ProcessedApps.Remove(app.Key); } } }); } foreach (var workaround in packages) { var package = workaround; Log.WriteInfo("PICSProductInfo", "{0}SubID: {1}", package.Value == null ? "Unknown " : "", package.Key); Task mostRecentItem; lock (ProcessedSubs) { ProcessedSubs.TryGetValue(package.Key, out mostRecentItem); } var workerItem = TaskManager.Run(async delegate { if (mostRecentItem != null && !mostRecentItem.IsCompleted) { Log.WriteDebug("PICSProductInfo", "Waiting for package {0} to finish processing", package.Key); await mostRecentItem; } using (var processor = new SubProcessor(package.Key)) { if (package.Value == null) { processor.ProcessUnknown(); } else { processor.Process(package.Value); } } }); if (Settings.IsFullRun) { continue; } lock (ProcessedSubs) { ProcessedSubs[package.Key] = workerItem; } workerItem.ContinueWith(task => { lock (ProcessedSubs) { if (ProcessedSubs.TryGetValue(package.Key, out mostRecentItem) && mostRecentItem.IsCompleted) { ProcessedSubs.Remove(package.Key); } } }); } }
public void Process(SteamApps.PICSProductInfoCallback.PICSProductInfo productInfo) { ChangeNumber = productInfo.ChangeNumber; #if !DEBUG if (Settings.IsFullRun) #endif { Log.WriteDebug("Sub Processor", "SubID: {0}", SubID); } var appAddedToThisPackage = false; var packageOwned = LicenseList.OwnedSubs.ContainsKey(SubID); var kv = productInfo.KeyValues.Children.FirstOrDefault(); var newPackageName = kv["name"].AsString(); var apps = DbConnection.Query <PackageApp>("SELECT `AppID`, `Type` FROM `SubsApps` WHERE `SubID` = @SubID", new { SubID }).ToDictionary(x => x.AppID, x => x.Type); // TODO: Ideally this should be SteamDB Unknown Package and proper checks like app processor does if (newPackageName == null) { newPackageName = string.Concat("Steam Sub ", SubID); } if (newPackageName != null) { if (string.IsNullOrEmpty(PackageName)) { DbConnection.Execute("INSERT INTO `Subs` (`SubID`, `Name`, `LastKnownName`) VALUES (@SubID, @Name, @Name) ON DUPLICATE KEY UPDATE `Name` = @Name", new { SubID, Name = newPackageName }); MakeHistory("created_sub"); MakeHistory("created_info", SteamDB.DATABASE_NAME_TYPE, string.Empty, newPackageName); } else if (!PackageName.Equals(newPackageName)) { if (newPackageName.StartsWith("Steam Sub ", StringComparison.Ordinal)) { DbConnection.Execute("UPDATE `Subs` SET `Name` = @Name WHERE `SubID` = @SubID", new { SubID, Name = newPackageName }); } else { DbConnection.Execute("UPDATE `Subs` SET `Name` = @Name, `LastKnownName` = @Name WHERE `SubID` = @SubID", new { SubID, Name = newPackageName }); } MakeHistory("modified_info", SteamDB.DATABASE_NAME_TYPE, PackageName, newPackageName); } } foreach (var section in kv.Children) { string sectionName = section.Name.ToLower(); if (string.IsNullOrEmpty(sectionName) || sectionName.Equals("packageid") || sectionName.Equals("name")) { // Ignore common keys continue; } if (sectionName.Equals("appids") || sectionName.Equals("depotids")) { // Remove "ids", so we get "app" from appids and "depot" from depotids string type = sectionName.Replace("ids", string.Empty); var isAppSection = type.Equals("app"); var typeID = (uint)(isAppSection ? 0 : 1); // 0 = app, 1 = depot; can't store as string because it's in the `key` field foreach (var childrenApp in section.Children) { uint appID = uint.Parse(childrenApp.Value); // Is this appid already in this package? if (apps.ContainsKey(appID)) { // Is this appid's type the same? if (apps[appID] != type) { DbConnection.Execute("UPDATE `SubsApps` SET `Type` = @Type WHERE `SubID` = @SubID AND `AppID` = @AppID", new { SubID, AppID = appID, Type = type }); MakeHistory("added_to_sub", typeID, apps[appID].Equals("app") ? "0" : "1", childrenApp.Value); appAddedToThisPackage = true; // TODO: Log relevant add/remove history for depot/app? } apps.Remove(appID); } else { DbConnection.Execute("INSERT INTO `SubsApps` (`SubID`, `AppID`, `Type`) VALUES(@SubID, @AppID, @Type) ON DUPLICATE KEY UPDATE `Type` = @Type", new { SubID, AppID = appID, Type = type }); MakeHistory("added_to_sub", typeID, string.Empty, childrenApp.Value); if (isAppSection) { DbConnection.Execute(AppProcessor.GetHistoryQuery(), new PICSHistory { ID = appID, ChangeID = ChangeNumber, NewValue = SubID.ToString(), Action = "added_to_sub" } ); } else { DbConnection.Execute(DepotProcessor.GetHistoryQuery(), new DepotHistory { DepotID = appID, ChangeID = ChangeNumber, NewValue = SubID, Action = "added_to_sub" } ); } appAddedToThisPackage = true; if (packageOwned && !LicenseList.OwnedApps.ContainsKey(appID)) { LicenseList.OwnedApps.Add(appID, (byte)1); } } } } else if (sectionName.Equals("extended")) { string keyName; foreach (var children in section.Children) { keyName = string.Format("{0}_{1}", sectionName, children.Name); if (children.Children.Count > 0) { ProcessKey(keyName, children.Name, Utils.JsonifyKeyValue(children), true); } else { ProcessKey(keyName, children.Name, children.Value); } } } else if (section.Children.Any()) { sectionName = string.Format("root_{0}", sectionName); ProcessKey(sectionName, sectionName, Utils.JsonifyKeyValue(section), true); } else if (!string.IsNullOrEmpty(section.Value)) { string keyName = string.Format("root_{0}", sectionName); ProcessKey(keyName, sectionName, section.Value); } } foreach (var data in CurrentData.Values) { if (!data.Processed && !data.KeyName.StartsWith("website", StringComparison.Ordinal)) { DbConnection.Execute("DELETE FROM `SubsInfo` WHERE `SubID` = @SubID AND `Key` = @Key", new { SubID, data.Key }); MakeHistory("removed_key", data.Key, data.Value); } } var appsRemoved = apps.Any(); foreach (var app in apps) { DbConnection.Execute("DELETE FROM `SubsApps` WHERE `SubID` = @SubID AND `AppID` = @AppID AND `Type` = @Type", new { SubID, AppID = app.Key, Type = app.Value }); var isAppSection = app.Value.Equals("app"); var typeID = (uint)(isAppSection ? 0 : 1); // 0 = app, 1 = depot; can't store as string because it's in the `key` field MakeHistory("removed_from_sub", typeID, app.Key.ToString()); if (isAppSection) { DbConnection.Execute(AppProcessor.GetHistoryQuery(), new PICSHistory { ID = app.Key, ChangeID = ChangeNumber, OldValue = SubID.ToString(), Action = "removed_from_sub" } ); } else { DbConnection.Execute(DepotProcessor.GetHistoryQuery(), new DepotHistory { DepotID = app.Key, ChangeID = ChangeNumber, OldValue = SubID, Action = "removed_from_sub" } ); } } if (appsRemoved) { LicenseList.RefreshApps(); } if (kv["billingtype"].AsInteger() == 12 && !packageOwned) // 12 == free on demand { Log.WriteDebug("Sub Processor", "Requesting apps in SubID {0} as a free license", SubID); JobManager.AddJob(() => SteamDB.RequestFreeLicense(kv["appids"].Children.Select(appid => (uint)appid.AsInteger()).ToList())); } // Re-queue apps in this package so we can update depots and whatnot if (appAddedToThisPackage && !Settings.IsFullRun && !string.IsNullOrEmpty(PackageName)) { JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(kv["appids"].Children.Select(x => (uint)x.AsInteger()), Enumerable.Empty <uint>())); } }
private static void OnPICSProductInfo(SteamApps.PICSProductInfoCallback callback) { JobManager.TryRemoveJob(callback.JobID); var apps = callback.Apps.Concat(callback.UnknownApps.ToDictionary(x => x, x => (SteamApps.PICSProductInfoCallback.PICSProductInfo)null)); var packages = callback.Packages.Concat(callback.UnknownPackages.ToDictionary(x => x, x => (SteamApps.PICSProductInfoCallback.PICSProductInfo)null)); foreach (var workaround in apps) { var app = workaround; Log.WriteInfo("PICSProductInfo", "{0}AppID: {1}", app.Value == null ? "Unknown " : "", app.Key); IWorkItemResult mostRecentItem; lock (ProcessedApps) { ProcessedApps.TryGetValue(app.Key, out mostRecentItem); } var workerItem = ProcessorThreadPool.QueueWorkItem(delegate { try { if (mostRecentItem != null && !mostRecentItem.IsCompleted) { Log.WriteDebug("PICSProductInfo", "Waiting for app {0} to finish processing", app.Key); SmartThreadPool.WaitAll(new IWaitableResult[] { mostRecentItem }); } using (var processor = new AppProcessor(app.Key)) { if (app.Value == null) { processor.ProcessUnknown(); } else { processor.Process(app.Value); } } } catch (MySqlException e) { Log.WriteError("PICSProductInfo", "App {0} faulted: {1}", app.Key, e); JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(app.Key, null)); } catch (Exception e) { Log.WriteError("PICSProductInfo", "App {0} faulted: {1}", app.Key, e); } finally { lock (ProcessedApps) { if (ProcessedApps.TryGetValue(app.Key, out mostRecentItem) && mostRecentItem.IsCompleted) { ProcessedApps.Remove(app.Key); } } } }); if (Settings.IsFullRun) { continue; } lock (ProcessedApps) { ProcessedApps[app.Key] = workerItem; } } foreach (var workaround in packages) { var package = workaround; Log.WriteInfo("PICSProductInfo", "{0}SubID: {1}", package.Value == null ? "Unknown " : "", package.Key); IWorkItemResult mostRecentItem; lock (ProcessedSubs) { ProcessedSubs.TryGetValue(package.Key, out mostRecentItem); } var workerItem = ProcessorThreadPool.QueueWorkItem(delegate { try { if (mostRecentItem != null && !mostRecentItem.IsCompleted) { Log.WriteDebug("PICSProductInfo", "Waiting for package {0} to finish processing", package.Key); SmartThreadPool.WaitAll(new IWaitableResult[] { mostRecentItem }); } using (var processor = new SubProcessor(package.Key)) { if (package.Value == null) { processor.ProcessUnknown(); } else { processor.Process(package.Value); } } } catch (MySqlException e) { Log.WriteError("PICSProductInfo", "Package {0} faulted: {1}", package.Key, e); JobManager.AddJob(() => Steam.Instance.Apps.PICSGetProductInfo(null, package.Key, false, false)); } catch (Exception e) { Log.WriteError("PICSProductInfo", "Package {0} faulted: {1}", package.Key, e); } finally { lock (ProcessedSubs) { if (ProcessedSubs.TryGetValue(package.Key, out mostRecentItem) && mostRecentItem.IsCompleted) { ProcessedSubs.Remove(package.Key); } } } }); if (Settings.IsFullRun) { continue; } lock (ProcessedSubs) { ProcessedSubs[package.Key] = workerItem; } } }
private void TryProcess(SteamApps.PICSProductInfoCallback.PICSProductInfo productInfo) { string packageName = string.Empty; var apps = new Dictionary <uint, string>(); using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `Name`, `Value` FROM `SubsInfo` INNER JOIN `KeyNamesSubs` ON `SubsInfo`.`Key` = `KeyNamesSubs`.`ID` WHERE `SubID` = @SubID", new MySqlParameter("@SubID", SubID))) { while (Reader.Read()) { CurrentData.Add(DbWorker.GetString("Name", Reader), DbWorker.GetString("Value", Reader)); } } using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `Name` FROM `Subs` WHERE `SubID` = @SubID LIMIT 1", new MySqlParameter("@SubID", SubID))) { if (Reader.Read()) { packageName = DbWorker.GetString("Name", Reader); } } using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `AppID`, `Type` FROM `SubsApps` WHERE `SubID` = @SubID", new MySqlParameter("@SubID", SubID))) { while (Reader.Read()) { apps.Add(Reader.GetUInt32("AppID"), Reader.GetString("Type")); } } var kv = productInfo.KeyValues.Children.FirstOrDefault(); if (kv["name"].Value != null) { if (string.IsNullOrEmpty(packageName)) { DbWorker.ExecuteNonQuery("INSERT INTO `Subs` (`SubID`, `Name`) VALUES (@SubID, @Name) ON DUPLICATE KEY UPDATE `Name` = @Name", new MySqlParameter("@SubID", SubID), new MySqlParameter("@Name", kv["name"].Value) ); MakeHistory("created_sub"); MakeHistory("created_info", DATABASE_NAME_TYPE, string.Empty, kv["name"].Value); } else if (!packageName.Equals(kv["name"].Value)) { DbWorker.ExecuteNonQuery("UPDATE `Subs` SET `Name` = @Name WHERE `SubID` = @SubID", new MySqlParameter("@SubID", SubID), new MySqlParameter("@Name", kv["name"].Value) ); MakeHistory("modified_info", DATABASE_NAME_TYPE, packageName, kv["name"].Value); } } foreach (KeyValue section in kv.Children) { string sectionName = section.Name.ToLower(); if (string.IsNullOrEmpty(sectionName) || sectionName.Equals("packageid") || sectionName.Equals("name")) { // Ignore common keys continue; } if (sectionName.Equals("appids") || sectionName.Equals("depotids")) { string type = sectionName.Replace("ids", string.Empty); // Remove "ids", so we get app from appids and depot from depotids uint typeID = (uint)(type.Equals("app") ? 0 : 1); // TODO: Remove legacy 0/1 and replace with type foreach (KeyValue childrenApp in section.Children) { uint appID = uint.Parse(childrenApp.Value); // Is this appid already in this package? if (apps.ContainsKey(appID)) { // Is this appid's type the same? if (apps[appID] != type) { DbWorker.ExecuteNonQuery("UPDATE `SubsApps` SET `Type` = @Type WHERE `SubID` = @SubID AND `AppID` = @AppID", new MySqlParameter("@SubID", SubID), new MySqlParameter("@AppID", appID), new MySqlParameter("@Type", type) ); MakeHistory("added_to_sub", typeID, apps[appID].Equals("app") ? "0" : "1", childrenApp.Value); } apps.Remove(appID); } else { DbWorker.ExecuteNonQuery("INSERT INTO `SubsApps` (`SubID`, `AppID`, `Type`) VALUES(@SubID, @AppID, @Type) ON DUPLICATE KEY UPDATE `Type` = @Type", new MySqlParameter("@SubID", SubID), new MySqlParameter("@AppID", appID), new MySqlParameter("@Type", type) ); MakeHistory("added_to_sub", typeID, string.Empty, childrenApp.Value); AppProcessor.MakeHistory(appID, ChangeNumber, "added_to_sub", typeID, string.Empty, SubID.ToString()); if (SteamProxy.Instance.ImportantApps.Contains(appID)) { IRC.SendMain("Important app {0}{1}{2} was added to package {3}{4}{5} -{6} {7}", Colors.OLIVE, SteamProxy.GetAppName(appID), Colors.NORMAL, Colors.OLIVE, packageName, Colors.NORMAL, Colors.DARK_BLUE, SteamDB.GetPackageURL(SubID, "history") ); } } } } else if (sectionName.Equals("extended")) { string keyName; foreach (KeyValue children in section.Children) { if (children.Children.Count > 0) { Log.WriteError("Sub Processor", "SubID {0} has childen in extended section", SubID); } keyName = string.Format("{0}_{1}", sectionName, children.Name); ProcessKey(keyName, children.Name, children.Value); } } else if (section.Children.Count > 0) { sectionName = string.Format("root_{0}", sectionName); ProcessKey(sectionName, sectionName, DbWorker.JsonifyKeyValue(section), true); } else if (!string.IsNullOrEmpty(section.Value)) { string keyName = string.Format("root_{0}", sectionName); ProcessKey(keyName, sectionName, section.Value); } } foreach (string keyName in CurrentData.Keys) { if (!keyName.StartsWith("website", StringComparison.Ordinal)) { uint ID = GetKeyNameID(keyName); DbWorker.ExecuteNonQuery("DELETE FROM `SubsInfo` WHERE `SubID` = @SubID AND `Key` = @KeyNameID", new MySqlParameter("@SubID", SubID), new MySqlParameter("@KeyNameID", ID) ); MakeHistory("removed_key", ID, CurrentData[keyName]); } } foreach (var app in apps) { DbWorker.ExecuteNonQuery("DELETE FROM `SubsApps` WHERE `SubID` = @SubID AND `AppID` = @AppID AND `Type` = @Type", new MySqlParameter("@SubID", SubID), new MySqlParameter("@AppID", app.Key), new MySqlParameter("@Type", app.Value) ); uint typeID = (uint)(app.Value.Equals("app") ? 0 : 1); // TODO: Remove legacy 0/1 and replace with type MakeHistory("removed_from_sub", typeID, app.Key.ToString()); AppProcessor.MakeHistory(app.Key, ChangeNumber, "removed_from_sub", typeID, SubID.ToString()); if (SteamProxy.Instance.ImportantApps.Contains(app.Key)) { IRC.SendMain("Important app {0}{1}{2} was removed from package {3}{4}{5} -{6} {7}", Colors.OLIVE, SteamProxy.GetAppName(app.Key), Colors.NORMAL, Colors.OLIVE, packageName, Colors.NORMAL, Colors.DARK_BLUE, SteamDB.GetPackageURL(SubID, "history") ); } } #if DEBUG if (kv["name"].Value == null) { if (string.IsNullOrEmpty(packageName)) // We don't have the package in our database yet { // Don't do anything then Log.WriteError("Sub Processor", "Got a package without a name, and we don't have it in our database: {0}", SubID); } else { ////MakeHistory("deleted_sub", "0", packageName, "", true); Log.WriteError("Sub Processor", "Got a package without a name, but we have it in our database: {0}", SubID); } } #endif }
private static void OnPICSProductInfo(SteamApps.PICSProductInfoCallback callback) { JobManager.TryRemoveJob(callback.JobID); var apps = callback.Apps.Concat(callback.UnknownApps.ToDictionary(x => x, x => (SteamApps.PICSProductInfoCallback.PICSProductInfo)null)); var packages = callback.Packages.Concat(callback.UnknownPackages.ToDictionary(x => x, x => (SteamApps.PICSProductInfoCallback.PICSProductInfo)null)); foreach (var workaround in apps) { var app = workaround; Log.WriteInfo("PICSProductInfo", "{0}AppID: {1}", app.Value == null ? "Unknown " : "", app.Key); Task mostRecentItem; lock (ProcessedApps) { ProcessedApps.TryGetValue(app.Key, out mostRecentItem); } var workerItem = TaskManager.Run(async() => { try { await ProcessorSemaphore.WaitAsync().ConfigureAwait(false); if (mostRecentItem != null && !mostRecentItem.IsCompleted) { Log.WriteDebug("PICSProductInfo", "Waiting for app {0} to finish processing", app.Key); await mostRecentItem.ConfigureAwait(false); } using (var processor = new AppProcessor(app.Key)) { if (app.Value == null) { processor.ProcessUnknown(); } else { processor.Process(app.Value); } } } catch (MySqlException e) { ErrorReporter.Notify($"App {app.Key}", e); JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(app.Key, null)); } catch (Exception e) { ErrorReporter.Notify($"App {app.Key}", e); } finally { lock (ProcessedApps) { if (ProcessedApps.TryGetValue(app.Key, out mostRecentItem) && mostRecentItem.IsCompleted) { ProcessedApps.Remove(app.Key); } } ProcessorSemaphore.Release(); } }); if (Settings.IsFullRun) { continue; } lock (ProcessedApps) { ProcessedApps[app.Key] = workerItem; } } foreach (var workaround in packages) { var package = workaround; Log.WriteInfo("PICSProductInfo", "{0}SubID: {1}", package.Value == null ? "Unknown " : "", package.Key); Task mostRecentItem; lock (ProcessedSubs) { ProcessedSubs.TryGetValue(package.Key, out mostRecentItem); } var workerItem = TaskManager.Run(async() => { try { await ProcessorSemaphore.WaitAsync().ConfigureAwait(false); if (mostRecentItem != null && !mostRecentItem.IsCompleted) { Log.WriteDebug("PICSProductInfo", "Waiting for package {0} to finish processing", package.Key); await mostRecentItem.ConfigureAwait(false); } using (var processor = new SubProcessor(package.Key)) { if (package.Value == null) { processor.ProcessUnknown(); } else { processor.Process(package.Value); } } } catch (MySqlException e) { ErrorReporter.Notify($"Package {package.Key}", e); JobManager.AddJob(() => Steam.Instance.Apps.PICSGetProductInfo(null, package.Key, false, false)); } catch (Exception e) { ErrorReporter.Notify($"Package {package.Key}", e); } finally { lock (ProcessedSubs) { if (ProcessedSubs.TryGetValue(package.Key, out mostRecentItem) && mostRecentItem.IsCompleted) { ProcessedSubs.Remove(package.Key); } } ProcessorSemaphore.Release(); } }); if (Settings.IsFullRun) { continue; } lock (ProcessedSubs) { ProcessedSubs[package.Key] = workerItem; } } }