protected override async Task ProcessData()
        {
            await LoadData();

            ChangeNumber = ProductInfo.ChangeNumber;

            if (Settings.IsFullRun)
            {
                await DbConnection.ExecuteAsync("INSERT INTO `Changelists` (`ChangeID`) VALUES (@ChangeNumber) ON DUPLICATE KEY UPDATE `Date` = `Date`", new { ProductInfo.ChangeNumber });

                await DbConnection.ExecuteAsync("INSERT INTO `ChangelistsSubs` (`ChangeID`, `SubID`) VALUES (@ChangeNumber, @SubID) ON DUPLICATE KEY UPDATE `SubID` = `SubID`", new { SubID, ProductInfo.ChangeNumber });
            }

            await ProcessKey("root_changenumber", "changenumber", ChangeNumber.ToString());

            var appAddedToThisPackage = false;
            var packageOwned          = LicenseList.OwnedSubs.ContainsKey(SubID);
            var newPackageName        = ProductInfo.KeyValues["name"].AsString();
            var apps = (await DbConnection.QueryAsync <PackageApp>("SELECT `AppID`, `Type` FROM `SubsApps` WHERE `SubID` = @SubID", new { SubID })).ToDictionary(x => x.AppID, x => x.Type);

            // TODO: Ideally this should be SteamDB Unknown Package and proper checks like app processor does
            if (newPackageName == null)
            {
                newPackageName = string.Concat("Steam Sub ", SubID);
            }

            if (newPackageName != null)
            {
                if (string.IsNullOrEmpty(PackageName))
                {
                    await DbConnection.ExecuteAsync("INSERT INTO `Subs` (`SubID`, `Name`, `LastKnownName`) VALUES (@SubID, @Name, @Name)", new { SubID, Name = newPackageName });

                    await MakeHistory("created_sub");
                    await MakeHistory("created_info", SteamDB.DATABASE_NAME_TYPE, string.Empty, newPackageName);
                }
                else if (!PackageName.Equals(newPackageName))
                {
                    if (newPackageName.StartsWith("Steam Sub ", StringComparison.Ordinal))
                    {
                        await DbConnection.ExecuteAsync("UPDATE `Subs` SET `Name` = @Name WHERE `SubID` = @SubID", new { SubID, Name = newPackageName });
                    }
                    else
                    {
                        await DbConnection.ExecuteAsync("UPDATE `Subs` SET `Name` = @Name, `LastKnownName` = @Name WHERE `SubID` = @SubID", new { SubID, Name = newPackageName });
                    }

                    await MakeHistory("modified_info", SteamDB.DATABASE_NAME_TYPE, PackageName, newPackageName);
                }
            }

            foreach (var section in ProductInfo.KeyValues.Children)
            {
                var sectionName = section.Name.ToLower();

                if (string.IsNullOrEmpty(sectionName) || sectionName.Equals("packageid") || sectionName.Equals("changenumber") || sectionName.Equals("name"))
                {
                    // Ignore common keys
                    continue;
                }

                if (sectionName.Equals("appids") || sectionName.Equals("depotids"))
                {
                    // Remove "ids", so we get "app" from appids and "depot" from depotids
                    var type         = sectionName.Replace("ids", string.Empty);
                    var isAppSection = type.Equals("app");
                    var typeID       = (uint)(isAppSection ? 0 : 1); // 0 = app, 1 = depot; can't store as string because it's in the `key` field

                    foreach (var childrenApp in section.Children)
                    {
                        var appID = uint.Parse(childrenApp.Value);

                        // Is this appid already in this package?
                        if (apps.ContainsKey(appID))
                        {
                            // Is this appid's type the same?
                            if (apps[appID] != type)
                            {
                                await DbConnection.ExecuteAsync("UPDATE `SubsApps` SET `Type` = @Type WHERE `SubID` = @SubID AND `AppID` = @AppID", new { SubID, AppID = appID, Type = type });
                                await MakeHistory("added_to_sub", typeID, apps[appID].Equals("app")? "0" : "1", childrenApp.Value);

                                appAddedToThisPackage = true;

                                // TODO: Log relevant add/remove history for depot/app?
                            }

                            apps.Remove(appID);
                        }
                        else
                        {
                            await DbConnection.ExecuteAsync("INSERT INTO `SubsApps` (`SubID`, `AppID`, `Type`) VALUES(@SubID, @AppID, @Type)", new { SubID, AppID = appID, Type = type });
                            await MakeHistory("added_to_sub", typeID, string.Empty, childrenApp.Value);

                            if (isAppSection)
                            {
                                await DbConnection.ExecuteAsync(AppProcessor.HistoryQuery,
                                                                new PICSHistory
                                {
                                    ID       = appID,
                                    ChangeID = ChangeNumber,
                                    NewValue = SubID.ToString(),
                                    Action   = "added_to_sub"
                                }
                                                                );
                            }
                            else
                            {
                                await DbConnection.ExecuteAsync(DepotProcessor.HistoryQuery,
                                                                new DepotHistory
                                {
                                    DepotID  = appID,
                                    ChangeID = ChangeNumber,
                                    NewValue = SubID,
                                    Action   = "added_to_sub"
                                }
                                                                );
                            }

                            appAddedToThisPackage = true;

                            if (packageOwned && !LicenseList.OwnedApps.ContainsKey(appID))
                            {
                                LicenseList.OwnedApps.Add(appID, 1);
                            }
                        }
                    }
                }
                else if (sectionName.Equals("extended"))
                {
                    foreach (var children in section.Children)
                    {
                        var keyName = string.Format("{0}_{1}", sectionName, children.Name);

                        if (children.Children.Count > 0)
                        {
                            await ProcessKey(keyName, children.Name, Utils.JsonifyKeyValue(children), true);
                        }
                        else
                        {
                            await ProcessKey(keyName, children.Name, children.Value);
                        }
                    }
                }
                else if (section.Children.Any())
                {
                    sectionName = string.Format("root_{0}", sectionName);

                    await ProcessKey(sectionName, sectionName, Utils.JsonifyKeyValue(section), true);
                }
                else if (!string.IsNullOrEmpty(section.Value))
                {
                    var keyName = string.Format("root_{0}", sectionName);

                    await ProcessKey(keyName, sectionName, section.Value);
                }
            }

            foreach (var data in CurrentData.Values.Where(data => !data.Processed && !data.KeyName.StartsWith("website", StringComparison.Ordinal)))
            {
                await DbConnection.ExecuteAsync("DELETE FROM `SubsInfo` WHERE `SubID` = @SubID AND `Key` = @Key", new { SubID, data.Key });
                await MakeHistory("removed_key", data.Key, data.Value);
            }

            var appsRemoved = apps.Any();

            foreach (var app in apps)
            {
                await DbConnection.ExecuteAsync("DELETE FROM `SubsApps` WHERE `SubID` = @SubID AND `AppID` = @AppID AND `Type` = @Type", new { SubID, AppID = app.Key, Type = app.Value });

                var isAppSection = app.Value.Equals("app");

                var typeID = (uint)(isAppSection ? 0 : 1); // 0 = app, 1 = depot; can't store as string because it's in the `key` field

                await MakeHistory("removed_from_sub", typeID, app.Key.ToString());

                if (isAppSection)
                {
                    await DbConnection.ExecuteAsync(AppProcessor.HistoryQuery,
                                                    new PICSHistory
                    {
                        ID       = app.Key,
                        ChangeID = ChangeNumber,
                        OldValue = SubID.ToString(),
                        Action   = "removed_from_sub"
                    }
                                                    );
                }
                else
                {
                    await DbConnection.ExecuteAsync(DepotProcessor.HistoryQuery,
                                                    new DepotHistory
                    {
                        DepotID  = app.Key,
                        ChangeID = ChangeNumber,
                        OldValue = SubID,
                        Action   = "removed_from_sub"
                    }
                                                    );
                }
            }

            if (appsRemoved)
            {
                LicenseList.RefreshApps();
            }

            if (!packageOwned && SubID != 17906)
            {
                FreeLicense.RequestFromPackage(SubID, ProductInfo.KeyValues);
            }

            // Re-queue apps in this package so we can update depots and whatnot
            if (appAddedToThisPackage && !Settings.IsFullRun && !string.IsNullOrEmpty(PackageName))
            {
                JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(ProductInfo.KeyValues["appids"].Children.Select(x => (uint)x.AsInteger()), Enumerable.Empty <uint>()));
            }

            // Maintain a list of anonymous content
            if (appAddedToThisPackage && SubID == 17906)
            {
                LicenseList.RefreshAnonymous();
            }
        }
        protected override async Task ProcessData()
        {
            await LoadData();

            ChangeNumber = ProductInfo.ChangeNumber;

            if (Settings.IsFullRun)
            {
                await DbConnection.ExecuteAsync("INSERT INTO `Changelists` (`ChangeID`) VALUES (@ChangeNumber) ON DUPLICATE KEY UPDATE `Date` = `Date`", new { ProductInfo.ChangeNumber });

                await DbConnection.ExecuteAsync("INSERT INTO `ChangelistsSubs` (`ChangeID`, `SubID`) VALUES (@ChangeNumber, @SubID) ON DUPLICATE KEY UPDATE `SubID` = `SubID`", new { SubID, ProductInfo.ChangeNumber });
            }

            await ProcessKey("root_changenumber", "changenumber", ChangeNumber.ToString());

            var appAddedToThisPackage = false;
            var hasPackageInfo        = ProductInfo.KeyValues.Children.Count > 0;
            var packageOwned          = LicenseList.OwnedSubs.ContainsKey(SubID);
            var newPackageName        = ProductInfo.KeyValues["name"].AsString() ?? string.Concat("Steam Sub ", SubID);
            var apps = (await DbConnection.QueryAsync <PackageApp>("SELECT `AppID`, `Type` FROM `SubsApps` WHERE `SubID` = @SubID", new { SubID })).ToDictionary(x => x.AppID, x => x.Type);
            var alreadySeenAppIds = new HashSet <uint>();

            if (!hasPackageInfo)
            {
                ProductInfo.KeyValues.Children.Add(new KeyValue("steamdb_requires_token", "1"));
            }

            if (string.IsNullOrEmpty(PackageName))
            {
                await DbConnection.ExecuteAsync("INSERT INTO `Subs` (`SubID`, `Name`, `LastKnownName`) VALUES (@SubID, @Name, @Name)", new { SubID, Name = newPackageName });

                await MakeHistory("created_sub");
                await MakeHistory("created_info", SteamDB.DatabaseNameType, string.Empty, newPackageName);
            }
            else if (PackageName != newPackageName)
            {
                if (newPackageName.StartsWith("Steam Sub ", StringComparison.Ordinal))
                {
                    await DbConnection.ExecuteAsync("UPDATE `Subs` SET `Name` = @Name WHERE `SubID` = @SubID", new { SubID, Name = newPackageName });
                }
                else
                {
                    await DbConnection.ExecuteAsync("UPDATE `Subs` SET `Name` = @Name, `LastKnownName` = @Name WHERE `SubID` = @SubID", new { SubID, Name = newPackageName });
                }

                await MakeHistory("modified_info", SteamDB.DatabaseNameType, PackageName, newPackageName);
            }

            foreach (var section in ProductInfo.KeyValues.Children)
            {
                var sectionName = section.Name.ToLowerInvariant();

                if (string.IsNullOrEmpty(sectionName) || sectionName == "packageid" || sectionName == "changenumber" || sectionName == "name")
                {
                    // Ignore common keys
                    continue;
                }

                if (sectionName == "appids" || sectionName == "depotids")
                {
                    // Remove "ids", so we get "app" from appids and "depot" from depotids
                    var type         = sectionName.Replace("ids", string.Empty);
                    var isAppSection = type == "app";
                    var typeID       = (uint)(isAppSection ? 0 : 1); // 0 = app, 1 = depot; can't store as string because it's in the `key` field

                    foreach (var childrenApp in section.Children)
                    {
                        if (!uint.TryParse(childrenApp.Value, out var appID))
                        {
                            Log.WriteWarn(nameof(SubProcessor), $"Package {SubID} has an invalid uint: {childrenApp.Value}");
                            continue;
                        }

                        if (alreadySeenAppIds.Contains(appID))
                        {
                            Log.WriteWarn(nameof(SubProcessor), $"Package {SubID} has a duplicate app: {appID}");
                            continue;
                        }

                        alreadySeenAppIds.Add(appID);

                        // Is this appid already in this package?
                        if (apps.ContainsKey(appID))
                        {
                            // Is this appid's type the same?
                            if (apps[appID] != type)
                            {
                                await DbConnection.ExecuteAsync("UPDATE `SubsApps` SET `Type` = @Type WHERE `SubID` = @SubID AND `AppID` = @AppID", new { SubID, AppID = appID, Type = type });
                                await MakeHistory("added_to_sub", typeID, apps[appID] == "app"? "0" : "1", childrenApp.Value);

                                appAddedToThisPackage = true;

                                // Log relevant add/remove history events for depot and app
                                var appHistory = new PICSHistory
                                {
                                    ID       = appID,
                                    ChangeID = ChangeNumber,
                                };

                                if (isAppSection)
                                {
                                    appHistory.NewValue = SubID.ToString();
                                    appHistory.Action   = "added_to_sub";
                                }
                                else
                                {
                                    appHistory.OldValue = SubID.ToString();
                                    appHistory.Action   = "removed_from_sub";
                                }

                                await DbConnection.ExecuteAsync(AppProcessor.HistoryQuery, appHistory);

                                var depotHistory = new DepotHistory
                                {
                                    DepotID    = appID,
                                    ManifestID = 0,
                                    ChangeID   = ChangeNumber,
                                    OldValue   = SubID,
                                    Action     = isAppSection ? "removed_from_sub" : "added_to_sub"
                                };

                                if (isAppSection)
                                {
                                    depotHistory.OldValue = SubID;
                                    depotHistory.Action   = "removed_from_sub";
                                }
                                else
                                {
                                    depotHistory.NewValue = SubID;
                                    depotHistory.Action   = "added_to_sub";
                                }

                                await DbConnection.ExecuteAsync(DepotProcessor.HistoryQuery, depotHistory);
                            }

                            apps.Remove(appID);
                        }
                        else
                        {
                            await DbConnection.ExecuteAsync("INSERT INTO `SubsApps` (`SubID`, `AppID`, `Type`) VALUES(@SubID, @AppID, @Type)", new { SubID, AppID = appID, Type = type });
                            await MakeHistory("added_to_sub", typeID, string.Empty, childrenApp.Value);

                            if (isAppSection)
                            {
                                await DbConnection.ExecuteAsync(AppProcessor.HistoryQuery,
                                                                new PICSHistory
                                {
                                    ID       = appID,
                                    ChangeID = ChangeNumber,
                                    NewValue = SubID.ToString(),
                                    Action   = "added_to_sub"
                                }
                                                                );
                            }
                            else
                            {
                                await DbConnection.ExecuteAsync(DepotProcessor.HistoryQuery,
                                                                new DepotHistory
                                {
                                    DepotID    = appID,
                                    ManifestID = 0,
                                    ChangeID   = ChangeNumber,
                                    NewValue   = SubID,
                                    Action     = "added_to_sub"
                                }
                                                                );
                            }

                            appAddedToThisPackage = true;

                            if (packageOwned && !LicenseList.OwnedApps.ContainsKey(appID))
                            {
                                LicenseList.OwnedApps.Add(appID, 1);
                            }
                        }
                    }
                }
                else if (sectionName == "extended")
                {
                    foreach (var children in section.Children)
                    {
                        var keyName = $"{sectionName}_{children.Name}";

                        if (children.Children.Count > 0)
                        {
                            await ProcessKey(keyName, children.Name, Utils.JsonifyKeyValue(children), true);
                        }
                        else
                        {
                            await ProcessKey(keyName, children.Name, children.Value);
                        }
                    }
                }
                else if (sectionName == "appitems" && section.Children.Count > 1)
                {
                    sectionName = $"root_{sectionName}";

                    var fixedAppItems = new KeyValue(section.Name);

                    // Valve for some reason creates a new children for each item,
                    // instead of actually making it an array.
                    // This causes json_decode in php override the key, thus lose data.
                    foreach (var item in section.Children)
                    {
                        var appItem = fixedAppItems.Children.Find(s => s.Name == item.Name);

                        if (appItem == default)
                        {
                            appItem = new KeyValue(item.Name);
                            fixedAppItems.Children.Add(appItem);
                        }

                        foreach (var itemId in item.Children)
                        {
                            appItem.Children.Add(new KeyValue(itemId.Name, itemId.Value));
                        }
                    }

                    await ProcessKey(sectionName, sectionName, Utils.JsonifyKeyValue(fixedAppItems), true);
                }
                else if (section.Children.Count > 0)
                {
                    sectionName = $"root_{sectionName}";

                    await ProcessKey(sectionName, sectionName, Utils.JsonifyKeyValue(section), true);
                }
                else if (!string.IsNullOrEmpty(section.Value))
                {
                    var keyName = $"root_{sectionName}";

                    await ProcessKey(keyName, sectionName, section.Value);
                }
            }

            // If this package no longer returns any package info, keep the existing info we have
            if (hasPackageInfo)
            {
                foreach (var data in CurrentData.Values.Where(data => !data.Processed && !data.KeyName.StartsWith("website", StringComparison.Ordinal)))
                {
                    await DbConnection.ExecuteAsync("DELETE FROM `SubsInfo` WHERE `SubID` = @SubID AND `Key` = @Key", new { SubID, data.Key });
                    await MakeHistory("removed_key", data.Key, data.Value);
                }

                var appsRemoved = apps.Count > 0;

                foreach (var(appid, type) in apps)
                {
                    await DbConnection.ExecuteAsync("DELETE FROM `SubsApps` WHERE `SubID` = @SubID AND `AppID` = @AppID AND `Type` = @Type", new { SubID, AppID = appid, Type = type });

                    var isAppSection = type == "app";

                    var typeID = (uint)(isAppSection ? 0 : 1);  // 0 = app, 1 = depot; can't store as string because it's in the `key` field

                    await MakeHistory("removed_from_sub", typeID, appid.ToString());

                    if (isAppSection)
                    {
                        await DbConnection.ExecuteAsync(AppProcessor.HistoryQuery,
                                                        new PICSHistory
                        {
                            ID       = appid,
                            ChangeID = ChangeNumber,
                            OldValue = SubID.ToString(),
                            Action   = "removed_from_sub"
                        }
                                                        );
                    }
                    else
                    {
                        await DbConnection.ExecuteAsync(DepotProcessor.HistoryQuery,
                                                        new DepotHistory
                        {
                            DepotID    = appid,
                            ManifestID = 0,
                            ChangeID   = ChangeNumber,
                            OldValue   = SubID,
                            Action     = "removed_from_sub"
                        }
                                                        );
                    }
                }

                if (appsRemoved)
                {
                    LicenseList.RefreshApps();
                }

                if (!packageOwned && SubID != 17906 && Settings.IsMillhaven)
                {
                    Steam.Instance.FreeLicense.RequestFromPackage(SubID, ProductInfo.KeyValues);
                }
            }

            // Re-queue apps in this package so we can update depots and whatnot
            if (appAddedToThisPackage && !Settings.IsFullRun && !string.IsNullOrEmpty(PackageName))
            {
                var appsToRequest = ProductInfo.KeyValues["appids"].Children.Select(x => (uint)x.AsInteger()).ToList();

                JobManager.AddJob(
                    () => Steam.Instance.Apps.PICSGetAccessTokens(appsToRequest, Enumerable.Empty <uint>()),
                    new PICSTokens.RequestedTokens
                {
                    Apps = appsToRequest
                });
            }

            if (ProductInfo.MissingToken && PICSTokens.HasPackageToken(SubID))
            {
                Log.WriteError(nameof(PICSTokens), $"Overridden token for subid {SubID} is invalid?");
                IRC.Instance.SendOps($"[Tokens] Looks like the overridden token for subid {SubID} ({newPackageName}) is invalid");
            }
        }
Example #3
0
        public void Process(SteamApps.PICSProductInfoCallback.PICSProductInfo productInfo)
        {
            ChangeNumber = productInfo.ChangeNumber;

#if !DEBUG
            if (Settings.IsFullRun)
#endif
            {
                Log.WriteDebug("Sub Processor", "SubID: {0}", SubID);
            }

            var appAddedToThisPackage = false;
            var packageOwned          = LicenseList.OwnedSubs.ContainsKey(SubID);
            var kv             = productInfo.KeyValues.Children.FirstOrDefault();
            var newPackageName = kv["name"].AsString();
            var apps           = DbConnection.Query <PackageApp>("SELECT `AppID`, `Type` FROM `SubsApps` WHERE `SubID` = @SubID", new { SubID }).ToDictionary(x => x.AppID, x => x.Type);

            // TODO: Ideally this should be SteamDB Unknown Package and proper checks like app processor does
            if (newPackageName == null)
            {
                newPackageName = string.Concat("Steam Sub ", SubID);
            }

            if (newPackageName != null)
            {
                if (string.IsNullOrEmpty(PackageName))
                {
                    DbConnection.Execute("INSERT INTO `Subs` (`SubID`, `Name`, `LastKnownName`) VALUES (@SubID, @Name, @Name) ON DUPLICATE KEY UPDATE `Name` = @Name", new { SubID, Name = newPackageName });

                    MakeHistory("created_sub");
                    MakeHistory("created_info", SteamDB.DATABASE_NAME_TYPE, string.Empty, newPackageName);
                }
                else if (!PackageName.Equals(newPackageName))
                {
                    if (newPackageName.StartsWith("Steam Sub ", StringComparison.Ordinal))
                    {
                        DbConnection.Execute("UPDATE `Subs` SET `Name` = @Name WHERE `SubID` = @SubID", new { SubID, Name = newPackageName });
                    }
                    else
                    {
                        DbConnection.Execute("UPDATE `Subs` SET `Name` = @Name, `LastKnownName` = @Name WHERE `SubID` = @SubID", new { SubID, Name = newPackageName });
                    }

                    MakeHistory("modified_info", SteamDB.DATABASE_NAME_TYPE, PackageName, newPackageName);
                }
            }

            foreach (var section in kv.Children)
            {
                string sectionName = section.Name.ToLower();

                if (string.IsNullOrEmpty(sectionName) || sectionName.Equals("packageid") || sectionName.Equals("name"))
                {
                    // Ignore common keys
                    continue;
                }

                if (sectionName.Equals("appids") || sectionName.Equals("depotids"))
                {
                    // Remove "ids", so we get "app" from appids and "depot" from depotids
                    string type = sectionName.Replace("ids", string.Empty);

                    var isAppSection = type.Equals("app");

                    var typeID = (uint)(isAppSection ? 0 : 1); // 0 = app, 1 = depot; can't store as string because it's in the `key` field

                    foreach (var childrenApp in section.Children)
                    {
                        uint appID = uint.Parse(childrenApp.Value);

                        // Is this appid already in this package?
                        if (apps.ContainsKey(appID))
                        {
                            // Is this appid's type the same?
                            if (apps[appID] != type)
                            {
                                DbConnection.Execute("UPDATE `SubsApps` SET `Type` = @Type WHERE `SubID` = @SubID AND `AppID` = @AppID", new { SubID, AppID = appID, Type = type });

                                MakeHistory("added_to_sub", typeID, apps[appID].Equals("app") ? "0" : "1", childrenApp.Value);

                                appAddedToThisPackage = true;

                                // TODO: Log relevant add/remove history for depot/app?
                            }

                            apps.Remove(appID);
                        }
                        else
                        {
                            DbConnection.Execute("INSERT INTO `SubsApps` (`SubID`, `AppID`, `Type`) VALUES(@SubID, @AppID, @Type) ON DUPLICATE KEY UPDATE `Type` = @Type", new { SubID, AppID = appID, Type = type });

                            MakeHistory("added_to_sub", typeID, string.Empty, childrenApp.Value);

                            if (isAppSection)
                            {
                                DbConnection.Execute(AppProcessor.GetHistoryQuery(),
                                                     new PICSHistory
                                {
                                    ID       = appID,
                                    ChangeID = ChangeNumber,
                                    NewValue = SubID.ToString(),
                                    Action   = "added_to_sub"
                                }
                                                     );
                            }
                            else
                            {
                                DbConnection.Execute(DepotProcessor.GetHistoryQuery(),
                                                     new DepotHistory
                                {
                                    DepotID  = appID,
                                    ChangeID = ChangeNumber,
                                    NewValue = SubID,
                                    Action   = "added_to_sub"
                                }
                                                     );
                            }

                            appAddedToThisPackage = true;

                            if (packageOwned && !LicenseList.OwnedApps.ContainsKey(appID))
                            {
                                LicenseList.OwnedApps.Add(appID, (byte)1);
                            }
                        }
                    }
                }
                else if (sectionName.Equals("extended"))
                {
                    string keyName;

                    foreach (var children in section.Children)
                    {
                        keyName = string.Format("{0}_{1}", sectionName, children.Name);

                        if (children.Children.Count > 0)
                        {
                            ProcessKey(keyName, children.Name, Utils.JsonifyKeyValue(children), true);
                        }
                        else
                        {
                            ProcessKey(keyName, children.Name, children.Value);
                        }
                    }
                }
                else if (section.Children.Any())
                {
                    sectionName = string.Format("root_{0}", sectionName);

                    ProcessKey(sectionName, sectionName, Utils.JsonifyKeyValue(section), true);
                }
                else if (!string.IsNullOrEmpty(section.Value))
                {
                    string keyName = string.Format("root_{0}", sectionName);

                    ProcessKey(keyName, sectionName, section.Value);
                }
            }

            foreach (var data in CurrentData.Values)
            {
                if (!data.Processed && !data.KeyName.StartsWith("website", StringComparison.Ordinal))
                {
                    DbConnection.Execute("DELETE FROM `SubsInfo` WHERE `SubID` = @SubID AND `Key` = @Key", new { SubID, data.Key });

                    MakeHistory("removed_key", data.Key, data.Value);
                }
            }

            var appsRemoved = apps.Any();

            foreach (var app in apps)
            {
                DbConnection.Execute("DELETE FROM `SubsApps` WHERE `SubID` = @SubID AND `AppID` = @AppID AND `Type` = @Type", new { SubID, AppID = app.Key, Type = app.Value });

                var isAppSection = app.Value.Equals("app");

                var typeID = (uint)(isAppSection ? 0 : 1); // 0 = app, 1 = depot; can't store as string because it's in the `key` field

                MakeHistory("removed_from_sub", typeID, app.Key.ToString());

                if (isAppSection)
                {
                    DbConnection.Execute(AppProcessor.GetHistoryQuery(),
                                         new PICSHistory
                    {
                        ID       = app.Key,
                        ChangeID = ChangeNumber,
                        OldValue = SubID.ToString(),
                        Action   = "removed_from_sub"
                    }
                                         );
                }
                else
                {
                    DbConnection.Execute(DepotProcessor.GetHistoryQuery(),
                                         new DepotHistory
                    {
                        DepotID  = app.Key,
                        ChangeID = ChangeNumber,
                        OldValue = SubID,
                        Action   = "removed_from_sub"
                    }
                                         );
                }
            }

            if (appsRemoved)
            {
                LicenseList.RefreshApps();
            }

            if (kv["billingtype"].AsInteger() == 12 && !packageOwned) // 12 == free on demand
            {
                Log.WriteDebug("Sub Processor", "Requesting apps in SubID {0} as a free license", SubID);

                JobManager.AddJob(() => SteamDB.RequestFreeLicense(kv["appids"].Children.Select(appid => (uint)appid.AsInteger()).ToList()));
            }

            // Re-queue apps in this package so we can update depots and whatnot
            if (appAddedToThisPackage && !Settings.IsFullRun && !string.IsNullOrEmpty(PackageName))
            {
                JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(kv["appids"].Children.Select(x => (uint)x.AsInteger()), Enumerable.Empty <uint>()));
            }
        }