Exemple #1
0
        private static async Task FullRun(HttpListenerContext context)
        {
            _ = TaskManager.Run(async() =>
            {
                await FullUpdateProcessor.FullUpdateAppsMetadata(true);
                await FullUpdateProcessor.FullUpdatePackagesMetadata();
            });

            await WriteJsonResponse("Triggered full run", context.Response);
        }
        public PICSChanges(CallbackManager manager)
        {
            if (Settings.IsFullRun)
            {
                var timer = new Timer
                {
                    Interval = TimeSpan.FromSeconds(10).TotalMilliseconds
                };
                timer.Elapsed += (sender, args) => FullUpdateProcessor.IsBusy();
                timer.Start();

                return;
            }

            manager.Subscribe <SteamApps.PICSChangesCallback>(OnPICSChanges);

            using (var db = Database.Get())
            {
                BillingTypeKey = KeyNameCache.GetSubKeyID("root_billingtype");

                if (LocalConfig.Current.ChangeNumber == 0)
                {
                    PreviousChangeNumber             = db.ExecuteScalar <uint>("SELECT `ChangeID` FROM `Changelists` ORDER BY `ChangeID` DESC LIMIT 1");
                    LocalConfig.Current.ChangeNumber = PreviousChangeNumber;
                }
                else
                {
                    PreviousChangeNumber = LocalConfig.Current.ChangeNumber;
                }

                Log.WriteInfo(nameof(PICSChanges), $"Previous changelist was {PreviousChangeNumber}");
            }

            if (PreviousChangeNumber == 0)
            {
                Log.WriteWarn(nameof(PICSChanges), "Looks like there are no changelists in the database.");
                Log.WriteWarn(nameof(PICSChanges), $"If you want to fill up your database first, restart with \"FullRun\" setting set to {(int)FullRunState.Enumerate}.");
            }
        }
        private void OnPICSChanges(SteamApps.PICSChangesCallback callback)
        {
            if (PreviousChangeNumber == callback.CurrentChangeNumber)
            {
                return;
            }

            Log.WriteInfo(nameof(PICSChanges), $"Changelist {PreviousChangeNumber} -> {callback.CurrentChangeNumber} ({callback.AppChanges.Count} apps, {callback.PackageChanges.Count} packages)");

            LocalConfig.Current.ChangeNumber = PreviousChangeNumber = callback.CurrentChangeNumber;

            TaskManager.Run(async() => await HandleChangeNumbers(callback));

            if (callback.RequiresFullAppUpdate || callback.RequiresFullPackageUpdate)
            {
                TaskManager.Run(async() =>
                {
                    if (callback.RequiresFullAppUpdate)
                    {
                        IRC.Instance.SendOps($"Changelist {callback.CurrentChangeNumber} has forced a full app update");

                        await FullUpdateProcessor.FullUpdateAppsMetadata();
                    }

                    if (callback.RequiresFullPackageUpdate)
                    {
                        IRC.Instance.SendOps($"Changelist {callback.CurrentChangeNumber} has forced a full package update");

                        await FullUpdateProcessor.FullUpdatePackagesMetadata();
                    }
                });
            }

            if (callback.AppChanges.Count == 0 && callback.PackageChanges.Count == 0)
            {
                IRC.Instance.SendAnnounce($"{Colors.RED}»{Colors.NORMAL} Changelist {Colors.BLUE}{PreviousChangeNumber}{Colors.DARKGRAY} (empty)");

                return;
            }

            const int appsPerJob = 50;

            if (callback.AppChanges.Count > appsPerJob)
            {
                foreach (var list in callback.AppChanges.Keys.Split(appsPerJob))
                {
                    JobManager.AddJob(
                        () => Steam.Instance.Apps.PICSGetAccessTokens(list, Enumerable.Empty <uint>()),
                        new PICSTokens.RequestedTokens
                    {
                        Apps = list.ToList()
                    });
                }
            }
            else if (callback.AppChanges.Count > 0)
            {
                JobManager.AddJob(
                    () => Steam.Instance.Apps.PICSGetAccessTokens(callback.AppChanges.Keys, Enumerable.Empty <uint>()),
                    new PICSTokens.RequestedTokens
                {
                    Apps = callback.AppChanges.Keys.ToList()
                });
            }

            if (callback.PackageChanges.Count > appsPerJob)
            {
                foreach (var list in callback.PackageChanges.Keys.Split(appsPerJob))
                {
                    JobManager.AddJob(
                        () => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), list),
                        new PICSTokens.RequestedTokens
                    {
                        Packages = list.ToList()
                    });
                }
            }
            else if (callback.PackageChanges.Count > 0)
            {
                JobManager.AddJob(
                    () => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), callback.PackageChanges.Keys),
                    new PICSTokens.RequestedTokens
                {
                    Packages = callback.PackageChanges.Keys.ToList()
                });
            }

            if (callback.AppChanges.Count > 0)
            {
                _ = TaskManager.Run(async() => await HandleApps(callback));
            }

            if (callback.PackageChanges.Count > 0)
            {
                _ = TaskManager.Run(async() => await HandlePackages(callback));
                _ = TaskManager.Run(async() => await HandlePackagesChangelists(callback));
            }

            _ = TaskManager.Run(async() => await SendChangelistsToIRC(callback));

            PrintImportants(callback);
        }
Exemple #4
0
        private static void OnPICSProductInfo(SteamApps.PICSProductInfoCallback callback)
        {
            JobManager.TryRemoveJob(callback.JobID);

            if (callback.MetaDataOnly)
            {
                TaskManager.Run(async() => await FullUpdateProcessor.HandleMetadataInfo(callback));
                return;
            }

            var processors = new List <BaseProcessor>(
                callback.Apps.Count +
                callback.Packages.Count +
                callback.UnknownApps.Count +
                callback.UnknownPackages.Count
                );

            processors.AddRange(callback.Apps.Select(app => new AppProcessor(app.Key, app.Value)));
            processors.AddRange(callback.Packages.Select(package => new SubProcessor(package.Key, package.Value)));
            processors.AddRange(callback.UnknownApps.Select(app => new AppProcessor(app, null)));
            processors.AddRange(callback.UnknownPackages.Select(package => new SubProcessor(package, null)));

            foreach (var workaround in processors)
            {
                var processor = workaround;

                Task mostRecentItem;

                lock (CurrentlyProcessing)
                {
                    CurrentlyProcessing.TryGetValue(processor.Id, out mostRecentItem);
                }

                var workerItem = TaskManager.Run(async() =>
                {
                    try
                    {
                        await Semaphore.WaitAsync(TaskManager.TaskCancellationToken.Token).ConfigureAwait(false);

                        if (mostRecentItem?.IsCompleted == false)
                        {
                            Log.WriteDebug(processor.ToString(), $"Waiting for previous task to finish processing ({CurrentlyProcessing.Count})");

                            await mostRecentItem.ConfigureAwait(false);

#if DEBUG
                            Log.WriteDebug(processor.ToString(), "Previous task lock ended");
#endif
                        }

                        await processor.Process().ConfigureAwait(false);
                    }
                    catch (Exception e)
                    {
                        ErrorReporter.Notify(processor.ToString(), e);
                    }
                    finally
                    {
                        Semaphore.Release();

                        processor.Dispose();
                    }

                    return(processor);
                });

                lock (CurrentlyProcessing)
                {
                    CurrentlyProcessing[processor.Id] = workerItem;
                }

                workerItem.ContinueWith(RemoveProcessorLock, TaskManager.TaskCancellationToken.Token);
            }
        }
Exemple #5
0
        private void OnPICSChanges(SteamApps.PICSChangesCallback callback)
        {
            if (PreviousChangeNumber == callback.CurrentChangeNumber)
            {
                return;
            }

            Log.WriteInfo(nameof(PICSChanges), $"Changelist {callback.LastChangeNumber} -> {callback.CurrentChangeNumber} ({callback.AppChanges.Count} apps, {callback.PackageChanges.Count} packages)");

            PreviousChangeNumber = callback.CurrentChangeNumber;

            TaskManager.Run(async() => await HandleChangeNumbers(callback));

            if (callback.RequiresFullAppUpdate || callback.RequiresFullPackageUpdate)
            {
                TaskManager.Run(async() =>
                {
                    if (callback.RequiresFullAppUpdate)
                    {
                        IRC.Instance.SendOps($"Changelist {callback.CurrentChangeNumber} has forced a full app update");

                        // When full update flag is set, presumably Steam client start hammering the servers
                        // and the PICS service just does not return any data for a while until it clears up
                        await FullUpdateProcessor.FullUpdateAppsMetadata(true);
                    }

                    if (callback.RequiresFullPackageUpdate)
                    {
                        IRC.Instance.SendOps($"Changelist {callback.CurrentChangeNumber} has forced a full package update");

                        await FullUpdateProcessor.FullUpdatePackagesMetadata();
                    }

                    IRC.Instance.SendOps($"Changelist {callback.CurrentChangeNumber} full update has finished");
                });
            }

            if (callback.AppChanges.Count == 0 && callback.PackageChanges.Count == 0)
            {
                return;
            }

            const int appsPerJob = 50;

            if (callback.AppChanges.Count > appsPerJob)
            {
                foreach (var list in callback.AppChanges.Keys.Split(appsPerJob))
                {
                    JobManager.AddJob(
                        () => Steam.Instance.Apps.PICSGetAccessTokens(list, Enumerable.Empty <uint>()),
                        new PICSTokens.RequestedTokens
                    {
                        Apps = list.ToList()
                    });
                }
            }
            else if (callback.AppChanges.Count > 0)
            {
                JobManager.AddJob(
                    () => Steam.Instance.Apps.PICSGetAccessTokens(callback.AppChanges.Keys, Enumerable.Empty <uint>()),
                    new PICSTokens.RequestedTokens
                {
                    Apps = callback.AppChanges.Keys.ToList()
                });
            }

            if (callback.PackageChanges.Count > appsPerJob)
            {
                foreach (var list in callback.PackageChanges.Keys.Split(appsPerJob))
                {
                    JobManager.AddJob(
                        () => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), list),
                        new PICSTokens.RequestedTokens
                    {
                        Packages = list.ToList()
                    });
                }
            }
            else if (callback.PackageChanges.Count > 0)
            {
                JobManager.AddJob(
                    () => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), callback.PackageChanges.Keys),
                    new PICSTokens.RequestedTokens
                {
                    Packages = callback.PackageChanges.Keys.ToList()
                });
            }

            if (callback.AppChanges.Count > 0)
            {
                _ = TaskManager.Run(async() => await HandleApps(callback));
            }

            if (callback.PackageChanges.Count > 0)
            {
                _ = TaskManager.Run(async() => await HandlePackages(callback));
                _ = TaskManager.Run(async() => await HandlePackagesChangelists(callback));
            }

            if (PreviousChangeNumber - LastStoredChangeNumber >= 1000)
            {
                LastStoredChangeNumber = PreviousChangeNumber;

                _ = TaskManager.Run(async() => await LocalConfig.Update("backend.changenumber", LastStoredChangeNumber.ToString()));
            }

            PrintImportants(callback);
        }
Exemple #6
0
        public override async Task OnCommand(CommandArguments command)
        {
            var s     = command.Message.Split(' ');
            var count = s.Length;

            if (count > 0)
            {
                uint id;
                switch (s[0])
                {
                case "reload":
                    await Application.ReloadImportant();

                    await PICSTokens.Reload();

                    command.Notice("Reloaded important apps and pics tokens");

                    return;

                case "fullrun":
                    _ = TaskManager.Run(async() =>
                    {
                        command.Reply("Started full metadata scan, this will take a while…");
                        await FullUpdateProcessor.FullUpdateAppsMetadata(true);
                        command.Reply("App full scan finished, starting packages, this will take even longer…");
                        await FullUpdateProcessor.FullUpdatePackagesMetadata();
                        command.Reply("Full metadata scan finished.");
                    });

                    return;

                case "add":
                    if (count < 3)
                    {
                        break;
                    }

                    if (!uint.TryParse(s[2], out id))
                    {
                        break;
                    }

                    switch (s[1])
                    {
                    case "app":
                        if (Application.ImportantApps.Contains(id))
                        {
                            command.Reply($"App {Colors.BLUE}{id}{Colors.NORMAL} ({Steam.GetAppName(id)}) is already important.");
                        }
                        else
                        {
                            Application.ImportantApps.Add(id);

                            await using (var db = await Database.GetConnectionAsync())
                            {
                                await db.ExecuteAsync("INSERT INTO `ImportantApps` (`AppID`) VALUES (@AppID)", new { AppID = id });
                            }

                            command.Reply($"Marked app {Colors.BLUE}{id}{Colors.NORMAL} ({Steam.GetAppName(id)}) as important.");
                        }

                        return;

                    case "sub":
                        if (Application.ImportantSubs.Contains(id))
                        {
                            command.Reply($"Package {Colors.BLUE}{id}{Colors.NORMAL} ({Steam.GetPackageName(id)}) is already important.");
                        }
                        else
                        {
                            Application.ImportantSubs.Add(id);

                            await using (var db = await Database.GetConnectionAsync())
                            {
                                await db.ExecuteAsync("INSERT INTO `ImportantSubs` (`SubID`) VALUES (@SubID)", new { SubID = id });
                            }

                            command.Reply($"Marked package {Colors.BLUE}{id}{Colors.NORMAL} ({Steam.GetPackageName(id)}) as important.");
                        }

                        return;
                    }

                    break;

                case "remove":
                    if (count < 3)
                    {
                        break;
                    }

                    if (!uint.TryParse(s[2], out id))
                    {
                        break;
                    }

                    switch (s[1])
                    {
                    case "app":
                        if (!Application.ImportantApps.Contains(id))
                        {
                            command.Reply($"App {Colors.BLUE}{id}{Colors.NORMAL} ({Steam.GetAppName(id)}) is not important.");
                        }
                        else
                        {
                            Application.ImportantApps.Remove(id);

                            await using (var db = await Database.GetConnectionAsync())
                            {
                                await db.ExecuteAsync("DELETE FROM `ImportantApps` WHERE `AppID` = @AppID", new { AppID = id });
                            }

                            command.Reply($"Removed app {Colors.BLUE}{id}{Colors.NORMAL} ({Steam.GetAppName(id)}) from the important list.");
                        }

                        return;

                    case "sub":
                        if (!Application.ImportantSubs.Contains(id))
                        {
                            command.Reply($"Package {Colors.BLUE}{id}{Colors.NORMAL} ({Steam.GetPackageName(id)}) is not important.");
                        }
                        else
                        {
                            Application.ImportantSubs.Remove(id);

                            await using (var db = await Database.GetConnectionAsync())
                            {
                                await db.ExecuteAsync("DELETE FROM `ImportantSubs` WHERE `SubID` = @SubID", new { SubID = id });
                            }

                            command.Reply($"Removed package {Colors.BLUE}{id}{Colors.NORMAL} ({Steam.GetPackageName(id)}) from the important list.");
                        }

                        return;
                    }

                    break;
                }
            }

            command.Reply($"Usage:{Colors.OLIVE} important reload {Colors.NORMAL}or{Colors.OLIVE} important <add/remove> <app/sub> <id> {Colors.NORMAL}or{Colors.OLIVE} important fullrun");
        }