Example #1
0
        private async Task DownloadDepots(uint appID, List <ManifestJob> depots)
        {
            Log.WriteDebug("Depot Downloader", "Will process {0} depots ({1} depot locks left)", depots.Count(), DepotLocks.Count);

            var  processTasks       = new List <Task <EResult> >();
            bool anyFilesDownloaded = false;

            foreach (var depot in depots)
            {
                depot.DepotKey = await GetDepotDecryptionKey(depot.DepotID, appID);

                if (depot.DepotKey == null)
                {
                    RemoveLock(depot.DepotID);

                    continue;
                }

                var cdnToken = await GetCDNAuthToken(depot.DepotID);

                if (cdnToken == null)
                {
                    RemoveLock(depot.DepotID);

                    Log.WriteDebug("Depot Downloader", "Got a depot key for depot {0} but no cdn auth token", depot.DepotID);

                    continue;
                }

                depot.CDNToken = cdnToken.Token;
                depot.Server   = cdnToken.Server;

                DepotManifest depotManifest = null;
                string        lastError     = string.Empty;

                for (var i = 0; i <= 5; i++)
                {
                    try
                    {
                        depotManifest = CDNClient.DownloadManifest(depot.DepotID, depot.ManifestID, depot.Server, depot.CDNToken, depot.DepotKey);

                        break;
                    }
                    catch (Exception e)
                    {
                        Log.WriteWarn("Depot Downloader", "{0} Manifest download failed: {1} - {2}", depot.DepotID, e.GetType(), e.Message);

                        lastError = e.Message;
                    }

                    // TODO: get new auth key if auth fails

                    if (depotManifest == null)
                    {
                        await Task.Delay(Utils.ExponentionalBackoff(i));
                    }
                }

                if (depotManifest == null)
                {
                    RemoveLock(depot.DepotID);

                    Log.WriteError("Depot Processor", "Failed to download depot manifest for depot {0} ({1}: {2})", depot.DepotID, depot.Server, lastError);

                    if (FileDownloader.IsImportantDepot(depot.DepotID))
                    {
                        IRC.Instance.SendOps("{0}[{1}]{2} Failed to download depot {3} manifest ({4}: {5})",
                                             Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL, depot.DepotID, depot.Server, lastError);
                    }

                    continue;
                }

                var task = TaskManager.Run(() =>
                {
                    using (var db = Database.GetConnection())
                    {
                        using (var transaction = db.BeginTransaction())
                        {
                            var result = ProcessDepotAfterDownload(db, depot, depotManifest);
                            transaction.Commit();
                            return(result);
                        }
                    }
                });

                processTasks.Add(task);

                if (FileDownloader.IsImportantDepot(depot.DepotID))
                {
                    task = TaskManager.Run(() =>
                    {
                        var result = FileDownloader.DownloadFilesFromDepot(appID, depot, depotManifest);

                        if (result == EResult.OK)
                        {
                            anyFilesDownloaded = true;
                        }

                        return(result);
                    }, TaskCreationOptions.LongRunning);

                    TaskManager.RegisterErrorHandler(task);

                    processTasks.Add(task);
                }
            }

            if (SaveLocalConfig)
            {
                SaveLocalConfig = false;

                LocalConfig.Save();
            }

            await Task.WhenAll(processTasks);

            Log.WriteDebug("Depot Downloader", "{0} depot downloads finished", depots.Count());

            // TODO: use ContinueWith on tasks
            if (!anyFilesDownloaded)
            {
                foreach (var depot in depots)
                {
                    RemoveLock(depot.DepotID);
                }

                return;
            }

            if (!File.Exists(UpdateScript))
            {
                return;
            }

            bool lockTaken = false;

            try
            {
                UpdateScriptLock.Enter(ref lockTaken);

                foreach (var depot in depots)
                {
                    if (depot.Result == EResult.OK)
                    {
                        RunUpdateScript(string.Format("{0} no-git", depot.DepotID));
                    }

                    RemoveLock(depot.DepotID);
                }

                // Only commit changes if all depots downloaded
                if (processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored))
                {
                    RunUpdateScript("0");
                }
                else
                {
                    Log.WriteDebug("Depot Processor", "Reprocessing the app {0} because some files failed to download", appID);

                    IRC.Instance.SendOps("{0}[{1}]{2} Reprocessing the app because some files in {3} depots failed",
                                         Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL,
                                         depots.Count(x => x.Result != EResult.OK && x.Result != EResult.Ignored)
                                         );

                    JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
                }
            }
            finally
            {
                if (lockTaken)
                {
                    UpdateScriptLock.Exit();
                }
            }
        }
Example #2
0
        public override async Task OnCommand(CommandArguments command)
        {
            if (command.CommandType != ECommandType.IRC || !IRC.IsRecipientChannel(command.Recipient))
            {
                command.Reply("This command is only available in channels.");

                return;
            }

            var channel = command.Recipient;

            var s     = command.Message.Split(' ');
            var count = s.Length;

            if (count > 0)
            {
                uint id;
                switch (s[0])
                {
                case "reload":
                    Application.ReloadImportant(command);
                    PICSTokens.Reload(command);
                    FileDownloader.ReloadFileList();

                    return;

                case "add":
                    if (count < 3)
                    {
                        break;
                    }

                    if (!uint.TryParse(s[2], out id))
                    {
                        break;
                    }

                    switch (s[1])
                    {
                    case "app":
                        var exists = Application.ImportantApps.TryGetValue(id, out var channels);

                        if (exists && channels.Contains(channel))
                        {
                            command.Reply("App {0}{1}{2} ({3}) is already important in {4}{5}{6}.", Colors.BLUE, id, Colors.NORMAL, Steam.GetAppName(id), Colors.BLUE, channel, Colors.NORMAL);
                        }
                        else
                        {
                            if (exists)
                            {
                                Application.ImportantApps[id].Add(channel);
                            }
                            else
                            {
                                Application.ImportantApps.Add(id, new List <string> {
                                    channel
                                });
                            }

                            using (var db = Database.Get())
                            {
                                await db.ExecuteAsync("INSERT INTO `ImportantApps` (`AppID`, `Channel`) VALUES (@AppID, @Channel)", new { AppID = id, Channel = channel });
                            }

                            command.Reply("Marked app {0}{1}{2} ({3}) as important in {4}{5}{6}.", Colors.BLUE, id, Colors.NORMAL, Steam.GetAppName(id), Colors.BLUE, channel, Colors.NORMAL);
                        }

                        return;

                    case "sub":
                        if (Application.ImportantSubs.ContainsKey(id))
                        {
                            command.Reply("Package {0}{1}{2} ({3}) is already important.", Colors.BLUE, id, Colors.NORMAL, Steam.GetPackageName(id));
                        }
                        else
                        {
                            Application.ImportantSubs.Add(id, 1);

                            using (var db = Database.Get())
                            {
                                await db.ExecuteAsync("INSERT INTO `ImportantSubs` (`SubID`) VALUES (@SubID)", new { SubID = id });
                            }

                            command.Reply("Marked package {0}{1}{2} ({3}) as important.", Colors.BLUE, id, Colors.NORMAL, Steam.GetPackageName(id));
                        }

                        return;
                    }

                    break;

                case "remove":
                    if (count < 3)
                    {
                        break;
                    }

                    if (!uint.TryParse(s[2], out id))
                    {
                        break;
                    }

                    switch (s[1])
                    {
                    case "app":
                        if (!Application.ImportantApps.TryGetValue(id, out var channels) || !channels.Contains(channel))
                        {
                            command.Reply("App {0}{1}{2} ({3}) is not important in {4}{5}{6}.", Colors.BLUE, id, Colors.NORMAL, Steam.GetAppName(id), Colors.BLUE, channel, Colors.NORMAL);
                        }
                        else
                        {
                            if (channels.Count > 1)
                            {
                                Application.ImportantApps[id].Remove(channel);
                            }
                            else
                            {
                                Application.ImportantApps.Remove(id);
                            }

                            using (var db = Database.Get())
                            {
                                db.Execute("DELETE FROM `ImportantApps` WHERE `AppID` = @AppID AND `Channel` = @Channel", new { AppID = id, Channel = channel });
                            }

                            command.Reply("Removed app {0}{1}{2} ({3}) from the important list in {4}{5}{6}.", Colors.BLUE, id, Colors.NORMAL, Steam.GetAppName(id), Colors.BLUE, channel, Colors.NORMAL);
                        }

                        return;

                    case "sub":
                        if (!Application.ImportantSubs.ContainsKey(id))
                        {
                            command.Reply("Package {0}{1}{2} ({3}) is not important.", Colors.BLUE, id, Colors.NORMAL, Steam.GetPackageName(id));
                        }
                        else
                        {
                            Application.ImportantSubs.Remove(id);

                            using (var db = Database.Get())
                            {
                                await db.ExecuteAsync("DELETE FROM `ImportantSubs` WHERE `SubID` = @SubID", new { SubID = id });
                            }

                            command.Reply("Removed package {0}{1}{2} ({3}) from the important list.", Colors.BLUE, id, Colors.NORMAL, Steam.GetPackageName(id));
                        }

                        return;
                    }

                    break;
                }
            }

            command.Reply("Usage:{0} important reload {1}or{2} important <add/remove> <app/sub> <id>", Colors.OLIVE, Colors.NORMAL, Colors.OLIVE);
        }
Example #3
0
        private async Task DownloadDepots(uint appID, List <ManifestJob> depots)
        {
            Log.WriteDebug("Depot Downloader", "Will process {0} depots ({1} depot locks left)", depots.Count, DepotLocks.Count);

            var processTasks       = new List <Task <EResult> >();
            var anyFilesDownloaded = false;
            var willDownloadFiles  = false;

            foreach (var depot in depots)
            {
                if (depot.DepotKey == null)
                {
                    await GetDepotDecryptionKey(Steam.Instance.Apps, depot, appID);

                    if (depot.DepotKey == null)
                    {
                        RemoveLock(depot.DepotID);

                        continue;
                    }
                }

                depot.Server = GetContentServer();

                DepotManifest depotManifest = null;
                var           lastError     = string.Empty;

                for (var i = 0; i <= 5; i++)
                {
                    try
                    {
                        depotManifest = await CDNClient.DownloadManifestAsync(depot.DepotID, depot.ManifestID, depot.Server, string.Empty, depot.DepotKey);

                        break;
                    }
                    catch (Exception e)
                    {
                        lastError = e.Message;

                        Log.WriteError("Depot Processor", "Failed to download depot manifest for app {0} depot {1} ({2}: {3}) (#{4})", appID, depot.DepotID, depot.Server, lastError, i);
                    }

                    depot.Server = GetContentServer();

                    if (depotManifest == null)
                    {
                        await Task.Delay(Utils.ExponentionalBackoff(i));
                    }
                }

                if (depotManifest == null)
                {
                    RemoveLock(depot.DepotID);

                    if (FileDownloader.IsImportantDepot(depot.DepotID))
                    {
                        IRC.Instance.SendOps("{0}[{1}]{2} Failed to download manifest ({3})",
                                             Colors.OLIVE, depot.DepotName, Colors.NORMAL, lastError);
                    }

                    if (!Settings.IsFullRun)
                    {
                        JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
                    }

                    continue;
                }

                var task = ProcessDepotAfterDownload(depot, depotManifest);

                processTasks.Add(task);

                if (!FileDownloader.IsImportantDepot(depot.DepotID))
                {
                    continue;
                }

                willDownloadFiles = true;

                task = TaskManager.Run(async() =>
                {
                    var result = EResult.Fail;

                    try
                    {
                        result = await FileDownloader.DownloadFilesFromDepot(depot, depotManifest);

                        if (result == EResult.OK)
                        {
                            anyFilesDownloaded = true;
                        }
                    }
                    catch (Exception e)
                    {
                        ErrorReporter.Notify($"Depot Processor {depot.DepotID}", e);
                    }

                    return(result);
                }).Unwrap();

                processTasks.Add(task);
            }

            if (SaveLocalConfig)
            {
                SaveLocalConfig = false;

                LocalConfig.Save();
            }

            await Task.WhenAll(processTasks).ConfigureAwait(false);

            Log.WriteDebug("Depot Downloader", "{0} depot downloads finished", depots.Count);

            // TODO: use ContinueWith on tasks
            if (!anyFilesDownloaded && !willDownloadFiles)
            {
                foreach (var depot in depots)
                {
                    RemoveLock(depot.DepotID);
                }

                return;
            }

            lock (UpdateScriptLock)
            {
                foreach (var depot in depots)
                {
                    if (depot.Result == EResult.OK)
                    {
                        RunUpdateScript(UpdateScript, string.Format("{0} no-git", depot.DepotID));
                    }
                    else if (depot.Result != EResult.Ignored)
                    {
                        Log.WriteWarn("Depot Processor", $"Download failed for {depot.DepotID}");

                        using (var db = Database.Get())
                        {
                            // Mark this depot for redownload
                            db.Execute("UPDATE `Depots` SET `LastManifestID` = 0 WHERE `DepotID` = @DepotID", new { depot.DepotID });
                        }
                    }

                    RemoveLock(depot.DepotID);
                }

                // Only commit changes if all depots downloaded
                if (processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored))
                {
                    if (!RunUpdateScriptForApp(appID, depots[0].BuildID))
                    {
                        RunUpdateScript(UpdateScript, "0");
                    }
                }
                else
                {
                    Log.WriteDebug("Depot Processor", "Reprocessing the app {0} because some files failed to download", appID);

                    IRC.Instance.SendOps("{0}[{1}]{2} Reprocessing the app due to download failures",
                                         Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL
                                         );

                    JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
                }
            }
        }