Пример #1
0
        private static void HandleClientRequestFreeLicenseResponse(IPacketMsg packetMsg)
        {
            var resp = new ClientMsgProtobuf <CMsgClientRequestFreeLicenseResponse>(packetMsg);

            JobAction job;

            JobManager.TryRemoveJob(packetMsg.TargetJobID, out job);

            var packageIDs = resp.Body.granted_packageids;
            var appIDs     = resp.Body.granted_appids;

            Log.WriteDebug("FreeLicense", "Received free license: {0} ({1} apps, {2} packages)", (EResult)resp.Body.eresult, appIDs.Count, packageIDs.Count);

            if (appIDs.Count > 0)
            {
                JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appIDs, Enumerable.Empty <uint>()));
            }

            if (packageIDs.Count > 0)
            {
                JobManager.AddJob(() => Steam.Instance.Apps.PICSGetProductInfo(Enumerable.Empty <uint>(), packageIDs));

                // TODO: We could re-queue apps in these packages as well

                // We don't want to block our main thread with web requests
                TaskManager.Run(() =>
                {
                    string data = null;

                    try
                    {
                        var response = WebAuth.PerformRequest("GET", "https://store.steampowered.com/account/");

                        using (var responseStream = response.GetResponseStream())
                        {
                            using (var reader = new StreamReader(responseStream))
                            {
                                data = reader.ReadToEnd();
                            }
                        }
                    }
                    catch (WebException e)
                    {
                        Log.WriteError("FreeLicense", "Failed to fetch account details page: {0}", e.Message);
                    }

                    foreach (var package in packageIDs)
                    {
                        Package packageData;

                        using (var db = Database.GetConnection())
                        {
                            packageData = db.Query <Package>("SELECT `SubID`, `Name`, `LastKnownName` FROM `Subs` WHERE `SubID` = @SubID", new { SubID = package }).FirstOrDefault();
                        }

                        if (!string.IsNullOrEmpty(data))
                        {
                            // Tell me all about using regex
                            var match = Regex.Match(data, string.Format("RemoveFreeLicense\\( ?{0}, ?'(.+)' ?\\)", package));

                            if (match.Success)
                            {
                                var grantedName = Encoding.UTF8.GetString(Convert.FromBase64String(match.Groups[1].Value));

                                // Update last known name if we can
                                if (packageData.SubID > 0 && (string.IsNullOrEmpty(packageData.LastKnownName) || packageData.LastKnownName.StartsWith("Steam Sub ", StringComparison.Ordinal)))
                                {
                                    using (var db = Database.GetConnection())
                                    {
                                        db.Execute("UPDATE `Subs` SET `LastKnownName` = @Name WHERE `SubID` = @SubID", new { SubID = package, Name = grantedName });

                                        db.Execute(SubProcessor.GetHistoryQuery(),
                                                   new PICSHistory
                                        {
                                            ID       = package,
                                            Key      = SteamDB.DATABASE_NAME_TYPE,
                                            OldValue = "free on demand; account page",
                                            NewValue = grantedName,
                                            Action   = "created_info"
                                        }
                                                   );

                                        // Add a app comment on each app in this package
                                        var comment = string.Format("This app is in a free on demand package called <b>{0}</b>", System.Security.SecurityElement.Escape(grantedName));
                                        var apps    = db.Query <PackageApp>("SELECT `AppID` FROM `SubsApps` WHERE `SubID` = @SubID", new { SubID = package }).ToList();
                                        var types   = db.Query <App>("SELECT `AppID` FROM `Apps` WHERE `AppType` > 0 AND `AppID` IN @Ids", new { Ids = apps.Select(x => x.AppID) }).ToDictionary(x => x.AppID, x => true);
                                        var key     = db.ExecuteScalar <uint>("SELECT `ID` FROM `KeyNames` WHERE `Name` = 'website_comment'");

                                        foreach (var app in apps)
                                        {
                                            if (types.ContainsKey(app.AppID))
                                            {
                                                continue;
                                            }

                                            db.Execute("INSERT INTO `AppsInfo` VALUES (@AppID, @Key, @Value) ON DUPLICATE KEY UPDATE `Key` = `Key`", new { AppID = app.AppID, Key = key, value = comment });
                                        }
                                    }
                                }

                                packageData.LastKnownName = grantedName;
                            }
                        }

                        IRC.Instance.SendMain("New free license granted: {0}{1}{2} -{3} {4}",
                                              Colors.BLUE, Steam.FormatPackageName(package, packageData), Colors.NORMAL,
                                              Colors.DARKBLUE, SteamDB.GetPackageURL(package)
                                              );
                    }
                });
            }
        }
Пример #2
0
 private void OnLoggedOn(SteamUser.LoggedOnCallback callback)
 {
     TaskManager.Run(AuthenticateUser);
 }
Пример #3
0
 public void StartTick()
 {
     TickerHash++;
     TaskManager.Run(Tick);
 }
Пример #4
0
        private static async void OnClanState(SteamFriends.ClanStateCallback callback)
        {
            if (callback.Events.Count == 0 && callback.Announcements.Count == 0)
            {
                return;
            }

            var groupName = callback.ClanName;

            if (string.IsNullOrEmpty(groupName))
            {
                groupName = Steam.Instance.Friends.GetClanName(callback.ClanID);

                // Check once more, because that can fail too
                if (string.IsNullOrEmpty(groupName))
                {
                    groupName = "Group";
                }
            }

            foreach (var announcement in callback.Announcements)
            {
                var url = $"https://steamcommunity.com/gid/{callback.ClanID.AccountID}/announcements/detail/{announcement.ID}";

                IRC.Instance.SendAnnounce($"{Colors.BLUE}{groupName}{Colors.NORMAL} announcement: {Colors.OLIVE}{announcement.Headline}{Colors.NORMAL} -{Colors.DARKBLUE} {url}");

                _ = TaskManager.Run(async() => await Utils.SendWebhook(new
                {
                    Type  = "GroupAnnouncement",
                    Title = announcement.Headline,
                    Group = groupName,
                    Url   = url,
                }));

                Log.WriteInfo(nameof(ClanState), $"{groupName} \"{announcement.Headline}\"");
            }

            await using var db = await Database.GetConnectionAsync();

            foreach (var groupEvent in callback.Events)
            {
                var link = $"https://steamcommunity.com/gid/{callback.ClanID.AccountID}/events/{groupEvent.ID}";
                var id   = await db.ExecuteScalarAsync <int>("SELECT `ID` FROM `RSS` WHERE `Link` = @Link", new { Link = link });

                if (id > 0)
                {
                    continue;
                }

                IRC.Instance.SendAnnounce(
                    $"{Colors.BLUE}{groupName}{Colors.NORMAL} event: {Colors.OLIVE}{groupEvent.Headline}{Colors.NORMAL} -{Colors.DARKBLUE} {link} {Colors.DARKGRAY}({groupEvent.EventTime.ToString("s", CultureInfo.InvariantCulture).Replace("T", " ")})"
                    );

                Log.WriteInfo(nameof(ClanState), $"{groupName} Event \"{groupEvent.Headline}\" {link}");

                await db.ExecuteAsync("INSERT INTO `RSS` (`Link`, `Title`) VALUES(@Link, @Title)", new { Link = link, Title = groupEvent.Headline });

                _ = TaskManager.Run(async() => await Utils.SendWebhook(new
                {
                    Type  = "GroupAnnouncement",
                    Title = groupEvent.Headline,
                    Group = groupName,
                    Url   = link,
                }));
            }
        }
Пример #5
0
        private static void OnPICSProductInfo(SteamApps.PICSProductInfoCallback callback)
        {
            JobManager.TryRemoveJob(callback.JobID);

            var apps     = callback.Apps.Concat(callback.UnknownApps.ToDictionary(x => x, x => (SteamApps.PICSProductInfoCallback.PICSProductInfo)null));
            var packages = callback.Packages.Concat(callback.UnknownPackages.ToDictionary(x => x, x => (SteamApps.PICSProductInfoCallback.PICSProductInfo)null));

            foreach (var workaround in apps)
            {
                var app = workaround;

                Log.WriteInfo("PICSProductInfo", "{0}AppID: {1}", app.Value == null ? "Unknown " : "", app.Key);

                Task mostRecentItem;

                lock (ProcessedApps)
                {
                    ProcessedApps.TryGetValue(app.Key, out mostRecentItem);
                }

                var workerItem = TaskManager.Run(async delegate
                {
                    if (mostRecentItem != null && !mostRecentItem.IsCompleted)
                    {
                        Log.WriteDebug("PICSProductInfo", "Waiting for app {0} to finish processing", app.Key);

                        await mostRecentItem;
                    }

                    using (var processor = new AppProcessor(app.Key))
                    {
                        if (app.Value == null)
                        {
                            processor.ProcessUnknown();
                        }
                        else
                        {
                            processor.Process(app.Value);
                        }
                    }
                });

                if (Settings.IsFullRun)
                {
                    continue;
                }


                lock (ProcessedApps)
                {
                    ProcessedApps[app.Key] = workerItem;
                }

                workerItem.ContinueWith(task =>
                {
                    lock (ProcessedApps)
                    {
                        if (ProcessedApps.TryGetValue(app.Key, out mostRecentItem) && mostRecentItem.IsCompleted)
                        {
                            ProcessedApps.Remove(app.Key);
                        }
                    }
                });
            }

            foreach (var workaround in packages)
            {
                var package = workaround;

                Log.WriteInfo("PICSProductInfo", "{0}SubID: {1}", package.Value == null ? "Unknown " : "", package.Key);

                Task mostRecentItem;

                lock (ProcessedSubs)
                {
                    ProcessedSubs.TryGetValue(package.Key, out mostRecentItem);
                }

                var workerItem = TaskManager.Run(async delegate
                {
                    if (mostRecentItem != null && !mostRecentItem.IsCompleted)
                    {
                        Log.WriteDebug("PICSProductInfo", "Waiting for package {0} to finish processing", package.Key);

                        await mostRecentItem;
                    }

                    using (var processor = new SubProcessor(package.Key))
                    {
                        if (package.Value == null)
                        {
                            processor.ProcessUnknown();
                        }
                        else
                        {
                            processor.Process(package.Value);
                        }
                    }
                });

                if (Settings.IsFullRun)
                {
                    continue;
                }

                lock (ProcessedSubs)
                {
                    ProcessedSubs[package.Key] = workerItem;
                }

                workerItem.ContinueWith(task =>
                {
                    lock (ProcessedSubs)
                    {
                        if (ProcessedSubs.TryGetValue(package.Key, out mostRecentItem) && mostRecentItem.IsCompleted)
                        {
                            ProcessedSubs.Remove(package.Key);
                        }
                    }
                });
            }
        }
        private async Task DownloadDepots(uint appID, List <ManifestJob> depots)
        {
            Log.WriteDebug(nameof(DepotProcessor), $"Will process {depots.Count} depots ({DepotLocks.Count} depot locks left)");

            var processTasks       = new List <Task <EResult> >();
            var anyFilesDownloaded = false;
            var willDownloadFiles  = false;

            foreach (var depot in depots)
            {
                if (depot.DepotKey == null)
                {
                    await GetDepotDecryptionKey(Steam.Instance.Apps, depot, appID);

                    if (depot.DepotKey == null && (depot.LastManifestID == depot.ManifestID || Settings.Current.OnlyOwnedDepots))
                    {
                        RemoveLock(depot.DepotID);

                        continue;
                    }
                }

                depot.Server = GetContentServer();

                DepotManifest depotManifest = null;
                var           lastError     = string.Empty;

                for (var i = 0; i <= 5; i++)
                {
                    try
                    {
                        await ManifestDownloadSemaphore.WaitAsync(TaskManager.TaskCancellationToken.Token).ConfigureAwait(false);

                        depotManifest = await CDNClient.DownloadManifestAsync(depot.DepotID, depot.ManifestID, depot.Server, string.Empty, depot.DepotKey);

                        break;
                    }
                    catch (Exception e)
                    {
                        lastError = e.Message;

                        Log.WriteError(nameof(DepotProcessor), $"Failed to download depot manifest for app {appID} depot {depot.DepotID} ({depot.Server}: {lastError}) (#{i})");
                    }
                    finally
                    {
                        ManifestDownloadSemaphore.Release();
                    }

                    if (depot.DepotKey != null)
                    {
                        RemoveErroredServer(depot.Server);
                    }

                    depot.Server = GetContentServer();

                    if (depotManifest == null && i < 5)
                    {
                        await Task.Delay(Utils.ExponentionalBackoff(i + 1));
                    }
                }

                if (depotManifest == null)
                {
                    RemoveLock(depot.DepotID);

                    if (FileDownloader.IsImportantDepot(depot.DepotID))
                    {
                        IRC.Instance.SendOps($"{Colors.OLIVE}[{depot.DepotName}]{Colors.NORMAL} Failed to download manifest ({lastError})");
                    }

                    if (!Settings.IsFullRun && depot.DepotKey != null)
                    {
                        JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
                    }

                    continue;
                }

                var task = ProcessDepotAfterDownload(depot, depotManifest);

                processTasks.Add(task);

                if (!FileDownloader.IsImportantDepot(depot.DepotID) || depot.DepotKey == null)
                {
                    depot.Result = EResult.Ignored;
                    continue;
                }

                willDownloadFiles = true;

                task = TaskManager.Run(async() =>
                {
                    var result = EResult.Fail;

                    try
                    {
                        result = await FileDownloader.DownloadFilesFromDepot(depot, depotManifest);

                        if (result == EResult.OK)
                        {
                            anyFilesDownloaded = true;
                        }
                    }
                    catch (Exception e)
                    {
                        ErrorReporter.Notify($"Depot Processor {depot.DepotID}", e);
                    }

                    return(result);
                });

                processTasks.Add(task);
            }

            await Task.WhenAll(processTasks).ConfigureAwait(false);

            Log.WriteDebug(nameof(DepotProcessor), $"{depots.Count} depot downloads finished for app {appID}");

            // TODO: use ContinueWith on tasks
            if (!anyFilesDownloaded && !willDownloadFiles)
            {
                foreach (var depot in depots)
                {
                    RemoveLock(depot.DepotID);
                }

                return;
            }

            lock (UpdateScriptLock)
            {
                foreach (var depot in depots)
                {
                    if (depot.Result == EResult.OK)
                    {
                        RunUpdateScript(UpdateScript, $"{depot.DepotID} no-git");
                    }
                    else if (depot.Result != EResult.Ignored)
                    {
                        Log.WriteWarn(nameof(DepotProcessor), $"Download failed for {depot.DepotID}: {depot.Result}");

                        // Mark this depot for redownload
                        using var db = Database.Get();
                        db.Execute("UPDATE `Depots` SET `LastManifestID` = 0 WHERE `DepotID` = @DepotID", new { depot.DepotID });
                    }

                    RemoveLock(depot.DepotID);
                }

                // Only commit changes if all depots downloaded
                if (processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored))
                {
                    if (!RunUpdateScriptForApp(appID, depots[0].BuildID))
                    {
                        RunUpdateScript(UpdateScript, "0");
                    }
                }
                else
                {
                    Log.WriteDebug(nameof(DepotProcessor), $"Reprocessing the app {appID} because some files failed to download");

                    IRC.Instance.SendOps($"{Colors.OLIVE}[{Steam.GetAppName(appID)}]{Colors.NORMAL} Reprocessing the app due to download failures");

                    JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
                }
            }
        }
Пример #7
0
        private async Task DownloadDepots(uint appID, List <ManifestJob> depots)
        {
            Log.WriteDebug("Depot Downloader", "Will process {0} depots ({1} depot locks left)", depots.Count, DepotLocks.Count);

            var  processTasks       = new List <Task <EResult> >();
            bool anyFilesDownloaded = false;

            foreach (var depot in depots)
            {
                var instance = depot.Anonymous ? Steam.Anonymous.Apps : Steam.Instance.Apps;

                depot.DepotKey = await GetDepotDecryptionKey(instance, depot.DepotID, appID);

                if (depot.DepotKey == null)
                {
                    RemoveLock(depot.DepotID);

                    continue;
                }

                var cdnToken = await GetCDNAuthToken(instance, appID, depot.DepotID);

                if (cdnToken == null)
                {
                    RemoveLock(depot.DepotID);

                    Log.WriteDebug("Depot Downloader", "Got a depot key for depot {0} but no cdn auth token", depot.DepotID);

                    continue;
                }

                depot.CDNToken = cdnToken.Token;
                depot.Server   = GetContentServer();

                DepotManifest depotManifest = null;
                string        lastError     = string.Empty;

                for (var i = 0; i <= 5; i++)
                {
                    try
                    {
                        depotManifest = await CDNClient.DownloadManifestAsync(depot.DepotID, depot.ManifestID, depot.Server, depot.CDNToken, depot.DepotKey);

                        break;
                    }
                    catch (Exception e)
                    {
                        lastError = e.Message;

                        Log.WriteError("Depot Processor", "Failed to download depot manifest for app {0} depot {1} ({2}: {3}) (#{4})", appID, depot.DepotID, depot.Server, lastError, i);
                    }

                    // TODO: get new auth key if auth fails
                    depot.Server = GetContentServer();

                    if (depotManifest == null)
                    {
                        await Task.Delay(Utils.ExponentionalBackoff(i));
                    }
                }

                if (depotManifest == null)
                {
                    LocalConfig.CDNAuthTokens.TryRemove(depot.DepotID, out _);

                    RemoveLock(depot.DepotID);

                    if (FileDownloader.IsImportantDepot(depot.DepotID))
                    {
                        IRC.Instance.SendOps("{0}[{1}]{2} Failed to download manifest ({3})",
                                             Colors.OLIVE, depot.DepotName, Colors.NORMAL, lastError);
                    }

                    if (!Settings.IsFullRun)
                    {
                        JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
                    }

                    continue;
                }

                var task = ProcessDepotAfterDownload(depot, depotManifest);

                processTasks.Add(task);

                if (!FileDownloader.IsImportantDepot(depot.DepotID))
                {
                    continue;
                }

                task = TaskManager.Run(async() =>
                {
                    var result = EResult.Fail;

                    try
                    {
                        result = await FileDownloader.DownloadFilesFromDepot(depot, depotManifest);

                        if (result == EResult.OK)
                        {
                            anyFilesDownloaded = true;
                        }
                    }
                    catch (Exception e)
                    {
                        ErrorReporter.Notify($"Depot Processor {depot.DepotID}", e);
                    }

                    return(result);
                }).Unwrap();

                processTasks.Add(task);
            }

            if (SaveLocalConfig)
            {
                SaveLocalConfig = false;

                LocalConfig.Save();
            }

            await Task.WhenAll(processTasks).ConfigureAwait(false);

            Log.WriteDebug("Depot Downloader", "{0} depot downloads finished", depots.Count);

            // TODO: use ContinueWith on tasks
            if (!anyFilesDownloaded)
            {
                foreach (var depot in depots)
                {
                    RemoveLock(depot.DepotID);
                }

                return;
            }

            if (!File.Exists(UpdateScript))
            {
                return;
            }

            lock (UpdateScriptLock)
            {
                foreach (var depot in depots)
                {
                    if (depot.Result == EResult.OK)
                    {
                        RunUpdateScript(UpdateScript, string.Format("{0} no-git", depot.DepotID));
                    }
                    else if (depot.Result != EResult.Ignored)
                    {
                        Log.WriteWarn("Depot Processor", "Dropping stored token for {0} due to download failures", depot.DepotID);

                        LocalConfig.CDNAuthTokens.TryRemove(depot.DepotID, out _);

                        using (var db = Database.Get())
                        {
                            // Mark this depot for redownload
                            db.Execute("UPDATE `Depots` SET `LastManifestID` = 0 WHERE `DepotID` = @DepotID", new { depot.DepotID });
                        }
                    }

                    RemoveLock(depot.DepotID);
                }

                // Only commit changes if all depots downloaded
                if (processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored))
                {
                    if (!RunUpdateScript(appID, depots[0].BuildID))
                    {
                        RunUpdateScript(UpdateScript, "0");
                    }
                }
                else
                {
                    Log.WriteDebug("Depot Processor", "Reprocessing the app {0} because some files failed to download", appID);

                    IRC.Instance.SendOps("{0}[{1}]{2} Reprocessing the app due to download failures",
                                         Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL
                                         );

                    JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
                }
            }
        }
        private static async Task <EResult> DownloadFile(DepotProcessor.ManifestJob job, DepotManifest.FileData file, byte[] hash)
        {
            var directory    = Path.Combine(Application.Path, "files", DownloadFolders[job.DepotID], Path.GetDirectoryName(file.FileName));
            var finalPath    = new FileInfo(Path.Combine(directory, Path.GetFileName(file.FileName)));
            var downloadPath = new FileInfo(Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".steamdb_tmp")));

            if (!Directory.Exists(directory))
            {
                Directory.CreateDirectory(directory);
            }
            else if (file.TotalSize == 0)
            {
                if (!finalPath.Exists)
                {
                    using (var _ = finalPath.Create())
                    {
                        // FileInfo.Create returns a stream but we don't need it
                    }

                    Log.WriteInfo("FileDownloader", "{0} created an empty file", file.FileName);

                    return(EResult.SameAsPreviousValue);
                }
                else if (finalPath.Length == 0)
                {
#if DEBUG
                    Log.WriteDebug("FileDownloader", "{0} is already empty", file.FileName);
#endif

                    return(EResult.SameAsPreviousValue);
                }
            }
            else if (hash != null && file.FileHash.SequenceEqual(hash))
            {
#if DEBUG
                Log.WriteDebug("FileDownloader", "{0} already matches the file we have", file.FileName);
#endif

                return(EResult.SameAsPreviousValue);
            }

            byte[] checksum;

            using (var sha = SHA1.Create())
            {
                checksum = sha.ComputeHash(Encoding.UTF8.GetBytes(file.FileName));
            }

            var neededChunks  = new List <DepotManifest.ChunkData>();
            var chunks        = file.Chunks.OrderBy(x => x.Offset).ToList();
            var oldChunksFile = Path.Combine(Application.Path, "files", ".support", "chunks", string.Format("{0}-{1}.json", job.DepotID, BitConverter.ToString(checksum)));

            using (var fs = downloadPath.Open(FileMode.OpenOrCreate, FileAccess.ReadWrite))
            {
                fs.SetLength((long)file.TotalSize);

                if (finalPath.Exists && File.Exists(oldChunksFile))
                {
                    var oldChunks = JsonConvert.DeserializeObject <List <DepotManifest.ChunkData> >(File.ReadAllText(oldChunksFile), JsonHandleAllReferences);

                    using (var fsOld = finalPath.Open(FileMode.Open, FileAccess.Read))
                    {
                        foreach (var chunk in chunks)
                        {
                            var oldChunk = oldChunks.Find(c => c.ChunkID.SequenceEqual(chunk.ChunkID));

                            if (oldChunk != null)
                            {
                                var oldData = new byte[oldChunk.UncompressedLength];
                                fsOld.Seek((long)oldChunk.Offset, SeekOrigin.Begin);
                                fsOld.Read(oldData, 0, oldData.Length);

                                var existingChecksum = Utils.AdlerHash(oldData);

                                if (existingChecksum.SequenceEqual(chunk.Checksum))
                                {
                                    fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
                                    fs.Write(oldData, 0, oldData.Length);

#if DEBUG
                                    Log.WriteDebug("FileDownloader", "{0} Found chunk ({1}), not downloading", file.FileName, chunk.Offset);
#endif
                                }
                                else
                                {
                                    neededChunks.Add(chunk);

#if DEBUG
                                    Log.WriteDebug("FileDownloader", "{0} Found chunk ({1}), but checksum differs", file.FileName, chunk.Offset);
#endif
                                }
                            }
                            else
                            {
                                neededChunks.Add(chunk);
                            }
                        }
                    }
                }
                else
                {
                    neededChunks = chunks;
                }
            }

            var downloadedSize    = file.TotalSize - (ulong)neededChunks.Sum(x => x.UncompressedLength);
            var chunkCancellation = new CancellationTokenSource();
            var chunkTasks        = new Task[neededChunks.Count];

            Log.WriteInfo("FileDownloader", "Downloading {0} ({1} bytes, {2} out of {3} chunks)", file.FileName, downloadedSize, neededChunks.Count, chunks.Count);

            for (var i = 0; i < chunkTasks.Length; i++)
            {
                var chunk = neededChunks[i];
                chunkTasks[i] = TaskManager.Run(async() =>
                {
                    try
                    {
                        chunkCancellation.Token.ThrowIfCancellationRequested();

                        await ChunkDownloadingSemaphore.WaitAsync(chunkCancellation.Token).ConfigureAwait(false);

                        var result = await DownloadChunk(job, chunk, downloadPath);

                        if (!result)
                        {
                            Log.WriteWarn("FileDownloader", "Failed to download chunk for {0}", file.FileName);

                            chunkCancellation.Cancel();
                        }
                        else
                        {
                            downloadedSize += chunk.UncompressedLength;

                            // Do not write progress info to log file
                            Console.WriteLine("{2} [{0,6:#00.00}%] {1}", downloadedSize / (float)file.TotalSize * 100.0f, file.FileName, job.DepotName);
                        }
                    }
                    finally
                    {
                        ChunkDownloadingSemaphore.Release();
                    }
                }).Unwrap();

                // Register error handler on inner task
                TaskManager.RegisterErrorHandler(chunkTasks[i]);
            }

            await Task.WhenAll(chunkTasks).ConfigureAwait(false);

            using (var fs = downloadPath.Open(FileMode.Open, FileAccess.ReadWrite))
            {
                fs.Seek(0, SeekOrigin.Begin);

                using (var sha = SHA1.Create())
                {
                    checksum = sha.ComputeHash(fs);
                }
            }

            if (!file.FileHash.SequenceEqual(checksum))
            {
                IRC.Instance.SendOps("{0}[{1}]{2} Failed to correctly download {3}{4}",
                                     Colors.OLIVE, job.DepotName, Colors.NORMAL, Colors.BLUE, file.FileName);

                Log.WriteWarn("FileDownloader", "Failed to download file {0} ({1})", file.FileName, job.Server);

                downloadPath.Delete();

                return(EResult.DataCorruption);
            }

            Log.WriteInfo("FileDownloader", "Downloaded {0} from {1}", file.FileName, job.DepotName);

            finalPath.Delete();

            downloadPath.MoveTo(finalPath.FullName);

            if (chunks.Count > 1)
            {
                File.WriteAllText(oldChunksFile, JsonConvert.SerializeObject(chunks, Formatting.None, JsonHandleAllReferences));
            }
            else if (File.Exists(oldChunksFile))
            {
                File.Delete(oldChunksFile);
            }

            return(EResult.OK);
        }
        private async Task DownloadDepots(uint appID, List <ManifestJob> depots)
        {
            Log.WriteDebug("Depot Downloader", "Will process {0} depots ({1} depot locks left)", depots.Count(), DepotLocks.Count);

            var  processTasks       = new List <Task <EResult> >();
            bool anyFilesDownloaded = false;

            foreach (var depot in depots)
            {
                var instance = depot.Anonymous ? Steam.Anonymous.Apps : Steam.Instance.Apps;

                depot.DepotKey = await GetDepotDecryptionKey(instance, depot.DepotID, appID);

                if (depot.DepotKey == null)
                {
                    RemoveLock(depot.DepotID);

                    continue;
                }

                var cdnToken = await GetCDNAuthToken(instance, appID, depot.DepotID);

                if (cdnToken == null)
                {
                    RemoveLock(depot.DepotID);

                    Log.WriteDebug("Depot Downloader", "Got a depot key for depot {0} but no cdn auth token", depot.DepotID);

                    continue;
                }

                depot.CDNToken = cdnToken.Token;
                depot.Server   = cdnToken.Server;

                DepotManifest depotManifest = null;
                string        lastError     = string.Empty;

                for (var i = 0; i <= 5; i++)
                {
                    try
                    {
                        depotManifest = CDNClient.DownloadManifest(depot.DepotID, depot.ManifestID, depot.Server, depot.CDNToken, depot.DepotKey);

                        break;
                    }
                    catch (Exception e)
                    {
                        Log.WriteWarn("Depot Downloader", "[{0}] Manifest download failed: {1} - {2}", depot.DepotID, e.GetType(), e.Message);

                        lastError = e.Message;
                    }

                    // TODO: get new auth key if auth fails

                    if (depotManifest == null)
                    {
                        await Task.Delay(Utils.ExponentionalBackoff(i));
                    }
                }

                if (depotManifest == null)
                {
                    RemoveLock(depot.DepotID);

                    Log.WriteError("Depot Processor", "Failed to download depot manifest for app {0} depot {1} ({2}: {3})", appID, depot.DepotID, depot.Server, lastError);

                    if (FileDownloader.IsImportantDepot(depot.DepotID))
                    {
                        IRC.Instance.SendOps("{0}[{1}]{2} Failed to download app {3} depot {4} manifest ({5}: {6})",
                                             Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL, appID, depot.DepotID, depot.Server, lastError);
                    }

                    continue;
                }

                var task = TaskManager.Run(() =>
                {
                    using (var db = Database.GetConnection())
                    {
                        using (var transaction = db.BeginTransaction())
                        {
                            var result = ProcessDepotAfterDownload(db, depot, depotManifest);
                            transaction.Commit();
                            return(result);
                        }
                    }
                });

                processTasks.Add(task);

                if (FileDownloader.IsImportantDepot(depot.DepotID))
                {
                    task = TaskManager.Run(() =>
                    {
                        var result = FileDownloader.DownloadFilesFromDepot(appID, depot, depotManifest);

                        if (result == EResult.OK)
                        {
                            anyFilesDownloaded = true;
                        }

                        return(result);
                    }, TaskCreationOptions.LongRunning);

                    TaskManager.RegisterErrorHandler(task);

                    processTasks.Add(task);
                }
            }

            if (SaveLocalConfig)
            {
                SaveLocalConfig = false;

                LocalConfig.Save();
            }

            await Task.WhenAll(processTasks);

            Log.WriteDebug("Depot Downloader", "{0} depot downloads finished", depots.Count());

            // TODO: use ContinueWith on tasks
            if (!anyFilesDownloaded)
            {
                foreach (var depot in depots)
                {
                    RemoveLock(depot.DepotID);
                }

                return;
            }

            if (!File.Exists(UpdateScript))
            {
                return;
            }

            bool lockTaken = false;

            try
            {
                UpdateScriptLock.Enter(ref lockTaken);

                foreach (var depot in depots)
                {
                    if (depot.Result == EResult.OK)
                    {
                        RunUpdateScript(string.Format("{0} no-git", depot.DepotID));
                    }

                    RemoveLock(depot.DepotID);
                }

                // Only commit changes if all depots downloaded
                if (processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored))
                {
                    if (!RunUpdateScript(appID))
                    {
                        RunUpdateScript("0");
                    }
                }
                else
                {
                    Log.WriteDebug("Depot Processor", "Reprocessing the app {0} because some files failed to download", appID);

                    IRC.Instance.SendOps("{0}[{1}]{2} Reprocessing the app due to download failures",
                                         Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL
                                         );

                    JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
                }
            }
            finally
            {
                if (lockTaken)
                {
                    UpdateScriptLock.Exit();
                }
            }
        }
Пример #10
0
        private void OnPICSChanges(SteamApps.PICSChangesCallback callback)
        {
            if (PreviousChangeNumber == callback.CurrentChangeNumber)
            {
                return;
            }

            Log.WriteInfo(nameof(PICSChanges), $"Changelist {callback.LastChangeNumber} -> {callback.CurrentChangeNumber} ({callback.AppChanges.Count} apps, {callback.PackageChanges.Count} packages)");

            PreviousChangeNumber = callback.CurrentChangeNumber;

            TaskManager.Run(async() => await HandleChangeNumbers(callback));

            if (callback.RequiresFullAppUpdate || callback.RequiresFullPackageUpdate)
            {
                TaskManager.Run(async() =>
                {
                    if (callback.RequiresFullAppUpdate)
                    {
                        IRC.Instance.SendOps($"Changelist {callback.CurrentChangeNumber} has forced a full app update");

                        // When full update flag is set, presumably Steam client start hammering the servers
                        // and the PICS service just does not return any data for a while until it clears up
                        await FullUpdateProcessor.FullUpdateAppsMetadata(true);
                    }

                    if (callback.RequiresFullPackageUpdate)
                    {
                        IRC.Instance.SendOps($"Changelist {callback.CurrentChangeNumber} has forced a full package update");

                        await FullUpdateProcessor.FullUpdatePackagesMetadata();
                    }

                    IRC.Instance.SendOps($"Changelist {callback.CurrentChangeNumber} full update has finished");
                });
            }

            if (callback.AppChanges.Count == 0 && callback.PackageChanges.Count == 0)
            {
                return;
            }

            const int appsPerJob = 50;

            if (callback.AppChanges.Count > appsPerJob)
            {
                foreach (var list in callback.AppChanges.Keys.Split(appsPerJob))
                {
                    JobManager.AddJob(
                        () => Steam.Instance.Apps.PICSGetAccessTokens(list, Enumerable.Empty <uint>()),
                        new PICSTokens.RequestedTokens
                    {
                        Apps = list.ToList()
                    });
                }
            }
            else if (callback.AppChanges.Count > 0)
            {
                JobManager.AddJob(
                    () => Steam.Instance.Apps.PICSGetAccessTokens(callback.AppChanges.Keys, Enumerable.Empty <uint>()),
                    new PICSTokens.RequestedTokens
                {
                    Apps = callback.AppChanges.Keys.ToList()
                });
            }

            if (callback.PackageChanges.Count > appsPerJob)
            {
                foreach (var list in callback.PackageChanges.Keys.Split(appsPerJob))
                {
                    JobManager.AddJob(
                        () => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), list),
                        new PICSTokens.RequestedTokens
                    {
                        Packages = list.ToList()
                    });
                }
            }
            else if (callback.PackageChanges.Count > 0)
            {
                JobManager.AddJob(
                    () => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), callback.PackageChanges.Keys),
                    new PICSTokens.RequestedTokens
                {
                    Packages = callback.PackageChanges.Keys.ToList()
                });
            }

            if (callback.AppChanges.Count > 0)
            {
                _ = TaskManager.Run(async() => await HandleApps(callback));
            }

            if (callback.PackageChanges.Count > 0)
            {
                _ = TaskManager.Run(async() => await HandlePackages(callback));
                _ = TaskManager.Run(async() => await HandlePackagesChangelists(callback));
            }

            if (PreviousChangeNumber - LastStoredChangeNumber >= 1000)
            {
                LastStoredChangeNumber = PreviousChangeNumber;

                _ = TaskManager.Run(async() => await LocalConfig.Update("backend.changenumber", LastStoredChangeNumber.ToString()));
            }

            PrintImportants(callback);
        }
Пример #11
0
        private void OnTimer(object sender, ElapsedEventArgs e)
        {
            if (!Steam.Instance.IsLoggedOn)
            {
                lock (FreeLicenseTimer)
                {
                    FreeLicenseTimer.Start();
                }

                return;
            }

            if (FreeLicensesToRequest.IsEmpty)
            {
                TaskManager.Run(RequestBetas);
                return;
            }

            var list = FreeLicensesToRequest.Take(REQUEST_RATE_LIMIT).ToList();
            var now  = DateUtils.DateTimeToUnixTime(DateTime.UtcNow) - 60;
            Dictionary <uint, ulong> startTimes;

            using (var db = Database.Get())
            {
                startTimes = db.Query(
                    "SELECT `SubID`, `Value` FROM `SubsInfo` WHERE `Key` = @Key AND `SubID` IN @Ids",
                    new
                {
                    Key = KeyNameCache.GetSubKeyID("extended_starttime"),
                    Ids = list.Select(x => x.Key)
                }
                    ).ToDictionary(x => (uint)x.SubID, x => Convert.ToUInt64((string)x.Value));
            }

            foreach (var(subId, _) in list)
            {
                if (startTimes.TryGetValue(subId, out var startTime) && startTime > now)
                {
                    // If start time has not been reached yet, don't remove this app from the list and keep trying to activate it
                    continue;
                }

                FreeLicensesToRequest.TryRemove(subId, out _);
            }

            TaskManager.Run(Save);
            TaskManager.Run(RequestBetas);

            var appids = list.Select(x => x.Value).Distinct();

            AppsRequestedInHour = appids.Count();

            Log.WriteDebug(nameof(FreeLicense), $"Requesting {AppsRequestedInHour} free apps as the rate limit timer ran: {string.Join(", ", appids)}");

            JobManager.AddJob(() => Steam.Instance.Apps.RequestFreeLicense(appids));

            if (!FreeLicensesToRequest.IsEmpty)
            {
                lock (FreeLicenseTimer)
                {
                    FreeLicenseTimer.Start();
                }
            }
        }
Пример #12
0
        public override async Task OnCommand(CommandArguments command)
        {
            var s     = command.Message.Split(' ');
            var count = s.Length;

            if (count > 0)
            {
                uint id;
                switch (s[0])
                {
                case "reload":
                    await Application.ReloadImportant();

                    await PICSTokens.Reload();

                    command.Notice("Reloaded important apps and pics tokens");

                    return;

                case "fullrun":
                    _ = TaskManager.Run(async() =>
                    {
                        command.Reply("Started full metadata scan, this will take a while…");
                        await FullUpdateProcessor.FullUpdateAppsMetadata(true);
                        command.Reply("App full scan finished, starting packages, this will take even longer…");
                        await FullUpdateProcessor.FullUpdatePackagesMetadata();
                        command.Reply("Full metadata scan finished.");
                    });

                    return;

                case "add":
                    if (count < 3)
                    {
                        break;
                    }

                    if (!uint.TryParse(s[2], out id))
                    {
                        break;
                    }

                    switch (s[1])
                    {
                    case "app":
                        if (Application.ImportantApps.Contains(id))
                        {
                            command.Reply($"App {Colors.BLUE}{id}{Colors.NORMAL} ({Steam.GetAppName(id)}) is already important.");
                        }
                        else
                        {
                            Application.ImportantApps.Add(id);

                            await using (var db = await Database.GetConnectionAsync())
                            {
                                await db.ExecuteAsync("INSERT INTO `ImportantApps` (`AppID`) VALUES (@AppID)", new { AppID = id });
                            }

                            command.Reply($"Marked app {Colors.BLUE}{id}{Colors.NORMAL} ({Steam.GetAppName(id)}) as important.");
                        }

                        return;

                    case "sub":
                        if (Application.ImportantSubs.Contains(id))
                        {
                            command.Reply($"Package {Colors.BLUE}{id}{Colors.NORMAL} ({Steam.GetPackageName(id)}) is already important.");
                        }
                        else
                        {
                            Application.ImportantSubs.Add(id);

                            await using (var db = await Database.GetConnectionAsync())
                            {
                                await db.ExecuteAsync("INSERT INTO `ImportantSubs` (`SubID`) VALUES (@SubID)", new { SubID = id });
                            }

                            command.Reply($"Marked package {Colors.BLUE}{id}{Colors.NORMAL} ({Steam.GetPackageName(id)}) as important.");
                        }

                        return;
                    }

                    break;

                case "remove":
                    if (count < 3)
                    {
                        break;
                    }

                    if (!uint.TryParse(s[2], out id))
                    {
                        break;
                    }

                    switch (s[1])
                    {
                    case "app":
                        if (!Application.ImportantApps.Contains(id))
                        {
                            command.Reply($"App {Colors.BLUE}{id}{Colors.NORMAL} ({Steam.GetAppName(id)}) is not important.");
                        }
                        else
                        {
                            Application.ImportantApps.Remove(id);

                            await using (var db = await Database.GetConnectionAsync())
                            {
                                await db.ExecuteAsync("DELETE FROM `ImportantApps` WHERE `AppID` = @AppID", new { AppID = id });
                            }

                            command.Reply($"Removed app {Colors.BLUE}{id}{Colors.NORMAL} ({Steam.GetAppName(id)}) from the important list.");
                        }

                        return;

                    case "sub":
                        if (!Application.ImportantSubs.Contains(id))
                        {
                            command.Reply($"Package {Colors.BLUE}{id}{Colors.NORMAL} ({Steam.GetPackageName(id)}) is not important.");
                        }
                        else
                        {
                            Application.ImportantSubs.Remove(id);

                            await using (var db = await Database.GetConnectionAsync())
                            {
                                await db.ExecuteAsync("DELETE FROM `ImportantSubs` WHERE `SubID` = @SubID", new { SubID = id });
                            }

                            command.Reply($"Removed package {Colors.BLUE}{id}{Colors.NORMAL} ({Steam.GetPackageName(id)}) from the important list.");
                        }

                        return;
                    }

                    break;
                }
            }

            command.Reply($"Usage:{Colors.OLIVE} important reload {Colors.NORMAL}or{Colors.OLIVE} important <add/remove> <app/sub> <id> {Colors.NORMAL}or{Colors.OLIVE} important fullrun");
        }
Пример #13
0
        /*
         * Here be dragons.
         */
        public static async Task <EResult> DownloadFilesFromDepot(DepotProcessor.ManifestJob job, DepotManifest depotManifest)
        {
            var filesRegex    = Files[job.DepotID];
            var files         = depotManifest.Files.Where(x => filesRegex.IsMatch(x.FileName.Replace('\\', '/'))).ToList();
            var downloadState = EResult.Fail;

            ConcurrentDictionary <string, ExistingFileData> existingFileData;

            await using (var db = await Database.GetConnectionAsync())
            {
                var data = db.ExecuteScalar <string>("SELECT `Value` FROM `LocalConfig` WHERE `ConfigKey` = @Key", new { Key = $"depot.{job.DepotID}" });

                if (data != null)
                {
                    existingFileData = JsonConvert.DeserializeObject <ConcurrentDictionary <string, ExistingFileData> >(data);
                }
                else
                {
                    existingFileData = new ConcurrentDictionary <string, ExistingFileData>();
                }
            }

            foreach (var file in existingFileData.Keys.Except(files.Select(x => x.FileName)))
            {
                Log.WriteWarn(nameof(FileDownloader), $"\"{file}\" no longer exists in manifest");
            }

            Log.WriteInfo($"FileDownloader {job.DepotID}", $"Will download {files.Count} files");

            var downloadedFiles = 0;
            var fileTasks       = new Task[files.Count];

            for (var i = 0; i < fileTasks.Length; i++)
            {
                var file = files[i];
                fileTasks[i] = TaskManager.Run(async() =>
                {
                    var existingFile = existingFileData.GetOrAdd(file.FileName, _ => new ExistingFileData());
                    EResult fileState;

                    try
                    {
                        await ChunkDownloadingSemaphore.WaitAsync().ConfigureAwait(false);

                        fileState = await DownloadFile(job, file, existingFile);
                    }
                    finally
                    {
                        ChunkDownloadingSemaphore.Release();
                    }

                    if (fileState == EResult.OK || fileState == EResult.SameAsPreviousValue)
                    {
                        existingFile.FileHash = file.FileHash;

                        downloadedFiles++;
                    }

                    if (fileState != EResult.SameAsPreviousValue)
                    {
                        // Do not write progress info to log file
                        Console.WriteLine($"{job.DepotName} [{downloadedFiles / (float) files.Count * 100.0f,6:#00.00}%] {files.Count - downloadedFiles} files left to download");
                    }

                    if (downloadState == EResult.DataCorruption)
                    {
                        return;
                    }

                    if (fileState == EResult.OK || fileState == EResult.DataCorruption)
                    {
                        downloadState = fileState;
                    }
                });
            }

            await Task.WhenAll(fileTasks).ConfigureAwait(false);

            await LocalConfig.Update($"depot.{job.DepotID}", JsonConvert.SerializeObject(existingFileData));

            job.Result = downloadState switch
            {
                EResult.OK => EResult.OK,
                EResult.DataCorruption => EResult.DataCorruption,
                _ => EResult.Ignored
            };

            return(job.Result);
        }
Пример #14
0
        private static async Task <EResult> DownloadFile(DepotProcessor.ManifestJob job, DepotManifest.FileData file, ExistingFileData existingFile)
        {
            var directory    = Path.Combine(Application.Path, "files", DownloadFolders[job.DepotID], Path.GetDirectoryName(file.FileName));
            var finalPath    = new FileInfo(Path.Combine(directory, Path.GetFileName(file.FileName)));
            var downloadPath = new FileInfo(Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".steamdb_tmp")));

            if (!Directory.Exists(directory))
            {
                Directory.CreateDirectory(directory);
            }
            else if (file.TotalSize == 0)
            {
                if (!finalPath.Exists)
                {
                    await using (var _ = finalPath.Create())
                    {
                        // FileInfo.Create returns a stream but we don't need it
                    }

                    Log.WriteInfo($"FileDownloader {job.DepotID}", $"{file.FileName} created an empty file");

                    return(EResult.SameAsPreviousValue);
                }
                else if (finalPath.Length == 0)
                {
#if DEBUG
                    Log.WriteDebug($"FileDownloader {job.DepotID}", $"{file.FileName} is already empty");
#endif

                    return(EResult.SameAsPreviousValue);
                }
            }
            else if (existingFile.FileHash != null && file.FileHash.SequenceEqual(existingFile.FileHash))
            {
#if DEBUG
                Log.WriteDebug($"FileDownloader {job.DepotID}", $"{file.FileName} already matches the file we have");
#endif

                return(EResult.SameAsPreviousValue);
            }

            using var sha = SHA1.Create();

            var neededChunks = new List <DepotManifest.ChunkData>();
            var chunks       = file.Chunks.OrderBy(x => x.Offset).ToList();

            await using (var fs = downloadPath.Open(FileMode.OpenOrCreate, FileAccess.ReadWrite))
            {
                fs.SetLength((long)file.TotalSize);

                if (finalPath.Exists)
                {
                    await using var fsOld = finalPath.Open(FileMode.Open, FileAccess.Read);

                    foreach (var chunk in chunks)
                    {
                        var oldChunk = existingFile.Chunks.FirstOrDefault(c => c.Value.SequenceEqual(chunk.ChunkID));

                        if (oldChunk.Value != null)
                        {
                            var oldData = new byte[chunk.UncompressedLength];
                            fsOld.Seek((long)oldChunk.Key, SeekOrigin.Begin);
                            fsOld.Read(oldData, 0, oldData.Length);

                            var existingChecksum = sha.ComputeHash(oldData);

                            if (existingChecksum.SequenceEqual(chunk.ChunkID))
                            {
                                fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
                                fs.Write(oldData, 0, oldData.Length);

#if DEBUG
                                Log.WriteDebug($"FileDownloader {job.DepotID}", $"{file.FileName} Found chunk ({chunk.Offset}), not downloading");
#endif
                            }
                            else
                            {
                                neededChunks.Add(chunk);

#if DEBUG
                                Log.WriteDebug($"FileDownloader {job.DepotID}", $"{file.FileName} Found chunk ({chunk.Offset}), but checksum differs");
#endif
                            }
                        }
                        else
                        {
                            neededChunks.Add(chunk);
                        }
                    }
                }
                else
                {
                    neededChunks = chunks;
                }
            }

            using var chunkCancellation = new CancellationTokenSource();
            var downloadedSize = file.TotalSize - (ulong)neededChunks.Sum(x => x.UncompressedLength);
            var chunkTasks     = new Task[neededChunks.Count];

            Log.WriteInfo($"FileDownloader {job.DepotID}", $"Downloading {file.FileName} ({neededChunks.Count} out of {chunks.Count} chunks to download)");

            for (var i = 0; i < chunkTasks.Length; i++)
            {
                var chunk = neededChunks[i];
                chunkTasks[i] = TaskManager.Run(async() =>
                {
                    var result = await DownloadChunk(job, chunk, downloadPath, chunkCancellation);

                    if (!result)
                    {
                        Log.WriteWarn($"FileDownloader {job.DepotID}", $"Failed to download chunk for {file.FileName} ({chunk.Offset})");

                        chunkCancellation.Cancel();
                    }
                    else
                    {
                        downloadedSize += chunk.UncompressedLength;

                        // Do not write progress info to log file
                        Console.WriteLine($"{job.DepotName} [{downloadedSize / (float) file.TotalSize * 100.0f,6:#00.00}%] {file.FileName}");
                    }
                });
            }

            await Task.WhenAll(chunkTasks).ConfigureAwait(false);

            byte[] checksum;

            await using (var fs = downloadPath.Open(FileMode.Open, FileAccess.ReadWrite))
            {
                checksum = sha.ComputeHash(fs);
            }

            if (!file.FileHash.SequenceEqual(checksum))
            {
                if (!job.DownloadCorrupted)
                {
                    job.DownloadCorrupted = true;

                    IRC.Instance.SendOps($"{Colors.OLIVE}[{job.DepotName}]{Colors.NORMAL} Failed to correctly download {Colors.BLUE}{file.FileName}");
                }

                Log.WriteWarn($"FileDownloader {job.DepotID}", $"Hash check failed for {file.FileName} ({job.Server})");

                downloadPath.Delete();
                existingFile.FileHash = null;
                existingFile.Chunks.Clear();

                return(EResult.DataCorruption);
            }

            Log.WriteInfo($"FileDownloader {job.DepotID}", $"Downloaded {file.FileName}");

            finalPath.Delete();

            downloadPath.MoveTo(finalPath.FullName);

            if (chunks.Count > 0)
            {
                existingFile.Chunks = chunks.ToDictionary(chunk => chunk.Offset, chunk => chunk.ChunkID);
            }
            else
            {
                existingFile.Chunks.Clear();
            }

            return(EResult.OK);
        }
 public void StartTick()
 {
     TaskManager.Run(Tick);
 }
Пример #16
0
 private void OnLoggedOn(SteamUser.LoggedOnCallback callback)
 {
     TaskManager.Run(UpdateContentServerList);
 }
        public async Task Process(uint appID, uint changeNumber, KeyValue depots)
        {
            var requests = new List <ManifestJob>();

            // Get data in format we want first
            foreach (var depot in depots.Children)
            {
                // Ignore these for now, parent app should be updated too anyway
                if (depot["depotfromapp"].Value != null)
                {
                    continue;
                }

                var request = new ManifestJob
                {
                    ChangeNumber = changeNumber,
                };

                // Ignore keys that aren't integers, for example "branches"
                if (!uint.TryParse(depot.Name, out request.DepotID))
                {
                    continue;
                }

                request.DepotName = depot["name"].AsString();

                if (string.IsNullOrEmpty(request.DepotName))
                {
                    request.DepotName = $"SteamDB Unnamed Depot {request.DepotID}";
                }

                // TODO: instead of locking we could wait for current process to finish
                if (DepotLocks.ContainsKey(request.DepotID))
                {
                    continue;
                }

                // SteamVR trickery
                if (appID == 250820 &&
                    depot["manifests"]["beta"].Value != null &&
                    depots["branches"]["beta"]["buildid"].AsInteger() > depots["branches"]["public"]["buildid"].AsInteger())
                {
                    request.BuildID    = depots["branches"]["beta"]["buildid"].AsInteger();
                    request.ManifestID = ulong.Parse(depot["manifests"]["beta"].Value);
                }
                else
                if (depot["manifests"]["public"].Value == null || !ulong.TryParse(depot["manifests"]["public"].Value, out request.ManifestID))
                {
                    var branch = depot["manifests"].Children.Find(x => x.Name != "local");

                    if (branch == null || !ulong.TryParse(branch.Value, out request.ManifestID))
                    {
                        await using var db = await Database.GetConnectionAsync();

                        await db.ExecuteAsync("INSERT INTO `Depots` (`DepotID`, `Name`) VALUES (@DepotID, @DepotName) ON DUPLICATE KEY UPDATE `DepotID` = VALUES(`DepotID`)", new { request.DepotID, request.DepotName });

                        continue;
                    }

                    Log.WriteDebug(nameof(DepotProcessor), $"Depot {request.DepotID} (from {appID}) has no public branch, but there is another one");

                    request.BuildID = depots["branches"][branch.Name]["buildid"].AsInteger();
                }
                else
                {
                    request.BuildID = depots["branches"]["public"]["buildid"].AsInteger();
                }

                requests.Add(request);
            }

            if (requests.Count == 0)
            {
                return;
            }

            var depotsToDownload = new List <ManifestJob>();

            await using (var db = await Database.GetConnectionAsync())
            {
                await db.ExecuteAsync("INSERT INTO `Builds` (`BuildID`, `ChangeID`, `AppID`) VALUES (@BuildID, @ChangeNumber, @AppID) ON DUPLICATE KEY UPDATE `AppID` = VALUES(`AppID`)",
                                      new
                {
                    requests[0].BuildID,
                    requests[0].ChangeNumber,
                    appID
                });

                var depotIds = requests.Select(x => x.DepotID).ToList();
                var dbDepots = (await db.QueryAsync <Depot>("SELECT `DepotID`, `Name`, `BuildID`, `ManifestID`, `LastManifestID`, `FilenamesEncrypted` FROM `Depots` WHERE `DepotID` IN @depotIds", new { depotIds }))
                               .ToDictionary(x => x.DepotID, x => x);

                var decryptionKeys = (await db.QueryAsync <DepotKey>("SELECT `DepotID`, `Key` FROM `DepotsKeys` WHERE `DepotID` IN @depotIds", new { depotIds }))
                                     .ToDictionary(x => x.DepotID, x => Utils.StringToByteArray(x.Key));

                foreach (var request in requests)
                {
                    Depot dbDepot;

                    decryptionKeys.TryGetValue(request.DepotID, out request.DepotKey);

                    if (dbDepots.ContainsKey(request.DepotID))
                    {
                        dbDepot = dbDepots[request.DepotID];

                        if (dbDepot.BuildID > request.BuildID)
                        {
                            // buildid went back in time? this either means a rollback, or a shared depot that isn't synced properly

                            Log.WriteDebug(nameof(DepotProcessor), $"Skipping depot {request.DepotID} due to old buildid: {dbDepot.BuildID} > {request.BuildID}");

                            continue;
                        }

                        if (dbDepot.LastManifestID == request.ManifestID &&
                            dbDepot.ManifestID == request.ManifestID &&
                            Settings.Current.FullRun != FullRunState.WithForcedDepots &&
                            !dbDepot.FilenamesEncrypted && request.DepotKey != null)
                        {
                            // Update depot name if changed
                            if (request.DepotName != dbDepot.Name)
                            {
                                await db.ExecuteAsync("UPDATE `Depots` SET `Name` = @DepotName WHERE `DepotID` = @DepotID", new { request.DepotID, request.DepotName });
                            }

                            continue;
                        }

                        request.StoredFilenamesEncrypted = dbDepot.FilenamesEncrypted;
                        request.LastManifestID           = dbDepot.LastManifestID;
                    }
                    else
                    {
                        dbDepot = new Depot();
                    }

                    if (dbDepot.BuildID != request.BuildID || dbDepot.ManifestID != request.ManifestID || request.DepotName != dbDepot.Name)
                    {
                        await db.ExecuteAsync(@"INSERT INTO `Depots` (`DepotID`, `Name`, `BuildID`, `ManifestID`) VALUES (@DepotID, @DepotName, @BuildID, @ManifestID)
                                    ON DUPLICATE KEY UPDATE `LastUpdated` = CURRENT_TIMESTAMP(), `Name` = VALUES(`Name`), `BuildID` = VALUES(`BuildID`), `ManifestID` = VALUES(`ManifestID`)",
                                              new
                        {
                            request.DepotID,
                            request.DepotName,
                            request.BuildID,
                            request.ManifestID
                        });
                    }

                    if (dbDepot.ManifestID != request.ManifestID)
                    {
                        await MakeHistory(db, null, request, string.Empty, "manifest_change", dbDepot.ManifestID, request.ManifestID);
                    }

                    lock (DepotLocks)
                    {
                        // This doesn't really save us from concurrency issues
                        if (DepotLocks.ContainsKey(request.DepotID))
                        {
                            Log.WriteWarn(nameof(DepotProcessor), $"Depot {request.DepotID} was locked in another thread");
                            continue;
                        }

                        DepotLocks.Add(request.DepotID, 1);
                    }

                    depotsToDownload.Add(request);
                }
            }

            if (depotsToDownload.Count > 0)
            {
                _ = TaskManager.Run(async() =>
                {
                    try
                    {
                        await DownloadDepots(appID, depotsToDownload);
                    }
                    catch (Exception e)
                    {
                        ErrorReporter.Notify(nameof(DepotProcessor), e);
                    }

                    foreach (var depot in depotsToDownload)
                    {
                        RemoveLock(depot.DepotID);
                    }
                });
            }
        }
Пример #18
0
        private static void OnPICSProductInfo(SteamApps.PICSProductInfoCallback callback)
        {
            JobManager.TryRemoveJob(callback.JobID);

            var apps     = callback.Apps.Concat(callback.UnknownApps.ToDictionary(x => x, x => (SteamApps.PICSProductInfoCallback.PICSProductInfo)null));
            var packages = callback.Packages.Concat(callback.UnknownPackages.ToDictionary(x => x, x => (SteamApps.PICSProductInfoCallback.PICSProductInfo)null));

            foreach (var workaround in apps)
            {
                var app = workaround;

                Log.WriteInfo("PICSProductInfo", "{0}AppID: {1}", app.Value == null ? "Unknown " : "", app.Key);

                Task mostRecentItem;

                lock (ProcessedApps)
                {
                    ProcessedApps.TryGetValue(app.Key, out mostRecentItem);
                }

                var workerItem = TaskManager.Run(async() =>
                {
                    try
                    {
                        await ProcessorSemaphore.WaitAsync().ConfigureAwait(false);

                        if (mostRecentItem != null && !mostRecentItem.IsCompleted)
                        {
                            Log.WriteDebug("PICSProductInfo", "Waiting for app {0} to finish processing", app.Key);

                            await mostRecentItem.ConfigureAwait(false);
                        }

                        using (var processor = new AppProcessor(app.Key))
                        {
                            if (app.Value == null)
                            {
                                processor.ProcessUnknown();
                            }
                            else
                            {
                                processor.Process(app.Value);
                            }
                        }
                    }
                    catch (MySqlException e)
                    {
                        ErrorReporter.Notify($"App {app.Key}", e);

                        JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(app.Key, null));
                    }
                    catch (Exception e)
                    {
                        ErrorReporter.Notify($"App {app.Key}", e);
                    }
                    finally
                    {
                        lock (ProcessedApps)
                        {
                            if (ProcessedApps.TryGetValue(app.Key, out mostRecentItem) && mostRecentItem.IsCompleted)
                            {
                                ProcessedApps.Remove(app.Key);
                            }
                        }

                        ProcessorSemaphore.Release();
                    }
                });

                if (Settings.IsFullRun)
                {
                    continue;
                }

                lock (ProcessedApps)
                {
                    ProcessedApps[app.Key] = workerItem;
                }
            }

            foreach (var workaround in packages)
            {
                var package = workaround;

                Log.WriteInfo("PICSProductInfo", "{0}SubID: {1}", package.Value == null ? "Unknown " : "", package.Key);

                Task mostRecentItem;

                lock (ProcessedSubs)
                {
                    ProcessedSubs.TryGetValue(package.Key, out mostRecentItem);
                }

                var workerItem = TaskManager.Run(async() =>
                {
                    try
                    {
                        await ProcessorSemaphore.WaitAsync().ConfigureAwait(false);

                        if (mostRecentItem != null && !mostRecentItem.IsCompleted)
                        {
                            Log.WriteDebug("PICSProductInfo", "Waiting for package {0} to finish processing", package.Key);

                            await mostRecentItem.ConfigureAwait(false);
                        }

                        using (var processor = new SubProcessor(package.Key))
                        {
                            if (package.Value == null)
                            {
                                processor.ProcessUnknown();
                            }
                            else
                            {
                                processor.Process(package.Value);
                            }
                        }
                    }
                    catch (MySqlException e)
                    {
                        ErrorReporter.Notify($"Package {package.Key}", e);

                        JobManager.AddJob(() => Steam.Instance.Apps.PICSGetProductInfo(null, package.Key, false, false));
                    }
                    catch (Exception e)
                    {
                        ErrorReporter.Notify($"Package {package.Key}", e);
                    }
                    finally
                    {
                        lock (ProcessedSubs)
                        {
                            if (ProcessedSubs.TryGetValue(package.Key, out mostRecentItem) && mostRecentItem.IsCompleted)
                            {
                                ProcessedSubs.Remove(package.Key);
                            }
                        }

                        ProcessorSemaphore.Release();
                    }
                });

                if (Settings.IsFullRun)
                {
                    continue;
                }

                lock (ProcessedSubs)
                {
                    ProcessedSubs[package.Key] = workerItem;
                }
            }
        }
Пример #19
0
        private void DownloadManifest(ManifestJob request)
        {
            Log.WriteInfo("Depot Processor", "DepotID: {0}", request.DepotID);

            DepotManifest depotManifest = null;
            string        lastError     = string.Empty;

            // CDN is very random, just keep trying
            for (var i = 0; i <= 5; i++)
            {
                try
                {
                    depotManifest = CDNClient.DownloadManifest(request.DepotID, request.ManifestID, request.Server, request.CDNToken, request.DepotKey);

                    break;
                }
                catch (Exception e)
                {
                    lastError = e.Message;
                }
            }

            if (depotManifest == null)
            {
                Log.WriteError("Depot Processor", "Failed to download depot manifest for depot {0} ({1}: {2}) (#{3})", request.DepotID, request.Server, lastError, request.Tries);

                if (--request.Tries >= 0)
                {
                    request.Server = GetContentServer(request.Tries);

                    JobManager.AddJob(() => Steam.Instance.Apps.GetCDNAuthToken(request.DepotID, request.Server), request);

                    return;
                }

                RemoveLock(request.DepotID); // TODO: Remove this once task in OnCDNAuthTokenCallback is used

                if (FileDownloader.IsImportantDepot(request.DepotID))
                {
                    IRC.Instance.SendOps("{0}[{1}]{2} Failed to download depot {3} manifest ({4}: {5})",
                                         Colors.OLIVE, Steam.GetAppName(request.ParentAppID), Colors.NORMAL, request.DepotID, request.Server, lastError);
                }

                return;
            }

            if (FileDownloader.IsImportantDepot(request.DepotID))
            {
                TaskManager.Run(() => FileDownloader.DownloadFilesFromDepot(request, depotManifest));
            }

            // TODO: Task here instead of in OnCDNAuthTokenCallback due to mono's silly threadpool
            TaskManager.Run(() =>
            {
                using (var db = Database.GetConnection())
                {
                    ProcessDepotAfterDownload(db, request, depotManifest);
                }
            }).ContinueWith(task =>
            {
                RemoveLock(request.DepotID);

                Log.WriteDebug("Depot Processor", "Processed depot {0} ({1} depot locks left)", request.DepotID, DepotLocks.Count);
            });
        }
Пример #20
0
        private static void OnPICSProductInfo(SteamApps.PICSProductInfoCallback callback)
        {
            JobManager.TryRemoveJob(callback.JobID);

            var processors = new List <BaseProcessor>(
                callback.Apps.Count +
                callback.Packages.Count +
                callback.UnknownApps.Count +
                callback.UnknownPackages.Count
                );

            processors.AddRange(callback.Apps.Select(app => new AppProcessor(app.Key, app.Value)));
            processors.AddRange(callback.Packages.Select(package => new SubProcessor(package.Key, package.Value)));
            processors.AddRange(callback.UnknownApps.Select(app => new AppProcessor(app, null)));
            processors.AddRange(callback.UnknownPackages.Select(package => new SubProcessor(package, null)));

            foreach (var workaround in processors)
            {
                var processor = workaround;

                Task mostRecentItem;

                lock (CurrentlyProcessing)
                {
                    CurrentlyProcessing.TryGetValue(processor.Id, out mostRecentItem);
                }

                var workerItem = TaskManager.Run(async() =>
                {
                    try
                    {
                        await Semaphore.WaitAsync(TaskManager.TaskCancellationToken.Token).ConfigureAwait(false);

                        if (mostRecentItem?.IsCompleted == false)
                        {
                            Log.WriteDebug(processor.ToString(), $"Waiting for previous task to finish processing ({CurrentlyProcessing.Count})");

                            await mostRecentItem.ConfigureAwait(false);

#if DEBUG
                            Log.WriteDebug(processor.ToString(), "Previous task lock ended");
#endif
                        }

                        await processor.Process().ConfigureAwait(false);
                    }
                    catch (Exception e)
                    {
                        ErrorReporter.Notify(processor.ToString(), e);
                    }
                    finally
                    {
                        Semaphore.Release();

                        processor.Dispose();
                    }

                    return(processor);
                }).Unwrap();

                lock (CurrentlyProcessing)
                {
                    CurrentlyProcessing[processor.Id] = workerItem;
                }

                // Register error handler on inner task and the continuation
                TaskManager.RegisterErrorHandler(workerItem);
                TaskManager.RegisterErrorHandler(workerItem.ContinueWith(RemoveProcessorLock, TaskManager.TaskCancellationToken.Token));
            }
        }
Пример #21
0
        public async Task Process(uint appID, uint changeNumber, KeyValue depots)
        {
            var requests = new List <ManifestJob>();

            // Get data in format we want first
            foreach (var depot in depots.Children)
            {
                // Ignore these for now, parent app should be updated too anyway
                if (depot["depotfromapp"].Value != null)
                {
                    continue;
                }

                var request = new ManifestJob
                {
                    ChangeNumber = changeNumber,
                    DepotName    = depot["name"].AsString()
                };

                // Ignore keys that aren't integers, for example "branches"
                if (!uint.TryParse(depot.Name, out request.DepotID))
                {
                    continue;
                }

                // TODO: instead of locking we could wait for current process to finish
                if (DepotLocks.ContainsKey(request.DepotID))
                {
                    continue;
                }

                if (depot["manifests"]["public"].Value == null || !ulong.TryParse(depot["manifests"]["public"].Value, out request.ManifestID))
                {
                    var branch = depot["manifests"].Children.Find(x => x.Name != "local");

                    if (branch == null || !ulong.TryParse(branch.Value, out request.ManifestID))
                    {
                        using (var db = Database.Get())
                        {
                            await db.ExecuteAsync("INSERT INTO `Depots` (`DepotID`, `Name`) VALUES (@DepotID, @DepotName) ON DUPLICATE KEY UPDATE `DepotID` = VALUES(`DepotID`)", new { request.DepotID, request.DepotName });
                        }

                        continue;
                    }

                    Log.WriteDebug("Depot Downloader", "Depot {0} (from {1}) has no public branch, but there is another one", request.DepotID, appID);

                    request.BuildID = depots["branches"][branch.Name]["buildid"].AsInteger();
                }
                else
                {
                    request.BuildID = depots["branches"]["public"]["buildid"].AsInteger();
                }

                requests.Add(request);
            }

            if (requests.Count == 0)
            {
                return;
            }

            var depotsToDownload = new List <ManifestJob>();

            using (var db = await Database.GetConnectionAsync())
            {
                await db.ExecuteAsync("INSERT INTO `Builds` (`BuildID`, `ChangeID`, `AppID`) VALUES (@BuildID, @ChangeNumber, @AppID) ON DUPLICATE KEY UPDATE `AppID` = VALUES(`AppID`)",
                                      new {
                    requests[0].BuildID,
                    requests[0].ChangeNumber,
                    appID
                });

                var dbDepots = (await db.QueryAsync <Depot>("SELECT `DepotID`, `Name`, `BuildID`, `ManifestID`, `LastManifestID` FROM `Depots` WHERE `DepotID` IN @Depots", new { Depots = requests.Select(x => x.DepotID) }))
                               .ToDictionary(x => x.DepotID, x => x);

                foreach (var request in requests)
                {
                    Depot dbDepot;

                    if (dbDepots.ContainsKey(request.DepotID))
                    {
                        dbDepot = dbDepots[request.DepotID];

                        if (dbDepot.BuildID > request.BuildID)
                        {
                            // buildid went back in time? this either means a rollback, or a shared depot that isn't synced properly

                            Log.WriteDebug("Depot Processor", "Skipping depot {0} due to old buildid: {1} > {2}", request.DepotID, dbDepot.BuildID, request.BuildID);

                            continue;
                        }

                        if (dbDepot.LastManifestID == request.ManifestID && dbDepot.ManifestID == request.ManifestID && Settings.Current.FullRun != FullRunState.WithForcedDepots)
                        {
                            // Update depot name if changed
                            if (!request.DepotName.Equals(dbDepot.Name))
                            {
                                await db.ExecuteAsync("UPDATE `Depots` SET `Name` = @DepotName WHERE `DepotID` = @DepotID", new { request.DepotID, request.DepotName });
                            }

                            continue;
                        }
                    }
                    else
                    {
                        dbDepot = new Depot();
                    }

                    if (dbDepot.BuildID != request.BuildID || dbDepot.ManifestID != request.ManifestID || !request.DepotName.Equals(dbDepot.Name))
                    {
                        await db.ExecuteAsync(@"INSERT INTO `Depots` (`DepotID`, `Name`, `BuildID`, `ManifestID`) VALUES (@DepotID, @DepotName, @BuildID, @ManifestID)
                                    ON DUPLICATE KEY UPDATE `LastUpdated` = CURRENT_TIMESTAMP(), `Name` = VALUES(`Name`), `BuildID` = VALUES(`BuildID`), `ManifestID` = VALUES(`ManifestID`)",
                                              new {
                            request.DepotID,
                            request.DepotName,
                            request.BuildID,
                            request.ManifestID
                        });
                    }

                    if (dbDepot.ManifestID != request.ManifestID)
                    {
                        await MakeHistory(db, null, request, string.Empty, "manifest_change", dbDepot.ManifestID, request.ManifestID);
                    }

                    var owned = LicenseList.OwnedApps.ContainsKey(request.DepotID);

                    if (!owned)
                    {
                        request.Anonymous = owned = LicenseList.AnonymousApps.ContainsKey(request.DepotID);

                        if (owned)
                        {
                            Log.WriteWarn("Depot Processor", "Will download depot {0} using anonymous account", request.DepotID);
                        }
                    }

                    if (owned)
                    {
                        lock (DepotLocks)
                        {
                            // This doesn't really save us from concurrency issues
                            if (DepotLocks.ContainsKey(request.DepotID))
                            {
                                Log.WriteWarn("Depot Processor", "Depot {0} was locked in another thread", request.DepotID);
                                continue;
                            }

                            DepotLocks.Add(request.DepotID, 1);
                        }

                        depotsToDownload.Add(request);
                    }
                }
            }

            if (depotsToDownload.Count > 0)
            {
                _ = TaskManager.Run(async() =>
                {
                    try
                    {
                        await DownloadDepots(appID, depotsToDownload);
                    }
                    catch (Exception e)
                    {
                        ErrorReporter.Notify("Depot Processor", e);
                    }

                    foreach (var depot in depotsToDownload)
                    {
                        RemoveLock(depot.DepotID);
                    }
                });
            }
        }
Пример #22
0
        private static async Task <DateTime> ProcessFeed(Uri feedUrl, DateTime lastPostDate)
        {
            var feed = await LoadRSS(feedUrl);

            if (feed == null)
            {
                return(DateTime.MinValue);
            }

            if (feed.Items.Count == 0)
            {
                Log.WriteError(nameof(RSS), $"Did not find any items in {feedUrl}");
                return(DateTime.MinValue);
            }

            await using var db = await Database.GetConnectionAsync();

            var items = (await db.QueryAsync <GenericFeedItem>("SELECT `Link` FROM `RSS` WHERE `Link` IN @Ids", new { Ids = feed.Items.Select(x => x.Link) })).ToDictionary(x => x.Link, _ => (byte)1);

            var newItems = feed.Items.Where(item => item.PubDate > lastPostDate && !items.ContainsKey(item.Link));
            var maxDate  = DateTime.MinValue;

            foreach (var item in newItems)
            {
                if (maxDate < item.PubDate)
                {
                    maxDate = item.PubDate;
                }

                Log.WriteInfo(nameof(RSS), $"[{feed.Title}] {item.Title}: {item.Link} ({item.PubDate})");

                IRC.Instance.SendAnnounce($"{Colors.BLUE}{feed.Title}{Colors.NORMAL}: {item.Title} -{Colors.DARKBLUE} {item.Link}");

                await db.ExecuteAsync("INSERT INTO `RSS` (`Link`, `Title`, `Date`) VALUES(@Link, @Title, @PubDate)", new
                {
                    item.Link,
                    item.Title,
                    item.PubDate,
                });

                _ = TaskManager.Run(async() => await Utils.SendWebhook(new
                {
                    Type = "RSS",
                    item.Title,
                    Url = item.Link,
                }));

                if (!Settings.IsMillhaven)
                {
                    continue;
                }

                uint appID = 0;

                if (feed.Title == "Steam RSS News Feed")
                {
                    if (item.Title.StartsWith("Dota 2 Update", StringComparison.Ordinal))
                    {
                        appID = 570;
                    }
                    else if (item.Title == "Team Fortress 2 Update Released")
                    {
                        appID = 440;

                        // tf2 changelog cleanup
                        item.Content = item.Content.Replace("<br/>", "\n");
                        item.Content = item.Content.Replace("<ul style=\"padding-bottom: 0px; margin-bottom: 0px;\">", "\n");
                        item.Content = item.Content.Replace("<ul style=\"padding-bottom: 0px; margin-bottom: 0px;\" >", "\n");
                        item.Content = item.Content.Replace("</ul>", "\n");
                        item.Content = item.Content.Replace("<li>", "* ");
                    }
                    else if (item.Title == "Left 4 Dead 2 - Update")
                    {
                        appID = 550;
                    }
                    else if (item.Title == "Left 4 Dead - Update")
                    {
                        appID = 500;
                    }
                    else if (item.Title == "Portal 2 - Update")
                    {
                        appID = 620;
                    }
                }
                else if (feed.Title.Contains("Counter-Strike: Global Offensive") && item.Title.StartsWith("Release Notes", StringComparison.Ordinal))
                {
                    appID = 730;

                    // csgo changelog cleanup
                    item.Content = item.Content.Replace("</p>", "\n");
                    item.Content = new Regex("<p>\\[\\s*(.+)\\s*\\]", RegexOptions.Multiline | RegexOptions.CultureInvariant).Replace(item.Content, "## $1");
                    item.Content = item.Content.Replace("<p>", "");
                }

                if (appID > 0)
                {
                    var build = (await db.QueryAsync <Build>(
                                     "SELECT `Builds`.`BuildID`, `Builds`.`ChangeID`, `Builds`.`AppID`, `Changelists`.`Date`, LENGTH(`Official`) as `Official` FROM `Builds` " +
                                     "LEFT JOIN `Patchnotes` ON `Patchnotes`.`BuildID` = `Builds`.`BuildID` " +
                                     "JOIN `Apps` ON `Apps`.`AppID` = `Builds`.`AppID` " +
                                     "JOIN `Changelists` ON `Builds`.`ChangeID` = `Changelists`.`ChangeID` " +
                                     "WHERE `Builds`.`AppID` = @AppID ORDER BY `Builds`.`BuildID` DESC LIMIT 1",
                                     new { appID }
                                     )).SingleOrDefault();

                    if (build == null)
                    {
                        continue;
                    }

                    if (DateTime.UtcNow > build.Date.AddMinutes(60))
                    {
                        Log.WriteDebug(nameof(RSS), $"Got {appID} update patch notes, but there is no build within last 10 minutes. {item.Link}");
                        IRC.Instance.SendOps($"{Colors.GREEN}[Patch notes]{Colors.NORMAL} Got {appID} update patch notes, but there is no build within last 10 minutes. {item.Link}");
                        continue;
                    }

                    if (build.Official > 0)
                    {
                        Log.WriteDebug(nameof(RSS), $"Got {appID} update patch notes, but official patch notes is already filled. {item.Link}");
                        IRC.Instance.SendOps($"{Colors.GREEN}[Patch notes]{Colors.NORMAL} Got {appID} update patch notes, but official patch notes is already filled. {item.Link}");
                        continue;
                    }

                    // breaks
                    item.Content = new Regex(@"<br( \/)?>\r?\n?", RegexOptions.Multiline | RegexOptions.CultureInvariant).Replace(item.Content, "\n");

                    // dashes (CS:GO mainly)
                    item.Content = new Regex("^(?:-|&#(?:8208|8209|8210|8211|8212|8213);|–|—) ?", RegexOptions.Multiline | RegexOptions.CultureInvariant).Replace(item.Content, "* ");

                    item.Content = WebUtility.HtmlDecode(item.Content);

                    Log.WriteDebug(nameof(RSS), $"Inserting {build.AppID} patchnotes for build {build.BuildID}:\n{item.Content}");

                    var accountId = Steam.Instance.Client?.SteamID?.AccountID ?? 0;

                    await db.ExecuteAsync(
                        "INSERT INTO `Patchnotes` (`BuildID`, `AppID`, `ChangeID`, `Date`, `Official`, `OfficialURL`) " +
                        "VALUES (@BuildID, @AppID, @ChangeID, @Date, @Content, @Link) ON DUPLICATE KEY UPDATE `Official` = VALUES(`Official`), `OfficialURL` = VALUES(`OfficialURL`), `LastEditor` = @AccountID",
                        new
                    {
                        build.BuildID,
                        build.AppID,
                        build.ChangeID,
                        Date = build.Date.AddSeconds(1).ToString("yyyy-MM-dd HH:mm:ss"),
                        item.Content,
                        item.Link,
                        accountId
                    }
                        );

                    IRC.Instance.SendAnnounce($"\u2699 Official patch notes:{Colors.BLUE} {Steam.GetAppName(build.AppID)}{Colors.NORMAL} -{Colors.DARKBLUE} {SteamDB.GetPatchnotesUrl(build.BuildID)}");
                }
            }

            return(maxDate);
        }