Пример #1
0
 private static void MakeHistory(ManifestJob request, string file, string action, ulong oldValue = 0, ulong newValue = 0)
 {
     DbWorker.ExecuteNonQuery("INSERT INTO `DepotsHistory` (`ChangeID`, `DepotID`, `File`, `Action`, `OldValue`, `NewValue`) VALUES (@ChangeID, @DepotID, @File, @Action, @OldValue, @NewValue)",
                              new MySqlParameter("@DepotID", request.DepotID),
                              new MySqlParameter("@ChangeID", request.ChangeNumber),
                              new MySqlParameter("@File", file),
                              new MySqlParameter("@Action", action),
                              new MySqlParameter("@OldValue", oldValue),
                              new MySqlParameter("@NewValue", newValue)
                              );
 }
Пример #2
0
        private async Task <EResult> ProcessDepotAfterDownload(ManifestJob request, DepotManifest depotManifest)
        {
            using (var db = await Database.GetConnectionAsync())
                using (var transaction = await db.BeginTransactionAsync())
                {
                    var result = await ProcessDepotAfterDownload(db, transaction, request, depotManifest);

                    await transaction.CommitAsync();

                    return(result);
                }
        }
 private static void MakeHistory(IDbConnection db, ManifestJob request, string file, string action, ulong oldValue = 0, ulong newValue = 0)
 {
     db.Execute(GetHistoryQuery(),
                new DepotHistory
     {
         DepotID  = request.DepotID,
         ChangeID = request.ChangeNumber,
         Action   = action,
         File     = file,
         OldValue = oldValue,
         NewValue = newValue
     }
                );
 }
Пример #4
0
        private static void TryDownloadManifest(ManifestJob request)
        {
            try
            {
                DownloadManifest(request);
            }
            catch (Exception e)
            {
                Log.WriteError("Depot Processor", "Caught exception while processing depot {0}: {1}\n{2}", request.DepotID, e.Message, e.StackTrace);
            }

            lock (ManifestJobs)
            {
                ManifestJobs.Remove(request);
            }
        }
Пример #5
0
        private static async Task <(uint, EResult)> ProcessDepotAfterDownload(ManifestJob request, DepotManifest depotManifest)
        {
            if (depotManifest.FilenamesEncrypted && request.DepotKey != null)
            {
                Log.WriteError(nameof(DepotProcessor), $"Depot key for depot {request.DepotID} is invalid?");
                IRC.Instance.SendOps($"[Tokens] Looks like the depot key for depot {request.DepotID} is invalid");
            }

            await using var db = await Database.GetConnectionAsync();

            await using var transaction = await db.BeginTransactionAsync();

            var result = await ProcessDepotAfterDownload(db, transaction, request, depotManifest);

            await transaction.CommitAsync();

            return(request.DepotID, result);
        }
        private static async Task GetDepotDecryptionKey(SteamApps instance, ManifestJob depot, uint appID)
        {
            if (!LicenseList.OwnedApps.ContainsKey(depot.DepotID))
            {
                return;
            }

            var task = instance.GetDepotDecryptionKey(depot.DepotID, appID);

            task.Timeout = TimeSpan.FromMinutes(15);

            SteamApps.DepotKeyCallback callback;

            try
            {
                callback = await task;
            }
            catch (TaskCanceledException)
            {
                Log.WriteWarn(nameof(DepotProcessor), $"Decryption key timed out for {depot.DepotID}");

                return;
            }

            if (callback.Result != EResult.OK)
            {
                if (callback.Result != EResult.AccessDenied)
                {
                    Log.WriteWarn(nameof(DepotProcessor), $"No access to depot {depot.DepotID} ({callback.Result})");
                }

                return;
            }

            Log.WriteDebug(nameof(DepotProcessor), $"Got a new depot key for depot {depot.DepotID}");

            await using (var db = await Database.GetConnectionAsync())
            {
                await db.ExecuteAsync("INSERT INTO `DepotsKeys` (`DepotID`, `Key`) VALUES (@DepotID, @Key) ON DUPLICATE KEY UPDATE `Key` = VALUES(`Key`)", new { depot.DepotID, Key = Utils.ByteArrayToString(callback.DepotKey) });
            }

            depot.DepotKey = callback.DepotKey;
        }
        public static void Process(uint appID, uint changeNumber, KeyValue depots)
        {
            foreach (KeyValue depot in depots.Children)
            {
                // Ignore these for now, parent app should be updated too anyway
                if (depot["depotfromapp"].Value != null)
                {
                    ////Log.WriteDebug("Depot Processor", "Ignoring depot {0} with depotfromapp value {1} (parent {2})", depot.Name, depot["depotfromapp"].AsString(), AppID);

                    continue;
                }

                uint depotID;

                if (!uint.TryParse(depot.Name, out depotID))
                {
                    // Ignore keys that aren't integers, for example "branches"
                    continue;
                }

                lock (ManifestJobs)
                {
                    if (ManifestJobs.Find(r => r.DepotID == depotID) != null)
                    {
                        // If we already have this depot in our job list, ignore it
                        continue;
                    }
                }

                ulong manifestID;

                if (depot["manifests"]["public"].Value == null || !ulong.TryParse(depot["manifests"]["public"].Value, out manifestID))
                {
            #if false
                    Log.WriteDebug("Depot Processor", "Failed to public branch for depot {0} (parent {1}) - {2}", DepotID, AppID);

                    // If there is no public manifest for this depot, it still could have some sort of open beta

                    var branch = depot["manifests"].Children.SingleOrDefault(x => x.Name != "local");

                    if (branch == null || !ulong.TryParse(branch.Value, out ManifestID))
                    {
                        continue;
                    }
            #endif

                    continue;
                }

                var request = new ManifestJob
                {
                    ChangeNumber = changeNumber,
                    ParentAppID = appID,
                    DepotID = depotID,
                    ManifestID = manifestID,
                    DepotName = depot["name"].AsString()
                };

                // Check if manifestid in our database is equal
                using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `ManifestID` FROM `Depots` WHERE `DepotID` = @DepotID AND `Files` != '' LIMIT 1", new MySqlParameter("DepotID", depotID)))
                {
                    if (Reader.Read())
                    {
                        request.PreviousManifestID = Reader.GetUInt64("ManifestID");

                        if (Settings.Current.FullRun != 3 && request.PreviousManifestID == manifestID)
                        {
                            continue;
                        }
                    }
                }

                Log.WriteInfo("Depot Processor", "DepotID: {0}", depotID);

                lock (ManifestJobs)
                {
                    ManifestJobs.Add(request);
                }

                request.JobID = Steam.Instance.Apps.GetAppOwnershipTicket(depotID);
            }
        }
        private static void DownloadManifest(ManifestJob request)
        {
            CDNClient cdnClient = new CDNClient(Steam.Instance.Client, request.DepotID, request.Ticket, request.DepotKey);
            List<CDNClient.Server> cdnServers;

            try
            {
                cdnServers = cdnClient.FetchServerList();

                if (cdnServers.Count == 0)
                {
                    throw new InvalidOperationException("No servers returned"); // Great programming!
                }
            }
            catch
            {
                Log.WriteError("Depot Processor", "Failed to get server list for depot {0}", request.DepotID);

                lock (ManifestJobs)
                {
                    ManifestJobs.Remove(request);
                }

                return;
            }

            DepotManifest depotManifest = null;

            foreach (var server in cdnServers)
            {
                try
                {
                    cdnClient.Connect(server);

                    depotManifest = cdnClient.DownloadManifest(request.ManifestID);

                    break;
                }
                catch { }
            }

            lock (ManifestJobs)
            {
                ManifestJobs.Remove(request);
            }

            if (depotManifest == null)
            {
                Log.WriteError("Depot Processor", "Failed to download depot manifest for depot {0} (parent {1}) (jobs still in queue: {2})", request.DepotID, request.ParentAppID, ManifestJobs.Count);

                if (SteamProxy.Instance.ImportantApps.Contains(request.ParentAppID))
                {
                    IRC.SendMain("Important manifest update: {0}{1}{2} {3}(parent {4}){5} -{6} manifest download failed from {7} servers", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.DARK_GRAY, request.ParentAppID, Colors.NORMAL, Colors.RED, cdnServers.Count);
                }

                return;
            }

            if (SteamProxy.Instance.ImportantApps.Contains(request.ParentAppID))
            {
                IRC.SendMain("Important manifest update: {0}{1}{2} {3}(parent {4}){5} -{6} {7}", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.DARK_GRAY, request.ParentAppID, Colors.NORMAL, Colors.DARK_BLUE, SteamDB.GetDepotURL(request.DepotID, "history"));
            }

            var sortedFiles = depotManifest.Files.OrderBy(f => f.FileName, StringComparer.OrdinalIgnoreCase);

            bool shouldHistorize = false;
            List<DepotFile> filesNew = new List<DepotFile>();
            List<DepotFile> filesOld = new List<DepotFile>();

            foreach (var file in sortedFiles)
            {
                System.Text.Encoding.UTF8.GetString(file.FileHash);

                filesNew.Add(new DepotFile
                {
                    Name = file.FileName.Replace('\\', '/'),
                    Size = file.TotalSize,
                    Chunks = file.Chunks.Count,
                    Flags = (int)file.Flags
                });
            }

            using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `Files` FROM `Depots` WHERE `DepotID` = @DepotID LIMIT 1", new MySqlParameter("DepotID", request.DepotID)))
            {
                if (Reader.Read())
                {
                    string files = Reader.GetString("Files");

                    if (!string.IsNullOrEmpty(files))
                    {
                        shouldHistorize = true;
                        filesOld = JsonConvert.DeserializeObject<List<DepotFile>>(files);
                    }
                }
            }

            DbWorker.ExecuteNonQuery("UPDATE `Depots` SET `Files` = @Files WHERE `DepotID` = @DepotID",
                                     new MySqlParameter("@DepotID", request.DepotID),
                                     new MySqlParameter("@Files", JsonConvert.SerializeObject(filesNew, new JsonSerializerSettings { DefaultValueHandling = DefaultValueHandling.Ignore }))
            );

            if (shouldHistorize)
            {
                List<string> filesAdded = new List<string>();

                foreach (var file in filesNew)
                {
                    var oldFile = filesOld.Find(x => x.Name == file.Name);

                    if (oldFile == null)
                    {
                        // We want to historize modifications first, and only then deletions and additions
                        filesAdded.Add(file.Name);
                    }
                    else
                    {
                        if (oldFile.Size != file.Size)
                        {
                            MakeHistory(request, file.Name, "modified", oldFile.Size, file.Size);
                        }

                        filesOld.Remove(oldFile);
                    }
                }

                foreach (var file in filesOld)
                {
                    MakeHistory(request, file.Name, "removed");
                }

                foreach (string file in filesAdded)
                {
                    MakeHistory(request, file, "added");
                }
            }

            lock (ManifestJobs)
            {
                Log.WriteDebug("Depot Processor", "DepotID: Processed {0} (jobs still in queue: {1})", request.DepotID, ManifestJobs.Count);
            }

            // Our job is done here, now download important files if any
            if (Settings.Current.ImportantFiles.ContainsKey(request.DepotID))
            {
                // TODO: Horribly inefficient
                var importantFiles = sortedFiles.Where(x => Settings.Current.ImportantFiles[request.DepotID].Contains(x.FileName.Replace('\\', '/')));

                ThreadPool.QueueWorkItem(DownloadFiles, importantFiles, request.DepotID, cdnClient);
            }
        }
        public async Task Process(uint appID, uint changeNumber, KeyValue depots)
        {
            var requests = new List <ManifestJob>();

            // Get data in format we want first
            foreach (var depot in depots.Children)
            {
                // Ignore these for now, parent app should be updated too anyway
                if (depot["depotfromapp"].Value != null)
                {
                    continue;
                }

                var request = new ManifestJob
                {
                    ChangeNumber = changeNumber,
                };

                // Ignore keys that aren't integers, for example "branches"
                if (!uint.TryParse(depot.Name, out request.DepotID))
                {
                    continue;
                }

                request.DepotName = depot["name"].AsString();

                if (string.IsNullOrEmpty(request.DepotName))
                {
                    request.DepotName = $"SteamDB Unnamed Depot {request.DepotID}";
                }

                // TODO: instead of locking we could wait for current process to finish
                if (DepotLocks.ContainsKey(request.DepotID))
                {
                    continue;
                }

                // SteamVR trickery
                if (appID == 250820 &&
                    depot["manifests"]["beta"].Value != null &&
                    depots["branches"]["beta"]["buildid"].AsInteger() > depots["branches"]["public"]["buildid"].AsInteger())
                {
                    request.BuildID    = depots["branches"]["beta"]["buildid"].AsInteger();
                    request.ManifestID = ulong.Parse(depot["manifests"]["beta"].Value);
                }
                else
                if (depot["manifests"]["public"].Value == null || !ulong.TryParse(depot["manifests"]["public"].Value, out request.ManifestID))
                {
                    var branch = depot["manifests"].Children.Find(x => x.Name != "local");

                    if (branch == null || !ulong.TryParse(branch.Value, out request.ManifestID))
                    {
                        await using var db = await Database.GetConnectionAsync();

                        await db.ExecuteAsync("INSERT INTO `Depots` (`DepotID`, `Name`) VALUES (@DepotID, @DepotName) ON DUPLICATE KEY UPDATE `DepotID` = VALUES(`DepotID`)", new { request.DepotID, request.DepotName });

                        continue;
                    }

                    Log.WriteDebug(nameof(DepotProcessor), $"Depot {request.DepotID} (from {appID}) has no public branch, but there is another one");

                    request.BuildID = depots["branches"][branch.Name]["buildid"].AsInteger();
                }
                else
                {
                    request.BuildID = depots["branches"]["public"]["buildid"].AsInteger();
                }

                requests.Add(request);
            }

            if (requests.Count == 0)
            {
                return;
            }

            var depotsToDownload = new List <ManifestJob>();

            await using (var db = await Database.GetConnectionAsync())
            {
                await db.ExecuteAsync("INSERT INTO `Builds` (`BuildID`, `ChangeID`, `AppID`) VALUES (@BuildID, @ChangeNumber, @AppID) ON DUPLICATE KEY UPDATE `AppID` = VALUES(`AppID`)",
                                      new
                {
                    requests[0].BuildID,
                    requests[0].ChangeNumber,
                    appID
                });

                var depotIds = requests.Select(x => x.DepotID).ToList();
                var dbDepots = (await db.QueryAsync <Depot>("SELECT `DepotID`, `Name`, `BuildID`, `ManifestID`, `LastManifestID`, `FilenamesEncrypted` FROM `Depots` WHERE `DepotID` IN @depotIds", new { depotIds }))
                               .ToDictionary(x => x.DepotID, x => x);

                var decryptionKeys = (await db.QueryAsync <DepotKey>("SELECT `DepotID`, `Key` FROM `DepotsKeys` WHERE `DepotID` IN @depotIds", new { depotIds }))
                                     .ToDictionary(x => x.DepotID, x => Utils.StringToByteArray(x.Key));

                foreach (var request in requests)
                {
                    Depot dbDepot;

                    decryptionKeys.TryGetValue(request.DepotID, out request.DepotKey);

                    if (dbDepots.ContainsKey(request.DepotID))
                    {
                        dbDepot = dbDepots[request.DepotID];

                        if (dbDepot.BuildID > request.BuildID)
                        {
                            // buildid went back in time? this either means a rollback, or a shared depot that isn't synced properly

                            Log.WriteDebug(nameof(DepotProcessor), $"Skipping depot {request.DepotID} due to old buildid: {dbDepot.BuildID} > {request.BuildID}");

                            continue;
                        }

                        if (dbDepot.LastManifestID == request.ManifestID &&
                            dbDepot.ManifestID == request.ManifestID &&
                            Settings.Current.FullRun != FullRunState.WithForcedDepots &&
                            !dbDepot.FilenamesEncrypted && request.DepotKey != null)
                        {
                            // Update depot name if changed
                            if (request.DepotName != dbDepot.Name)
                            {
                                await db.ExecuteAsync("UPDATE `Depots` SET `Name` = @DepotName WHERE `DepotID` = @DepotID", new { request.DepotID, request.DepotName });
                            }

                            continue;
                        }

                        request.StoredFilenamesEncrypted = dbDepot.FilenamesEncrypted;
                        request.LastManifestID           = dbDepot.LastManifestID;
                    }
                    else
                    {
                        dbDepot = new Depot();
                    }

                    if (dbDepot.BuildID != request.BuildID || dbDepot.ManifestID != request.ManifestID || request.DepotName != dbDepot.Name)
                    {
                        await db.ExecuteAsync(@"INSERT INTO `Depots` (`DepotID`, `Name`, `BuildID`, `ManifestID`) VALUES (@DepotID, @DepotName, @BuildID, @ManifestID)
                                    ON DUPLICATE KEY UPDATE `LastUpdated` = CURRENT_TIMESTAMP(), `Name` = VALUES(`Name`), `BuildID` = VALUES(`BuildID`), `ManifestID` = VALUES(`ManifestID`)",
                                              new
                        {
                            request.DepotID,
                            request.DepotName,
                            request.BuildID,
                            request.ManifestID
                        });
                    }

                    if (dbDepot.ManifestID != request.ManifestID)
                    {
                        await MakeHistory(db, null, request, string.Empty, "manifest_change", dbDepot.ManifestID, request.ManifestID);
                    }

                    lock (DepotLocks)
                    {
                        // This doesn't really save us from concurrency issues
                        if (DepotLocks.ContainsKey(request.DepotID))
                        {
                            Log.WriteWarn(nameof(DepotProcessor), $"Depot {request.DepotID} was locked in another thread");
                            continue;
                        }

                        DepotLocks.Add(request.DepotID, 1);
                    }

                    depotsToDownload.Add(request);
                }
            }

            if (depotsToDownload.Count > 0)
            {
                _ = TaskManager.Run(async() =>
                {
                    try
                    {
                        await DownloadDepots(appID, depotsToDownload);
                    }
                    catch (Exception e)
                    {
                        ErrorReporter.Notify(nameof(DepotProcessor), e);
                    }

                    foreach (var depot in depotsToDownload)
                    {
                        RemoveLock(depot.DepotID);
                    }
                });
            }
        }
Пример #10
0
 private static Task MakeHistory(IDbConnection db, IDbTransaction transaction, ManifestJob request, string file, string action, ulong oldValue = 0, ulong newValue = 0)
 {
     return(db.ExecuteAsync(HistoryQuery,
                            new DepotHistory
     {
         DepotID = request.DepotID,
         ChangeID = request.ChangeNumber,
         Action = action,
         File = file,
         OldValue = oldValue,
         NewValue = newValue
     },
                            transaction: transaction
                            ));
 }
        private void DownloadManifest(ManifestJob request)
        {
            Log.WriteInfo("Depot Processor", "DepotID: {0}", request.DepotID);

            DepotManifest depotManifest = null;
            string lastError = string.Empty;

            // CDN is very random, just keep trying
            for (var i = 0; i <= 5; i++)
            {
                try
                {
                    depotManifest = CDNClient.DownloadManifest(request.DepotID, request.ManifestID, request.Server, request.CDNToken, request.DepotKey);

                    break;
                }
                catch (Exception e)
                {
                    lastError = e.Message;
                }
            }

            if (depotManifest == null)
            {
                Log.WriteError("Depot Processor", "Failed to download depot manifest for depot {0} ({1}: {2}) (#{3})", request.DepotID, request.Server, lastError, request.Tries);

                if (--request.Tries >= 0)
                {
                    request.Server = GetContentServer(request.Tries);

                    JobManager.AddJob(() => Steam.Instance.Apps.GetCDNAuthToken(request.DepotID, request.Server), request);

                    return;
                }

                RemoveLock(request.DepotID); // TODO: Remove this once task in OnCDNAuthTokenCallback is used

                if (FileDownloader.IsImportantDepot(request.DepotID))
                {
                    IRC.Instance.SendOps("{0}[{1}]{2} Failed to download depot {3} manifest ({4}: {5})",
                        Colors.OLIVE, Steam.GetAppName(request.ParentAppID), Colors.NORMAL, request.DepotID, request.Server, lastError);
                }

                return;
            }

            if (FileDownloader.IsImportantDepot(request.DepotID))
            {
                TaskManager.Run(() => FileDownloader.DownloadFilesFromDepot(request, depotManifest));
            }

            // TODO: Task here instead of in OnCDNAuthTokenCallback due to mono's silly threadpool
            TaskManager.Run(() =>
            {
                using(var db = Database.GetConnection())
                {
                    ProcessDepotAfterDownload(db, request, depotManifest);
                }
            }).ContinueWith(task =>
            {
                RemoveLock(request.DepotID);

                Log.WriteDebug("Depot Processor", "Processed depot {0} ({1} depot locks left)", request.DepotID, DepotLocks.Count);
            });
        }
 private void MakeHistory(ManifestJob request, string file, string action, ulong oldValue = 0, ulong newValue = 0)
 {
     DbWorker.ExecuteNonQuery(
         "INSERT INTO `DepotsHistory` (`ChangeID`, `DepotID`, `File`, `Action`, `OldValue`, `NewValue`) VALUES (@ChangeID, @DepotID, @File, @Action, @OldValue, @NewValue)",
         new MySqlParameter("@DepotID", request.DepotID),
         new MySqlParameter("@ChangeID", request.ChangeNumber),
         new MySqlParameter("@File", file),
         new MySqlParameter("@Action", action),
         new MySqlParameter("@OldValue", oldValue),
         new MySqlParameter("@NewValue", newValue)
     );
 }
 private static void MakeHistory(IDbConnection db, ManifestJob request, string file, string action, ulong oldValue = 0, ulong newValue = 0)
 {
     db.Execute(GetHistoryQuery(),
         new DepotHistory
         {
             DepotID  = request.DepotID,
             ChangeID = request.ChangeNumber,
             Action   = action,
             File     = file,
             OldValue = oldValue,
             NewValue = newValue
         }
     );
 }
        private void TryDownloadManifest(ManifestJob request)
        {
            try
            {
                DownloadManifest(request);
            }
            catch (Exception e)
            {
                Log.WriteError("Depot Processor", "Caught exception while processing depot {0}: {1}\n{2}", request.DepotID, e.Message, e.StackTrace);
            }

            RemoveLock(request.DepotID);

            Log.WriteDebug("Depot Processor", "Processed depot {0} ({1} depot locks left)", request.DepotID, DepotLocks.Count);
        }
Пример #15
0
        public async Task Process(IDbConnection db, uint appID, uint changeNumber, KeyValue depots)
        {
            var requests = new List <ManifestJob>();
            var dlcNames = new Dictionary <uint, string>();

            // Get data in format we want first
            foreach (var depot in depots.Children)
            {
                var request = new ManifestJob
                {
                    ChangeNumber = changeNumber,
                };

                // Ignore keys that aren't integers, for example "branches"
                if (!uint.TryParse(depot.Name, out request.DepotID))
                {
                    continue;
                }

                // Ignore these for now, parent app should be updated too anyway
                if (depot["depotfromapp"].Value != null)
                {
                    continue;
                }

                request.DepotName = depot["name"].AsString();

                if (string.IsNullOrEmpty(request.DepotName))
                {
                    request.DepotName = $"SteamDB Unnamed Depot {request.DepotID}";
                }
                else if (depot["dlcappid"].Value != null)
                {
                    if (uint.TryParse(depot["dlcappid"].Value, out var dlcAppId))
                    {
                        dlcNames[dlcAppId] = request.DepotName;
                    }
                }

                // TODO: instead of locking we could wait for current process to finish
                if (DepotLocks.ContainsKey(request.DepotID))
                {
                    continue;
                }

                // SteamVR trickery
                if (appID == 250820 &&
                    depot["manifests"]["beta"].Value != null &&
                    depots["branches"]["beta"]["buildid"].AsInteger() > depots["branches"]["public"]["buildid"].AsInteger())
                {
                    request.BuildID    = depots["branches"]["beta"]["buildid"].AsInteger();
                    request.ManifestID = ulong.Parse(depot["manifests"]["beta"].Value);
                }
                else
                if (depot["manifests"]["public"].Value == null || !ulong.TryParse(depot["manifests"]["public"].Value, out request.ManifestID))
                {
                    var branch = depot["manifests"].Children.Find(x => x.Name != "local");

                    if (branch == null || !ulong.TryParse(branch.Value, out request.ManifestID))
                    {
                        await db.ExecuteAsync("INSERT INTO `Depots` (`DepotID`, `Name`) VALUES (@DepotID, @DepotName) ON DUPLICATE KEY UPDATE `DepotID` = VALUES(`DepotID`)", new { request.DepotID, request.DepotName });

                        continue;
                    }

                    Log.WriteDebug(nameof(DepotProcessor), $"Depot {request.DepotID} (from {appID}) has no public branch, but there is another one");

                    request.BuildID = depots["branches"][branch.Name]["buildid"].AsInteger();
                }
                else
                {
                    request.BuildID = depots["branches"]["public"]["buildid"].AsInteger();
                }

                requests.Add(request);
            }

            if (dlcNames.Any())
            {
                await ProcessDlcNames(appID, changeNumber, dlcNames);
            }

            foreach (var branch in depots["branches"].Children)
            {
                var buildId = branch["buildid"].AsInteger();

                if (buildId < 1)
                {
                    continue;
                }

                var isPublic = branch.Name != null && branch.Name.Equals("public", StringComparison.OrdinalIgnoreCase);

                await db.ExecuteAsync(isPublic?
                                      "INSERT INTO `Builds` (`BuildID`, `ChangeID`, `AppID`, `Public`) VALUES (@BuildID, @ChangeNumber, @AppID, 1) ON DUPLICATE KEY UPDATE `ChangeID` = IF(`Public` = 0, VALUES(`ChangeID`), `ChangeID`), `Public` = 1"
                                      :
                                      "INSERT INTO `Builds` (`BuildID`, `ChangeID`, `AppID`) VALUES (@BuildID, @ChangeNumber, @AppID) ON DUPLICATE KEY UPDATE `AppID` = `AppID`",
                                      new
                {
                    BuildID      = buildId,
                    ChangeNumber = changeNumber,
                    AppID        = appID,
                });
            }

            if (requests.Count == 0)
            {
                return;
            }

            var depotsToDownload = new List <ManifestJob>();

            var depotIds = requests.Select(x => x.DepotID).ToList();
            var dbDepots = (await db.QueryAsync <Depot>("SELECT `DepotID`, `Name`, `BuildID`, `ManifestID`, `LastManifestID`, `FilenamesEncrypted` FROM `Depots` WHERE `DepotID` IN @depotIds", new { depotIds }))
                           .ToDictionary(x => x.DepotID, x => x);

            var decryptionKeys = (await db.QueryAsync <DepotKey>("SELECT `DepotID`, `Key` FROM `DepotsKeys` WHERE `DepotID` IN @depotIds", new { depotIds }))
                                 .ToDictionary(x => x.DepotID, x => Utils.StringToByteArray(x.Key));

            foreach (var request in requests)
            {
                Depot dbDepot;

                decryptionKeys.TryGetValue(request.DepotID, out request.DepotKey);

                if (dbDepots.ContainsKey(request.DepotID))
                {
                    dbDepot = dbDepots[request.DepotID];

                    if (dbDepot.BuildID > request.BuildID)
                    {
                        // buildid went back in time? this either means a rollback, or a shared depot that isn't synced properly

                        Log.WriteDebug(nameof(DepotProcessor), $"Skipping depot {request.DepotID} due to old buildid: {dbDepot.BuildID} > {request.BuildID}");

                        continue;
                    }

                    if (dbDepot.LastManifestID == request.ManifestID &&
                        dbDepot.ManifestID == request.ManifestID &&
                        Settings.FullRun != FullRunState.WithForcedDepots &&
                        !dbDepot.FilenamesEncrypted && request.DepotKey != null)
                    {
                        // Update depot name if changed
                        if (request.DepotName != dbDepot.Name)
                        {
                            await db.ExecuteAsync("UPDATE `Depots` SET `Name` = @DepotName WHERE `DepotID` = @DepotID", new { request.DepotID, request.DepotName });
                        }

                        continue;
                    }

                    request.StoredFilenamesEncrypted = dbDepot.FilenamesEncrypted;
                    request.LastManifestID           = dbDepot.LastManifestID;
                }
                else
                {
                    dbDepot = new Depot();
                }

                if (dbDepot.BuildID != request.BuildID || dbDepot.ManifestID != request.ManifestID || request.DepotName != dbDepot.Name)
                {
                    await db.ExecuteAsync(@"INSERT INTO `Depots` (`DepotID`, `Name`, `BuildID`, `ManifestID`) VALUES (@DepotID, @DepotName, @BuildID, @ManifestID)
                                ON DUPLICATE KEY UPDATE `LastUpdated` = CURRENT_TIMESTAMP(), `Name` = VALUES(`Name`), `BuildID` = VALUES(`BuildID`), `ManifestID` = VALUES(`ManifestID`)",
                                          new
                    {
                        request.DepotID,
                        request.DepotName,
                        request.BuildID,
                        request.ManifestID
                    });
                }

                if (dbDepot.ManifestID != request.ManifestID)
                {
                    await MakeHistory(db, null, request, string.Empty, "manifest_change", dbDepot.ManifestID, request.ManifestID);
                }

                if (Settings.Current.OnlyOwnedDepots && !LicenseList.OwnedDepots.ContainsKey(request.DepotID))
                {
                    continue;
                }

                lock (DepotLocks)
                {
                    DepotLocks.Add(request.DepotID, 1);
                }

                depotsToDownload.Add(request);
            }

            if (depotsToDownload.Count > 0)
            {
                _ = TaskManager.Run(async() => await DownloadDepots(appID, depotsToDownload)).ContinueWith(task =>
                {
                    Log.WriteError(nameof(DepotProcessor), $"An exception occured when processing depots from app {appID}, removing locks");

                    foreach (var depot in depotsToDownload)
                    {
                        RemoveLock(depot.DepotID);
                    }
                }, TaskContinuationOptions.OnlyOnFaulted);
            }
        }
Пример #16
0
        private static void DownloadManifest(ManifestJob request)
        {
            var cdnClient = new CDNClient(Steam.Instance.Client, request.DepotID, request.Ticket, request.DepotKey);
            List<CDNClient.Server> cdnServers = null;

            for (var i = 0; i <= 5; i++)
            {
                try
                {
                    cdnServers = cdnClient.FetchServerList();

                    if (cdnServers.Count > 0)
                    {
                        break;
                    }
                }
                catch { }
            }

            if (cdnServers == null || cdnServers.Count == 0)
            {
                Log.WriteError("Depot Processor", "Failed to get server list for depot {0}", request.DepotID);

                if (SteamProxy.Instance.ImportantApps.Contains(request.ParentAppID))
                {
                    IRC.SendMain("Important manifest update: {0}{1}{2} {3}(parent {4}){5} -{6} failed to fetch server list", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.DARK_GRAY, request.ParentAppID, Colors.NORMAL, Colors.RED);
                }

                lock (ManifestJobs)
                {
                    ManifestJobs.Remove(request);
                }

                return;
            }

            DepotManifest depotManifest = null;

            foreach (var server in cdnServers)
            {
                try
                {
                    cdnClient.Connect(server);

                    depotManifest = cdnClient.DownloadManifest(request.ManifestID);

                    break;
                }
                catch { }
            }

            lock (ManifestJobs)
            {
                ManifestJobs.Remove(request);
            }

            if (depotManifest == null)
            {
                Log.WriteError("Depot Processor", "Failed to download depot manifest for depot {0} (parent {1}) (jobs still in queue: {2})", request.DepotID, request.ParentAppID, ManifestJobs.Count);

                if (SteamProxy.Instance.ImportantApps.Contains(request.ParentAppID))
                {
                    IRC.SendMain("Important manifest update: {0}{1}{2} {3}(parent {4}){5} -{6} failed to download depot manifest from {7} servers", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.DARK_GRAY, request.ParentAppID, Colors.NORMAL, Colors.RED, cdnServers.Count);
                }

                return;
            }

            if (SteamProxy.Instance.ImportantApps.Contains(request.ParentAppID))
            {
                IRC.SendMain("Important manifest update: {0}{1}{2} {3}(parent {4}){5} -{6} {7}", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.DARK_GRAY, request.ParentAppID, Colors.NORMAL, Colors.DARK_BLUE, SteamDB.GetDepotURL(request.DepotID, "history"));
            }

            var sortedFiles = depotManifest.Files.OrderBy(f => f.FileName, StringComparer.OrdinalIgnoreCase);

            bool shouldHistorize = false;
            var filesNew = new List<DepotFile>();
            var filesOld = new List<DepotFile>();

            foreach (var file in sortedFiles)
            {
                System.Text.Encoding.UTF8.GetString(file.FileHash);

                var depotFile = new DepotFile
                {
                    Name = file.FileName.Replace('\\', '/'),
                    Size = file.TotalSize,
                    Chunks = file.Chunks.Count,
                    Flags = (int)file.Flags
                };

                // TODO: Ideally we would check if filehash is not empty
                if (!file.Flags.HasFlag(EDepotFileFlag.Directory))
                {
                    depotFile.Hash = string.Concat(Array.ConvertAll(file.FileHash, x => x.ToString("X2")));
                }

                filesNew.Add(depotFile);
            }

            using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `Files` FROM `Depots` WHERE `DepotID` = @DepotID LIMIT 1", new MySqlParameter("DepotID", request.DepotID)))
            {
                if (Reader.Read())
                {
                    string files = Reader.GetString("Files");

                    if (!string.IsNullOrEmpty(files))
                    {
                        shouldHistorize = true;
                        filesOld = JsonConvert.DeserializeObject<List<DepotFile>>(files);
                    }
                }
            }

            DbWorker.ExecuteNonQuery("UPDATE `Depots` SET `Files` = @Files WHERE `DepotID` = @DepotID",
                                     new MySqlParameter("@DepotID", request.DepotID),
                                     new MySqlParameter("@Files", JsonConvert.SerializeObject(filesNew, new JsonSerializerSettings { DefaultValueHandling = DefaultValueHandling.Ignore }))
            );

            if (shouldHistorize)
            {
                var filesAdded = new List<string>();

                foreach (var file in filesNew)
                {
                    var oldFile = filesOld.Find(x => x.Name == file.Name);

                    if (oldFile == null)
                    {
                        // We want to historize modifications first, and only then deletions and additions
                        filesAdded.Add(file.Name);
                    }
                    else
                    {
                        if (oldFile.Size != file.Size)
                        {
                            MakeHistory(request, file.Name, "modified", oldFile.Size, file.Size);
                        }
                        else if (file.Hash != null && oldFile.Hash != null && !file.Hash.Equals(oldFile.Hash))
                        {
                            MakeHistory(request, file.Name, "modified", oldFile.Size, file.Size);
                        }

                        filesOld.Remove(oldFile);
                    }
                }

                foreach (var file in filesOld)
                {
                    MakeHistory(request, file.Name, "removed");
                }

                foreach (string file in filesAdded)
                {
                    MakeHistory(request, file, "added");
                }
            }

            lock (ManifestJobs)
            {
                Log.WriteDebug("Depot Processor", "DepotID: Processed {0} (jobs still in queue: {1})", request.DepotID, ManifestJobs.Count);
            }
        }
Пример #17
0
        public static void Process(uint appID, uint changeNumber, KeyValue depots)
        {
            foreach (KeyValue depot in depots.Children)
            {
                // Ignore these for now, parent app should be updated too anyway
                if (depot["depotfromapp"].Value != null)
                {
                    ////Log.WriteDebug("Depot Processor", "Ignoring depot {0} with depotfromapp value {1} (parent {2})", depot.Name, depot["depotfromapp"].AsString(), AppID);

                    continue;
                }

                uint depotID;

                if (!uint.TryParse(depot.Name, out depotID))
                {
                    // Ignore keys that aren't integers, for example "branches"
                    continue;
                }

                lock (ManifestJobs)
                {
                    if (ManifestJobs.Find(r => r.DepotID == depotID) != null)
                    {
                        // If we already have this depot in our job list, ignore it
                        continue;
                    }
                }

                ulong manifestID;

                if (depot["manifests"]["public"].Value == null || !ulong.TryParse(depot["manifests"]["public"].Value, out manifestID))
                {
#if false
                    Log.WriteDebug("Depot Processor", "Failed to public branch for depot {0} (parent {1}) - {2}", DepotID, AppID);

                    // If there is no public manifest for this depot, it still could have some sort of open beta

                    var branch = depot["manifests"].Children.SingleOrDefault(x => x.Name != "local");

                    if (branch == null || !ulong.TryParse(branch.Value, out ManifestID))
                    {
                        continue;
                    }
#endif

                    continue;
                }

                var request = new ManifestJob
                {
                    ChangeNumber = changeNumber,
                    ParentAppID  = appID,
                    DepotID      = depotID,
                    ManifestID   = manifestID,
                    DepotName    = depot["name"].AsString()
                };

                // Check if manifestid in our database is equal
                using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `ManifestID` FROM `Depots` WHERE `DepotID` = @DepotID AND `Files` != '' LIMIT 1", new MySqlParameter("DepotID", depotID)))
                {
                    if (Reader.Read())
                    {
                        request.PreviousManifestID = Reader.GetUInt64("ManifestID");

                        if (request.PreviousManifestID == manifestID && Settings.Current.FullRun < 2)
                        {
                            continue;
                        }
                    }
                }

                lock (ManifestJobs)
                {
                    ManifestJobs.Add(request);
                }

                request.Server = CDNServers[new Random().Next(CDNServers.Count)];
                request.JobID  = Steam.Instance.Apps.GetCDNAuthToken(depotID, request.Server);
            }
        }
Пример #18
0
        private static void DownloadManifest(ManifestJob request)
        {
            DepotManifest depotManifest = null;
            string        lastError     = string.Empty;

            // CDN is very random, just keep trying
            for (var i = 0; i <= 5; i++)
            {
                try
                {
                    depotManifest = CDNClient.DownloadManifest(request.DepotID, request.ManifestID, request.Server, request.DepotKey, request.CDNToken);

                    break;
                }
                catch (Exception e)
                {
                    lastError = e.Message;
                }
            }

            if (depotManifest == null)
            {
                Log.WriteError("Depot Processor", "Failed to download depot manifest for depot {0} (jobs still in queue: {1}) ({2}: {3})", request.DepotID, ManifestJobs.Count - 1, request.Server, lastError);

                if (SteamProxy.Instance.ImportantApps.Contains(request.ParentAppID))
                {
                    IRC.SendMain("Important depot update: {0}{1}{2} -{3} failed to download depot manifest", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.RED);
                }

                return;
            }

            if (SteamProxy.Instance.ImportantApps.Contains(request.ParentAppID))
            {
                IRC.SendMain("Important depot update: {0}{1}{2} -{3} {4}", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.DARK_BLUE, SteamDB.GetDepotURL(request.DepotID, "history"));
            }

            var sortedFiles = depotManifest.Files.OrderBy(f => f.FileName, StringComparer.OrdinalIgnoreCase);

            bool shouldHistorize = false;
            var  filesNew        = new List <DepotFile>();
            var  filesOld        = new Dictionary <string, DepotFile>();

            foreach (var file in sortedFiles)
            {
                System.Text.Encoding.UTF8.GetString(file.FileHash);

                var depotFile = new DepotFile
                {
                    Name   = file.FileName.Replace('\\', '/'),
                    Size   = file.TotalSize,
                    Chunks = file.Chunks.Count,
                    Flags  = (int)file.Flags
                };

                // TODO: Ideally we would check if filehash is not empty
                if (!file.Flags.HasFlag(EDepotFileFlag.Directory))
                {
                    depotFile.Hash = string.Concat(Array.ConvertAll(file.FileHash, x => x.ToString("X2")));
                }

                filesNew.Add(depotFile);
            }

            using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `Files` FROM `Depots` WHERE `DepotID` = @DepotID LIMIT 1", new MySqlParameter("DepotID", request.DepotID)))
            {
                if (Reader.Read())
                {
                    string files = Reader.GetString("Files");

                    if (!string.IsNullOrEmpty(files))
                    {
                        shouldHistorize = true;

                        var _filesOld = JsonConvert.DeserializeObject <List <DepotFile> >(files);

                        filesOld = _filesOld.ToDictionary(x => x.Name);
                    }
                }
            }

            DbWorker.ExecuteNonQuery("UPDATE `Depots` SET `Files` = @Files WHERE `DepotID` = @DepotID",
                                     new MySqlParameter("@DepotID", request.DepotID),
                                     new MySqlParameter("@Files", JsonConvert.SerializeObject(filesNew, new JsonSerializerSettings {
                DefaultValueHandling = DefaultValueHandling.Ignore
            }))
                                     );

            if (shouldHistorize)
            {
                var filesAdded = new List <string>();

                foreach (var file in filesNew)
                {
                    if (filesOld.ContainsKey(file.Name))
                    {
                        var oldFile = filesOld[file.Name];

                        if (oldFile.Size != file.Size)
                        {
                            MakeHistory(request, file.Name, "modified", oldFile.Size, file.Size);
                        }
                        else if (file.Hash != null && oldFile.Hash != null && !file.Hash.Equals(oldFile.Hash))
                        {
                            MakeHistory(request, file.Name, "modified", oldFile.Size, file.Size);
                        }

                        filesOld.Remove(file.Name);
                    }
                    else
                    {
                        // We want to historize modifications first, and only then deletions and additions
                        filesAdded.Add(file.Name);
                    }
                }

                foreach (var file in filesOld)
                {
                    MakeHistory(request, file.Value.Name, "removed");
                }

                foreach (string file in filesAdded)
                {
                    MakeHistory(request, file, "added");
                }
            }

            lock (ManifestJobs)
            {
                Log.WriteDebug("Depot Processor", "DepotID: Processed {0} (jobs still in queue: {1})", request.DepotID, ManifestJobs.Count - 1);
            }
        }
        private void DownloadManifest(ManifestJob request)
        {
            DepotManifest depotManifest = null;
            string lastError = string.Empty;

            // CDN is very random, just keep trying
            for (var i = 0; i <= 5; i++)
            {
                try
                {
                    depotManifest = CDNClient.DownloadManifest(request.DepotID, request.ManifestID, request.Server, request.CDNToken, request.DepotKey);

                    break;
                }
                catch (Exception e)
                {
                    lastError = e.Message;
                }
            }

            if (depotManifest == null)
            {
                Log.WriteError("Depot Processor", "Failed to download depot manifest for depot {0} ({1}: {2})", request.DepotID, request.Server, lastError);

                if (Application.ImportantApps.ContainsKey(request.ParentAppID))
                {
                    IRC.Instance.SendMain("Important depot update: {0}{1}{2} -{3} failed to download depot manifest", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.RED);
                }

                return;
            }

            if (Application.ImportantApps.ContainsKey(request.ParentAppID))
            {
                IRC.Instance.SendMain("Important depot update: {0}{1}{2} -{3} {4}", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.DARKBLUE, SteamDB.GetDepotURL(request.DepotID, "history"));
            }

            var sortedFiles = depotManifest.Files.OrderBy(f => f.FileName, StringComparer.OrdinalIgnoreCase);

            bool shouldHistorize = false;
            var filesNew = new List<DepotFile>();
            var filesOld = new Dictionary<string, DepotFile>();

            foreach (var file in sortedFiles)
            {
                System.Text.Encoding.UTF8.GetString(file.FileHash);

                var depotFile = new DepotFile
                {
                    Name = file.FileName.Replace('\\', '/'),
                    Size = file.TotalSize,
                    Chunks = file.Chunks.Count,
                    Flags = (int)file.Flags
                };

                // TODO: Ideally we would check if filehash is not empty
                if (!file.Flags.HasFlag(EDepotFileFlag.Directory))
                {
                    depotFile.Hash = string.Concat(Array.ConvertAll(file.FileHash, x => x.ToString("X2")));
                }

                filesNew.Add(depotFile);
            }

            using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `Files` FROM `Depots` WHERE `DepotID` = @DepotID LIMIT 1", new MySqlParameter("DepotID", request.DepotID)))
            {
                if (Reader.Read())
                {
                    string files = Reader.GetString("Files");

                    if (!string.IsNullOrEmpty(files))
                    {
                        shouldHistorize = true;

                        filesOld = JsonConvert.DeserializeObject<List<DepotFile>>(files).ToDictionary(x => x.Name);
                    }
                }
            }

            DbWorker.ExecuteNonQuery(
                "UPDATE `Depots` SET `Files` = @Files WHERE `DepotID` = @DepotID",
                new MySqlParameter("@DepotID", request.DepotID),
                new MySqlParameter("@Files", JsonConvert.SerializeObject(filesNew, new JsonSerializerSettings { DefaultValueHandling = DefaultValueHandling.Ignore }))
            );

            if (shouldHistorize)
            {
                var filesAdded = new List<string>();

                foreach (var file in filesNew)
                {
                    if (filesOld.ContainsKey(file.Name))
                    {
                        var oldFile = filesOld[file.Name];

                        if (oldFile.Size != file.Size)
                        {
                            MakeHistory(request, file.Name, "modified", oldFile.Size, file.Size);
                        }
                        else if (file.Hash != null && oldFile.Hash != null && !file.Hash.Equals(oldFile.Hash))
                        {
                            MakeHistory(request, file.Name, "modified", oldFile.Size, file.Size);
                        }

                        filesOld.Remove(file.Name);
                    }
                    else
                    {
                        // We want to historize modifications first, and only then deletions and additions
                        filesAdded.Add(file.Name);
                    }
                }

                foreach (var file in filesOld)
                {
                    MakeHistory(request, file.Value.Name, "removed");
                }

                foreach (string file in filesAdded)
                {
                    MakeHistory(request, file, "added");
                }
            }
        }
Пример #20
0
        private static async Task <EResult> ProcessDepotAfterDownload(IDbConnection db, IDbTransaction transaction, ManifestJob request, DepotManifest depotManifest)
        {
            var filesOld        = (await db.QueryAsync <DepotFile>("SELECT `File`, `Hash`, `Size`, `Flags` FROM `DepotsFiles` WHERE `DepotID` = @DepotID", new { request.DepotID }, transaction)).ToDictionary(x => x.File, x => x);
            var filesAdded      = new List <DepotFile>();
            var shouldHistorize = filesOld.Count > 0 && !depotManifest.FilenamesEncrypted; // Don't historize file additions if we didn't have any data before

            if (request.StoredFilenamesEncrypted && !depotManifest.FilenamesEncrypted)
            {
                Log.WriteInfo(nameof(DepotProcessor), $"Depot {request.DepotID} will decrypt stored filenames");

                var decryptedFilesOld = new Dictionary <string, DepotFile>();

                foreach (var file in filesOld.Values)
                {
                    var oldFile = file.File;
                    file.File = DecryptFilename(oldFile, request.DepotKey);

                    decryptedFilesOld.Add(file.File, file);

                    await db.ExecuteAsync("UPDATE `DepotsFiles` SET `File` = @File WHERE `DepotID` = @DepotID AND `File` = @OldFile", new
                    {
                        request.DepotID,
                        file.File,
                        OldFile = oldFile
                    }, transaction);
                }

                await MakeHistory(db, transaction, request, string.Empty, "files_decrypted");

                filesOld = decryptedFilesOld;
            }

            foreach (var file in depotManifest.Files.OrderByDescending(x => x.FileName))
            {
                var name = depotManifest.FilenamesEncrypted ? file.FileName.Replace("\n", "") : file.FileName.Replace('\\', '/');

                byte[] hash = null;

                // Store empty hashes as NULL (e.g. an empty file)
                if ((file.Flags & EDepotFileFlag.Directory) == 0 && file.FileHash.Length > 0 && file.FileHash.Any(c => c != 0))
                {
                    hash = file.FileHash;
                }

                // Limit path names to 260 characters (default windows max length)
                // File column is varchar(260) and not higher to prevent reducing performance
                // See https://stackoverflow.com/questions/1962310/importance-of-varchar-length-in-mysql-table/1962329#1962329
                // Until 2019 there hasn't been a single file that went over this limit, so far there has been only one
                // game with a big node_modules path, so we're safeguarding by limiting it.
                if (name.Length > 260)
                {
                    if (depotManifest.FilenamesEncrypted)
                    {
                        continue;
                    }

                    using var sha = SHA1.Create();
                    var nameHash = Utils.ByteArrayToString(sha.ComputeHash(Encoding.UTF8.GetBytes(name)));
                    name = $"{{SteamDB file name is too long}}/{nameHash}/...{name.Substring(name.Length - 150)}";
                }

                if (filesOld.ContainsKey(name))
                {
                    var oldFile    = filesOld[name];
                    var updateFile = false;

                    if (oldFile.Size != file.TotalSize || !Utils.ByteArrayEquals(hash, oldFile.Hash))
                    {
                        await MakeHistory(db, transaction, request, name, "modified", oldFile.Size, file.TotalSize);

                        updateFile = true;
                    }

                    if (oldFile.Flags != file.Flags)
                    {
                        await MakeHistory(db, transaction, request, name, "modified_flags", (ulong)oldFile.Flags, (ulong)file.Flags);

                        updateFile = true;
                    }

                    if (updateFile)
                    {
                        await db.ExecuteAsync("UPDATE `DepotsFiles` SET `Hash` = @Hash, `Size` = @Size, `Flags` = @Flags WHERE `DepotID` = @DepotID AND `File` = @File", new DepotFile
                        {
                            DepotID = request.DepotID,
                            File    = name,
                            Hash    = hash,
                            Size    = file.TotalSize,
                            Flags   = file.Flags
                        }, transaction);
                    }

                    filesOld.Remove(name);
                }
                else
                {
                    // We want to historize modifications first, and only then deletions and additions
                    filesAdded.Add(new DepotFile
                    {
                        DepotID = request.DepotID,
                        Hash    = hash,
                        File    = name,
                        Size    = file.TotalSize,
                        Flags   = file.Flags
                    });
                }
            }

            if (filesOld.Count > 0)
            {
                // Chunk file deletion queries so it doesn't go over max_allowed_packet
                var filesOldChunks = filesOld.Select(x => x.Value.File).Split(1000);

                foreach (var filesOldChunk in filesOldChunks)
                {
                    await db.ExecuteAsync("DELETE FROM `DepotsFiles` WHERE `DepotID` = @DepotID AND `File` IN @Files",
                                          new
                    {
                        request.DepotID,
                        Files = filesOldChunk,
                    }, transaction);
                }

                if (shouldHistorize)
                {
                    await db.ExecuteAsync(HistoryQuery, filesOld.Select(x => new DepotHistory
                    {
                        DepotID    = request.DepotID,
                        ManifestID = request.ManifestID,
                        ChangeID   = request.ChangeNumber,
                        Action     = "removed",
                        File       = x.Value.File,
                        OldValue   = x.Value.Size
                    }), transaction);
                }
            }

            if (filesAdded.Count > 0)
            {
                await db.ExecuteAsync("INSERT INTO `DepotsFiles` (`DepotID`, `File`, `Hash`, `Size`, `Flags`) VALUES (@DepotID, @File, @Hash, @Size, @Flags)", filesAdded, transaction);

                if (shouldHistorize)
                {
                    await db.ExecuteAsync(HistoryQuery, filesAdded.Select(x => new DepotHistory
                    {
                        DepotID    = request.DepotID,
                        ManifestID = request.ManifestID,
                        ChangeID   = request.ChangeNumber,
                        Action     = "added",
                        File       = x.File,
                        NewValue   = x.Size
                    }), transaction);
                }
            }

            await db.ExecuteAsync(
                request.LastManifestID == request.ManifestID?
                "UPDATE `Depots` SET `LastManifestID` = @ManifestID, `ManifestDate` = @ManifestDate, `FilenamesEncrypted` = @FilenamesEncrypted, `SizeOriginal` = @SizeOriginal, `SizeCompressed` = @SizeCompressed WHERE `DepotID` = @DepotID" :
                "UPDATE `Depots` SET `LastManifestID` = @ManifestID, `ManifestDate` = @ManifestDate, `FilenamesEncrypted` = @FilenamesEncrypted, `SizeOriginal` = @SizeOriginal, `SizeCompressed` = @SizeCompressed, `LastUpdated` = CURRENT_TIMESTAMP() WHERE `DepotID` = @DepotID",
                new
            {
                request.DepotID,
                request.ManifestID,
                depotManifest.FilenamesEncrypted,
                ManifestDate   = depotManifest.CreationTime,
                SizeOriginal   = depotManifest.TotalUncompressedSize,
                SizeCompressed = depotManifest.TotalCompressedSize,
            }, transaction);

            return(EResult.OK);
        }
        public void Process(uint appID, uint changeNumber, KeyValue depots)
        {
            var buildID = depots["branches"]["public"]["buildid"].AsInteger();

            foreach (KeyValue depot in depots.Children)
            {
                // Ignore these for now, parent app should be updated too anyway
                if (depot["depotfromapp"].Value != null)
                {
                    ////Log.WriteDebug("Depot Processor", "Ignoring depot {0} with depotfromapp value {1} (parent {2})", depot.Name, depot["depotfromapp"].AsString(), AppID);

                    continue;
                }

                uint depotID;

                if (!uint.TryParse(depot.Name, out depotID))
                {
                    // Ignore keys that aren't integers, for example "branches"
                    continue;
                }

                // TODO: instead of locking we could wait for current process to finish
                if (DepotLocks.ContainsKey(depotID))
                {
                    continue;
                }

                ulong manifestID;

                var depotName = depot["name"].AsString();

                if (depot["manifests"]["public"].Value == null || !ulong.TryParse(depot["manifests"]["public"].Value, out manifestID))
                {
#if false
                    Log.WriteDebug("Depot Processor", "Failed to public branch for depot {0} (parent {1}) - {2}", DepotID, AppID);

                    // If there is no public manifest for this depot, it still could have some sort of open beta

                    var branch = depot["manifests"].Children.SingleOrDefault(x => x.Name != "local");

                    if (branch == null || !ulong.TryParse(branch.Value, out ManifestID))
                    {
                        continue;
                    }
#endif

                    DbWorker.ExecuteNonQuery("INSERT INTO `Depots` (`DepotID`, `Name`) VALUES (@DepotID, @Name) ON DUPLICATE KEY UPDATE `LastUpdated` = CURRENT_TIMESTAMP(), `Name` = @Name",
                        new MySqlParameter("@DepotID", depotID),
                        new MySqlParameter("@Name", depotName)
                    );

                    continue;
                }
                    
                var request = new ManifestJob
                {
                    ChangeNumber = changeNumber,
                    ParentAppID = appID,
                    DepotID = depotID,
                    ManifestID = manifestID,
                    DepotName = depotName
                };

                int currentBuildID = 0;

                // Check if manifestid in our database is equal
                using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `Name`, `ManifestID`, `BuildID` FROM `Depots` WHERE `DepotID` = @DepotID LIMIT 1", new MySqlParameter("DepotID", depotID)))
                {
                    if (Reader.Read())
                    {
                        currentBuildID = Reader.GetInt32("buildID");

                        request.PreviousManifestID = Reader.GetUInt64("ManifestID");

                        if (request.PreviousManifestID == manifestID && Settings.Current.FullRun < 2)
                        {
                            // Update depot name if changed
                            if(!depotName.Equals(Reader.GetString("Name")))
                            {
                                DbWorker.ExecuteNonQuery("UPDATE `Depots` SET `Name` = @Name WHERE `DepotID` = @DepotID",
                                    new MySqlParameter("@DepotID", request.DepotID),
                                    new MySqlParameter("@Name", request.DepotName)
                                );
                            }

                            continue;
                        }

                        if (currentBuildID > buildID)
                        {
                            Log.WriteDebug("Depot Processor", "Skipping depot {0} due to old buildid: {1} > {2}", depotID, currentBuildID, buildID);
                            continue;
                        }
                    }
                }

                DepotLocks.TryAdd(depotID, 1);

                // Update/insert depot information straight away
                if (currentBuildID != buildID || request.PreviousManifestID != request.ManifestID)
                {
                    DbWorker.ExecuteNonQuery("INSERT INTO `Depots` (`DepotID`, `Name`, `BuildID`, `ManifestID`) VALUES (@DepotID, @Name, @BuildID, @ManifestID) ON DUPLICATE KEY UPDATE `LastUpdated` = CURRENT_TIMESTAMP(), `Name` = @Name, `BuildID` = @BuildID, `ManifestID` = @ManifestID",
                        new MySqlParameter("@DepotID", request.DepotID),
                        new MySqlParameter("@BuildID", buildID),
                        new MySqlParameter("@ManifestID", request.ManifestID),
                        new MySqlParameter("@Name", request.DepotName)
                    );

                    MakeHistory(request, string.Empty, "manifest_change", request.PreviousManifestID, request.ManifestID);
                }

                request.Server = CDNServers[new Random().Next(CDNServers.Count)];

                JobManager.AddJob(() => Steam.Instance.Apps.GetCDNAuthToken(depotID, request.Server), request);
            }
        }
        private void DownloadManifest(ManifestJob request)
        {
            Log.WriteInfo("Depot Processor", "DepotID: {0}", request.DepotID);

            DepotManifest depotManifest = null;
            string        lastError     = string.Empty;

            // CDN is very random, just keep trying
            for (var i = 0; i <= 5; i++)
            {
                try
                {
                    depotManifest = CDNClient.DownloadManifest(request.DepotID, request.ManifestID, request.Server, request.CDNToken, request.DepotKey);

                    break;
                }
                catch (Exception e)
                {
                    lastError = e.Message;
                }
            }

            if (depotManifest == null)
            {
                Log.WriteError("Depot Processor", "Failed to download depot manifest for depot {0} ({1}: {2}) (#{3})", request.DepotID, request.Server, lastError, request.Tries);

                if (--request.Tries >= 0)
                {
                    request.Server = GetContentServer(request.Tries);

                    JobManager.AddJob(() => Steam.Instance.Apps.GetCDNAuthToken(request.DepotID, request.Server), request);

                    return;
                }

                RemoveLock(request.DepotID); // TODO: Remove this once task in OnCDNAuthTokenCallback is used

                if (FileDownloader.IsImportantDepot(request.DepotID))
                {
                    IRC.Instance.SendOps("{0}[{1}]{2} Failed to download depot {3} manifest ({4}: {5})",
                                         Colors.OLIVE, Steam.GetAppName(request.ParentAppID), Colors.NORMAL, request.DepotID, request.Server, lastError);
                }

                return;
            }

            if (FileDownloader.IsImportantDepot(request.DepotID))
            {
                TaskManager.Run(() => FileDownloader.DownloadFilesFromDepot(request, depotManifest));
            }

            // TODO: Task here instead of in OnCDNAuthTokenCallback due to mono's silly threadpool
            TaskManager.Run(() =>
            {
                using (var db = Database.GetConnection())
                {
                    ProcessDepotAfterDownload(db, request, depotManifest);
                }
            }).ContinueWith(task =>
            {
                RemoveLock(request.DepotID);

                Log.WriteDebug("Depot Processor", "Processed depot {0} ({1} depot locks left)", request.DepotID, DepotLocks.Count);
            });
        }
        private static void ProcessDepotAfterDownload(IDbConnection db, ManifestJob request, DepotManifest depotManifest)
        {
            var filesOld = db.Query<DepotFile>("SELECT `ID`, `File`, `Hash`, `Size`, `Flags` FROM `DepotsFiles` WHERE `DepotID` = @DepotID", new { request.DepotID }).ToDictionary(x => x.File, x => x);
            var filesNew = new List<DepotFile>();
            var filesAdded = new List<DepotFile>();
            var shouldHistorize = filesOld.Any(); // Don't historize file additions if we didn't have any data before

            foreach (var file in depotManifest.Files)
            {
                var name = file.FileName.Replace('\\', '/');

                // safe guard
                if (name.Length > 255)
                {
                    ErrorReporter.Notify(new OverflowException(string.Format("File \"{0}\" in depot {1} is too long", name, request.DepotID)));

                    continue;
                }

                var depotFile = new DepotFile
                {
                    DepotID = request.DepotID,
                    File    = name,
                    Size    = file.TotalSize,
                    Flags   = file.Flags
                };

                if (file.FileHash.Length > 0 && !file.Flags.HasFlag(EDepotFileFlag.Directory))
                {
                    depotFile.Hash = Utils.ByteArrayToString(file.FileHash);
                }
                else
                {
                    depotFile.Hash = "0000000000000000000000000000000000000000";
                }

                filesNew.Add(depotFile);
            }

            foreach (var file in filesNew)
            {
                if (filesOld.ContainsKey(file.File))
                {
                    var oldFile = filesOld[file.File];
                    var updateFile = false;

                    if (oldFile.Size != file.Size || !file.Hash.Equals(oldFile.Hash))
                    {
                        MakeHistory(db, request, file.File, "modified", oldFile.Size, file.Size);

                        updateFile = true;
                    }

                    if (oldFile.Flags != file.Flags)
                    {
                        MakeHistory(db, request, file.File, "modified_flags", (ulong)oldFile.Flags, (ulong)file.Flags);

                        updateFile = true;
                    }

                    if (updateFile)
                    {
                        file.ID = oldFile.ID;

                        db.Execute("UPDATE `DepotsFiles` SET `Hash` = @Hash, `Size` = @Size, `Flags` = @Flags WHERE `DepotID` = @DepotID AND `ID` = @ID", file);
                    }

                    filesOld.Remove(file.File);
                }
                else
                {
                    // We want to historize modifications first, and only then deletions and additions
                    filesAdded.Add(file);
                }
            }

            if (filesOld.Any())
            {
                db.Execute("DELETE FROM `DepotsFiles` WHERE `DepotID` = @DepotID AND `ID` IN @Files", new { request.DepotID, Files = filesOld.Select(x => x.Value.ID) });

                db.Execute(GetHistoryQuery(), filesOld.Select(x => new DepotHistory
                {
                    DepotID  = request.DepotID,
                    ChangeID = request.ChangeNumber,
                    Action   = "removed",
                    File     = x.Value.File
                }));
            }

            if (filesAdded.Any())
            {
                db.Execute("INSERT INTO `DepotsFiles` (`DepotID`, `File`, `Hash`, `Size`, `Flags`) VALUES (@DepotID, @File, @Hash, @Size, @Flags)", filesAdded);

                if (shouldHistorize)
                {
                    db.Execute(GetHistoryQuery(), filesAdded.Select(x => new DepotHistory
                    {
                        DepotID  = request.DepotID,
                        ChangeID = request.ChangeNumber,
                        Action   = "added",
                        File     = x.File
                    }));
                }
            }

            db.Execute("UPDATE `Depots` SET `LastManifestID` = @ManifestID WHERE `DepotID` = @DepotID", new { request.DepotID, request.ManifestID });
        }
        public void Process(uint appID, uint changeNumber, KeyValue depots)
        {
            var requests = new List <ManifestJob>();

            // Get data in format we want first
            foreach (var depot in depots.Children)
            {
                // Ignore these for now, parent app should be updated too anyway
                if (depot["depotfromapp"].Value != null)
                {
                    continue;
                }

                var request = new ManifestJob
                {
                    ChangeNumber = changeNumber,
                    ParentAppID  = appID,
                    DepotName    = depot["name"].AsString()
                };

                // Ignore keys that aren't integers, for example "branches"
                if (!uint.TryParse(depot.Name, out request.DepotID))
                {
                    continue;
                }

                // TODO: instead of locking we could wait for current process to finish
                if (DepotLocks.ContainsKey(request.DepotID))
                {
                    continue;
                }

                // If there is no public manifest for this depot, it still could have some sort of open beta
                if (depot["manifests"]["public"].Value == null || !ulong.TryParse(depot["manifests"]["public"].Value, out request.ManifestID))
                {
                    var branch = depot["manifests"].Children.FirstOrDefault(x => x.Name != "local");

                    if (branch == null || !ulong.TryParse(branch.Value, out request.ManifestID))
                    {
                        using (var db = Database.GetConnection())
                        {
                            db.Execute("INSERT INTO `Depots` (`DepotID`, `Name`) VALUES (@DepotID, @DepotName) ON DUPLICATE KEY UPDATE `Name` = @DepotName", new { request.DepotID, request.DepotName });
                        }

                        continue;
                    }

                    request.BuildID = branch["build"].AsInteger();
                }
                else
                {
                    request.BuildID = depots["branches"]["public"]["buildid"].AsInteger();
                }

                requests.Add(request);
            }

            if (!requests.Any())
            {
                return;
            }

            using (var db = Database.GetConnection())
            {
                var dbDepots = db.Query <Depot>("SELECT `DepotID`, `Name`, `BuildID`, `ManifestID`, `LastManifestID` FROM `Depots` WHERE `DepotID` IN @Depots", new { Depots = requests.Select(x => x.DepotID) })
                               .ToDictionary(x => x.DepotID, x => x);

                foreach (var request in requests)
                {
                    Depot dbDepot;

                    if (dbDepots.ContainsKey(request.DepotID))
                    {
                        dbDepot = dbDepots[request.DepotID];

                        if (dbDepot.BuildID > request.BuildID)
                        {
                            // buildid went back in time? this either means a rollback, or a shared depot that isn't synced properly

                            Log.WriteDebug("Depot Processor", "Skipping depot {0} due to old buildid: {1} > {2}", request.DepotID, dbDepot.BuildID, request.BuildID);

                            continue;
                        }

                        if (dbDepot.LastManifestID == request.ManifestID && dbDepot.ManifestID == request.ManifestID && Settings.Current.FullRun < 2)
                        {
                            // Update depot name if changed
                            if (!request.DepotName.Equals(dbDepot.Name))
                            {
                                db.Execute("UPDATE `Depots` SET `Name` = @DepotName WHERE `DepotID` = @DepotID", new { request.DepotID, request.DepotName });
                            }

                            continue;
                        }
                    }
                    else
                    {
                        dbDepot = new Depot();
                    }

                    if (dbDepot.BuildID != request.BuildID || dbDepot.ManifestID != request.ManifestID || !request.DepotName.Equals(dbDepot.Name))
                    {
                        db.Execute(@"INSERT INTO `Depots` (`DepotID`, `Name`, `BuildID`, `ManifestID`) VALUES (@DepotID, @DepotName, @BuildID, @ManifestID)
                                    ON DUPLICATE KEY UPDATE `LastUpdated` = CURRENT_TIMESTAMP(), `Name` = @DepotName, `BuildID` = @BuildID, `ManifestID` = @ManifestID",
                                   new {
                            request.DepotID,
                            request.DepotName,
                            request.BuildID,
                            request.ManifestID
                        });
                    }

                    if (dbDepot.ManifestID != request.ManifestID)
                    {
                        MakeHistory(db, request, string.Empty, "manifest_change", dbDepot.ManifestID, request.ManifestID);
                    }

                    if (LicenseList.OwnedApps.ContainsKey(request.DepotID) || Settings.Current.FullRun > 1)
                    {
                        DepotLocks.TryAdd(request.DepotID, 1);

                        JobManager.AddJob(() => Steam.Instance.Apps.GetDepotDecryptionKey(request.DepotID, request.ParentAppID), request);
                    }
#if DEBUG
                    else
                    {
                        Log.WriteDebug("Depot Processor", "Skipping depot {0} from app {1} because we don't own it", request.DepotID, request.ParentAppID);
                    }
#endif
                }
            }
        }
        public void Process(uint appID, uint changeNumber, KeyValue depots)
        {
            var requests = new List<ManifestJob>();

            // Get data in format we want first
            foreach (var depot in depots.Children)
            {
                // Ignore these for now, parent app should be updated too anyway
                if (depot["depotfromapp"].Value != null)
                {
                    continue;
                }

                var request = new ManifestJob
                {
                    ChangeNumber = changeNumber,
                    ParentAppID  = appID,
                    DepotName    = depot["name"].AsString()
                };

                // Ignore keys that aren't integers, for example "branches"
                if (!uint.TryParse(depot.Name, out request.DepotID))
                {
                    continue;
                }

                // TODO: instead of locking we could wait for current process to finish
                if (DepotLocks.ContainsKey(request.DepotID))
                {
                    continue;
                }

                // If there is no public manifest for this depot, it still could have some sort of open beta
                if (depot["manifests"]["public"].Value == null || !ulong.TryParse(depot["manifests"]["public"].Value, out request.ManifestID))
                {
                    var branch = depot["manifests"].Children.FirstOrDefault(x => x.Name != "local");

                    if (branch == null || !ulong.TryParse(branch.Value, out request.ManifestID))
                    {
                        using (var db = Database.GetConnection())
                        {
                            db.Execute("INSERT INTO `Depots` (`DepotID`, `Name`) VALUES (@DepotID, @DepotName) ON DUPLICATE KEY UPDATE `Name` = VALUES(`Name`)", new { request.DepotID, request.DepotName });
                        }

                        continue;
                    }

                    request.BuildID = branch["build"].AsInteger();
                }
                else
                {
                    request.BuildID = depots["branches"]["public"]["buildid"].AsInteger();
                }

                requests.Add(request);
            }

            if (!requests.Any())
            {
                return;
            }

            using (var db = Database.GetConnection())
            {
                var dbDepots = db.Query<Depot>("SELECT `DepotID`, `Name`, `BuildID`, `ManifestID`, `LastManifestID` FROM `Depots` WHERE `DepotID` IN @Depots", new { Depots = requests.Select(x => x.DepotID) })
                    .ToDictionary(x => x.DepotID, x => x);

                foreach (var request in requests)
                {
                    Depot dbDepot;

                    if (dbDepots.ContainsKey(request.DepotID))
                    {
                        dbDepot = dbDepots[request.DepotID];

                        if (dbDepot.BuildID > request.BuildID)
                        {
                            // buildid went back in time? this either means a rollback, or a shared depot that isn't synced properly

                            Log.WriteDebug("Depot Processor", "Skipping depot {0} due to old buildid: {1} > {2}", request.DepotID, dbDepot.BuildID, request.BuildID);

                            continue;
                        }

                        if (dbDepot.LastManifestID == request.ManifestID && dbDepot.ManifestID == request.ManifestID && Settings.Current.FullRun < 2)
                        {
                            // Update depot name if changed
                            if (!request.DepotName.Equals(dbDepot.Name))
                            {
                                db.Execute("UPDATE `Depots` SET `Name` = @DepotName WHERE `DepotID` = @DepotID", new { request.DepotID, request.DepotName });
                            }

                            continue;
                        }
                    }
                    else
                    {
                        dbDepot = new Depot();
                    }

                    if (dbDepot.BuildID != request.BuildID || dbDepot.ManifestID != request.ManifestID || !request.DepotName.Equals(dbDepot.Name))
                    {
                        db.Execute(@"INSERT INTO `Depots` (`DepotID`, `Name`, `BuildID`, `ManifestID`) VALUES (@DepotID, @DepotName, @BuildID, @ManifestID)
                                    ON DUPLICATE KEY UPDATE `LastUpdated` = CURRENT_TIMESTAMP(), `Name` = VALUES(`Name`), `BuildID` = VALUES(`BuildID`), `ManifestID` = VALUES(`ManifestID`)",
                        new {
                            request.DepotID,
                            request.DepotName,
                            request.BuildID,
                            request.ManifestID
                        });
                    }

                    if (dbDepot.ManifestID != request.ManifestID)
                    {
                        MakeHistory(db, request, string.Empty, "manifest_change", dbDepot.ManifestID, request.ManifestID);
                    }

                    if (LicenseList.OwnedApps.ContainsKey(request.DepotID) || Settings.Current.FullRun > 1)
                    {
                        DepotLocks.TryAdd(request.DepotID, 1);

                        JobManager.AddJob(() => Steam.Instance.Apps.GetDepotDecryptionKey(request.DepotID, request.ParentAppID), request);
                    }
            #if DEBUG
                    else
                    {
                        Log.WriteDebug("Depot Processor", "Skipping depot {0} from app {1} because we don't own it", request.DepotID, request.ParentAppID);
                    }
            #endif
                }
            }
        }
        private EResult ProcessDepotAfterDownload(IDbConnection db, ManifestJob request, DepotManifest depotManifest)
        {
            var filesOld        = db.Query <DepotFile>("SELECT `ID`, `File`, `Hash`, `Size`, `Flags` FROM `DepotsFiles` WHERE `DepotID` = @DepotID", new { request.DepotID }).ToDictionary(x => x.File, x => x);
            var filesNew        = new List <DepotFile>();
            var filesAdded      = new List <DepotFile>();
            var shouldHistorize = filesOld.Any(); // Don't historize file additions if we didn't have any data before

            foreach (var file in depotManifest.Files)
            {
                var name = file.FileName.Replace('\\', '/');

                // safe guard
                if (name.Length > 255)
                {
                    ErrorReporter.Notify(new OverflowException(string.Format("File \"{0}\" in depot {1} is too long", name, request.DepotID)));

                    continue;
                }

                var depotFile = new DepotFile
                {
                    DepotID = request.DepotID,
                    File    = name,
                    Size    = file.TotalSize,
                    Flags   = file.Flags
                };

                if (file.FileHash.Length > 0 && !file.Flags.HasFlag(EDepotFileFlag.Directory))
                {
                    depotFile.Hash = Utils.ByteArrayToString(file.FileHash);
                }
                else
                {
                    depotFile.Hash = "0000000000000000000000000000000000000000";
                }

                filesNew.Add(depotFile);
            }

            foreach (var file in filesNew)
            {
                if (filesOld.ContainsKey(file.File))
                {
                    var oldFile    = filesOld[file.File];
                    var updateFile = false;

                    if (oldFile.Size != file.Size || !file.Hash.Equals(oldFile.Hash))
                    {
                        MakeHistory(db, request, file.File, "modified", oldFile.Size, file.Size);

                        updateFile = true;
                    }

                    if (oldFile.Flags != file.Flags)
                    {
                        MakeHistory(db, request, file.File, "modified_flags", (ulong)oldFile.Flags, (ulong)file.Flags);

                        updateFile = true;
                    }

                    if (updateFile)
                    {
                        file.ID = oldFile.ID;

                        db.Execute("UPDATE `DepotsFiles` SET `Hash` = @Hash, `Size` = @Size, `Flags` = @Flags WHERE `DepotID` = @DepotID AND `ID` = @ID", file);
                    }

                    filesOld.Remove(file.File);
                }
                else
                {
                    // We want to historize modifications first, and only then deletions and additions
                    filesAdded.Add(file);
                }
            }

            if (filesOld.Any())
            {
                db.Execute("DELETE FROM `DepotsFiles` WHERE `DepotID` = @DepotID AND `ID` IN @Files", new { request.DepotID, Files = filesOld.Select(x => x.Value.ID) });

                db.Execute(GetHistoryQuery(), filesOld.Select(x => new DepotHistory
                {
                    DepotID  = request.DepotID,
                    ChangeID = request.ChangeNumber,
                    Action   = "removed",
                    File     = x.Value.File
                }));
            }

            if (filesAdded.Any())
            {
                db.Execute("INSERT INTO `DepotsFiles` (`DepotID`, `File`, `Hash`, `Size`, `Flags`) VALUES (@DepotID, @File, @Hash, @Size, @Flags)", filesAdded);

                if (shouldHistorize)
                {
                    db.Execute(GetHistoryQuery(), filesAdded.Select(x => new DepotHistory
                    {
                        DepotID  = request.DepotID,
                        ChangeID = request.ChangeNumber,
                        Action   = "added",
                        File     = x.File
                    }));
                }
            }

            db.Execute("UPDATE `Depots` SET `LastManifestID` = @ManifestID WHERE `DepotID` = @DepotID", new { request.DepotID, request.ManifestID });

            return(EResult.OK);
        }
Пример #27
0
        private async Task <EResult> ProcessDepotAfterDownload(IDbConnection db, IDbTransaction transaction, ManifestJob request, DepotManifest depotManifest)
        {
            var filesOld        = (await db.QueryAsync <DepotFile>("SELECT `ID`, `File`, `Hash`, `Size`, `Flags` FROM `DepotsFiles` WHERE `DepotID` = @DepotID", new { request.DepotID }, transaction: transaction)).ToDictionary(x => x.File, x => x);
            var filesAdded      = new List <DepotFile>();
            var shouldHistorize = filesOld.Any(); // Don't historize file additions if we didn't have any data before

            foreach (var file in depotManifest.Files)
            {
                var    name = file.FileName.Replace('\\', '/');
                byte[] hash = null;

                // Store empty hashes as NULL (e.g. an empty file)
                if (file.FileHash.Length > 0 && !file.Flags.HasFlag(EDepotFileFlag.Directory))
                {
                    for (int i = 0; i < file.FileHash.Length; ++i)
                    {
                        if (file.FileHash[i] != 0)
                        {
                            hash = file.FileHash;
                            break;
                        }
                    }
                }

                // safe guard
                if (name.Length > 255)
                {
                    ErrorReporter.Notify("Depot Processor", new OverflowException(string.Format("File \"{0}\" in depot {1} is too long", name, request.DepotID)));

                    continue;
                }

                if (filesOld.ContainsKey(name))
                {
                    var oldFile    = filesOld[name];
                    var updateFile = false;

                    if (oldFile.Size != file.TotalSize || !Utils.IsEqualSHA1(hash, oldFile.Hash))
                    {
                        await MakeHistory(db, transaction, request, name, "modified", oldFile.Size, file.TotalSize);

                        updateFile = true;
                    }

                    if (oldFile.Flags != file.Flags)
                    {
                        await MakeHistory(db, transaction, request, name, "modified_flags", (ulong)oldFile.Flags, (ulong)file.Flags);

                        updateFile = true;
                    }

                    if (updateFile)
                    {
                        await db.ExecuteAsync("UPDATE `DepotsFiles` SET `Hash` = @Hash, `Size` = @Size, `Flags` = @Flags WHERE `DepotID` = @DepotID AND `ID` = @ID", new DepotFile
                        {
                            ID      = oldFile.ID,
                            DepotID = request.DepotID,
                            Hash    = hash,
                            Size    = file.TotalSize,
                            Flags   = file.Flags
                        }, transaction : transaction);
                    }

                    filesOld.Remove(name);
                }
                else
                {
                    // We want to historize modifications first, and only then deletions and additions
                    filesAdded.Add(new DepotFile
                    {
                        DepotID = request.DepotID,
                        Hash    = hash,
                        File    = name,
                        Size    = file.TotalSize,
                        Flags   = file.Flags
                    });
                }
            }

            if (filesOld.Any())
            {
                await db.ExecuteAsync("DELETE FROM `DepotsFiles` WHERE `DepotID` = @DepotID AND `ID` IN @Files", new { request.DepotID, Files = filesOld.Select(x => x.Value.ID) }, transaction : transaction);

                await db.ExecuteAsync(HistoryQuery, filesOld.Select(x => new DepotHistory
                {
                    DepotID  = request.DepotID,
                    ChangeID = request.ChangeNumber,
                    Action   = "removed",
                    File     = x.Value.File
                }), transaction : transaction);
            }

            if (filesAdded.Any())
            {
                await db.ExecuteAsync("INSERT INTO `DepotsFiles` (`DepotID`, `File`, `Hash`, `Size`, `Flags`) VALUES (@DepotID, @File, @Hash, @Size, @Flags)", filesAdded, transaction : transaction);

                if (shouldHistorize)
                {
                    await db.ExecuteAsync(HistoryQuery, filesAdded.Select(x => new DepotHistory
                    {
                        DepotID  = request.DepotID,
                        ChangeID = request.ChangeNumber,
                        Action   = "added",
                        File     = x.File
                    }), transaction : transaction);
                }
            }

            await db.ExecuteAsync("UPDATE `Depots` SET `LastManifestID` = @ManifestID, `LastUpdated` = CURRENT_TIMESTAMP() WHERE `DepotID` = @DepotID", new { request.DepotID, request.ManifestID }, transaction : transaction);

            return(EResult.OK);
        }
        public void Process(uint appID, uint changeNumber, KeyValue depots)
        {
            var requests = new List <ManifestJob>();

            // Get data in format we want first
            foreach (var depot in depots.Children)
            {
                // Ignore these for now, parent app should be updated too anyway
                if (depot["depotfromapp"].Value != null)
                {
                    continue;
                }

                var request = new ManifestJob
                {
                    ChangeNumber = changeNumber,
                    DepotName    = depot["name"].AsString()
                };

                // Ignore keys that aren't integers, for example "branches"
                if (!uint.TryParse(depot.Name, out request.DepotID))
                {
                    continue;
                }

                // TODO: instead of locking we could wait for current process to finish
                if (DepotLocks.ContainsKey(request.DepotID))
                {
                    continue;
                }

                // If there is no public manifest for this depot, it still could have some sort of open beta
                if (depot["manifests"]["public"].Value == null || !ulong.TryParse(depot["manifests"]["public"].Value, out request.ManifestID))
                {
                    var branch = depot["manifests"].Children.FirstOrDefault(x => x.Name != "local");

                    if (branch == null || !ulong.TryParse(branch.Value, out request.ManifestID))
                    {
                        using (var db = Database.GetConnection())
                        {
                            db.Execute("INSERT INTO `Depots` (`DepotID`, `Name`) VALUES (@DepotID, @DepotName) ON DUPLICATE KEY UPDATE `Name` = VALUES(`Name`)", new { request.DepotID, request.DepotName });
                        }

                        continue;
                    }

                    request.BuildID = branch["build"].AsInteger();
                }
                else
                {
                    request.BuildID = depots["branches"]["public"]["buildid"].AsInteger();
                }

                requests.Add(request);
            }

            if (!requests.Any())
            {
                return;
            }

            var depotsToDownload = new List <ManifestJob>();

            using (var db = Database.GetConnection())
            {
                var dbDepots = db.Query <Depot>("SELECT `DepotID`, `Name`, `BuildID`, `ManifestID`, `LastManifestID` FROM `Depots` WHERE `DepotID` IN @Depots", new { Depots = requests.Select(x => x.DepotID) })
                               .ToDictionary(x => x.DepotID, x => x);

                foreach (var request in requests)
                {
                    Depot dbDepot;

                    if (dbDepots.ContainsKey(request.DepotID))
                    {
                        dbDepot = dbDepots[request.DepotID];

                        if (dbDepot.BuildID > request.BuildID)
                        {
                            // buildid went back in time? this either means a rollback, or a shared depot that isn't synced properly

                            Log.WriteDebug("Depot Processor", "Skipping depot {0} due to old buildid: {1} > {2}", request.DepotID, dbDepot.BuildID, request.BuildID);

                            continue;
                        }

                        if (dbDepot.LastManifestID == request.ManifestID && dbDepot.ManifestID == request.ManifestID && Settings.Current.FullRun <= FullRunState.Normal)
                        {
                            // Update depot name if changed
                            if (!request.DepotName.Equals(dbDepot.Name))
                            {
                                db.Execute("UPDATE `Depots` SET `Name` = @DepotName WHERE `DepotID` = @DepotID", new { request.DepotID, request.DepotName });
                            }

                            continue;
                        }
                    }
                    else
                    {
                        dbDepot = new Depot();
                    }

                    if (dbDepot.BuildID != request.BuildID || dbDepot.ManifestID != request.ManifestID || !request.DepotName.Equals(dbDepot.Name))
                    {
                        db.Execute(@"INSERT INTO `Depots` (`DepotID`, `Name`, `BuildID`, `ManifestID`) VALUES (@DepotID, @DepotName, @BuildID, @ManifestID)
                                    ON DUPLICATE KEY UPDATE `LastUpdated` = unix_timestamp(), `Name` = VALUES(`Name`), `BuildID` = VALUES(`BuildID`), `ManifestID` = VALUES(`ManifestID`)",
                                   new {
                            request.DepotID,
                            request.DepotName,
                            request.BuildID,
                            request.ManifestID
                        });
                    }

                    if (dbDepot.ManifestID != request.ManifestID)
                    {
                        MakeHistory(db, request, string.Empty, "manifest_change", dbDepot.ManifestID, request.ManifestID);
                    }

                    var owned = LicenseList.OwnedApps.ContainsKey(request.DepotID);

                    if (!owned)
                    {
                        request.Anonymous = owned = LicenseList.AnonymousApps.ContainsKey(request.DepotID);

                        if (owned)
                        {
                            Log.WriteWarn("Depot Processor", "Will download depot {0} using anonymous account", request.DepotID);
                        }
                    }

                    if (owned || Settings.Current.FullRun == FullRunState.WithForcedDepots || Settings.Current.FullRun == FullRunState.ImportantOnly)
                    {
                        lock (DepotLocks)
                        {
                            // This doesn't really save us from concurrency issues
                            if (DepotLocks.ContainsKey(request.DepotID))
                            {
                                Log.WriteWarn("Depot Processor", "Depot {0} was locked in another thread", request.DepotID);
                                continue;
                            }

                            DepotLocks.Add(request.DepotID, 1);
                        }

                        depotsToDownload.Add(request);
                    }
#if DEBUG
                    else
                    {
                        Log.WriteDebug("Depot Processor", "Skipping depot {0} from app {1} because we don't own it", request.DepotID, appID);
                    }
#endif
                }
            }

            if (depotsToDownload.Any())
            {
                PICSProductInfo.ProcessorThreadPool.QueueWorkItem(async() =>
                {
                    try
                    {
                        await DownloadDepots(appID, depotsToDownload);
                    }
                    catch (Exception e)
                    {
                        ErrorReporter.Notify(e);
                    }

                    foreach (var depot in depotsToDownload)
                    {
                        RemoveLock(depot.DepotID);
                    }
                });
            }
        }
Пример #29
0
        public async Task Process(uint appID, uint changeNumber, KeyValue depots)
        {
            var requests = new List <ManifestJob>();

            // Get data in format we want first
            foreach (var depot in depots.Children)
            {
                // Ignore these for now, parent app should be updated too anyway
                if (depot["depotfromapp"].Value != null)
                {
                    continue;
                }

                var request = new ManifestJob
                {
                    ChangeNumber = changeNumber,
                    DepotName    = depot["name"].AsString()
                };

                // Ignore keys that aren't integers, for example "branches"
                if (!uint.TryParse(depot.Name, out request.DepotID))
                {
                    continue;
                }

                // TODO: instead of locking we could wait for current process to finish
                if (DepotLocks.ContainsKey(request.DepotID))
                {
                    continue;
                }

                if (depot["manifests"]["public"].Value == null || !ulong.TryParse(depot["manifests"]["public"].Value, out request.ManifestID))
                {
                    var branch = depot["manifests"].Children.FirstOrDefault(x => x.Name != "local");

                    if (branch == null || !ulong.TryParse(branch.Value, out request.ManifestID))
                    {
                        using (var db = Database.Get())
                        {
                            await db.ExecuteAsync("INSERT INTO `Depots` (`DepotID`, `Name`) VALUES (@DepotID, @DepotName) ON DUPLICATE KEY UPDATE `Name` = VALUES(`Name`)", new { request.DepotID, request.DepotName });
                        }

                        continue;
                    }

                    Log.WriteDebug("Depot Downloader", "Depot {0} (from {1}) has no public branch, but there is another one", request.DepotID, appID);

                    request.BuildID = depots["branches"][branch.Name]["buildid"].AsInteger();
                }
                else
                {
                    request.BuildID = depots["branches"]["public"]["buildid"].AsInteger();
                }

                requests.Add(request);
            }

            if (!requests.Any())
            {
                return;
            }

            var depotsToDownload = new List <ManifestJob>();

            using (var db = await Database.GetConnectionAsync())
            {
                var firstRequest = requests.First();
                await db.ExecuteAsync("INSERT INTO `Builds` (`BuildID`, `ChangeID`, `AppID`) VALUES (@BuildID, @ChangeNumber, @AppID) ON DUPLICATE KEY UPDATE `AppID` = VALUES(`AppID`)",
                                      new {
                    firstRequest.BuildID,
                    firstRequest.ChangeNumber,
                    appID
                });

                var dbDepots = (await db.QueryAsync <Depot>("SELECT `DepotID`, `Name`, `BuildID`, `ManifestID`, `LastManifestID` FROM `Depots` WHERE `DepotID` IN @Depots", new { Depots = requests.Select(x => x.DepotID) }))
                               .ToDictionary(x => x.DepotID, x => x);

                foreach (var request in requests)
                {
                    Depot dbDepot;

                    if (dbDepots.ContainsKey(request.DepotID))
                    {
                        dbDepot = dbDepots[request.DepotID];

                        if (dbDepot.BuildID > request.BuildID)
                        {
                            // buildid went back in time? this either means a rollback, or a shared depot that isn't synced properly

                            Log.WriteDebug("Depot Processor", "Skipping depot {0} due to old buildid: {1} > {2}", request.DepotID, dbDepot.BuildID, request.BuildID);

                            continue;
                        }

                        if (dbDepot.LastManifestID == request.ManifestID && dbDepot.ManifestID == request.ManifestID && Settings.Current.FullRun != FullRunState.WithForcedDepots)
                        {
                            // Update depot name if changed
                            if (!request.DepotName.Equals(dbDepot.Name))
                            {
                                await db.ExecuteAsync("UPDATE `Depots` SET `Name` = @DepotName WHERE `DepotID` = @DepotID", new { request.DepotID, request.DepotName });
                            }

                            continue;
                        }
                    }
                    else
                    {
                        dbDepot = new Depot();
                    }

                    if (dbDepot.BuildID != request.BuildID || dbDepot.ManifestID != request.ManifestID || !request.DepotName.Equals(dbDepot.Name))
                    {
                        await db.ExecuteAsync(@"INSERT INTO `Depots` (`DepotID`, `Name`, `BuildID`, `ManifestID`) VALUES (@DepotID, @DepotName, @BuildID, @ManifestID)
                                    ON DUPLICATE KEY UPDATE `LastUpdated` = CURRENT_TIMESTAMP(), `Name` = VALUES(`Name`), `BuildID` = VALUES(`BuildID`), `ManifestID` = VALUES(`ManifestID`)",
                                              new {
                            request.DepotID,
                            request.DepotName,
                            request.BuildID,
                            request.ManifestID
                        });
                    }

                    if (dbDepot.ManifestID != request.ManifestID)
                    {
                        await MakeHistory(db, null, request, string.Empty, "manifest_change", dbDepot.ManifestID, request.ManifestID);
                    }

                    var owned = LicenseList.OwnedApps.ContainsKey(request.DepotID);

                    if (!owned)
                    {
                        request.Anonymous = owned = LicenseList.AnonymousApps.ContainsKey(request.DepotID);

                        if (owned)
                        {
                            Log.WriteWarn("Depot Processor", "Will download depot {0} using anonymous account", request.DepotID);
                        }
                    }

                    if (owned)
                    {
                        lock (DepotLocks)
                        {
                            // This doesn't really save us from concurrency issues
                            if (DepotLocks.ContainsKey(request.DepotID))
                            {
                                Log.WriteWarn("Depot Processor", "Depot {0} was locked in another thread", request.DepotID);
                                continue;
                            }

                            DepotLocks.Add(request.DepotID, 1);
                        }

                        depotsToDownload.Add(request);
                    }
                }
            }

            if (depotsToDownload.Any())
            {
#pragma warning disable 4014
                if (FileDownloader.IsImportantDepot(appID) && !Settings.IsFullRun && !string.IsNullOrEmpty(Settings.Current.PatchnotesNotifyURL))
                {
                    TaskManager.Run(() => NotifyPatchnote(appID, depotsToDownload.First().BuildID));
                }

                TaskManager.Run(async() =>
                {
                    try
                    {
                        await DownloadDepots(appID, depotsToDownload);
                    }
                    catch (Exception e)
                    {
                        ErrorReporter.Notify("Depot Processor", e);
                    }

                    foreach (var depot in depotsToDownload)
                    {
                        RemoveLock(depot.DepotID);
                    }
                });
#pragma warning restore 4014
            }
        }
Пример #30
0
        private static async Task <EResult> ProcessDepotAfterDownload(IDbConnection db, IDbTransaction transaction, ManifestJob request, DepotManifest depotManifest)
        {
            var filesOld        = (await db.QueryAsync <DepotFile>("SELECT `ID`, `File`, `Hash`, `Size`, `Flags` FROM `DepotsFiles` WHERE `DepotID` = @DepotID", new { request.DepotID }, transaction: transaction)).ToDictionary(x => x.File, x => x);
            var filesAdded      = new List <DepotFile>();
            var shouldHistorize = filesOld.Count > 0; // Don't historize file additions if we didn't have any data before

            foreach (var file in depotManifest.Files.OrderByDescending(x => x.FileName))
            {
                var    name = file.FileName.Replace('\\', '/');
                byte[] hash = null;

                // Store empty hashes as NULL (e.g. an empty file)
                if (file.FileHash.Length > 0 && (file.Flags & EDepotFileFlag.Directory) == 0)
                {
                    for (var i = 0; i < file.FileHash.Length; ++i)
                    {
                        if (file.FileHash[i] != 0)
                        {
                            hash = file.FileHash;
                            break;
                        }
                    }
                }

                // Limit path names to 260 characters (default windows max length)
                // File column is varchar(260) and not higher to prevent reducing performance
                // See https://stackoverflow.com/questions/1962310/importance-of-varchar-length-in-mysql-table/1962329#1962329
                // Until 2019 there hasn't been a single file that went over this limit, so far there has been only one
                // game with a big node_modules path, so we're safeguarding by limiting it.
                if (name.Length > 260)
                {
                    using var sha = new System.Security.Cryptography.SHA1Managed();
                    var nameHash = Utils.ByteArrayToString(sha.ComputeHash(System.Text.Encoding.UTF8.GetBytes(name)));
                    name = $"{{SteamDB file name is too long}}/{nameHash}/...{name.Substring(name.Length - 150)}";
                }

                if (filesOld.ContainsKey(name))
                {
                    var oldFile    = filesOld[name];
                    var updateFile = false;

                    if (oldFile.Size != file.TotalSize || !Utils.IsEqualSHA1(hash, oldFile.Hash))
                    {
                        await MakeHistory(db, transaction, request, name, "modified", oldFile.Size, file.TotalSize);

                        updateFile = true;
                    }

                    if (oldFile.Flags != file.Flags)
                    {
                        await MakeHistory(db, transaction, request, name, "modified_flags", (ulong)oldFile.Flags, (ulong)file.Flags);

                        updateFile = true;
                    }

                    if (updateFile)
                    {
                        await db.ExecuteAsync("UPDATE `DepotsFiles` SET `Hash` = @Hash, `Size` = @Size, `Flags` = @Flags WHERE `DepotID` = @DepotID AND `ID` = @ID", new DepotFile
                        {
                            ID      = oldFile.ID,
                            DepotID = request.DepotID,
                            Hash    = hash,
                            Size    = file.TotalSize,
                            Flags   = file.Flags
                        }, transaction : transaction);
                    }

                    filesOld.Remove(name);
                }
                else
                {
                    // We want to historize modifications first, and only then deletions and additions
                    filesAdded.Add(new DepotFile
                    {
                        DepotID = request.DepotID,
                        Hash    = hash,
                        File    = name,
                        Size    = file.TotalSize,
                        Flags   = file.Flags
                    });
                }
            }

            if (filesOld.Count > 0)
            {
                await db.ExecuteAsync("DELETE FROM `DepotsFiles` WHERE `DepotID` = @DepotID AND `ID` IN @Files", new { request.DepotID, Files = filesOld.Select(x => x.Value.ID) }, transaction : transaction);

                await db.ExecuteAsync(HistoryQuery, filesOld.Select(x => new DepotHistory
                {
                    DepotID    = request.DepotID,
                    ManifestID = request.ManifestID,
                    ChangeID   = request.ChangeNumber,
                    Action     = "removed",
                    File       = x.Value.File,
                    OldValue   = x.Value.Size
                }), transaction : transaction);
            }

            if (filesAdded.Count > 0)
            {
                await db.ExecuteAsync("INSERT INTO `DepotsFiles` (`DepotID`, `File`, `Hash`, `Size`, `Flags`) VALUES (@DepotID, @File, @Hash, @Size, @Flags)", filesAdded, transaction : transaction);

                if (shouldHistorize)
                {
                    await db.ExecuteAsync(HistoryQuery, filesAdded.Select(x => new DepotHistory
                    {
                        DepotID    = request.DepotID,
                        ManifestID = request.ManifestID,
                        ChangeID   = request.ChangeNumber,
                        Action     = "added",
                        File       = x.File,
                        NewValue   = x.Size
                    }), transaction : transaction);
                }
            }

            await db.ExecuteAsync("UPDATE `Depots` SET `LastManifestID` = @ManifestID, `LastUpdated` = CURRENT_TIMESTAMP() WHERE `DepotID` = @DepotID", new { request.DepotID, request.ManifestID }, transaction : transaction);

            return(EResult.OK);
        }