private static void DownloadManifest(ManifestJob request) { CDNClient cdnClient = new CDNClient(Steam.Instance.Client, request.DepotID, request.Ticket, request.DepotKey); List<CDNClient.Server> cdnServers; try { cdnServers = cdnClient.FetchServerList(); if (cdnServers.Count == 0) { throw new InvalidOperationException("No servers returned"); // Great programming! } } catch { Log.WriteError("Depot Processor", "Failed to get server list for depot {0}", request.DepotID); lock (ManifestJobs) { ManifestJobs.Remove(request); } return; } DepotManifest depotManifest = null; foreach (var server in cdnServers) { try { cdnClient.Connect(server); depotManifest = cdnClient.DownloadManifest(request.ManifestID); break; } catch { } } lock (ManifestJobs) { ManifestJobs.Remove(request); } if (depotManifest == null) { Log.WriteError("Depot Processor", "Failed to download depot manifest for depot {0} (parent {1}) (jobs still in queue: {2})", request.DepotID, request.ParentAppID, ManifestJobs.Count); if (SteamProxy.Instance.ImportantApps.Contains(request.ParentAppID)) { IRC.SendMain("Important manifest update: {0}{1}{2} {3}(parent {4}){5} -{6} manifest download failed from {7} servers", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.DARK_GRAY, request.ParentAppID, Colors.NORMAL, Colors.RED, cdnServers.Count); } return; } if (SteamProxy.Instance.ImportantApps.Contains(request.ParentAppID)) { IRC.SendMain("Important manifest update: {0}{1}{2} {3}(parent {4}){5} -{6} {7}", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.DARK_GRAY, request.ParentAppID, Colors.NORMAL, Colors.DARK_BLUE, SteamDB.GetDepotURL(request.DepotID, "history")); } var sortedFiles = depotManifest.Files.OrderBy(f => f.FileName, StringComparer.OrdinalIgnoreCase); bool shouldHistorize = false; List<DepotFile> filesNew = new List<DepotFile>(); List<DepotFile> filesOld = new List<DepotFile>(); foreach (var file in sortedFiles) { System.Text.Encoding.UTF8.GetString(file.FileHash); filesNew.Add(new DepotFile { Name = file.FileName.Replace('\\', '/'), Size = file.TotalSize, Chunks = file.Chunks.Count, Flags = (int)file.Flags }); } using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `Files` FROM `Depots` WHERE `DepotID` = @DepotID LIMIT 1", new MySqlParameter("DepotID", request.DepotID))) { if (Reader.Read()) { string files = Reader.GetString("Files"); if (!string.IsNullOrEmpty(files)) { shouldHistorize = true; filesOld = JsonConvert.DeserializeObject<List<DepotFile>>(files); } } } DbWorker.ExecuteNonQuery("UPDATE `Depots` SET `Files` = @Files WHERE `DepotID` = @DepotID", new MySqlParameter("@DepotID", request.DepotID), new MySqlParameter("@Files", JsonConvert.SerializeObject(filesNew, new JsonSerializerSettings { DefaultValueHandling = DefaultValueHandling.Ignore })) ); if (shouldHistorize) { List<string> filesAdded = new List<string>(); foreach (var file in filesNew) { var oldFile = filesOld.Find(x => x.Name == file.Name); if (oldFile == null) { // We want to historize modifications first, and only then deletions and additions filesAdded.Add(file.Name); } else { if (oldFile.Size != file.Size) { MakeHistory(request, file.Name, "modified", oldFile.Size, file.Size); } filesOld.Remove(oldFile); } } foreach (var file in filesOld) { MakeHistory(request, file.Name, "removed"); } foreach (string file in filesAdded) { MakeHistory(request, file, "added"); } } lock (ManifestJobs) { Log.WriteDebug("Depot Processor", "DepotID: Processed {0} (jobs still in queue: {1})", request.DepotID, ManifestJobs.Count); } // Our job is done here, now download important files if any if (Settings.Current.ImportantFiles.ContainsKey(request.DepotID)) { // TODO: Horribly inefficient var importantFiles = sortedFiles.Where(x => Settings.Current.ImportantFiles[request.DepotID].Contains(x.FileName.Replace('\\', '/'))); ThreadPool.QueueWorkItem(DownloadFiles, importantFiles, request.DepotID, cdnClient); } }
private static void DownloadManifest(ManifestJob request) { var cdnClient = new CDNClient(Steam.Instance.Client, request.DepotID, request.Ticket, request.DepotKey); List<CDNClient.Server> cdnServers = null; for (var i = 0; i <= 5; i++) { try { cdnServers = cdnClient.FetchServerList(); if (cdnServers.Count > 0) { break; } } catch { } } if (cdnServers == null || cdnServers.Count == 0) { Log.WriteError("Depot Processor", "Failed to get server list for depot {0}", request.DepotID); if (SteamProxy.Instance.ImportantApps.Contains(request.ParentAppID)) { IRC.SendMain("Important manifest update: {0}{1}{2} {3}(parent {4}){5} -{6} failed to fetch server list", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.DARK_GRAY, request.ParentAppID, Colors.NORMAL, Colors.RED); } lock (ManifestJobs) { ManifestJobs.Remove(request); } return; } DepotManifest depotManifest = null; foreach (var server in cdnServers) { try { cdnClient.Connect(server); depotManifest = cdnClient.DownloadManifest(request.ManifestID); break; } catch { } } lock (ManifestJobs) { ManifestJobs.Remove(request); } if (depotManifest == null) { Log.WriteError("Depot Processor", "Failed to download depot manifest for depot {0} (parent {1}) (jobs still in queue: {2})", request.DepotID, request.ParentAppID, ManifestJobs.Count); if (SteamProxy.Instance.ImportantApps.Contains(request.ParentAppID)) { IRC.SendMain("Important manifest update: {0}{1}{2} {3}(parent {4}){5} -{6} failed to download depot manifest from {7} servers", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.DARK_GRAY, request.ParentAppID, Colors.NORMAL, Colors.RED, cdnServers.Count); } return; } if (SteamProxy.Instance.ImportantApps.Contains(request.ParentAppID)) { IRC.SendMain("Important manifest update: {0}{1}{2} {3}(parent {4}){5} -{6} {7}", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.DARK_GRAY, request.ParentAppID, Colors.NORMAL, Colors.DARK_BLUE, SteamDB.GetDepotURL(request.DepotID, "history")); } var sortedFiles = depotManifest.Files.OrderBy(f => f.FileName, StringComparer.OrdinalIgnoreCase); bool shouldHistorize = false; var filesNew = new List<DepotFile>(); var filesOld = new List<DepotFile>(); foreach (var file in sortedFiles) { System.Text.Encoding.UTF8.GetString(file.FileHash); var depotFile = new DepotFile { Name = file.FileName.Replace('\\', '/'), Size = file.TotalSize, Chunks = file.Chunks.Count, Flags = (int)file.Flags }; // TODO: Ideally we would check if filehash is not empty if (!file.Flags.HasFlag(EDepotFileFlag.Directory)) { depotFile.Hash = string.Concat(Array.ConvertAll(file.FileHash, x => x.ToString("X2"))); } filesNew.Add(depotFile); } using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `Files` FROM `Depots` WHERE `DepotID` = @DepotID LIMIT 1", new MySqlParameter("DepotID", request.DepotID))) { if (Reader.Read()) { string files = Reader.GetString("Files"); if (!string.IsNullOrEmpty(files)) { shouldHistorize = true; filesOld = JsonConvert.DeserializeObject<List<DepotFile>>(files); } } } DbWorker.ExecuteNonQuery("UPDATE `Depots` SET `Files` = @Files WHERE `DepotID` = @DepotID", new MySqlParameter("@DepotID", request.DepotID), new MySqlParameter("@Files", JsonConvert.SerializeObject(filesNew, new JsonSerializerSettings { DefaultValueHandling = DefaultValueHandling.Ignore })) ); if (shouldHistorize) { var filesAdded = new List<string>(); foreach (var file in filesNew) { var oldFile = filesOld.Find(x => x.Name == file.Name); if (oldFile == null) { // We want to historize modifications first, and only then deletions and additions filesAdded.Add(file.Name); } else { if (oldFile.Size != file.Size) { MakeHistory(request, file.Name, "modified", oldFile.Size, file.Size); } else if (file.Hash != null && oldFile.Hash != null && !file.Hash.Equals(oldFile.Hash)) { MakeHistory(request, file.Name, "modified", oldFile.Size, file.Size); } filesOld.Remove(oldFile); } } foreach (var file in filesOld) { MakeHistory(request, file.Name, "removed"); } foreach (string file in filesAdded) { MakeHistory(request, file, "added"); } } lock (ManifestJobs) { Log.WriteDebug("Depot Processor", "DepotID: Processed {0} (jobs still in queue: {1})", request.DepotID, ManifestJobs.Count); } }