/// <summary> /// Downloads the depot manifest specified by the given manifest ID, and optionally decrypts the manifest's filenames if the depot decryption key has been provided. /// </summary> /// <param name="depotId">The id of the depot being accessed.</param> /// <param name="manifestId">The unique identifier of the manifest to be downloaded.</param> /// <param name="manifestRequestCode">The manifest request code for the manifest that is being downloaded.</param> /// <param name="server">The content server to connect to.</param> /// <param name="depotKey"> /// The depot decryption key for the depot that will be downloaded. /// This is used for decrypting filenames (if needed) in depot manifests, and processing depot chunks. /// </param> /// <param name="proxyServer">Optional content server marked as UseAsProxy which transforms the request.</param> /// <returns>A <see cref="DepotManifest"/> instance that contains information about the files present within a depot.</returns> /// <exception cref="System.ArgumentNullException"><see ref="server"/> was null.</exception> /// <exception cref="HttpRequestException">An network error occurred when performing the request.</exception> /// <exception cref="SteamKitWebRequestException">A network error occurred when performing the request.</exception> public async Task <DepotManifest> DownloadManifestAsync(uint depotId, ulong manifestId, ulong manifestRequestCode, Server server, byte[]?depotKey = null, Server?proxyServer = null) { if (server == null) { throw new ArgumentNullException(nameof(server)); } const uint MANIFEST_VERSION = 5; string url; if (manifestRequestCode > 0) { url = $"depot/{depotId}/manifest/{manifestId}/{MANIFEST_VERSION}/{manifestRequestCode}"; } else { url = $"depot/{depotId}/manifest/{manifestId}/{MANIFEST_VERSION}"; } var manifestData = await DoRawCommandAsync(server, url, proxyServer).ConfigureAwait(false); manifestData = ZipUtil.Decompress(manifestData); var depotManifest = new DepotManifest(manifestData); if (depotKey != null) { // if we have the depot key, decrypt the manifest filenames depotManifest.DecryptFilenames(depotKey); } return(depotManifest); }
private async Task <EResult> ProcessDepotAfterDownload(ManifestJob request, DepotManifest depotManifest) { using (var db = await Database.GetConnectionAsync()) using (var transaction = await db.BeginTransactionAsync()) { var result = await ProcessDepotAfterDownload(db, transaction, request, depotManifest); await transaction.CommitAsync(); return(result); } }
private static async Task <DepotManifest> DownloadManifest(uint appId, uint depotId, ulong manifestId, byte[] depotKey) { DepotManifest depotManifest = null; while (depotManifest == null) { Tuple <CDNClient.Server, string> connection = null; try { connection = await cdnPool.GetConnectionForDepot(appId, depotId, CancellationToken.None); depotManifest = await cdnPool.CDNClient.DownloadManifestAsync(depotId, manifestId, connection.Item1, connection.Item2, depotKey).ConfigureAwait(false); cdnPool.ReturnConnection(connection); } catch (SteamKitWebRequestException e) { cdnPool.ReturnBrokenConnection(connection); if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden) { Console.WriteLine("Encountered 401 for depot manifest {0} {1}. Aborting.", depotId, manifestId); break; } else { Console.WriteLine("Encountered error downloading depot manifest {0} {1}: {2}", depotId, manifestId, e.StatusCode); } } catch (Exception e) { cdnPool.ReturnBrokenConnection(connection); Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depotId, manifestId, e.Message); } } if (depotManifest == null) { Console.WriteLine("\nUnable to download manifest {0} for depot {1}", manifestId, depotId); return(null); } return(depotManifest); }
private static void HandleManifest() { if (!string.IsNullOrEmpty(inputFile) && File.Exists(inputFile)) { byte[] fileBytes = File.ReadAllBytes(inputFile); BindingFlags flags = BindingFlags.NonPublic | BindingFlags.Instance; DepotManifest depotManifest = null; try { // DepotManifest constructor is internal, so use Activator to call it depotManifest = (DepotManifest)Activator.CreateInstance(typeof(DepotManifest), flags, null, new object[] { fileBytes }, System.Globalization.CultureInfo.InvariantCulture); } catch (Exception ex) { Console.WriteLine("Something bad happened! Exception message:\r\n{0}", ex.Message); } if (depotManifest != null) { FCIVExport exporter = new FCIVExport(outputFile); foreach (DepotManifest.FileData filedata in depotManifest.Files) { bool isDir = filedata.Flags == EDepotFileFlag.Directory; string strHash = filedata.FileHash.Aggregate(new StringBuilder(), (sb, v) => sb.Append(v.ToString("x2"))).ToString().ToUpper(); if (string.IsNullOrEmpty(outputFile)) { Console.WriteLine("{0} {1}", filedata.FileName, isDir ? "" : strHash); } if (!isDir) { exporter.AddEntry(filedata.FileName, strHash); } } exporter.Finalize(); } } else { Console.WriteLine("File {0} doesn't exists!", inputFile); } }
private static void DownloadSteam3(uint appId, List <DepotDownloadInfo> depots) { ulong TotalBytesCompressed = 0; ulong TotalBytesUncompressed = 0; foreach (var depot in depots) { ulong DepotBytesCompressed = 0; ulong DepotBytesUncompressed = 0; Console.WriteLine("Downloading depot {0} - {1}", depot.id, depot.contentName); CancellationTokenSource cts = new CancellationTokenSource(); ProtoManifest oldProtoManifest = null; ProtoManifest newProtoManifest = null; string configDir = Path.Combine(depot.installDir, CONFIG_DIR); ulong lastManifestId = INVALID_MANIFEST_ID; ConfigStore.TheConfig.LastManifests.TryGetValue(depot.id, out lastManifestId); // In case we have an early exit, this will force equiv of verifyall next run. ConfigStore.TheConfig.LastManifests[depot.id] = INVALID_MANIFEST_ID; ConfigStore.Save(); if (lastManifestId != INVALID_MANIFEST_ID) { var oldManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", lastManifestId)); if (File.Exists(oldManifestFileName)) { oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName); } } if (lastManifestId == depot.manifestId && oldProtoManifest != null) { newProtoManifest = oldProtoManifest; Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id); } else { var newManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", depot.manifestId)); if (newManifestFileName != null) { newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName); } if (newProtoManifest != null) { Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id); } else { Console.Write("Downloading depot manifest..."); DepotManifest depotManifest = null; while (depotManifest == null) { CDNClient client = null; try { client = cdnPool.GetConnectionForDepot(appId, depot.id, depot.depotKey, CancellationToken.None); depotManifest = client.DownloadManifest(depot.id, depot.manifestId); cdnPool.ReturnConnection(client); } catch (WebException e) { cdnPool.ReturnBrokenConnection(client); if (e.Status == WebExceptionStatus.ProtocolError) { var response = e.Response as HttpWebResponse; if (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden) { Console.WriteLine("Encountered 401 for depot manifest {0} {1}. Aborting.", depot.id, depot.manifestId); break; } else { Console.WriteLine("Encountered error downloading depot manifest {0} {1}: {2}", depot.id, depot.manifestId, response.StatusCode); } } else { Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depot.id, depot.manifestId, e.Status); } } catch (Exception e) { cdnPool.ReturnBrokenConnection(client); Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depot.id, depot.manifestId, e.Message); } } if (depotManifest == null) { Console.WriteLine("\nUnable to download manifest {0} for depot {1}", depot.manifestId, depot.id); return; } newProtoManifest = new ProtoManifest(depotManifest, depot.manifestId); newProtoManifest.SaveToFile(newManifestFileName); Console.WriteLine("Successfully downloaded manifest {0} for depot {1}", depot.manifestId, depot.id); Console.WriteLine(" Done!"); } } newProtoManifest.Files.Sort((x, y) => { return(x.FileName.CompareTo(y.FileName)); }); if (Config.DownloadManifestOnly) { StringBuilder manifestBuilder = new StringBuilder(); string txtManifest = Path.Combine(depot.installDir, string.Format("manifest_{0}.txt", depot.id)); foreach (var file in newProtoManifest.Files) { if (file.Flags.HasFlag(EDepotFileFlag.Directory)) { continue; } manifestBuilder.Append(string.Format("{0}\n", file.FileName)); } File.WriteAllText(txtManifest, manifestBuilder.ToString()); continue; } ulong complete_download_size = 0; ulong size_downloaded = 0; string stagingDir = Path.Combine(depot.installDir, STAGING_DIR); var filesAfterExclusions = newProtoManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).ToList(); // Pre-process filesAfterExclusions.ForEach(file => { var fileFinalPath = Path.Combine(depot.installDir, file.FileName); var fileStagingPath = Path.Combine(stagingDir, file.FileName); if (file.Flags.HasFlag(EDepotFileFlag.Directory)) { Directory.CreateDirectory(fileFinalPath); Directory.CreateDirectory(fileStagingPath); } else { // Some manifests don't explicitly include all necessary directories Directory.CreateDirectory(Path.GetDirectoryName(fileFinalPath)); Directory.CreateDirectory(Path.GetDirectoryName(fileStagingPath)); complete_download_size += file.TotalSize; } }); filesAfterExclusions.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory)) .AsParallel().WithCancellation(cts.Token).WithDegreeOfParallelism(Config.MaxDownloads) .ForAll(file => { string fileFinalPath = Path.Combine(depot.installDir, file.FileName); string fileStagingPath = Path.Combine(stagingDir, file.FileName); // This may still exist if the previous run exited before cleanup if (File.Exists(fileStagingPath)) { File.Delete(fileStagingPath); } FileStream fs = null; List <ProtoManifest.ChunkData> neededChunks; FileInfo fi = new FileInfo(fileFinalPath); if (!fi.Exists) { // create new file. need all chunks fs = File.Create(fileFinalPath); fs.SetLength(( long )file.TotalSize); neededChunks = new List <ProtoManifest.ChunkData>(file.Chunks); } else { // open existing ProtoManifest.FileData oldManifestFile = null; if (oldProtoManifest != null) { oldManifestFile = oldProtoManifest.Files.SingleOrDefault(f => f.FileName == file.FileName); } if (oldManifestFile != null) { neededChunks = new List <ProtoManifest.ChunkData>(); if (Config.VerifyAll || !oldManifestFile.FileHash.SequenceEqual(file.FileHash)) { // we have a version of this file, but it doesn't fully match what we want var matchingChunks = new List <ChunkMatch>(); foreach (var chunk in file.Chunks) { var oldChunk = oldManifestFile.Chunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID)); if (oldChunk != null) { matchingChunks.Add(new ChunkMatch(oldChunk, chunk)); } else { neededChunks.Add(chunk); } } File.Move(fileFinalPath, fileStagingPath); fs = File.Open(fileFinalPath, FileMode.Create); fs.SetLength(( long )file.TotalSize); using (var fsOld = File.Open(fileStagingPath, FileMode.Open)) { foreach (var match in matchingChunks) { fsOld.Seek(( long )match.OldChunk.Offset, SeekOrigin.Begin); byte[] tmp = new byte[match.OldChunk.UncompressedLength]; fsOld.Read(tmp, 0, tmp.Length); byte[] adler = Util.AdlerHash(tmp); if (!adler.SequenceEqual(match.OldChunk.Checksum)) { neededChunks.Add(match.NewChunk); } else { fs.Seek(( long )match.NewChunk.Offset, SeekOrigin.Begin); fs.Write(tmp, 0, tmp.Length); } } } File.Delete(fileStagingPath); } } else { // No old manifest or file not in old manifest. We must validate. fs = File.Open(fileFinalPath, FileMode.Open); if (( ulong )fi.Length != file.TotalSize) { fs.SetLength(( long )file.TotalSize); } neededChunks = Util.ValidateSteam3FileChecksums(fs, file.Chunks.OrderBy(x => x.Offset).ToArray()); } if (neededChunks.Count() == 0) { size_downloaded += file.TotalSize; Console.WriteLine("{0,6:#00.00}% {1}", (( float )size_downloaded / ( float )complete_download_size) * 100.0f, fileFinalPath); if (fs != null) { fs.Close(); } return; } else { size_downloaded += (file.TotalSize - ( ulong )neededChunks.Select(x => ( long )x.UncompressedLength).Sum()); } } foreach (var chunk in neededChunks) { if (cts.IsCancellationRequested) { break; } string chunkID = Util.EncodeHexString(chunk.ChunkID); CDNClient.DepotChunk chunkData = null; while (!cts.IsCancellationRequested) { CDNClient client; try { client = cdnPool.GetConnectionForDepot(appId, depot.id, depot.depotKey, cts.Token); } catch (OperationCanceledException) { break; } DepotManifest.ChunkData data = new DepotManifest.ChunkData(); data.ChunkID = chunk.ChunkID; data.Checksum = chunk.Checksum; data.Offset = chunk.Offset; data.CompressedLength = chunk.CompressedLength; data.UncompressedLength = chunk.UncompressedLength; try { chunkData = client.DownloadDepotChunk(depot.id, data); cdnPool.ReturnConnection(client); break; } catch (WebException e) { cdnPool.ReturnBrokenConnection(client); if (e.Status == WebExceptionStatus.ProtocolError) { var response = e.Response as HttpWebResponse; if (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden) { Console.WriteLine("Encountered 401 for chunk {0}. Aborting.", chunkID); cts.Cancel(); break; } else { Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, response.StatusCode); } } else { Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, e.Status); } } catch (Exception e) { cdnPool.ReturnBrokenConnection(client); Console.WriteLine("Encountered unexpected error downloading chunk {0}: {1}", chunkID, e.Message); } } if (chunkData == null) { Console.WriteLine("Failed to find any server with chunk {0} for depot {1}. Aborting.", chunkID, depot.id); return; } TotalBytesCompressed += chunk.CompressedLength; DepotBytesCompressed += chunk.CompressedLength; TotalBytesUncompressed += chunk.UncompressedLength; DepotBytesUncompressed += chunk.UncompressedLength; fs.Seek(( long )chunk.Offset, SeekOrigin.Begin); fs.Write(chunkData.Data, 0, chunkData.Data.Length); size_downloaded += chunk.UncompressedLength; } fs.Close(); Console.WriteLine("{0,6:#00.00}% {1}", (( float )size_downloaded / ( float )complete_download_size) * 100.0f, fileFinalPath); }); ConfigStore.TheConfig.LastManifests[depot.id] = depot.manifestId; ConfigStore.Save(); Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.id, DepotBytesCompressed, DepotBytesUncompressed); } Console.WriteLine("Total downloaded: {0} bytes ({1} bytes uncompressed) from {2} depots", TotalBytesCompressed, TotalBytesUncompressed, depots.Count); }
/* * Here be dragons. */ public static async Task <EResult> DownloadFilesFromDepot(DepotProcessor.ManifestJob job, DepotManifest depotManifest) { var files = depotManifest.Files.Where(x => IsFileNameMatching(job.DepotID, x.FileName)).ToList(); var downloadState = EResult.Fail; var hashesFile = Path.Combine(Application.Path, "files", ".support", "hashes", string.Format("{0}.json", job.DepotID)); ConcurrentDictionary <string, byte[]> hashes; if (File.Exists(hashesFile)) { hashes = JsonConvert.DeserializeObject <ConcurrentDictionary <string, byte[]> >(File.ReadAllText(hashesFile)); } else { hashes = new ConcurrentDictionary <string, byte[]>(); } foreach (var file in hashes.Keys.Except(files.Select(x => x.FileName))) { Log.WriteWarn(nameof(FileDownloader), $"\"{file}\" no longer exists in manifest"); } Log.WriteInfo("FileDownloader", "Will download {0} files from depot {1}", files.Count, job.DepotID); var downloadedFiles = 0; var fileTasks = new Task[files.Count]; for (var i = 0; i < fileTasks.Length; i++) { var file = files[i]; fileTasks[i] = TaskManager.Run(async() => { hashes.TryGetValue(file.FileName, out var hash); var fileState = await DownloadFile(job, file, hash); if (fileState == EResult.OK || fileState == EResult.SameAsPreviousValue) { hashes[file.FileName] = file.FileHash; downloadedFiles++; } if (fileState != EResult.SameAsPreviousValue) { // Do not write progress info to log file Console.WriteLine("{1} [{0,6:#00.00}%] {2} files left to download", downloadedFiles / (float)files.Count * 100.0f, job.DepotName, files.Count - downloadedFiles); } if (downloadState == EResult.DataCorruption) { return; } if (fileState == EResult.OK || fileState == EResult.DataCorruption) { downloadState = fileState; } }).Unwrap(); // Register error handler on inner task TaskManager.RegisterErrorHandler(fileTasks[i]); } await Task.WhenAll(fileTasks).ConfigureAwait(false); if (downloadState == EResult.OK) { File.WriteAllText(hashesFile, JsonConvert.SerializeObject(hashes)); job.Result = EResult.OK; } else { job.Result = EResult.Ignored; } return(job.Result); }
private async Task DownloadDepots(uint appID, List <ManifestJob> depots) { Log.WriteDebug("Depot Downloader", "Will process {0} depots ({1} depot locks left)", depots.Count(), DepotLocks.Count); var processTasks = new List <Task <EResult> >(); bool anyFilesDownloaded = false; foreach (var depot in depots) { var instance = depot.Anonymous ? Steam.Anonymous.Apps : Steam.Instance.Apps; depot.DepotKey = await GetDepotDecryptionKey(instance, depot.DepotID, appID); if (depot.DepotKey == null) { RemoveLock(depot.DepotID); continue; } var cdnToken = await GetCDNAuthToken(instance, appID, depot.DepotID); if (cdnToken == null) { RemoveLock(depot.DepotID); Log.WriteDebug("Depot Downloader", "Got a depot key for depot {0} but no cdn auth token", depot.DepotID); continue; } depot.CDNToken = cdnToken.Token; depot.Server = cdnToken.Server; DepotManifest depotManifest = null; string lastError = string.Empty; for (var i = 0; i <= 5; i++) { try { depotManifest = CDNClient.DownloadManifest(depot.DepotID, depot.ManifestID, depot.Server, depot.CDNToken, depot.DepotKey); break; } catch (Exception e) { Log.WriteWarn("Depot Downloader", "[{0}] Manifest download failed: {1} - {2}", depot.DepotID, e.GetType(), e.Message); lastError = e.Message; } // TODO: get new auth key if auth fails if (depotManifest == null) { await Task.Delay(Utils.ExponentionalBackoff(i)); } } if (depotManifest == null) { RemoveLock(depot.DepotID); Log.WriteError("Depot Processor", "Failed to download depot manifest for app {0} depot {1} ({2}: {3})", appID, depot.DepotID, depot.Server, lastError); if (FileDownloader.IsImportantDepot(depot.DepotID)) { IRC.Instance.SendOps("{0}[{1}]{2} Failed to download app {3} depot {4} manifest ({5}: {6})", Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL, appID, depot.DepotID, depot.Server, lastError); } continue; } var task = TaskManager.Run(() => { using (var db = Database.GetConnection()) { using (var transaction = db.BeginTransaction()) { var result = ProcessDepotAfterDownload(db, depot, depotManifest); transaction.Commit(); return(result); } } }); processTasks.Add(task); if (FileDownloader.IsImportantDepot(depot.DepotID)) { task = TaskManager.Run(() => { var result = FileDownloader.DownloadFilesFromDepot(appID, depot, depotManifest); if (result == EResult.OK) { anyFilesDownloaded = true; } return(result); }, TaskCreationOptions.LongRunning); TaskManager.RegisterErrorHandler(task); processTasks.Add(task); } } if (SaveLocalConfig) { SaveLocalConfig = false; LocalConfig.Save(); } await Task.WhenAll(processTasks); Log.WriteDebug("Depot Downloader", "{0} depot downloads finished", depots.Count()); // TODO: use ContinueWith on tasks if (!anyFilesDownloaded) { foreach (var depot in depots) { RemoveLock(depot.DepotID); } return; } if (!File.Exists(UpdateScript)) { return; } bool lockTaken = false; try { UpdateScriptLock.Enter(ref lockTaken); foreach (var depot in depots) { if (depot.Result == EResult.OK) { RunUpdateScript(string.Format("{0} no-git", depot.DepotID)); } RemoveLock(depot.DepotID); } // Only commit changes if all depots downloaded if (processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored)) { if (!RunUpdateScript(appID)) { RunUpdateScript("0"); } } else { Log.WriteDebug("Depot Processor", "Reprocessing the app {0} because some files failed to download", appID); IRC.Instance.SendOps("{0}[{1}]{2} Reprocessing the app due to download failures", Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL ); JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null)); } } finally { if (lockTaken) { UpdateScriptLock.Exit(); } } }
private async Task DownloadDepots(uint appID, List <ManifestJob> depots) { Log.WriteDebug(nameof(DepotProcessor), $"Will process {depots.Count} depots from app {appID} ({DepotLocks.Count} depot locks left)"); var processTasks = new List <Task <(uint DepotID, EResult Result)> >(); var anyFilesDownloaded = false; var willDownloadFiles = false; foreach (var depot in depots) { if (depot.DepotKey == null) { await GetDepotDecryptionKey(Steam.Instance.Apps, depot, appID); if (depot.DepotKey == null && depot.LastManifestID == depot.ManifestID && Settings.FullRun != FullRunState.WithForcedDepots) { RemoveLock(depot.DepotID); continue; } } depot.Server = GetContentServer(); DepotManifest depotManifest = null; var lastError = string.Empty; for (var i = 0; i <= 5; i++) { try { await ManifestDownloadSemaphore.WaitAsync(TaskManager.TaskCancellationToken.Token).ConfigureAwait(false); depotManifest = await CDNClient.DownloadManifestAsync(depot.DepotID, depot.ManifestID, depot.Server, string.Empty, depot.DepotKey); break; } catch (Exception e) { lastError = e.Message; Log.WriteError(nameof(DepotProcessor), $"Failed to download depot manifest for app {appID} depot {depot.DepotID} ({depot.Server}: {lastError}) (#{i})"); } finally { ManifestDownloadSemaphore.Release(); } if (depot.DepotKey != null) { RemoveErroredServer(depot.Server); } if (depotManifest == null && i < 5) { await Task.Delay(Utils.ExponentionalBackoff(i + 1)); depot.Server = GetContentServer(); } } if (depotManifest == null) { RemoveLock(depot.DepotID); if (FileDownloader.IsImportantDepot(depot.DepotID)) { IRC.Instance.SendOps($"{Colors.OLIVE}[{depot.DepotName}]{Colors.NORMAL} Failed to download manifest ({lastError})"); } if (!Settings.IsFullRun && depot.DepotKey != null) { JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null)); } continue; } var task = ProcessDepotAfterDownload(depot, depotManifest); processTasks.Add(task); if (!FileDownloader.IsImportantDepot(depot.DepotID) || depot.DepotKey == null) { depot.Result = EResult.Ignored; continue; } willDownloadFiles = true; task = TaskManager.Run(async() => { var result = EResult.Fail; try { result = await FileDownloader.DownloadFilesFromDepot(depot, depotManifest); if (result == EResult.OK) { anyFilesDownloaded = true; } } catch (Exception e) { ErrorReporter.Notify($"Depot Processor {depot.DepotID}", e); } return(depot.DepotID, result); }); processTasks.Add(task); } if (!anyFilesDownloaded && !willDownloadFiles) { foreach (var task in processTasks) { _ = task.ContinueWith(result => { RemoveLock(result.Result.DepotID); }, TaskManager.TaskCancellationToken.Token); } await Task.WhenAll(processTasks).ConfigureAwait(false); return; } await Task.WhenAll(processTasks).ConfigureAwait(false); Log.WriteDebug(nameof(DepotProcessor), $"{depots.Count} depot downloads finished for app {appID}"); lock (UpdateScriptLock) { foreach (var depot in depots) { if (depot.Result == EResult.OK) { RunUpdateScript(UpdateScript, $"{depot.DepotID} no-git"); } else if (depot.Result != EResult.Ignored) { Log.WriteWarn(nameof(DepotProcessor), $"Download failed for {depot.DepotID}: {depot.Result}"); RemoveErroredServer(depot.Server); // Mark this depot for redownload using var db = Database.Get(); db.Execute("UPDATE `Depots` SET `LastManifestID` = 0 WHERE `DepotID` = @DepotID", new { depot.DepotID }); } RemoveLock(depot.DepotID); } // Only commit changes if all depots downloaded if (processTasks.All(x => x.Result.Result == EResult.OK || x.Result.Result == EResult.Ignored)) { if (!RunUpdateScriptForApp(appID, depots[0].BuildID)) { RunUpdateScript(UpdateScript, "0"); } } else { Log.WriteDebug(nameof(DepotProcessor), $"Reprocessing the app {appID} because some files failed to download"); IRC.Instance.SendOps($"{Colors.OLIVE}[{Steam.GetAppName(appID)}]{Colors.NORMAL} Reprocessing the app due to download failures"); JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null)); } } }
private static async Task <(uint, EResult)> ProcessDepotAfterDownload(ManifestJob request, DepotManifest depotManifest) { if (depotManifest.FilenamesEncrypted && request.DepotKey != null) { Log.WriteError(nameof(DepotProcessor), $"Depot key for depot {request.DepotID} is invalid?"); IRC.Instance.SendOps($"[Tokens] Looks like the depot key for depot {request.DepotID} is invalid"); } await using var db = await Database.GetConnectionAsync(); await using var transaction = await db.BeginTransactionAsync(); var result = await ProcessDepotAfterDownload(db, transaction, request, depotManifest); await transaction.CommitAsync(); return(request.DepotID, result); }
private static void DownloadManifest(ManifestJob request) { DepotManifest depotManifest = null; string lastError = string.Empty; // CDN is very random, just keep trying for (var i = 0; i <= 5; i++) { try { depotManifest = CDNClient.DownloadManifest(request.DepotID, request.ManifestID, request.Server, request.DepotKey, request.CDNToken); break; } catch (Exception e) { lastError = e.Message; } } if (depotManifest == null) { Log.WriteError("Depot Processor", "Failed to download depot manifest for depot {0} (jobs still in queue: {1}) ({2}: {3})", request.DepotID, ManifestJobs.Count - 1, request.Server, lastError); if (SteamProxy.Instance.ImportantApps.Contains(request.ParentAppID)) { IRC.SendMain("Important depot update: {0}{1}{2} -{3} failed to download depot manifest", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.RED); } return; } if (SteamProxy.Instance.ImportantApps.Contains(request.ParentAppID)) { IRC.SendMain("Important depot update: {0}{1}{2} -{3} {4}", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.DARK_BLUE, SteamDB.GetDepotURL(request.DepotID, "history")); } var sortedFiles = depotManifest.Files.OrderBy(f => f.FileName, StringComparer.OrdinalIgnoreCase); bool shouldHistorize = false; var filesNew = new List <DepotFile>(); var filesOld = new Dictionary <string, DepotFile>(); foreach (var file in sortedFiles) { System.Text.Encoding.UTF8.GetString(file.FileHash); var depotFile = new DepotFile { Name = file.FileName.Replace('\\', '/'), Size = file.TotalSize, Chunks = file.Chunks.Count, Flags = (int)file.Flags }; // TODO: Ideally we would check if filehash is not empty if (!file.Flags.HasFlag(EDepotFileFlag.Directory)) { depotFile.Hash = string.Concat(Array.ConvertAll(file.FileHash, x => x.ToString("X2"))); } filesNew.Add(depotFile); } using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `Files` FROM `Depots` WHERE `DepotID` = @DepotID LIMIT 1", new MySqlParameter("DepotID", request.DepotID))) { if (Reader.Read()) { string files = Reader.GetString("Files"); if (!string.IsNullOrEmpty(files)) { shouldHistorize = true; var _filesOld = JsonConvert.DeserializeObject <List <DepotFile> >(files); filesOld = _filesOld.ToDictionary(x => x.Name); } } } DbWorker.ExecuteNonQuery("UPDATE `Depots` SET `Files` = @Files WHERE `DepotID` = @DepotID", new MySqlParameter("@DepotID", request.DepotID), new MySqlParameter("@Files", JsonConvert.SerializeObject(filesNew, new JsonSerializerSettings { DefaultValueHandling = DefaultValueHandling.Ignore })) ); if (shouldHistorize) { var filesAdded = new List <string>(); foreach (var file in filesNew) { if (filesOld.ContainsKey(file.Name)) { var oldFile = filesOld[file.Name]; if (oldFile.Size != file.Size) { MakeHistory(request, file.Name, "modified", oldFile.Size, file.Size); } else if (file.Hash != null && oldFile.Hash != null && !file.Hash.Equals(oldFile.Hash)) { MakeHistory(request, file.Name, "modified", oldFile.Size, file.Size); } filesOld.Remove(file.Name); } else { // We want to historize modifications first, and only then deletions and additions filesAdded.Add(file.Name); } } foreach (var file in filesOld) { MakeHistory(request, file.Value.Name, "removed"); } foreach (string file in filesAdded) { MakeHistory(request, file, "added"); } } lock (ManifestJobs) { Log.WriteDebug("Depot Processor", "DepotID: Processed {0} (jobs still in queue: {1})", request.DepotID, ManifestJobs.Count - 1); } }
/* * Here be dragons. */ public static EResult DownloadFilesFromDepot(uint appID, DepotProcessor.ManifestJob job, DepotManifest depotManifest) { var files = depotManifest.Files.Where(x => IsFileNameMatching(job.DepotID, x.FileName)).ToList(); var filesUpdated = false; var filesAnyFailed = false; var hashesFile = Path.Combine(Application.Path, "files", ".support", "hashes", string.Format("{0}.json", job.DepotID)); var hashes = new Dictionary <string, byte[]>(); if (File.Exists(hashesFile)) { hashes = JsonConvert.DeserializeObject <Dictionary <string, byte[]> >(File.ReadAllText(hashesFile)); } Log.WriteDebug("FileDownloader", "Will download {0} files from depot {1}", files.Count, job.DepotID); Parallel.ForEach(files, new ParallelOptions { MaxDegreeOfParallelism = 2 }, (file, state2) => { string directory = Path.Combine(Application.Path, "files", DownloadFolders[job.DepotID], Path.GetDirectoryName(file.FileName)); string finalPath = Path.Combine(directory, Path.GetFileName(file.FileName)); string downloadPath = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".steamdb_tmp")); if (!Directory.Exists(directory)) { Directory.CreateDirectory(directory); } else if (file.TotalSize == 0) { if (File.Exists(finalPath)) { var f = new FileInfo(finalPath); if (f.Length == 0) { #if DEBUG Log.WriteDebug("FileDownloader", "{0} is already empty", file.FileName); #endif return; } } else { File.Create(finalPath); Log.WriteInfo("FileDownloader", "{0} created an empty file", file.FileName); return; } } else if (hashes.ContainsKey(file.FileName) && file.FileHash.SequenceEqual(hashes[file.FileName])) { #if DEBUG Log.WriteDebug("FileDownloader", "{0} already matches the file we have", file.FileName); #endif return; } var chunks = file.Chunks.OrderBy(x => x.Offset).ToList(); Log.WriteInfo("FileDownloader", "Downloading {0} ({1} bytes, {2} chunks)", file.FileName, file.TotalSize, chunks.Count); uint count = 0; byte[] checksum; string lastError = "or checksum failed"; string oldChunksFile; using (var sha = new SHA1Managed()) { oldChunksFile = Path.Combine(Application.Path, "files", ".support", "chunks", string.Format("{0}-{1}.json", job.DepotID, BitConverter.ToString(sha.ComputeHash(Encoding.UTF8.GetBytes(file.FileName)))) ); } using (var fs = File.Open(downloadPath, FileMode.OpenOrCreate, FileAccess.ReadWrite)) { fs.SetLength((long)file.TotalSize); var lockObject = new object(); var neededChunks = new List <DepotManifest.ChunkData>(); if (File.Exists(oldChunksFile) && File.Exists(finalPath)) { var oldChunks = JsonConvert.DeserializeObject <List <DepotManifest.ChunkData> >( File.ReadAllText(oldChunksFile), new JsonSerializerSettings { PreserveReferencesHandling = PreserveReferencesHandling.All } ); using (var fsOld = File.Open(finalPath, FileMode.Open, FileAccess.Read)) { foreach (var chunk in chunks) { var oldChunk = oldChunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID)); if (oldChunk != null) { var oldData = new byte[oldChunk.UncompressedLength]; fsOld.Seek((long)oldChunk.Offset, SeekOrigin.Begin); fsOld.Read(oldData, 0, oldData.Length); var existingChecksum = Utils.AdlerHash(oldData); if (existingChecksum.SequenceEqual(chunk.Checksum)) { fs.Seek((long)chunk.Offset, SeekOrigin.Begin); fs.Write(oldData, 0, oldData.Length); #if DEBUG Log.WriteDebug("FileDownloader", "{0} Found chunk ({1}), not downloading ({2}/{3})", file.FileName, chunk.Offset, ++count, chunks.Count); #endif } else { neededChunks.Add(chunk); #if DEBUG Log.WriteDebug("FileDownloader", "{0} Found chunk ({1}), but checksum differs", file.FileName, chunk.Offset); #endif } } else { neededChunks.Add(chunk); } } } } else { neededChunks = chunks; } Parallel.ForEach(neededChunks, new ParallelOptions { MaxDegreeOfParallelism = 3 }, (chunk, state) => { var downloaded = false; for (var i = 0; i <= 5; i++) { try { var chunkData = CDNClient.DownloadDepotChunk(job.DepotID, chunk, job.Server, job.CDNToken, job.DepotKey); lock (lockObject) { fs.Seek((long)chunk.Offset, SeekOrigin.Begin); fs.Write(chunkData.Data, 0, chunkData.Data.Length); Log.WriteDebug("FileDownloader", "Downloaded {0} ({1}/{2})", file.FileName, ++count, chunks.Count); } downloaded = true; break; } catch (Exception e) { lastError = e.Message; } Task.Delay(Utils.ExponentionalBackoff(i)).Wait(); } if (!downloaded) { state.Stop(); } }); fs.Seek(0, SeekOrigin.Begin); using (var sha = new SHA1Managed()) { checksum = sha.ComputeHash(fs); } } if (file.FileHash.SequenceEqual(checksum)) { Log.WriteInfo("FileDownloader", "Downloaded {0} from {1}", file.FileName, Steam.GetAppName(appID)); hashes[file.FileName] = checksum; if (File.Exists(finalPath)) { File.Delete(finalPath); } File.Move(downloadPath, finalPath); if (chunks.Count > 1) { File.WriteAllText(oldChunksFile, JsonConvert.SerializeObject( chunks, Formatting.None, new JsonSerializerSettings { PreserveReferencesHandling = PreserveReferencesHandling.All } ) ); } else if (File.Exists(oldChunksFile)) { File.Delete(oldChunksFile); } filesUpdated = true; } else { filesAnyFailed = true; IRC.Instance.SendOps("{0}[{1}]{2} Failed to download {3}: Only {4} out of {5} chunks downloaded ({6})", Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL, file.FileName, count, chunks.Count, lastError); Log.WriteError("FileDownloader", "Failed to download {0}: Only {1} out of {2} chunks downloaded from {3} ({4})", file.FileName, count, chunks.Count, job.Server, lastError); File.Delete(downloadPath); } }); if (filesAnyFailed) { using (var db = Database.GetConnection()) { // Mark this depot for redownload db.Execute("UPDATE `Depots` SET `LastManifestID` = 0 WHERE `DepotID` = @DepotID", new { job.DepotID }); } IRC.Instance.SendOps("{0}[{1}]{2} Failed to download some files, not running update script to prevent broken diffs.", Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL); } else if (filesUpdated) { File.WriteAllText(hashesFile, JsonConvert.SerializeObject(hashes)); job.Result = EResult.OK; } else { job.Result = EResult.Ignored; } return(job.Result); }
private static async Task <DepotFilesData> ProcessDepotManifestAndFiles(CancellationTokenSource cts, UInt32 appId, DepotDownloadInfo depot, CDNClientPool cdnPool) { DepotDownloadCounter depotCounter = new DepotDownloadCounter(); FileLog.LogMessage("Processing depot {0} - {1}", depot.id, depot.contentName); ProtoManifest oldProtoManifest = null; ProtoManifest newProtoManifest = null; string configDir = Path.Combine(depot.installDir, CONFIG_DIR); ulong lastManifestId = INVALID_MANIFEST_ID; //DepotConfigStore.Instance.InstalledManifestIDs.TryGetValue(depot.id, out lastManifestId); //// In case we have an early exit, this will force equiv of verifyall next run. //DepotConfigStore.Instance.InstalledManifestIDs[depot.id] = INVALID_MANIFEST_ID; //DepotConfigStore.Save(); //if (lastManifestId != INVALID_MANIFEST_ID) //{ // var oldManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.id, lastManifestId)); // if (File.Exists(oldManifestFileName)) // { // byte[] expectedChecksum, currentChecksum; // try // { // expectedChecksum = File.ReadAllBytes(oldManifestFileName + ".sha"); // } // catch (IOException) // { // expectedChecksum = null; // } // oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName, out currentChecksum); // if (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum)) // { // // We only have to show this warning if the old manifest ID was different // if (lastManifestId != depot.manifestId) // FileLog.LogMessage("Manifest {0} on disk did not match the expected checksum.", lastManifestId); // oldProtoManifest = null; // } // } //} if (lastManifestId == depot.manifestId && oldProtoManifest != null) { newProtoManifest = oldProtoManifest; FileLog.LogMessage("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id); } else { var newManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.id, depot.manifestId)); if (newManifestFileName != null) { byte[] expectedChecksum, currentChecksum; try { expectedChecksum = File.ReadAllBytes(newManifestFileName + ".sha"); } catch (IOException) { expectedChecksum = null; } newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName, out currentChecksum); if (newProtoManifest != null && (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum))) { FileLog.LogMessage("Manifest {0} on disk did not match the expected checksum.", depot.manifestId); newProtoManifest = null; } } if (newProtoManifest != null) { FileLog.LogMessage("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id); } else { Console.Write("Downloading depot manifest..."); DepotManifest depotManifest = null; do { cts.Token.ThrowIfCancellationRequested(); CDNClient.Server connection = null; try { connection = cdnPool.GetConnection(cts.Token); var cdnToken = await cdnPool.AuthenticateConnection(appId, depot.id, connection); #if STEAMKIT_UNRELEASED depotManifest = await cdnPool.CDNClient.DownloadManifestAsync(depot.id, depot.manifestId, connection, cdnToken, depot.depotKey, proxyServer : cdnPool.ProxyServer).ConfigureAwait(false); #else depotManifest = await cdnPool.CDNClient.DownloadManifestAsync(depot.id, depot.manifestId, connection, cdnToken, depot.depotKey).ConfigureAwait(false); #endif cdnPool.ReturnConnection(connection); } catch (TaskCanceledException) { FileLog.LogMessage("Connection timeout downloading depot manifest {0} {1}", depot.id, depot.manifestId); } catch (SteamKitWebRequestException e) { cdnPool.ReturnBrokenConnection(connection); if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden) { FileLog.LogMessage("Encountered 401 for depot manifest {0} {1}. Aborting.", depot.id, depot.manifestId); break; } else if (e.StatusCode == HttpStatusCode.NotFound) { FileLog.LogMessage("Encountered 404 for depot manifest {0} {1}. Aborting.", depot.id, depot.manifestId); break; } else { FileLog.LogMessage("Encountered error downloading depot manifest {0} {1}: {2}", depot.id, depot.manifestId, e.StatusCode); } } catch (OperationCanceledException) { break; } catch (Exception e) { cdnPool.ReturnBrokenConnection(connection); FileLog.LogMessage("Encountered error downloading manifest for depot {0} {1}: {2}", depot.id, depot.manifestId, e.Message); } }while (depotManifest == null); if (depotManifest == null) { FileLog.LogMessage("\nUnable to download manifest {0} for depot {1}", depot.manifestId, depot.id); cts.Cancel(); } // Throw the cancellation exception if requested so that this task is marked failed cts.Token.ThrowIfCancellationRequested(); byte[] checksum; newProtoManifest = new ProtoManifest(depotManifest, depot.manifestId); newProtoManifest.SaveToFile(newManifestFileName, out checksum); File.WriteAllBytes(newManifestFileName + ".sha", checksum); FileLog.LogMessage(" Done!"); } } newProtoManifest.Files.Sort((x, y) => string.Compare(x.FileName, y.FileName, StringComparison.Ordinal)); FileLog.LogMessage("Manifest {0} ({1})", depot.manifestId, newProtoManifest.CreationTime); //if (Config.DownloadManifestOnly) //{ // StringBuilder manifestBuilder = new StringBuilder(); // string txtManifest = Path.Combine(depot.installDir, string.Format("manifest_{0}_{1}.txt", depot.id, depot.manifestId)); // manifestBuilder.Append(string.Format("{0}\n\n", newProtoManifest.CreationTime)); // foreach (var file in newProtoManifest.Files) // { // if (file.Flags.HasFlag(EDepotFileFlag.Directory)) // continue; // manifestBuilder.Append(string.Format("{0}\n", file.FileName)); // manifestBuilder.Append(string.Format("\t{0}\n", file.TotalSize)); // manifestBuilder.Append(string.Format("\t{0}\n", BitConverter.ToString(file.FileHash).Replace("-", ""))); // } // File.WriteAllText(txtManifest, manifestBuilder.ToString()); // return null; //} string stagingDir = Path.Combine(depot.installDir, STAGING_DIR); var filesAfterExclusions = newProtoManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).ToList(); var allFileNames = new HashSet <string>(filesAfterExclusions.Count); // Pre-process filesAfterExclusions.ForEach(file => { allFileNames.Add(file.FileName); var fileFinalPath = Path.Combine(depot.installDir, file.FileName); var fileStagingPath = Path.Combine(stagingDir, file.FileName); if (file.Flags.HasFlag(EDepotFileFlag.Directory)) { Directory.CreateDirectory(fileFinalPath); Directory.CreateDirectory(fileStagingPath); } else { // Some manifests don't explicitly include all necessary directories Directory.CreateDirectory(Path.GetDirectoryName(fileFinalPath)); Directory.CreateDirectory(Path.GetDirectoryName(fileStagingPath)); depotCounter.CompleteDownloadSize += file.TotalSize; } }); return(new DepotFilesData { depotDownloadInfo = depot, depotCounter = depotCounter, stagingDir = stagingDir, manifest = newProtoManifest, previousManifest = oldProtoManifest, filteredFiles = filesAfterExclusions, allFileNames = allFileNames }); }
private async Task DownloadDepots(uint appID, List <ManifestJob> depots) { Log.WriteDebug("Depot Downloader", "Will process {0} depots ({1} depot locks left)", depots.Count, DepotLocks.Count); var processTasks = new List <Task <EResult> >(); var anyFilesDownloaded = false; var willDownloadFiles = false; foreach (var depot in depots) { if (depot.DepotKey == null) { await GetDepotDecryptionKey(Steam.Instance.Apps, depot, appID); if (depot.DepotKey == null && depot.LastManifestID == depot.ManifestID) { RemoveLock(depot.DepotID); continue; } } depot.Server = GetContentServer(); DepotManifest depotManifest = null; var lastError = string.Empty; for (var i = 0; i <= 5; i++) { try { depotManifest = await CDNClient.DownloadManifestAsync(depot.DepotID, depot.ManifestID, depot.Server, string.Empty, depot.DepotKey); break; } catch (Exception e) { lastError = e.Message; Log.WriteError("Depot Processor", "Failed to download depot manifest for app {0} depot {1} ({2}: {3}) (#{4})", appID, depot.DepotID, depot.Server, lastError, i); } if (depot.DepotKey != null) { RemoveErroredServer(depot.Server); } depot.Server = GetContentServer(); if (depotManifest == null) { await Task.Delay(Utils.ExponentionalBackoff(i)); } } if (depotManifest == null) { RemoveLock(depot.DepotID); if (FileDownloader.IsImportantDepot(depot.DepotID)) { IRC.Instance.SendOps("{0}[{1}]{2} Failed to download manifest ({3})", Colors.OLIVE, depot.DepotName, Colors.NORMAL, lastError); } if (!Settings.IsFullRun && depot.DepotKey != null) { JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null)); } continue; } var task = ProcessDepotAfterDownload(depot, depotManifest); processTasks.Add(task); if (!FileDownloader.IsImportantDepot(depot.DepotID) || depot.DepotKey == null) { continue; } willDownloadFiles = true; task = TaskManager.Run(async() => { var result = EResult.Fail; try { result = await FileDownloader.DownloadFilesFromDepot(depot, depotManifest); if (result == EResult.OK) { anyFilesDownloaded = true; } } catch (Exception e) { ErrorReporter.Notify($"Depot Processor {depot.DepotID}", e); } return(result); }).Unwrap(); processTasks.Add(task); } if (SaveLocalConfig) { SaveLocalConfig = false; LocalConfig.Save(); } await Task.WhenAll(processTasks).ConfigureAwait(false); Log.WriteDebug("Depot Downloader", $"{depots.Count} depot downloads finished for app {appID}"); // TODO: use ContinueWith on tasks if (!anyFilesDownloaded && !willDownloadFiles) { foreach (var depot in depots) { RemoveLock(depot.DepotID); } return; } lock (UpdateScriptLock) { foreach (var depot in depots) { if (depot.Result == EResult.OK) { RunUpdateScript(UpdateScript, string.Format("{0} no-git", depot.DepotID)); } else if (depot.Result != EResult.Ignored) { Log.WriteWarn("Depot Processor", $"Download failed for {depot.DepotID}"); // Mark this depot for redownload var db = Database.Get(); db.Execute("UPDATE `Depots` SET `LastManifestID` = 0 WHERE `DepotID` = @DepotID", new { depot.DepotID }); } RemoveLock(depot.DepotID); } // Only commit changes if all depots downloaded if (processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored)) { if (!RunUpdateScriptForApp(appID, depots[0].BuildID)) { RunUpdateScript(UpdateScript, "0"); } } else { Log.WriteDebug("Depot Processor", "Reprocessing the app {0} because some files failed to download", appID); IRC.Instance.SendOps("{0}[{1}]{2} Reprocessing the app due to download failures", Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL ); JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null)); } } }
public async Task DownloadDepot(String path, uint appId, uint depotId, string branch, Func <String, bool> includeFile) { Directory.CreateDirectory(path); // retrieve app info steam3.RequestAppInfo(appId); var depotIDs = new List <uint>(); KeyValue depots = GetSteam3AppSection(appId, EAppInfoSection.Depots); Console.WriteLine("Using app branch: '{0}'.", branch); ulong manifestId = GetSteam3DepotManifest(depotId, appId, branch); steam3.RequestDepotKey(depotId, appId); byte[] depotKey = steam3.DepotKeys[depotId]; string contentName = GetAppOrDepotName(depotId, appId); Console.WriteLine("Downloading depot {0} - {1}", depotId, contentName); DepotManifest depotManifest = null; CancellationTokenSource cts = new CancellationTokenSource(); CDNClientPool cdnPool = new CDNClientPool(steam3); cdnPool.ExhaustedToken = cts; while (depotManifest == null) { Tuple <CDNClient.Server, string> connection = null; try { connection = await cdnPool.GetConnectionForDepot(appId, depotId, CancellationToken.None); depotManifest = await cdnPool.CDNClient.DownloadManifestAsync(depotId, manifestId, connection.Item1, connection.Item2, depotKey).ConfigureAwait(false); cdnPool.ReturnConnection(connection); } catch (SteamKitWebRequestException e) { cdnPool.ReturnBrokenConnection(connection); if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden) { Console.WriteLine("Encountered 401 for depot manifest {0} {1}. Aborting.", depotId, manifestId); break; } else { Console.WriteLine("Encountered error downloading depot manifest {0} {1}: {2}", depotId, manifestId, e.StatusCode); } } catch (Exception e) { cdnPool.ReturnBrokenConnection(connection); Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depotId, manifestId, e.Message); } } if (depotManifest == null) { Console.WriteLine("\nUnable to download manifest {0} for depot {1}", manifestId, depotId); return; } ulong size_downloaded = 0; foreach (var folder in depotManifest.Files.AsParallel().Where(f => f.Flags.HasFlag(EDepotFileFlag.Directory)) .ToList()) { Directory.CreateDirectory(Path.Join(path, folder.FileName)); } ulong TotalBytesCompressed = 0; ulong TotalBytesUncompressed = 0; ulong DepotBytesCompressed = 0; ulong DepotBytesUncompressed = 0; var semaphore = new SemaphoreSlim(10); var files = depotManifest.Files.AsParallel().Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory) && includeFile(f.FileName)) .ToArray(); ulong complete_download_size = 0; foreach (var file in files) { complete_download_size += file.TotalSize; } var tasks = new Task[files.Length]; for (var i = 0; i < files.Length; i++) { var file = files[i]; var task = Task.Run(async() => { cts.Token.ThrowIfCancellationRequested(); try { await semaphore.WaitAsync().ConfigureAwait(false); cts.Token.ThrowIfCancellationRequested(); string fileFinalPath = Path.Combine(path, file.FileName); FileStream fs = null; List <DepotManifest.ChunkData> neededChunks = new List <DepotManifest.ChunkData>(); FileInfo fi = new FileInfo(fileFinalPath); if (!fi.Exists) { // create new file. need all chunks fs = File.Create(fileFinalPath); fs.SetLength((long)file.TotalSize); neededChunks = new List <DepotManifest.ChunkData>(file.Chunks); } else { fs = File.Open(fileFinalPath, FileMode.Truncate); fs.SetLength((long)file.TotalSize); neededChunks = new List <DepotManifest.ChunkData>(file.Chunks); } foreach (var chunk in neededChunks) { if (cts.IsCancellationRequested) { break; } string chunkID = Util.EncodeHexString(chunk.ChunkID); CDNClient.DepotChunk chunkData = null; while (!cts.IsCancellationRequested) { Tuple <CDNClient.Server, string> connection; try { connection = await cdnPool.GetConnectionForDepot(appId, depotId, cts.Token); } catch (OperationCanceledException) { break; } DepotManifest.ChunkData data = new DepotManifest.ChunkData(); data.ChunkID = chunk.ChunkID; data.Checksum = chunk.Checksum; data.Offset = chunk.Offset; data.CompressedLength = chunk.CompressedLength; data.UncompressedLength = chunk.UncompressedLength; try { chunkData = await cdnPool.CDNClient.DownloadDepotChunkAsync(depotId, data, connection.Item1, connection.Item2, depotKey).ConfigureAwait(false); cdnPool.ReturnConnection(connection); break; } catch (SteamKitWebRequestException e) { cdnPool.ReturnBrokenConnection(connection); if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden) { Console.WriteLine("Encountered 401 for chunk {0}. Aborting.", chunkID); cts.Cancel(); break; } else { Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, e.StatusCode); } } catch (Exception e) { cdnPool.ReturnBrokenConnection(connection); Console.WriteLine("Encountered unexpected error downloading chunk {0}: {1}", chunkID, e.Message); } } if (chunkData == null) { Console.WriteLine("Failed to find any server with chunk {0} for depot {1}. Aborting.", chunkID, depotId); cts.Cancel(); } // Throw the cancellation exception if requested so that this task is marked failed cts.Token.ThrowIfCancellationRequested(); TotalBytesCompressed += chunk.CompressedLength; DepotBytesCompressed += chunk.CompressedLength; TotalBytesUncompressed += chunk.UncompressedLength; DepotBytesUncompressed += chunk.UncompressedLength; fs.Seek((long)chunk.Offset, SeekOrigin.Begin); fs.Write(chunkData.Data, 0, chunkData.Data.Length); size_downloaded += chunk.UncompressedLength; } fs.Dispose(); Console.WriteLine("{0,6:#00.00}% {1}", ((float)size_downloaded / (float)complete_download_size) * 100.0f, fileFinalPath); } finally { semaphore.Release(); } }); tasks[i] = task; } await Task.WhenAll(tasks).ConfigureAwait(false); Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depotId, DepotBytesCompressed, DepotBytesUncompressed); }
public static void DownloadFilesFromDepot(DepotProcessor.ManifestJob job, DepotManifest depotManifest) { var files = depotManifest.Files.Where(x => IsFileNameMatching(job.DepotID, x.FileName)).ToList(); var filesUpdated = false; Log.WriteDebug("FileDownloader", "Will download {0} files from depot {1}", files.Count(), job.DepotID); foreach (var file in files) { string directory = Path.Combine(Application.Path, FILES_DIRECTORY, job.DepotID.ToString(), Path.GetDirectoryName(file.FileName)); string finalPath = Path.Combine(directory, Path.GetFileName(file.FileName)); string downloadPath = Path.GetTempFileName(); if (!Directory.Exists(directory)) { Directory.CreateDirectory(directory); } else if (File.Exists(finalPath)) { using (var fs = File.Open(finalPath, FileMode.Open)) { using (var sha = new SHA1Managed()) { if (file.FileHash.SequenceEqual(sha.ComputeHash(fs))) { Log.WriteDebug("FileDownloader", "{0} already matches the file we have", file.FileName); continue; } } } } Log.WriteInfo("FileDownloader", "Downloading {0} ({1} bytes, {2} chunks)", file.FileName, file.TotalSize, file.Chunks.Count); uint count = 0; byte[] checksum; string lastError = "or checksum failed"; using (var fs = File.Open(downloadPath, FileMode.OpenOrCreate)) { fs.SetLength((long)file.TotalSize); var lockObject = new object(); // TODO: We *could* verify each chunk and only download needed ones Parallel.ForEach(file.Chunks, (chunk, state) => { var downloaded = false; for (var i = 0; i <= 5; i++) { try { var chunkData = CDNClient.DownloadDepotChunk(job.DepotID, chunk, job.Server, job.CDNToken, job.DepotKey); lock (lockObject) { fs.Seek((long)chunk.Offset, SeekOrigin.Begin); fs.Write(chunkData.Data, 0, chunkData.Data.Length); Log.WriteDebug("FileDownloader", "Downloaded {0} ({1}/{2})", file.FileName, ++count, file.Chunks.Count); } downloaded = true; break; } catch (Exception e) { lastError = e.Message; } } if (!downloaded) { state.Stop(); } }); fs.Seek(0, SeekOrigin.Begin); using (var sha = new SHA1Managed()) { checksum = sha.ComputeHash(fs); } } if (file.Chunks.Count == 0 || file.FileHash.SequenceEqual(checksum)) { Log.WriteInfo("FileDownloader", "Downloaded {0} from {1}", file.FileName, Steam.GetAppName(job.ParentAppID)); if (File.Exists(finalPath)) { File.Delete(finalPath); } File.Move(downloadPath, finalPath); filesUpdated = true; } else { IRC.Instance.SendOps("{0}[{1}]{2} Failed to download {3}: Only {4} out of {5} chunks downloaded ({6})", Colors.OLIVE, Steam.GetAppName(job.ParentAppID), Colors.NORMAL, file.FileName, count, file.Chunks.Count, lastError); Log.WriteError("FileDownloader", "Failed to download {0}: Only {1} out of {2} chunks downloaded from {3} ({4})", file.FileName, count, file.Chunks.Count, job.Server, lastError); File.Delete(downloadPath); } } if (filesUpdated) { var updateScript = Path.Combine(Application.Path, "files", "update.sh"); if (File.Exists(updateScript)) { // YOLO Process.Start(updateScript, job.DepotID.ToString()); } } }
private static async Task <EResult> ProcessDepotAfterDownload(IDbConnection db, IDbTransaction transaction, ManifestJob request, DepotManifest depotManifest) { var filesOld = (await db.QueryAsync <DepotFile>("SELECT `File`, `Hash`, `Size`, `Flags` FROM `DepotsFiles` WHERE `DepotID` = @DepotID", new { request.DepotID }, transaction)).ToDictionary(x => x.File, x => x); var filesAdded = new List <DepotFile>(); var shouldHistorize = filesOld.Count > 0 && !depotManifest.FilenamesEncrypted; // Don't historize file additions if we didn't have any data before if (request.StoredFilenamesEncrypted && !depotManifest.FilenamesEncrypted) { Log.WriteInfo(nameof(DepotProcessor), $"Depot {request.DepotID} will decrypt stored filenames"); var decryptedFilesOld = new Dictionary <string, DepotFile>(); foreach (var file in filesOld.Values) { var oldFile = file.File; file.File = DecryptFilename(oldFile, request.DepotKey); decryptedFilesOld.Add(file.File, file); await db.ExecuteAsync("UPDATE `DepotsFiles` SET `File` = @File WHERE `DepotID` = @DepotID AND `File` = @OldFile", new { request.DepotID, file.File, OldFile = oldFile }, transaction); } await MakeHistory(db, transaction, request, string.Empty, "files_decrypted"); filesOld = decryptedFilesOld; } foreach (var file in depotManifest.Files.OrderByDescending(x => x.FileName)) { var name = depotManifest.FilenamesEncrypted ? file.FileName.Replace("\n", "") : file.FileName.Replace('\\', '/'); byte[] hash = null; // Store empty hashes as NULL (e.g. an empty file) if ((file.Flags & EDepotFileFlag.Directory) == 0 && file.FileHash.Length > 0 && file.FileHash.Any(c => c != 0)) { hash = file.FileHash; } // Limit path names to 260 characters (default windows max length) // File column is varchar(260) and not higher to prevent reducing performance // See https://stackoverflow.com/questions/1962310/importance-of-varchar-length-in-mysql-table/1962329#1962329 // Until 2019 there hasn't been a single file that went over this limit, so far there has been only one // game with a big node_modules path, so we're safeguarding by limiting it. if (name.Length > 260) { if (depotManifest.FilenamesEncrypted) { continue; } using var sha = SHA1.Create(); var nameHash = Utils.ByteArrayToString(sha.ComputeHash(Encoding.UTF8.GetBytes(name))); name = $"{{SteamDB file name is too long}}/{nameHash}/...{name.Substring(name.Length - 150)}"; } if (filesOld.ContainsKey(name)) { var oldFile = filesOld[name]; var updateFile = false; if (oldFile.Size != file.TotalSize || !Utils.ByteArrayEquals(hash, oldFile.Hash)) { await MakeHistory(db, transaction, request, name, "modified", oldFile.Size, file.TotalSize); updateFile = true; } if (oldFile.Flags != file.Flags) { await MakeHistory(db, transaction, request, name, "modified_flags", (ulong)oldFile.Flags, (ulong)file.Flags); updateFile = true; } if (updateFile) { await db.ExecuteAsync("UPDATE `DepotsFiles` SET `Hash` = @Hash, `Size` = @Size, `Flags` = @Flags WHERE `DepotID` = @DepotID AND `File` = @File", new DepotFile { DepotID = request.DepotID, File = name, Hash = hash, Size = file.TotalSize, Flags = file.Flags }, transaction); } filesOld.Remove(name); } else { // We want to historize modifications first, and only then deletions and additions filesAdded.Add(new DepotFile { DepotID = request.DepotID, Hash = hash, File = name, Size = file.TotalSize, Flags = file.Flags }); } } if (filesOld.Count > 0) { // Chunk file deletion queries so it doesn't go over max_allowed_packet var filesOldChunks = filesOld.Select(x => x.Value.File).Split(1000); foreach (var filesOldChunk in filesOldChunks) { await db.ExecuteAsync("DELETE FROM `DepotsFiles` WHERE `DepotID` = @DepotID AND `File` IN @Files", new { request.DepotID, Files = filesOldChunk, }, transaction); } if (shouldHistorize) { await db.ExecuteAsync(HistoryQuery, filesOld.Select(x => new DepotHistory { DepotID = request.DepotID, ManifestID = request.ManifestID, ChangeID = request.ChangeNumber, Action = "removed", File = x.Value.File, OldValue = x.Value.Size }), transaction); } } if (filesAdded.Count > 0) { await db.ExecuteAsync("INSERT INTO `DepotsFiles` (`DepotID`, `File`, `Hash`, `Size`, `Flags`) VALUES (@DepotID, @File, @Hash, @Size, @Flags)", filesAdded, transaction); if (shouldHistorize) { await db.ExecuteAsync(HistoryQuery, filesAdded.Select(x => new DepotHistory { DepotID = request.DepotID, ManifestID = request.ManifestID, ChangeID = request.ChangeNumber, Action = "added", File = x.File, NewValue = x.Size }), transaction); } } await db.ExecuteAsync( request.LastManifestID == request.ManifestID? "UPDATE `Depots` SET `LastManifestID` = @ManifestID, `ManifestDate` = @ManifestDate, `FilenamesEncrypted` = @FilenamesEncrypted, `SizeOriginal` = @SizeOriginal, `SizeCompressed` = @SizeCompressed WHERE `DepotID` = @DepotID" : "UPDATE `Depots` SET `LastManifestID` = @ManifestID, `ManifestDate` = @ManifestDate, `FilenamesEncrypted` = @FilenamesEncrypted, `SizeOriginal` = @SizeOriginal, `SizeCompressed` = @SizeCompressed, `LastUpdated` = CURRENT_TIMESTAMP() WHERE `DepotID` = @DepotID", new { request.DepotID, request.ManifestID, depotManifest.FilenamesEncrypted, ManifestDate = depotManifest.CreationTime, SizeOriginal = depotManifest.TotalUncompressedSize, SizeCompressed = depotManifest.TotalCompressedSize, }, transaction); return(EResult.OK); }
private async Task DownloadDepots(IEnumerable <ManifestJob> depots) { Log.WriteDebug("Depot Downloader", "Will process {0} depots ({1} depot locks left)", depots.Count(), DepotLocks.Count); var processTasks = new List <Task <EResult> >(); bool anyFilesDownloaded = false; foreach (var depot in depots) { depot.DepotKey = await GetDepotDecryptionKey(depot.DepotID, depot.ParentAppID); if (depot.DepotKey == null) { RemoveLock(depot.DepotID); continue; } var cdnToken = await GetCDNAuthToken(depot.DepotID); if (cdnToken == null) { RemoveLock(depot.DepotID); Log.WriteDebug("Depot Downloader", "Got a depot key for depot {0} but no cdn auth token", depot.DepotID); continue; } depot.CDNToken = cdnToken.Token; depot.Server = cdnToken.Server; DepotManifest depotManifest = null; string lastError = string.Empty; for (var i = 0; i <= 5; i++) { try { depotManifest = CDNClient.DownloadManifest(depot.DepotID, depot.ManifestID, depot.Server, depot.CDNToken, depot.DepotKey); break; } catch (Exception e) { Log.WriteWarn("Depot Downloader", "{0} Manifest download failed: {1} - {2}", depot.DepotID, e.GetType(), e.Message); lastError = e.Message; } // TODO: get new auth key if auth fails if (depotManifest == null) { await Task.Delay(Utils.ExponentionalBackoff(i)); } } if (depotManifest == null) { RemoveLock(depot.DepotID); Log.WriteError("Depot Processor", "Failed to download depot manifest for depot {0} ({1}: {2})", depot.DepotID, depot.Server, lastError); if (FileDownloader.IsImportantDepot(depot.DepotID)) { IRC.Instance.SendOps("{0}[{1}]{2} Failed to download depot {3} manifest ({4}: {5})", Colors.OLIVE, Steam.GetAppName(depot.ParentAppID), Colors.NORMAL, depot.DepotID, depot.Server, lastError); } continue; } var task = TaskManager.Run(() => { using (var db = Database.GetConnection()) { return(ProcessDepotAfterDownload(db, depot, depotManifest)); } }); processTasks.Add(task); if (FileDownloader.IsImportantDepot(depot.DepotID)) { task = TaskManager.Run(() => { var result = FileDownloader.DownloadFilesFromDepot(depot, depotManifest); if (result == EResult.OK) { anyFilesDownloaded = true; } return(result); }, TaskCreationOptions.LongRunning); TaskManager.RegisterErrorHandler(task); processTasks.Add(task); } } if (SaveLocalConfig) { SaveLocalConfig = false; LocalConfig.Save(); } await Task.WhenAll(processTasks); // TODO: use ContinueWith on tasks if (!anyFilesDownloaded) { Log.WriteDebug("Depot Downloader", "Tasks awaited for {0} depot downloads", depots.Count()); foreach (var depot in depots) { RemoveLock(depot.DepotID); } return; } var canUpdate = processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored) && File.Exists(UpdateScript); #if true Log.WriteDebug("Depot Downloader", "Tasks awaited for {0} depot downloads (will run script: {1})", depots.Count(), canUpdate); #endif bool lockTaken = false; try { UpdateScriptLock.Enter(ref lockTaken); foreach (var depot in depots) { // TODO: this only needs to run if any downloaded files changed if (canUpdate && FileDownloader.IsImportantDepot(depot.DepotID)) { RunUpdateScript(string.Format("{0} no-git", depot.DepotID)); } RemoveLock(depot.DepotID); } if (canUpdate) { RunUpdateScript("0"); } } finally { if (lockTaken) { UpdateScriptLock.Exit(); } } }
private void DownloadManifest(ManifestJob request) { Log.WriteInfo("Depot Processor", "DepotID: {0}", request.DepotID); DepotManifest depotManifest = null; string lastError = string.Empty; // CDN is very random, just keep trying for (var i = 0; i <= 5; i++) { try { depotManifest = CDNClient.DownloadManifest(request.DepotID, request.ManifestID, request.Server, request.CDNToken, request.DepotKey); break; } catch (Exception e) { lastError = e.Message; } } if (depotManifest == null) { Log.WriteError("Depot Processor", "Failed to download depot manifest for depot {0} ({1}: {2}) (#{3})", request.DepotID, request.Server, lastError, request.Tries); if (--request.Tries >= 0) { request.Server = GetContentServer(request.Tries); JobManager.AddJob(() => Steam.Instance.Apps.GetCDNAuthToken(request.DepotID, request.Server), request); return; } RemoveLock(request.DepotID); // TODO: Remove this once task in OnCDNAuthTokenCallback is used if (FileDownloader.IsImportantDepot(request.DepotID)) { IRC.Instance.SendOps("{0}[{1}]{2} Failed to download depot {3} manifest ({4}: {5})", Colors.OLIVE, Steam.GetAppName(request.ParentAppID), Colors.NORMAL, request.DepotID, request.Server, lastError); } return; } if (FileDownloader.IsImportantDepot(request.DepotID)) { TaskManager.Run(() => FileDownloader.DownloadFilesFromDepot(request, depotManifest)); } // TODO: Task here instead of in OnCDNAuthTokenCallback due to mono's silly threadpool TaskManager.Run(() => { using (var db = Database.GetConnection()) { ProcessDepotAfterDownload(db, request, depotManifest); } }).ContinueWith(task => { RemoveLock(request.DepotID); Log.WriteDebug("Depot Processor", "Processed depot {0} ({1} depot locks left)", request.DepotID, DepotLocks.Count); }); }
private EResult ProcessDepotAfterDownload(IDbConnection db, ManifestJob request, DepotManifest depotManifest) { var filesOld = db.Query <DepotFile>("SELECT `ID`, `File`, `Hash`, `Size`, `Flags` FROM `DepotsFiles` WHERE `DepotID` = @DepotID", new { request.DepotID }).ToDictionary(x => x.File, x => x); var filesNew = new List <DepotFile>(); var filesAdded = new List <DepotFile>(); var shouldHistorize = filesOld.Any(); // Don't historize file additions if we didn't have any data before foreach (var file in depotManifest.Files) { var name = file.FileName.Replace('\\', '/'); // safe guard if (name.Length > 255) { ErrorReporter.Notify(new OverflowException(string.Format("File \"{0}\" in depot {1} is too long", name, request.DepotID))); continue; } var depotFile = new DepotFile { DepotID = request.DepotID, File = name, Size = file.TotalSize, Flags = file.Flags }; if (file.FileHash.Length > 0 && !file.Flags.HasFlag(EDepotFileFlag.Directory)) { depotFile.Hash = Utils.ByteArrayToString(file.FileHash); } else { depotFile.Hash = "0000000000000000000000000000000000000000"; } filesNew.Add(depotFile); } foreach (var file in filesNew) { if (filesOld.ContainsKey(file.File)) { var oldFile = filesOld[file.File]; var updateFile = false; if (oldFile.Size != file.Size || !file.Hash.Equals(oldFile.Hash)) { MakeHistory(db, request, file.File, "modified", oldFile.Size, file.Size); updateFile = true; } if (oldFile.Flags != file.Flags) { MakeHistory(db, request, file.File, "modified_flags", (ulong)oldFile.Flags, (ulong)file.Flags); updateFile = true; } if (updateFile) { file.ID = oldFile.ID; db.Execute("UPDATE `DepotsFiles` SET `Hash` = @Hash, `Size` = @Size, `Flags` = @Flags WHERE `DepotID` = @DepotID AND `ID` = @ID", file); } filesOld.Remove(file.File); } else { // We want to historize modifications first, and only then deletions and additions filesAdded.Add(file); } } if (filesOld.Any()) { db.Execute("DELETE FROM `DepotsFiles` WHERE `DepotID` = @DepotID AND `ID` IN @Files", new { request.DepotID, Files = filesOld.Select(x => x.Value.ID) }); db.Execute(GetHistoryQuery(), filesOld.Select(x => new DepotHistory { DepotID = request.DepotID, ChangeID = request.ChangeNumber, Action = "removed", File = x.Value.File })); } if (filesAdded.Any()) { db.Execute("INSERT INTO `DepotsFiles` (`DepotID`, `File`, `Hash`, `Size`, `Flags`) VALUES (@DepotID, @File, @Hash, @Size, @Flags)", filesAdded); if (shouldHistorize) { db.Execute(GetHistoryQuery(), filesAdded.Select(x => new DepotHistory { DepotID = request.DepotID, ChangeID = request.ChangeNumber, Action = "added", File = x.File })); } } db.Execute("UPDATE `Depots` SET `LastManifestID` = @ManifestID WHERE `DepotID` = @DepotID", new { request.DepotID, request.ManifestID }); return(EResult.OK); }
public List <DLC> GetInstalledDLCFiles() { List <DLC> dlcList = new List <DLC>(); if (AppManifestPath != null && File.Exists(AppManifestPath)) { KeyValue appManifest = KeyValue.LoadAsText(AppManifestPath); Dictionary <string, string> depotManifests = new Dictionary <string, string>(); foreach (KeyValue mountedDepot in appManifest["MountedDepots"].Children) { depotManifests[mountedDepot.Name] = mountedDepot.Value; } foreach (KeyValue mountedDepot in appManifest["InstalledDepots"].Children) { depotManifests[mountedDepot.Name] = mountedDepot["manifest"].Value; } foreach (KeyValuePair <string, string> depotManifest in depotManifests) { uint dlcappid = Convert.ToUInt32(depotManifest.Key); string manifestPath = Path.Combine(SteamPath, "depotcache", $"{dlcappid}_{depotManifest.Value}.manifest"); if (File.Exists(manifestPath)) { DLC dlc = new DLC(dlcappid); DepotManifest manifest = DepotManifest.Deserialize(File.ReadAllBytes(manifestPath)); foreach (DepotManifest.FileData file in manifest.Files) { if (file.Flags.HasFlag(EDepotFileFlag.Directory)) { continue; } string fileName = file.FileName.ToLower(); string extension = Path.GetExtension(fileName).ToLower(); if (fileName.Contains("assets")) { if (extension.Contains("xml") || extension.Contains("bin")) { dlc.IncludedFiles.Add(NormalizePath(GetRelativePath(Path.Combine(RWPath, "Assets"), Path.Combine(RWPath, fileName)))); } if (extension == ".ap") { string absoluteFileName = Path.Combine(RWPath, fileName); try { ZipArchive zipFile = ZipFile.OpenRead(absoluteFileName); dlc.IncludedFiles.AddRange(from x in zipFile.Entries where (x.FullName.Contains(".xml") || x.FullName.Contains(".bin")) select NormalizePath(GetRelativePath(Path.Combine(RWPath, "Assets"), Path.Combine(Path.GetDirectoryName(absoluteFileName), x.FullName)))); } catch { } } } } dlcList.Add(dlc); } } } return(dlcList); }
private static async Task DownloadSteam3Async(uint appId, List <DepotDownloadInfo> depots, CancellationToken cancellationToken) { ulong TotalBytesCompressed = 0; ulong TotalBytesUncompressed = 0; foreach (var depot in depots) { ulong DepotBytesCompressed = 0; ulong DepotBytesUncompressed = 0; Console.WriteLine("Downloading depot {0} - {1}", depot.Id, depot.ContentName); CancellationTokenSource cts = new CancellationTokenSource(); cdnPool.ExhaustedToken = cts; ProtoManifest oldProtoManifest = null; ProtoManifest newProtoManifest = null; string configDir = Path.Combine(depot.InstallDir, CONFIG_DIR); ulong lastManifestId = INVALID_MANIFEST_ID; DepotConfigStore.Instance.InstalledManifestIDs.TryGetValue(depot.Id, out lastManifestId); // In case we have an early exit, this will force equiv of verifyall next run. DepotConfigStore.Instance.InstalledManifestIDs[depot.Id] = INVALID_MANIFEST_ID; DepotConfigStore.Save(); if (lastManifestId != INVALID_MANIFEST_ID) { var oldManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", lastManifestId)); if (File.Exists(oldManifestFileName)) { byte[] expectedChecksum, currentChecksum; try { expectedChecksum = File.ReadAllBytes(oldManifestFileName + ".sha"); } catch (IOException) { expectedChecksum = null; } oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName, out currentChecksum); if (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum)) { // We only have to show this warning if the old manifest ID was different if (lastManifestId != depot.ManifestId) { Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", lastManifestId); } oldProtoManifest = null; } } } if (lastManifestId == depot.ManifestId && oldProtoManifest != null) { newProtoManifest = oldProtoManifest; Console.WriteLine("Already have manifest {0} for depot {1}.", depot.ManifestId, depot.Id); } else { var newManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.Id, depot.ManifestId)); if (newManifestFileName != null) { byte[] expectedChecksum, currentChecksum; try { expectedChecksum = File.ReadAllBytes(newManifestFileName + ".sha"); } catch (IOException) { expectedChecksum = null; } newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName, out currentChecksum); if (newProtoManifest != null && (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum))) { Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", depot.ManifestId); newProtoManifest = null; } } if (newProtoManifest != null) { Console.WriteLine("Already have manifest {0} for depot {1}.", depot.ManifestId, depot.Id); } else { Console.Write("Downloading depot manifest..."); DepotManifest depotManifest = await DownloadManifest(appId, depot.Id, depot.ManifestId, depot.DepotKey).ConfigureAwait(false); byte[] checksum; newProtoManifest = new ProtoManifest(depotManifest, depot.ManifestId); newProtoManifest.SaveToFile(newManifestFileName, out checksum); File.WriteAllBytes(newManifestFileName + ".sha", checksum); Console.WriteLine(" Done!"); } } newProtoManifest.Files.Sort((x, y) => string.Compare(x.FileName, y.FileName, StringComparison.Ordinal)); Console.WriteLine("Manifest {0} ({1})", depot.ManifestId, newProtoManifest.CreationTime); ulong complete_download_size = 0; ulong size_downloaded = 0; var downloadChanges = new ConcurrentQueue <(DateTime, ulong)>(); object downloadChangesLock = new object(); void reportProgress(string currentFilePath) { lock (downloadChangesLock) { while (downloadChanges.Count > 50) // Only average last 50 deltas { downloadChanges.TryDequeue(out _); } } ulong bytesPerSecond = 0; if (downloadChanges.Count > 0) { lock (downloadChangesLock) { TimeSpan timeSpan = downloadChanges.Last().Item1 - downloadChanges.First().Item1; ulong totalBytes = downloadChanges.Aggregate(0ul, (a, b) => a + b.Item2); double rate = totalBytes / timeSpan.TotalSeconds; if (double.IsNaN(rate) || double.IsInfinity(rate)) { rate = 0; } bytesPerSecond = (ulong)rate; } } Globals.UiDispatcher.Invoke(() => { Globals.AppState.DownloadState.DownloadPercentageComplete = ((double)size_downloaded / complete_download_size) * 100; Globals.AppState.DownloadState.DownloadCurrentFile = currentFilePath; Globals.AppState.DownloadState.DownloadedBytes = size_downloaded; Globals.AppState.DownloadState.BytesPerSecond = bytesPerSecond; }); } string stagingDir = Path.Combine(depot.InstallDir, STAGING_DIR); // Delete files that are removed in new manifest if (oldProtoManifest != null) { var directoriesToCheck = new HashSet <string>(); // Directories to check if they're empty and delete foreach (ProtoManifest.FileData oldFile in oldProtoManifest.Files) { if (!newProtoManifest.Files.Any(newFile => newFile.FileName == oldFile.FileName)) { Console.WriteLine($"Deleting file: {oldFile.FileName}"); var fileFinalPath = Path.Combine(depot.InstallDir, oldFile.FileName); if (File.Exists(fileFinalPath)) { File.Delete(fileFinalPath); directoriesToCheck.Add(Path.GetDirectoryName(fileFinalPath)); } } } // Delete the now empty directories foreach (string directoryPath in directoriesToCheck) { if (Directory.GetDirectories(directoryPath).Length == 0 && Directory.GetFiles(directoryPath).Length == 0) { Directory.Delete(directoryPath); } } } // Pre-process newProtoManifest.Files.ForEach(file => { var fileFinalPath = Path.Combine(depot.InstallDir, file.FileName); var fileStagingPath = Path.Combine(stagingDir, file.FileName); if (file.Flags.HasFlag(EDepotFileFlag.Directory)) { Directory.CreateDirectory(fileFinalPath); Directory.CreateDirectory(fileStagingPath); } else { // Some manifests don't explicitly include all necessary directories Directory.CreateDirectory(Path.GetDirectoryName(fileFinalPath)); Directory.CreateDirectory(Path.GetDirectoryName(fileStagingPath)); complete_download_size += file.TotalSize; } }); Globals.UiDispatcher.Invoke(() => { Globals.AppState.DownloadState.TotalBytes = complete_download_size; }); cancellationToken.ThrowIfCancellationRequested(); var semaphore = new SemaphoreSlim(Config.MaxDownloads); var files = newProtoManifest.Files.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory)).ToArray(); var tasks = new Task[files.Length]; for (var i = 0; i < files.Length; i++) { var file = files[i]; var task = Task.Run(async() => { cts.Token.ThrowIfCancellationRequested(); try { await semaphore.WaitAsync().ConfigureAwait(false); cts.Token.ThrowIfCancellationRequested(); cancellationToken.ThrowIfCancellationRequested(); string fileFinalPath = Path.Combine(depot.InstallDir, file.FileName); string fileStagingPath = Path.Combine(stagingDir, file.FileName); // This may still exist if the previous run exited before cleanup if (File.Exists(fileStagingPath)) { File.Delete(fileStagingPath); } FileStream fs = null; List <ProtoManifest.ChunkData> neededChunks; FileInfo fi = new FileInfo(fileFinalPath); if (!fi.Exists) { // create new file. need all chunks fs = File.Create(fileFinalPath); fs.SetLength((long)file.TotalSize); neededChunks = new List <ProtoManifest.ChunkData>(file.Chunks); } else { // open existing ProtoManifest.FileData oldManifestFile = null; if (oldProtoManifest != null) { oldManifestFile = oldProtoManifest.Files.SingleOrDefault(f => f.FileName == file.FileName); } if (oldManifestFile != null) { neededChunks = new List <ProtoManifest.ChunkData>(); if (Config.VerifyAll || !oldManifestFile.FileHash.SequenceEqual(file.FileHash)) { // we have a version of this file, but it doesn't fully match what we want var matchingChunks = new List <ChunkMatch>(); foreach (var chunk in file.Chunks) { var oldChunk = oldManifestFile.Chunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID)); if (oldChunk != null) { matchingChunks.Add(new ChunkMatch(oldChunk, chunk)); } else { neededChunks.Add(chunk); } } File.Move(fileFinalPath, fileStagingPath); fs = File.Open(fileFinalPath, FileMode.Create); fs.SetLength((long)file.TotalSize); using (var fsOld = File.Open(fileStagingPath, FileMode.Open)) { foreach (var match in matchingChunks) { fsOld.Seek((long)match.OldChunk.Offset, SeekOrigin.Begin); byte[] tmp = new byte[match.OldChunk.UncompressedLength]; fsOld.Read(tmp, 0, tmp.Length); byte[] adler = Util.AdlerHash(tmp); if (!adler.SequenceEqual(match.OldChunk.Checksum)) { neededChunks.Add(match.NewChunk); } else { fs.Seek((long)match.NewChunk.Offset, SeekOrigin.Begin); fs.Write(tmp, 0, tmp.Length); } } } File.Delete(fileStagingPath); } } else { // No old manifest or file not in old manifest. We must validate. fs = File.Open(fileFinalPath, FileMode.Open); if ((ulong)fi.Length != file.TotalSize) { fs.SetLength((long)file.TotalSize); } neededChunks = Util.ValidateSteam3FileChecksums(fs, file.Chunks.OrderBy(x => x.Offset).ToArray()); } if (neededChunks.Count() == 0) { size_downloaded += file.TotalSize; reportProgress(file.FileName); if (fs != null) { fs.Dispose(); } return; } else { ulong sizeDelta = (file.TotalSize - (ulong)neededChunks.Select(x => (long)x.UncompressedLength).Sum()); size_downloaded += sizeDelta; reportProgress(file.FileName); } } foreach (var chunk in neededChunks) { if (cts.IsCancellationRequested) { break; } string chunkID = Util.EncodeHexString(chunk.ChunkID); CDNClient.DepotChunk chunkData = null; while (!cts.IsCancellationRequested) { Tuple <CDNClient.Server, string> connection; try { connection = await cdnPool.GetConnectionForDepot(appId, depot.Id, cts.Token); } catch (OperationCanceledException) { break; } DepotManifest.ChunkData data = new DepotManifest.ChunkData(); data.ChunkID = chunk.ChunkID; data.Checksum = chunk.Checksum; data.Offset = chunk.Offset; data.CompressedLength = chunk.CompressedLength; data.UncompressedLength = chunk.UncompressedLength; try { chunkData = await cdnPool.CDNClient.DownloadDepotChunkAsync(depot.Id, data, connection.Item1, connection.Item2, depot.DepotKey).ConfigureAwait(false); cdnPool.ReturnConnection(connection); break; } catch (SteamKitWebRequestException e) { cdnPool.ReturnBrokenConnection(connection); if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden) { Console.WriteLine("Encountered 401 for chunk {0}. Aborting.", chunkID); cts.Cancel(); break; } else { Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, e.StatusCode); } } catch (TaskCanceledException) { Console.WriteLine("Connection timeout downloading chunk {0}", chunkID); } catch (Exception e) { cdnPool.ReturnBrokenConnection(connection); Console.WriteLine("Encountered unexpected error downloading chunk {0}: {1}", chunkID, e.Message); } } if (chunkData == null) { Console.WriteLine("Failed to find any server with chunk {0} for depot {1}. Aborting.", chunkID, depot.Id); cts.Cancel(); } // Throw the cancellation exception if requested so that this task is marked failed cts.Token.ThrowIfCancellationRequested(); cancellationToken.ThrowIfCancellationRequested(); TotalBytesCompressed += chunk.CompressedLength; DepotBytesCompressed += chunk.CompressedLength; TotalBytesUncompressed += chunk.UncompressedLength; DepotBytesUncompressed += chunk.UncompressedLength; fs.Seek((long)chunk.Offset, SeekOrigin.Begin); fs.Write(chunkData.Data, 0, chunkData.Data.Length); size_downloaded += chunk.UncompressedLength; downloadChanges.Enqueue((DateTime.Now, chunk.CompressedLength)); reportProgress(file.FileName); } fs.Dispose(); reportProgress(file.FileName); } finally { semaphore.Release(); } }); tasks[i] = task; } await Task.WhenAll(tasks).ConfigureAwait(false); DepotConfigStore.Instance.InstalledManifestIDs[depot.Id] = depot.ManifestId; DepotConfigStore.Save(); Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.Id, DepotBytesCompressed, DepotBytesUncompressed); reportProgress(null); } Console.WriteLine("Total downloaded: {0} bytes ({1} bytes uncompressed) from {2} depots", TotalBytesCompressed, TotalBytesUncompressed, depots.Count); }
private static void VerifyApp(SteamClientUtils steamClient, string appManifestPath, bool deleteUnknownFiles) { Console.WriteLine($"Parsing {appManifestPath}"); var kv = KeyValue.LoadAsText(appManifestPath); var mountedDepots = kv["MountedDepots"]; var gamePath = Path.Join(Path.GetDirectoryName(appManifestPath), "common", kv["installdir"].Value); if (!Directory.Exists(gamePath)) { throw new DirectoryNotFoundException($"Game not found: {gamePath}"); } Console.WriteLine($"Scanning {gamePath}"); var allKnownDepotFiles = new Dictionary <string, ulong>(); foreach (var mountedDepot in mountedDepots.Children) { var manifestPath = Path.Join(steamClient.SteamPath, "depotcache", $"{mountedDepot.Name}_{mountedDepot.Value}.manifest"); if (!File.Exists(manifestPath)) { Console.Error.WriteLine($"Manifest does not exist: {manifestPath}"); continue; } var manifest = DepotManifest.Deserialize(File.ReadAllBytes(manifestPath)); foreach (var file in manifest.Files) { if (file.Flags.HasFlag(EDepotFileFlag.Directory)) { continue; } allKnownDepotFiles[file.FileName] = file.TotalSize; } Console.WriteLine($"{manifestPath} - {manifest.Files.Count} files"); } Console.WriteLine($"{allKnownDepotFiles.Count} files in depot manifests"); var filesOnDisk = Directory.GetFiles(gamePath, "*", SearchOption.AllDirectories).ToList(); filesOnDisk.Sort(); Console.WriteLine($"{filesOnDisk.Count} files on disk"); Console.WriteLine(); Console.ForegroundColor = ConsoleColor.Red; var filenamesOnDisk = new HashSet <string>(); foreach (var file in filesOnDisk) { var unprefixedPath = file.Substring(gamePath.Length + 1); filenamesOnDisk.Add(unprefixedPath); if (unprefixedPath == "installscript.vdf") { continue; } if (!allKnownDepotFiles.ContainsKey(unprefixedPath)) { Console.Write($"Unknown file: {unprefixedPath}"); if (deleteUnknownFiles && !InsideWhitelist(file)) { try { File.Delete(file); Console.Write(" -> [Deleted]"); } catch (Exception ex) { Console.WriteLine("Unable to delete!"); Console.WriteLine(ex); } } Console.WriteLine(); continue; } var length = new FileInfo(file).Length; if (allKnownDepotFiles[unprefixedPath] != (ulong)length) { Console.WriteLine($"Mismatching file size: {unprefixedPath} (is {length} bytes, should be {allKnownDepotFiles[unprefixedPath]})"); continue; } } foreach (var file in allKnownDepotFiles.Keys.Where(file => !filenamesOnDisk.Contains(file))) { Console.WriteLine($"Missing file: {file}"); } Console.ResetColor(); }
public ProtoManifest(DepotManifest sourceManifest, ulong id) : this() { sourceManifest.Files.ForEach(f => Files.Add(new FileData(f))); ID = id; }
/* * Here be dragons. */ public static async Task <EResult> DownloadFilesFromDepot(DepotProcessor.ManifestJob job, DepotManifest depotManifest) { var filesRegex = Files[job.DepotID]; var files = depotManifest.Files.Where(x => filesRegex.IsMatch(x.FileName.Replace('\\', '/'))).ToList(); var downloadState = EResult.Fail; ConcurrentDictionary <string, ExistingFileData> existingFileData; await using (var db = await Database.GetConnectionAsync()) { var data = db.ExecuteScalar <string>("SELECT `Value` FROM `LocalConfig` WHERE `ConfigKey` = @Key", new { Key = $"depot.{job.DepotID}" }); if (data != null) { existingFileData = JsonConvert.DeserializeObject <ConcurrentDictionary <string, ExistingFileData> >(data); } else { existingFileData = new ConcurrentDictionary <string, ExistingFileData>(); } } foreach (var file in existingFileData.Keys.Except(files.Select(x => x.FileName))) { Log.WriteWarn(nameof(FileDownloader), $"\"{file}\" no longer exists in manifest"); } Log.WriteInfo($"FileDownloader {job.DepotID}", $"Will download {files.Count} files"); var downloadedFiles = 0; var fileTasks = new Task[files.Count]; for (var i = 0; i < fileTasks.Length; i++) { var file = files[i]; fileTasks[i] = TaskManager.Run(async() => { var existingFile = existingFileData.GetOrAdd(file.FileName, _ => new ExistingFileData()); EResult fileState; try { await ChunkDownloadingSemaphore.WaitAsync().ConfigureAwait(false); fileState = await DownloadFile(job, file, existingFile); } finally { ChunkDownloadingSemaphore.Release(); } if (fileState == EResult.OK || fileState == EResult.SameAsPreviousValue) { existingFile.FileHash = file.FileHash; downloadedFiles++; } if (fileState != EResult.SameAsPreviousValue) { // Do not write progress info to log file Console.WriteLine($"{job.DepotName} [{downloadedFiles / (float) files.Count * 100.0f,6:#00.00}%] {files.Count - downloadedFiles} files left to download"); } if (downloadState == EResult.DataCorruption) { return; } if (fileState == EResult.OK || fileState == EResult.DataCorruption) { downloadState = fileState; } }); } await Task.WhenAll(fileTasks).ConfigureAwait(false); await LocalConfig.Update($"depot.{job.DepotID}", JsonConvert.SerializeObject(existingFileData)); job.Result = downloadState switch { EResult.OK => EResult.OK, EResult.DataCorruption => EResult.DataCorruption, _ => EResult.Ignored }; return(job.Result); }
private static async Task DownloadSteam3Async(uint appId, List <DepotDownloadInfo> depots) { ulong TotalBytesCompressed = 0; ulong TotalBytesUncompressed = 0; foreach (var depot in depots) { ulong DepotBytesCompressed = 0; ulong DepotBytesUncompressed = 0; Console.WriteLine("Downloading depot {0} - {1}", depot.id, depot.contentName); CancellationTokenSource cts = new CancellationTokenSource(); cdnPool.ExhaustedToken = cts; ProtoManifest oldProtoManifest = null; ProtoManifest newProtoManifest = null; string configDir = Path.Combine(depot.installDir, CONFIG_DIR); ulong lastManifestId = INVALID_MANIFEST_ID; DepotConfigStore.Instance.InstalledManifestIDs.TryGetValue(depot.id, out lastManifestId); // In case we have an early exit, this will force equiv of verifyall next run. DepotConfigStore.Instance.InstalledManifestIDs[depot.id] = INVALID_MANIFEST_ID; DepotConfigStore.Save(); if (lastManifestId != INVALID_MANIFEST_ID) { var oldManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", lastManifestId)); if (File.Exists(oldManifestFileName)) { byte[] expectedChecksum, currentChecksum; try { expectedChecksum = File.ReadAllBytes(oldManifestFileName + ".sha"); } catch (IOException) { expectedChecksum = null; } oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName, out currentChecksum); if (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum)) { // We only have to show this warning if the old manifest ID was different if (lastManifestId != depot.manifestId) { Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", lastManifestId); } oldProtoManifest = null; } } } if (lastManifestId == depot.manifestId && oldProtoManifest != null) { newProtoManifest = oldProtoManifest; Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id); } else { var newManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", depot.manifestId)); if (newManifestFileName != null) { byte[] expectedChecksum, currentChecksum; try { expectedChecksum = File.ReadAllBytes(newManifestFileName + ".sha"); } catch (IOException) { expectedChecksum = null; } newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName, out currentChecksum); if (newProtoManifest != null && (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum))) { Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", depot.manifestId); newProtoManifest = null; } } if (newProtoManifest != null) { Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id); } else { Console.Write("Downloading depot manifest..."); DepotManifest depotManifest = null; while (depotManifest == null) { Tuple <CDNClient.Server, string> connection = null; try { connection = await cdnPool.GetConnectionForDepot(appId, depot.id, CancellationToken.None); depotManifest = await cdnPool.CDNClient.DownloadManifestAsync(depot.id, depot.manifestId, connection.Item1, connection.Item2, depot.depotKey).ConfigureAwait(false); cdnPool.ReturnConnection(connection); } catch (SteamKitWebRequestException e) { cdnPool.ReturnBrokenConnection(connection); if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden) { Console.WriteLine("Encountered 401 for depot manifest {0} {1}. Aborting.", depot.id, depot.manifestId); break; } else { Console.WriteLine("Encountered error downloading depot manifest {0} {1}: {2}", depot.id, depot.manifestId, e.StatusCode); } } catch (Exception e) { cdnPool.ReturnBrokenConnection(connection); Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depot.id, depot.manifestId, e.Message); } } if (depotManifest == null) { Console.WriteLine("\nUnable to download manifest {0} for depot {1}", depot.manifestId, depot.id); return; } byte[] checksum; newProtoManifest = new ProtoManifest(depotManifest, depot.manifestId); newProtoManifest.SaveToFile(newManifestFileName, out checksum); File.WriteAllBytes(newManifestFileName + ".sha", checksum); Console.WriteLine(" Done!"); } } newProtoManifest.Files.Sort((x, y) => string.Compare(x.FileName, y.FileName, StringComparison.Ordinal)); if (Config.DownloadManifestOnly) { StringBuilder manifestBuilder = new StringBuilder(); string txtManifest = Path.Combine(depot.installDir, string.Format("manifest_{0}.txt", depot.id)); foreach (var file in newProtoManifest.Files) { if (file.Flags.HasFlag(EDepotFileFlag.Directory)) { continue; } manifestBuilder.Append(string.Format("{0}\n", file.FileName)); } File.WriteAllText(txtManifest, manifestBuilder.ToString()); continue; } ulong complete_download_size = 0; ulong size_downloaded = 0; string stagingDir = Path.Combine(depot.installDir, STAGING_DIR); var filesAfterExclusions = newProtoManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).ToList(); // Pre-process filesAfterExclusions.ForEach(file => { var fileFinalPath = Path.Combine(depot.installDir, file.FileName); var fileStagingPath = Path.Combine(stagingDir, file.FileName); if (file.Flags.HasFlag(EDepotFileFlag.Directory)) { Directory.CreateDirectory(fileFinalPath); Directory.CreateDirectory(fileStagingPath); } else { // Some manifests don't explicitly include all necessary directories Directory.CreateDirectory(Path.GetDirectoryName(fileFinalPath)); Directory.CreateDirectory(Path.GetDirectoryName(fileStagingPath)); complete_download_size += file.TotalSize; } }); var semaphore = new SemaphoreSlim(Config.MaxDownloads); var files = filesAfterExclusions.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory)).ToArray(); var tasks = new Task[files.Length]; for (var i = 0; i < files.Length; i++) { var file = files[i]; var task = Task.Run(async() => { cts.Token.ThrowIfCancellationRequested(); try { await semaphore.WaitAsync().ConfigureAwait(false); cts.Token.ThrowIfCancellationRequested(); string fileFinalPath = Path.Combine(depot.installDir, file.FileName); string fileStagingPath = Path.Combine(stagingDir, file.FileName); // This may still exist if the previous run exited before cleanup if (File.Exists(fileStagingPath)) { File.Delete(fileStagingPath); } FileStream fs = null; List <ProtoManifest.ChunkData> neededChunks; FileInfo fi = new FileInfo(fileFinalPath); if (!fi.Exists) { // create new file. need all chunks fs = File.Create(fileFinalPath); fs.SetLength(( long )file.TotalSize); neededChunks = new List <ProtoManifest.ChunkData>(file.Chunks); } else { // open existing ProtoManifest.FileData oldManifestFile = null; if (oldProtoManifest != null) { oldManifestFile = oldProtoManifest.Files.SingleOrDefault(f => f.FileName == file.FileName); } if (oldManifestFile != null) { neededChunks = new List <ProtoManifest.ChunkData>(); if (Config.VerifyAll || !oldManifestFile.FileHash.SequenceEqual(file.FileHash)) { // we have a version of this file, but it doesn't fully match what we want var matchingChunks = new List <ChunkMatch>(); foreach (var chunk in file.Chunks) { var oldChunk = oldManifestFile.Chunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID)); if (oldChunk != null) { matchingChunks.Add(new ChunkMatch(oldChunk, chunk)); } else { neededChunks.Add(chunk); } } File.Move(fileFinalPath, fileStagingPath); fs = File.Open(fileFinalPath, FileMode.Create); fs.SetLength(( long )file.TotalSize); using (var fsOld = File.Open(fileStagingPath, FileMode.Open)) { foreach (var match in matchingChunks) { fsOld.Seek(( long )match.OldChunk.Offset, SeekOrigin.Begin); byte[] tmp = new byte[match.OldChunk.UncompressedLength]; fsOld.Read(tmp, 0, tmp.Length); byte[] adler = Util.AdlerHash(tmp); if (!adler.SequenceEqual(match.OldChunk.Checksum)) { neededChunks.Add(match.NewChunk); } else { fs.Seek(( long )match.NewChunk.Offset, SeekOrigin.Begin); fs.Write(tmp, 0, tmp.Length); } } } File.Delete(fileStagingPath); } } else { // No old manifest or file not in old manifest. We must validate. fs = File.Open(fileFinalPath, FileMode.Open); if (( ulong )fi.Length != file.TotalSize) { fs.SetLength(( long )file.TotalSize); } neededChunks = Util.ValidateSteam3FileChecksums(fs, file.Chunks.OrderBy(x => x.Offset).ToArray()); } if (neededChunks.Count() == 0) { size_downloaded += file.TotalSize; Console.WriteLine("{0,6:#00.00}% {1}", (( float )size_downloaded / ( float )complete_download_size) * 100.0f, fileFinalPath); if (fs != null) { fs.Dispose(); } return; } else { size_downloaded += (file.TotalSize - ( ulong )neededChunks.Select(x => ( long )x.UncompressedLength).Sum()); } } foreach (var chunk in neededChunks) { if (cts.IsCancellationRequested) { break; } string chunkID = Util.EncodeHexString(chunk.ChunkID); CDNClient.DepotChunk chunkData = null; while (!cts.IsCancellationRequested) { Tuple <CDNClient.Server, string> connection; try { connection = await cdnPool.GetConnectionForDepot(appId, depot.id, cts.Token); } catch (OperationCanceledException) { break; } DepotManifest.ChunkData data = new DepotManifest.ChunkData(); data.ChunkID = chunk.ChunkID; data.Checksum = chunk.Checksum; data.Offset = chunk.Offset; data.CompressedLength = chunk.CompressedLength; data.UncompressedLength = chunk.UncompressedLength; try { chunkData = await cdnPool.CDNClient.DownloadDepotChunkAsync(depot.id, data, connection.Item1, connection.Item2, depot.depotKey).ConfigureAwait(false); cdnPool.ReturnConnection(connection); break; } catch (SteamKitWebRequestException e) { cdnPool.ReturnBrokenConnection(connection); if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden) { Console.WriteLine("Encountered 401 for chunk {0}. Aborting.", chunkID); cts.Cancel(); break; } else { Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, e.StatusCode); } } catch (Exception e) { cdnPool.ReturnBrokenConnection(connection); Console.WriteLine("Encountered unexpected error downloading chunk {0}: {1}", chunkID, e.Message); } } if (chunkData == null) { Console.WriteLine("Failed to find any server with chunk {0} for depot {1}. Aborting.", chunkID, depot.id); cts.Cancel(); } // Throw the cancellation exception if requested so that this task is marked failed cts.Token.ThrowIfCancellationRequested(); TotalBytesCompressed += chunk.CompressedLength; DepotBytesCompressed += chunk.CompressedLength; TotalBytesUncompressed += chunk.UncompressedLength; DepotBytesUncompressed += chunk.UncompressedLength; fs.Seek(( long )chunk.Offset, SeekOrigin.Begin); fs.Write(chunkData.Data, 0, chunkData.Data.Length); size_downloaded += chunk.UncompressedLength; } fs.Dispose(); Console.WriteLine("{0,6:#00.00}% {1}", (( float )size_downloaded / ( float )complete_download_size) * 100.0f, fileFinalPath); } finally { semaphore.Release(); } }); tasks[i] = task; } await Task.WhenAll(tasks).ConfigureAwait(false); DepotConfigStore.Instance.InstalledManifestIDs[depot.id] = depot.manifestId; DepotConfigStore.Save(); Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.id, DepotBytesCompressed, DepotBytesUncompressed); } Console.WriteLine("Total downloaded: {0} bytes ({1} bytes uncompressed) from {2} depots", TotalBytesCompressed, TotalBytesUncompressed, depots.Count); }
private static void DownloadSteam3(List <DepotDownloadInfo3> depots) { foreach (var depot in depots) { int depotId = depot.id; ulong depot_manifest = depot.manifestId; byte[] depotKey = depot.depotKey; string installDir = depot.installDir; Console.WriteLine("Downloading depot {0} - {1}", depot.id, depot.contentName); Console.Write("Finding content servers..."); List <IPEndPoint> serverList = steam3.steamClient.GetServersOfType(EServerType.CS); List <CDNClient.ClientEndPoint> cdnServers = null; int counterDeferred = 0; for (int i = 0; ; i++) { IPEndPoint endpoint = serverList[i % serverList.Count]; cdnServers = CDNClient.FetchServerList(new CDNClient.ClientEndPoint(endpoint.Address.ToString(), endpoint.Port), Config.CellID); if (cdnServers == null) { counterDeferred++; } if (cdnServers != null && cdnServers.Count((ep) => { return(ep.Type == "CS"); }) > 0) { break; } if (((i + 1) % serverList.Count) == 0) { Console.WriteLine("Unable to find any Steam3 content servers"); return; } } Console.WriteLine(" Done!"); Console.Write("Downloading depot manifest..."); List <CDNClient.ClientEndPoint> cdnEndpoints = cdnServers.Where((ep) => { return(ep.Type == "CDN"); }).ToList(); List <CDNClient.ClientEndPoint> csEndpoints = cdnServers.Where((ep) => { return(ep.Type == "CS"); }).ToList(); List <CDNClient> cdnClients = new List <CDNClient>(); byte[] appTicket = steam3.AppTickets[(uint)depotId]; foreach (var server in csEndpoints) { CDNClient client; if (appTicket == null) { client = new CDNClient(server, (uint)depotId, steam3.steamUser.SteamID); } else { client = new CDNClient(server, appTicket); } if (client.Connect()) { cdnClients.Add(client); if (cdnClients.Count >= NUM_STEAM3_CONNECTIONS) { break; } } } if (cdnClients.Count == 0) { Console.WriteLine("\nCould not initialize connection with CDN."); return; } DepotManifest depotManifest = cdnClients[0].DownloadDepotManifest(depotId, depot_manifest); if (depotManifest == null) { // TODO: check for 401s for (int i = 1; i < cdnClients.Count && depotManifest == null; i++) { depotManifest = cdnClients[i].DownloadDepotManifest(depotId, depot_manifest); } if (depotManifest == null) { Console.WriteLine("\nUnable to download manifest {0} for depot {1}", depot_manifest, depotId); return; } } if (!depotManifest.DecryptFilenames(depotKey)) { Console.WriteLine("\nUnable to decrypt manifest for depot {0}", depotId); return; } Console.WriteLine(" Done!"); ulong complete_download_size = 0; ulong size_downloaded = 0; depotManifest.Files.Sort((x, y) => { return(x.FileName.CompareTo(y.FileName)); }); if (Config.DownloadManifestOnly) { StringBuilder manifestBuilder = new StringBuilder(); string txtManifest = Path.Combine(depot.installDir, string.Format("manifest_{0}.txt", depot.id)); foreach (var file in depotManifest.Files) { if (file.Flags.HasFlag(EDepotFileFlag.Directory)) { continue; } manifestBuilder.Append(string.Format("{0}\n", file.FileName)); } File.WriteAllText(txtManifest, manifestBuilder.ToString()); continue; } depotManifest.Files.RemoveAll((x) => !TestIsFileIncluded(x.FileName)); foreach (var file in depotManifest.Files) { complete_download_size += file.TotalSize; } foreach (var file in depotManifest.Files) { string download_path = Path.Combine(installDir, file.FileName); if (file.Flags.HasFlag(EDepotFileFlag.Directory)) { if (!Directory.Exists(download_path)) { Directory.CreateDirectory(download_path); } continue; } string dir_path = Path.GetDirectoryName(download_path); if (!Directory.Exists(dir_path)) { Directory.CreateDirectory(dir_path); } FileStream fs; DepotManifest.ChunkData[] neededChunks; FileInfo fi = new FileInfo(download_path); if (!fi.Exists) { // create new file. need all chunks fs = File.Create(download_path); neededChunks = file.Chunks.ToArray(); } else { // open existing fs = File.Open(download_path, FileMode.Open); if ((ulong)fi.Length != file.TotalSize) { fs.SetLength((long)file.TotalSize); } // find which chunks we need, in order so that we aren't seeking every which way neededChunks = Util.ValidateSteam3FileChecksums(fs, file.Chunks.OrderBy(x => x.Offset).ToArray()); if (neededChunks.Count() == 0) { size_downloaded += file.TotalSize; Console.WriteLine("{0,6:#00.00}% {1}", ((float)size_downloaded / (float)complete_download_size) * 100.0f, download_path); fs.Close(); continue; } else { size_downloaded += (file.TotalSize - (ulong)neededChunks.Select(x => (int)x.UncompressedLength).Sum()); } } Console.Write("{0,6:#00.00}% {1}", ((float)size_downloaded / (float)complete_download_size) * 100.0f, download_path); foreach (var chunk in neededChunks) { string chunkID = EncodeHexString(chunk.ChunkID); byte[] encrypted_chunk = cdnClients[0].DownloadDepotChunk(depotId, chunkID); if (encrypted_chunk == null) { for (int i = 1; i < cdnClients.Count && encrypted_chunk == null; i++) { encrypted_chunk = cdnClients[i].DownloadDepotChunk(depotId, chunkID); } if (encrypted_chunk == null) { Console.WriteLine("Unable to download chunk id {0} for depot {1}", chunkID, depotId); fs.Close(); return; } } byte[] chunk_data = CDNClient.ProcessChunk(encrypted_chunk, depotKey); fs.Seek((long)chunk.Offset, SeekOrigin.Begin); fs.Write(chunk_data, 0, chunk_data.Length); size_downloaded += chunk.UncompressedLength; Console.CursorLeft = 0; Console.Write("{0,6:#00.00}%", ((float)size_downloaded / (float)complete_download_size) * 100.0f); } fs.Close(); Console.WriteLine(); } } }
private async Task DownloadDepots(uint appID, List <ManifestJob> depots) { Log.WriteDebug("Depot Downloader", "Will process {0} depots ({1} depot locks left)", depots.Count(), DepotLocks.Count); var processTasks = new List <Task <EResult> >(); bool anyFilesDownloaded = false; foreach (var depot in depots) { var instance = depot.Anonymous ? Steam.Anonymous.Apps : Steam.Instance.Apps; depot.DepotKey = await GetDepotDecryptionKey(instance, depot.DepotID, appID); if (depot.DepotKey == null) { RemoveLock(depot.DepotID); continue; } var cdnToken = await GetCDNAuthToken(instance, appID, depot.DepotID); if (cdnToken == null) { RemoveLock(depot.DepotID); Log.WriteDebug("Depot Downloader", "Got a depot key for depot {0} but no cdn auth token", depot.DepotID); continue; } depot.CDNToken = cdnToken.Token; depot.Server = GetContentServer(); DepotManifest depotManifest = null; string lastError = string.Empty; for (var i = 0; i <= 5; i++) { try { depotManifest = await CDNClient.DownloadManifestAsync(depot.DepotID, depot.ManifestID, depot.Server, depot.CDNToken, depot.DepotKey); break; } catch (Exception e) { lastError = e.Message; Log.WriteError("Depot Processor", "Failed to download depot manifest for app {0} depot {1} ({2}: {3}) (#{4})", appID, depot.DepotID, depot.Server, lastError, i); } // TODO: get new auth key if auth fails depot.Server = GetContentServer(); if (depotManifest == null) { await Task.Delay(Utils.ExponentionalBackoff(i)); } } if (depotManifest == null) { LocalConfig.CDNAuthTokens.TryRemove(depot.DepotID, out _); RemoveLock(depot.DepotID); if (FileDownloader.IsImportantDepot(depot.DepotID)) { IRC.Instance.SendOps("{0}[{1}]{2} Failed to download manifest ({3})", Colors.OLIVE, depot.DepotName, Colors.NORMAL, lastError); } if (!Settings.IsFullRun) { JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null)); } continue; } var task = ProcessDepotAfterDownload(depot, depotManifest); processTasks.Add(task); if (!FileDownloader.IsImportantDepot(depot.DepotID)) { continue; } task = TaskManager.Run(async() => { var result = EResult.Fail; try { result = await FileDownloader.DownloadFilesFromDepot(depot, depotManifest); if (result == EResult.OK) { anyFilesDownloaded = true; } } catch (Exception e) { ErrorReporter.Notify($"Depot Processor {depot.DepotID}", e); } return(result); }).Unwrap(); processTasks.Add(task); } if (SaveLocalConfig) { SaveLocalConfig = false; LocalConfig.Save(); } await Task.WhenAll(processTasks).ConfigureAwait(false); Log.WriteDebug("Depot Downloader", "{0} depot downloads finished", depots.Count()); // TODO: use ContinueWith on tasks if (!anyFilesDownloaded) { foreach (var depot in depots) { RemoveLock(depot.DepotID); } return; } if (!File.Exists(UpdateScript)) { return; } lock (UpdateScriptLock) { foreach (var depot in depots) { if (depot.Result == EResult.OK) { RunUpdateScript(string.Format("{0} no-git", depot.DepotID)); } else if (depot.Result != EResult.Ignored) { Log.WriteWarn("Depot Processor", "Dropping stored token for {0} due to download failures", depot.DepotID); LocalConfig.CDNAuthTokens.TryRemove(depot.DepotID, out _); using (var db = Database.Get()) { // Mark this depot for redownload db.Execute("UPDATE `Depots` SET `LastManifestID` = 0 WHERE `DepotID` = @DepotID", new { depot.DepotID }); } } RemoveLock(depot.DepotID); } // Only commit changes if all depots downloaded if (processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored)) { if (!RunUpdateScript(appID, depots.First().BuildID)) { RunUpdateScript("0"); } } else { Log.WriteDebug("Depot Processor", "Reprocessing the app {0} because some files failed to download", appID); IRC.Instance.SendOps("{0}[{1}]{2} Reprocessing the app due to download failures", Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL ); JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null)); } } }
private static async Task DownloadSteam3Async(uint appId, List <DepotDownloadInfo> depots, Action downloadCompleteAction = null) { ulong TotalBytesCompressed = 0; ulong TotalBytesUncompressed = 0; foreach (var depot in depots) { ulong DepotBytesCompressed = 0; ulong DepotBytesUncompressed = 0; DebugLog.WriteLine("ContentDownloader", "Downloading depot " + depot.id + " - " + depot.contentName); CancellationTokenSource cts = new CancellationTokenSource(); cdnPool.ExhaustedToken = cts; ProtoManifest oldProtoManifest = null; ProtoManifest downloadManifest = null; string configDir = Path.Combine(depot.installDir, CONFIG_DIR); ulong lastManifestId = INVALID_MANIFEST_ID; ConfigStore.TheConfig.LastManifests.TryGetValue(depot.id, out lastManifestId); // In case we have an early exit, this will force equiv of verifyall next run. ConfigStore.TheConfig.LastManifests[depot.id] = INVALID_MANIFEST_ID; ConfigStore.Save(); if (lastManifestId != INVALID_MANIFEST_ID) { var oldManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", lastManifestId)); DebugLog.WriteLine(DEBUG_NAME_FILES, "Checking if " + oldManifestFileName + " exists"); if (File.Exists(oldManifestFileName)) { DebugLog.WriteLine(DEBUG_NAME_FILES, oldManifestFileName + " exists, reading!"); oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName); } } if (lastManifestId == depot.manifestId && oldProtoManifest != null) { downloadManifest = oldProtoManifest; DebugLog.WriteLine("ContentDownloader", "Already have manifest " + depot.manifestId + " for depot " + depot.id + "."); } else { var newManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", depot.manifestId)); if (newManifestFileName != null) { downloadManifest = ProtoManifest.LoadFromFile(newManifestFileName); } if (downloadManifest != null) { DebugLog.WriteLine("ContentDownloader", "Already have manifest " + depot.manifestId + " for depot " + depot.id + "."); } else { DebugLog.WriteLine("ContentDownloader", "Downloading depot manifest..."); DepotManifest depotManifest = null; while (depotManifest == null) { CDNClient client = null; try { client = await cdnPool.GetConnectionForDepotAsync(appId, depot.id, depot.depotKey, CancellationToken.None).ConfigureAwait(false); //client = await cdnPool.GetConnectionForDepotAsync(appId, depot.id, depot.depotKey, CancellationToken.None); depotManifest = await client.DownloadManifestAsync(depot.id, depot.manifestId).ConfigureAwait(false); //depotManifest = await client.DownloadManifestAsync(depot.id, depot.manifestId); cdnPool.ReturnConnection(client); } catch (WebException e) { cdnPool.ReturnBrokenConnection(client); if (e.Status == WebExceptionStatus.ProtocolError) { var response = e.Response as HttpWebResponse; if (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden) { DebugLog.WriteLine("ContentDownloader", "Encountered 401 for depot manifest " + depot.id + " " + depot.manifestId + ". Aborting."); break; } else { DebugLog.WriteLine("ContentDownloader", "Encountered error downloading depot manifest " + depot.id + " " + depot.manifestId + ": " + response.StatusCode); } } else { DebugLog.WriteLine("ContentDownloader", "Encountered error downloading manifest for depot " + depot.id + " " + depot.manifestId + ": " + e.Status); } } catch (Exception e) { cdnPool.ReturnBrokenConnection(client); DebugLog.WriteLine("ContentDownloader", "Encountered error downloading manifest for depot " + depot.id + " " + depot.manifestId + ": " + e.Message); } } if (depotManifest == null) { DebugLog.WriteLine("ContentDownloader", "\nUnable to download manifest " + depot.manifestId + " for depot " + depot.id); return; } downloadManifest = new ProtoManifest(depotManifest, depot.manifestId); downloadManifest.SaveToFile(newManifestFileName); DebugLog.WriteLine("ContentDownloader", "Done!"); } } downloadManifest.Files.Sort((x, y) => { return(x.FileName.CompareTo(y.FileName)); }); if (downloadManifest != null) { onManifestReceived?.Invoke(appId, depot.id, depot.contentName, downloadManifest); } if (Config.DownloadManifestOnly) { continue; } complete_download_size = 0; size_downloaded = 0; string stagingDir = Path.Combine(depot.installDir, STAGING_DIR); var filesAfterExclusions = downloadManifest.Files.Where(f => TestIsFileIncluded(f.FileName)).ToList(); // Pre-process filesAfterExclusions.ForEach(file => { var fileFinalPath = Path.Combine(depot.installDir, file.FileName); var fileStagingPath = Path.Combine(stagingDir, file.FileName); if (file.Flags.HasFlag(EDepotFileFlag.Directory)) { Directory.CreateDirectory(fileFinalPath); Directory.CreateDirectory(fileStagingPath); } else { // Some manifests don't explicitly include all necessary directories Directory.CreateDirectory(Path.GetDirectoryName(fileFinalPath)); Directory.CreateDirectory(Path.GetDirectoryName(fileStagingPath)); complete_download_size += file.TotalSize; } }); var semaphore = new SemaphoreSlim(Config.MaxDownloads); var files = filesAfterExclusions.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory)).ToArray(); var tasks = new Task[files.Length]; for (var i = 0; i < files.Length; i++) { var file = files[i]; var task = Task.Run(async() => { cts.Token.ThrowIfCancellationRequested(); try { await semaphore.WaitAsync().ConfigureAwait(false); //await semaphore.WaitAsync(); cts.Token.ThrowIfCancellationRequested(); string fileFinalPath = Path.Combine(depot.installDir, file.FileName); string fileStagingPath = Path.Combine(stagingDir, file.FileName); // This may still exist if the previous run exited before cleanup DebugLog.WriteLine(DEBUG_NAME_FILES, "Checking if " + fileStagingPath + " exists"); if (File.Exists(fileStagingPath)) { DebugLog.WriteLine(DEBUG_NAME_FILES, fileStagingPath + " exists, deleting!"); File.Delete(fileStagingPath); } List <ProtoManifest.ChunkData> neededChunks; FileInfo fi = new FileInfo(fileFinalPath); DebugLog.WriteLine(DEBUG_NAME_FILES, "Checking if " + fileFinalPath + " exists"); if (!fi.Exists) { // create new file. need all chunks DebugLog.WriteLine(DEBUG_NAME_FILES, fileFinalPath + " does not exist, creating!"); using (FileStream fs = File.Create(fileFinalPath)) { fs.SetLength((long)file.TotalSize); neededChunks = new List <ProtoManifest.ChunkData>(file.Chunks); } } else { // open existing ProtoManifest.FileData oldManifestFile = null; if (oldProtoManifest != null) { oldManifestFile = oldProtoManifest.Files.SingleOrDefault(f => f.FileName == file.FileName); } if (oldManifestFile != null) { neededChunks = new List <ProtoManifest.ChunkData>(); if (Config.VerifyAll || !oldManifestFile.FileHash.SequenceEqual(file.FileHash)) { // we have a version of this file, but it doesn't fully match what we want var matchingChunks = new List <ChunkMatch>(); foreach (var chunk in file.Chunks) { var oldChunk = oldManifestFile.Chunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID)); if (oldChunk != null) { matchingChunks.Add(new ChunkMatch(oldChunk, chunk)); } else { neededChunks.Add(chunk); } } DebugLog.WriteLine(DEBUG_NAME_FILES, "Moving file " + fileFinalPath + " to " + fileStagingPath); File.Move(fileFinalPath, fileStagingPath); DebugLog.WriteLine(DEBUG_NAME_FILES, "Creating file " + fileFinalPath); using (FileStream fs = File.Open(fileFinalPath, FileMode.Create)) { fs.SetLength((long)file.TotalSize); DebugLog.WriteLine(DEBUG_NAME_FILES, "Opening file " + fileStagingPath); using (var fsOld = File.Open(fileStagingPath, FileMode.Open)) { foreach (var match in matchingChunks) { fsOld.Seek((long)match.OldChunk.Offset, SeekOrigin.Begin); byte[] tmp = new byte[match.OldChunk.UncompressedLength]; fsOld.Read(tmp, 0, tmp.Length); byte[] adler = Util.AdlerHash(tmp); if (!adler.SequenceEqual(match.OldChunk.Checksum)) { neededChunks.Add(match.NewChunk); } else { fs.Seek((long)match.NewChunk.Offset, SeekOrigin.Begin); fs.Write(tmp, 0, tmp.Length); } } } } DebugLog.WriteLine(DEBUG_NAME_FILES, "Deleting file " + fileStagingPath); File.Delete(fileStagingPath); } } else { // No old manifest or file not in old manifest. We must validate. DebugLog.WriteLine(DEBUG_NAME_FILES, "Opening file " + fileFinalPath); using (FileStream fs = File.Open(fileFinalPath, FileMode.Open)) { if ((ulong)fi.Length != file.TotalSize) { fs.SetLength((long)file.TotalSize); } neededChunks = Util.ValidateSteam3FileChecksums(fs, file.Chunks.OrderBy(x => x.Offset).ToArray()); } } if (neededChunks.Count() == 0) { size_downloaded += file.TotalSize; DebugLog.WriteLine("ContentDownloader", DownloadPercent * 100.0f + "% " + fileFinalPath); return; } else { size_downloaded += (file.TotalSize - ( ulong )neededChunks.Select(x => ( long )x.UncompressedLength).Sum()); } } DebugLog.WriteLine(DEBUG_NAME_FILES, "Opening file " + fileFinalPath); using (FileStream fs = File.Open(fileFinalPath, FileMode.Open)) { foreach (var chunk in neededChunks) { if (cts.IsCancellationRequested) { break; } string chunkID = Util.EncodeHexString(chunk.ChunkID); CDNClient.DepotChunk chunkData = null; while (!cts.IsCancellationRequested) { CDNClient client; try { client = await cdnPool.GetConnectionForDepotAsync(appId, depot.id, depot.depotKey, cts.Token).ConfigureAwait(false); //client = await cdnPool.GetConnectionForDepotAsync(appId, depot.id, depot.depotKey, cts.Token); } catch (OperationCanceledException) { break; } DepotManifest.ChunkData data = new DepotManifest.ChunkData(); data.ChunkID = chunk.ChunkID; data.Checksum = chunk.Checksum; data.Offset = chunk.Offset; data.CompressedLength = chunk.CompressedLength; data.UncompressedLength = chunk.UncompressedLength; try { //chunkData = await client.DownloadDepotChunkAsStreamAsync(depot.id, data).ConfigureAwait(false); chunkData = client.DownloadDepotChunkAsStreamAsync(depot.id, data).Result; //chunkData = await client.DownloadDepotChunkAsync(depot.id, data); cdnPool.ReturnConnection(client); break; } catch (WebException e) { cdnPool.ReturnBrokenConnection(client); if (e.Status == WebExceptionStatus.ProtocolError) { var response = e.Response as HttpWebResponse; if (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden) { DebugLog.WriteLine("ContentDownloader", "Encountered 401 for chunk " + chunkID + ". Aborting."); cts.Cancel(); break; } else { DebugLog.WriteLine("ContentDownloader", "Encountered error downloading chunk " + chunkID + ": " + response.StatusCode); } } else { DebugLog.WriteLine("ContentDownloader", "Encountered error downloading chunk " + chunkID + ": " + e.Status); } } catch (Exception e) { cdnPool.ReturnBrokenConnection(client); DebugLog.WriteLine("ContentDownloader", "Encountered unexpected error downloading chunk " + chunkID + ": " + e.Message); } } if (chunkData == null) { DebugLog.WriteLine("ContentDownloader", "Failed to find any server with chunk " + chunkID + " for depot " + depot.id + ". Aborting."); cts.Cancel(); return; } TotalBytesCompressed += chunk.CompressedLength; DepotBytesCompressed += chunk.CompressedLength; TotalBytesUncompressed += chunk.UncompressedLength; DepotBytesUncompressed += chunk.UncompressedLength; using (chunkData.DataStream) { fs.Seek((long)chunk.Offset, SeekOrigin.Begin); chunkData.DataStream.CopyTo(fs); //fs.Write(chunkData.Data, 0, chunkData.Data.Length); } size_downloaded += chunk.UncompressedLength; } } DebugLog.WriteLine("ContentDownloader", DownloadPercent * 100.0f + "% " + fileFinalPath); } finally { semaphore.Release(); } }); tasks[i] = task; } Task.WaitAll(tasks); ConfigStore.TheConfig.LastManifests[depot.id] = depot.manifestId; ConfigStore.Save(); DebugLog.WriteLine("ContentDownloader", "Depot " + depot.id + " - Downloaded " + DepotBytesCompressed + " bytes (" + DepotBytesUncompressed + " bytes uncompressed)"); } IsDownloading = false; downloadCompleteAction?.Invoke(); onDownloadCompleted?.Invoke(); DebugLog.WriteLine("ContentDownloader", "Total downloaded: " + TotalBytesCompressed + " bytes (" + TotalBytesUncompressed + " bytes uncompressed) from " + depots.Count + " depots"); }
private async Task <EResult> ProcessDepotAfterDownload(IDbConnection db, IDbTransaction transaction, ManifestJob request, DepotManifest depotManifest) { var filesOld = (await db.QueryAsync <DepotFile>("SELECT `ID`, `File`, `Hash`, `Size`, `Flags` FROM `DepotsFiles` WHERE `DepotID` = @DepotID", new { request.DepotID }, transaction: transaction)).ToDictionary(x => x.File, x => x); var filesAdded = new List <DepotFile>(); var shouldHistorize = filesOld.Any(); // Don't historize file additions if we didn't have any data before foreach (var file in depotManifest.Files) { var name = file.FileName.Replace('\\', '/'); byte[] hash = null; // Store empty hashes as NULL (e.g. an empty file) if (file.FileHash.Length > 0 && !file.Flags.HasFlag(EDepotFileFlag.Directory)) { for (int i = 0; i < file.FileHash.Length; ++i) { if (file.FileHash[i] != 0) { hash = file.FileHash; break; } } } // safe guard if (name.Length > 255) { ErrorReporter.Notify("Depot Processor", new OverflowException(string.Format("File \"{0}\" in depot {1} is too long", name, request.DepotID))); continue; } if (filesOld.ContainsKey(name)) { var oldFile = filesOld[name]; var updateFile = false; if (oldFile.Size != file.TotalSize || !Utils.IsEqualSHA1(hash, oldFile.Hash)) { await MakeHistory(db, transaction, request, name, "modified", oldFile.Size, file.TotalSize); updateFile = true; } if (oldFile.Flags != file.Flags) { await MakeHistory(db, transaction, request, name, "modified_flags", (ulong)oldFile.Flags, (ulong)file.Flags); updateFile = true; } if (updateFile) { await db.ExecuteAsync("UPDATE `DepotsFiles` SET `Hash` = @Hash, `Size` = @Size, `Flags` = @Flags WHERE `DepotID` = @DepotID AND `ID` = @ID", new DepotFile { ID = oldFile.ID, DepotID = request.DepotID, Hash = hash, Size = file.TotalSize, Flags = file.Flags }, transaction : transaction); } filesOld.Remove(name); } else { // We want to historize modifications first, and only then deletions and additions filesAdded.Add(new DepotFile { DepotID = request.DepotID, Hash = hash, File = name, Size = file.TotalSize, Flags = file.Flags }); } } if (filesOld.Any()) { await db.ExecuteAsync("DELETE FROM `DepotsFiles` WHERE `DepotID` = @DepotID AND `ID` IN @Files", new { request.DepotID, Files = filesOld.Select(x => x.Value.ID) }, transaction : transaction); await db.ExecuteAsync(HistoryQuery, filesOld.Select(x => new DepotHistory { DepotID = request.DepotID, ChangeID = request.ChangeNumber, Action = "removed", File = x.Value.File }), transaction : transaction); } if (filesAdded.Any()) { await db.ExecuteAsync("INSERT INTO `DepotsFiles` (`DepotID`, `File`, `Hash`, `Size`, `Flags`) VALUES (@DepotID, @File, @Hash, @Size, @Flags)", filesAdded, transaction : transaction); if (shouldHistorize) { await db.ExecuteAsync(HistoryQuery, filesAdded.Select(x => new DepotHistory { DepotID = request.DepotID, ChangeID = request.ChangeNumber, Action = "added", File = x.File }), transaction : transaction); } } await db.ExecuteAsync("UPDATE `Depots` SET `LastManifestID` = @ManifestID, `LastUpdated` = CURRENT_TIMESTAMP() WHERE `DepotID` = @DepotID", new { request.DepotID, request.ManifestID }, transaction : transaction); return(EResult.OK); }
private static async Task <EResult> ProcessDepotAfterDownload(IDbConnection db, IDbTransaction transaction, ManifestJob request, DepotManifest depotManifest) { var filesOld = (await db.QueryAsync <DepotFile>("SELECT `ID`, `File`, `Hash`, `Size`, `Flags` FROM `DepotsFiles` WHERE `DepotID` = @DepotID", new { request.DepotID }, transaction: transaction)).ToDictionary(x => x.File, x => x); var filesAdded = new List <DepotFile>(); var shouldHistorize = filesOld.Count > 0; // Don't historize file additions if we didn't have any data before foreach (var file in depotManifest.Files.OrderByDescending(x => x.FileName)) { var name = file.FileName.Replace('\\', '/'); byte[] hash = null; // Store empty hashes as NULL (e.g. an empty file) if (file.FileHash.Length > 0 && (file.Flags & EDepotFileFlag.Directory) == 0) { for (var i = 0; i < file.FileHash.Length; ++i) { if (file.FileHash[i] != 0) { hash = file.FileHash; break; } } } // Limit path names to 260 characters (default windows max length) // File column is varchar(260) and not higher to prevent reducing performance // See https://stackoverflow.com/questions/1962310/importance-of-varchar-length-in-mysql-table/1962329#1962329 // Until 2019 there hasn't been a single file that went over this limit, so far there has been only one // game with a big node_modules path, so we're safeguarding by limiting it. if (name.Length > 260) { using var sha = new System.Security.Cryptography.SHA1Managed(); var nameHash = Utils.ByteArrayToString(sha.ComputeHash(System.Text.Encoding.UTF8.GetBytes(name))); name = $"{{SteamDB file name is too long}}/{nameHash}/...{name.Substring(name.Length - 150)}"; } if (filesOld.ContainsKey(name)) { var oldFile = filesOld[name]; var updateFile = false; if (oldFile.Size != file.TotalSize || !Utils.IsEqualSHA1(hash, oldFile.Hash)) { await MakeHistory(db, transaction, request, name, "modified", oldFile.Size, file.TotalSize); updateFile = true; } if (oldFile.Flags != file.Flags) { await MakeHistory(db, transaction, request, name, "modified_flags", (ulong)oldFile.Flags, (ulong)file.Flags); updateFile = true; } if (updateFile) { await db.ExecuteAsync("UPDATE `DepotsFiles` SET `Hash` = @Hash, `Size` = @Size, `Flags` = @Flags WHERE `DepotID` = @DepotID AND `ID` = @ID", new DepotFile { ID = oldFile.ID, DepotID = request.DepotID, Hash = hash, Size = file.TotalSize, Flags = file.Flags }, transaction : transaction); } filesOld.Remove(name); } else { // We want to historize modifications first, and only then deletions and additions filesAdded.Add(new DepotFile { DepotID = request.DepotID, Hash = hash, File = name, Size = file.TotalSize, Flags = file.Flags }); } } if (filesOld.Count > 0) { await db.ExecuteAsync("DELETE FROM `DepotsFiles` WHERE `DepotID` = @DepotID AND `ID` IN @Files", new { request.DepotID, Files = filesOld.Select(x => x.Value.ID) }, transaction : transaction); await db.ExecuteAsync(HistoryQuery, filesOld.Select(x => new DepotHistory { DepotID = request.DepotID, ManifestID = request.ManifestID, ChangeID = request.ChangeNumber, Action = "removed", File = x.Value.File, OldValue = x.Value.Size }), transaction : transaction); } if (filesAdded.Count > 0) { await db.ExecuteAsync("INSERT INTO `DepotsFiles` (`DepotID`, `File`, `Hash`, `Size`, `Flags`) VALUES (@DepotID, @File, @Hash, @Size, @Flags)", filesAdded, transaction : transaction); if (shouldHistorize) { await db.ExecuteAsync(HistoryQuery, filesAdded.Select(x => new DepotHistory { DepotID = request.DepotID, ManifestID = request.ManifestID, ChangeID = request.ChangeNumber, Action = "added", File = x.File, NewValue = x.Size }), transaction : transaction); } } await db.ExecuteAsync("UPDATE `Depots` SET `LastManifestID` = @ManifestID, `LastUpdated` = CURRENT_TIMESTAMP() WHERE `DepotID` = @DepotID", new { request.DepotID, request.ManifestID }, transaction : transaction); return(EResult.OK); }