private static void DownloadSteam3(uint appId, List <DepotDownloadInfo> depots) { ulong TotalBytesCompressed = 0; ulong TotalBytesUncompressed = 0; foreach (var depot in depots) { ulong DepotBytesCompressed = 0; ulong DepotBytesUncompressed = 0; Console.WriteLine("Downloading depot {0} - {1}", depot.id, depot.contentName); CancellationTokenSource cts = new CancellationTokenSource(); ProtoManifest oldProtoManifest = null; ProtoManifest newProtoManifest = null; string configDir = Path.Combine(depot.installDir, CONFIG_DIR); ulong lastManifestId = INVALID_MANIFEST_ID; ConfigStore.TheConfig.LastManifests.TryGetValue(depot.id, out lastManifestId); // In case we have an early exit, this will force equiv of verifyall next run. ConfigStore.TheConfig.LastManifests[depot.id] = INVALID_MANIFEST_ID; ConfigStore.Save(); if (lastManifestId != INVALID_MANIFEST_ID) { var oldManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", lastManifestId)); if (File.Exists(oldManifestFileName)) { oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName); } } if (lastManifestId == depot.manifestId && oldProtoManifest != null) { newProtoManifest = oldProtoManifest; Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id); } else { var newManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", depot.manifestId)); if (newManifestFileName != null) { newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName); } if (newProtoManifest != null) { Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id); } else { Console.Write("Downloading depot manifest..."); DepotManifest depotManifest = null; while (depotManifest == null) { CDNClient client = null; try { client = cdnPool.GetConnectionForDepot(appId, depot.id, depot.depotKey, CancellationToken.None); depotManifest = client.DownloadManifest(depot.id, depot.manifestId); cdnPool.ReturnConnection(client); } catch (WebException e) { cdnPool.ReturnBrokenConnection(client); if (e.Status == WebExceptionStatus.ProtocolError) { var response = e.Response as HttpWebResponse; if (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden) { Console.WriteLine("Encountered 401 for depot manifest {0} {1}. Aborting.", depot.id, depot.manifestId); break; } else { Console.WriteLine("Encountered error downloading depot manifest {0} {1}: {2}", depot.id, depot.manifestId, response.StatusCode); } } else { Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depot.id, depot.manifestId, e.Status); } } catch (Exception e) { cdnPool.ReturnBrokenConnection(client); Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depot.id, depot.manifestId, e.Message); } } if (depotManifest == null) { Console.WriteLine("\nUnable to download manifest {0} for depot {1}", depot.manifestId, depot.id); return; } newProtoManifest = new ProtoManifest(depotManifest, depot.manifestId); newProtoManifest.SaveToFile(newManifestFileName); Console.WriteLine("Successfully downloaded manifest {0} for depot {1}", depot.manifestId, depot.id); Console.WriteLine(" Done!"); } } newProtoManifest.Files.Sort((x, y) => { return(x.FileName.CompareTo(y.FileName)); }); if (Config.DownloadManifestOnly) { StringBuilder manifestBuilder = new StringBuilder(); string txtManifest = Path.Combine(depot.installDir, string.Format("manifest_{0}.txt", depot.id)); foreach (var file in newProtoManifest.Files) { if (file.Flags.HasFlag(EDepotFileFlag.Directory)) { continue; } manifestBuilder.Append(string.Format("{0}\n", file.FileName)); } File.WriteAllText(txtManifest, manifestBuilder.ToString()); continue; } ulong complete_download_size = 0; ulong size_downloaded = 0; string stagingDir = Path.Combine(depot.installDir, STAGING_DIR); var filesAfterExclusions = newProtoManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).ToList(); // Pre-process filesAfterExclusions.ForEach(file => { var fileFinalPath = Path.Combine(depot.installDir, file.FileName); var fileStagingPath = Path.Combine(stagingDir, file.FileName); if (file.Flags.HasFlag(EDepotFileFlag.Directory)) { Directory.CreateDirectory(fileFinalPath); Directory.CreateDirectory(fileStagingPath); } else { // Some manifests don't explicitly include all necessary directories Directory.CreateDirectory(Path.GetDirectoryName(fileFinalPath)); Directory.CreateDirectory(Path.GetDirectoryName(fileStagingPath)); complete_download_size += file.TotalSize; } }); filesAfterExclusions.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory)) .AsParallel().WithCancellation(cts.Token).WithDegreeOfParallelism(Config.MaxDownloads) .ForAll(file => { string fileFinalPath = Path.Combine(depot.installDir, file.FileName); string fileStagingPath = Path.Combine(stagingDir, file.FileName); // This may still exist if the previous run exited before cleanup if (File.Exists(fileStagingPath)) { File.Delete(fileStagingPath); } FileStream fs = null; List <ProtoManifest.ChunkData> neededChunks; FileInfo fi = new FileInfo(fileFinalPath); if (!fi.Exists) { // create new file. need all chunks fs = File.Create(fileFinalPath); fs.SetLength(( long )file.TotalSize); neededChunks = new List <ProtoManifest.ChunkData>(file.Chunks); } else { // open existing ProtoManifest.FileData oldManifestFile = null; if (oldProtoManifest != null) { oldManifestFile = oldProtoManifest.Files.SingleOrDefault(f => f.FileName == file.FileName); } if (oldManifestFile != null) { neededChunks = new List <ProtoManifest.ChunkData>(); if (Config.VerifyAll || !oldManifestFile.FileHash.SequenceEqual(file.FileHash)) { // we have a version of this file, but it doesn't fully match what we want var matchingChunks = new List <ChunkMatch>(); foreach (var chunk in file.Chunks) { var oldChunk = oldManifestFile.Chunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID)); if (oldChunk != null) { matchingChunks.Add(new ChunkMatch(oldChunk, chunk)); } else { neededChunks.Add(chunk); } } File.Move(fileFinalPath, fileStagingPath); fs = File.Open(fileFinalPath, FileMode.Create); fs.SetLength(( long )file.TotalSize); using (var fsOld = File.Open(fileStagingPath, FileMode.Open)) { foreach (var match in matchingChunks) { fsOld.Seek(( long )match.OldChunk.Offset, SeekOrigin.Begin); byte[] tmp = new byte[match.OldChunk.UncompressedLength]; fsOld.Read(tmp, 0, tmp.Length); byte[] adler = Util.AdlerHash(tmp); if (!adler.SequenceEqual(match.OldChunk.Checksum)) { neededChunks.Add(match.NewChunk); } else { fs.Seek(( long )match.NewChunk.Offset, SeekOrigin.Begin); fs.Write(tmp, 0, tmp.Length); } } } File.Delete(fileStagingPath); } } else { // No old manifest or file not in old manifest. We must validate. fs = File.Open(fileFinalPath, FileMode.Open); if (( ulong )fi.Length != file.TotalSize) { fs.SetLength(( long )file.TotalSize); } neededChunks = Util.ValidateSteam3FileChecksums(fs, file.Chunks.OrderBy(x => x.Offset).ToArray()); } if (neededChunks.Count() == 0) { size_downloaded += file.TotalSize; Console.WriteLine("{0,6:#00.00}% {1}", (( float )size_downloaded / ( float )complete_download_size) * 100.0f, fileFinalPath); if (fs != null) { fs.Close(); } return; } else { size_downloaded += (file.TotalSize - ( ulong )neededChunks.Select(x => ( long )x.UncompressedLength).Sum()); } } foreach (var chunk in neededChunks) { if (cts.IsCancellationRequested) { break; } string chunkID = Util.EncodeHexString(chunk.ChunkID); CDNClient.DepotChunk chunkData = null; while (!cts.IsCancellationRequested) { CDNClient client; try { client = cdnPool.GetConnectionForDepot(appId, depot.id, depot.depotKey, cts.Token); } catch (OperationCanceledException) { break; } DepotManifest.ChunkData data = new DepotManifest.ChunkData(); data.ChunkID = chunk.ChunkID; data.Checksum = chunk.Checksum; data.Offset = chunk.Offset; data.CompressedLength = chunk.CompressedLength; data.UncompressedLength = chunk.UncompressedLength; try { chunkData = client.DownloadDepotChunk(depot.id, data); cdnPool.ReturnConnection(client); break; } catch (WebException e) { cdnPool.ReturnBrokenConnection(client); if (e.Status == WebExceptionStatus.ProtocolError) { var response = e.Response as HttpWebResponse; if (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden) { Console.WriteLine("Encountered 401 for chunk {0}. Aborting.", chunkID); cts.Cancel(); break; } else { Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, response.StatusCode); } } else { Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, e.Status); } } catch (Exception e) { cdnPool.ReturnBrokenConnection(client); Console.WriteLine("Encountered unexpected error downloading chunk {0}: {1}", chunkID, e.Message); } } if (chunkData == null) { Console.WriteLine("Failed to find any server with chunk {0} for depot {1}. Aborting.", chunkID, depot.id); return; } TotalBytesCompressed += chunk.CompressedLength; DepotBytesCompressed += chunk.CompressedLength; TotalBytesUncompressed += chunk.UncompressedLength; DepotBytesUncompressed += chunk.UncompressedLength; fs.Seek(( long )chunk.Offset, SeekOrigin.Begin); fs.Write(chunkData.Data, 0, chunkData.Data.Length); size_downloaded += chunk.UncompressedLength; } fs.Close(); Console.WriteLine("{0,6:#00.00}% {1}", (( float )size_downloaded / ( float )complete_download_size) * 100.0f, fileFinalPath); }); ConfigStore.TheConfig.LastManifests[depot.id] = depot.manifestId; ConfigStore.Save(); Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.id, DepotBytesCompressed, DepotBytesUncompressed); } Console.WriteLine("Total downloaded: {0} bytes ({1} bytes uncompressed) from {2} depots", TotalBytesCompressed, TotalBytesUncompressed, depots.Count); }
private static async Task DownloadSteam3Async(uint appId, List <DepotDownloadInfo> depots, Action downloadCompleteAction = null) { ulong TotalBytesCompressed = 0; ulong TotalBytesUncompressed = 0; foreach (var depot in depots) { ulong DepotBytesCompressed = 0; ulong DepotBytesUncompressed = 0; DebugLog.WriteLine("ContentDownloader", "Downloading depot " + depot.id + " - " + depot.contentName); CancellationTokenSource cts = new CancellationTokenSource(); cdnPool.ExhaustedToken = cts; ProtoManifest oldProtoManifest = null; ProtoManifest downloadManifest = null; string configDir = Path.Combine(depot.installDir, CONFIG_DIR); ulong lastManifestId = INVALID_MANIFEST_ID; ConfigStore.TheConfig.LastManifests.TryGetValue(depot.id, out lastManifestId); // In case we have an early exit, this will force equiv of verifyall next run. ConfigStore.TheConfig.LastManifests[depot.id] = INVALID_MANIFEST_ID; ConfigStore.Save(); if (lastManifestId != INVALID_MANIFEST_ID) { var oldManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", lastManifestId)); DebugLog.WriteLine(DEBUG_NAME_FILES, "Checking if " + oldManifestFileName + " exists"); if (File.Exists(oldManifestFileName)) { DebugLog.WriteLine(DEBUG_NAME_FILES, oldManifestFileName + " exists, reading!"); oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName); } } if (lastManifestId == depot.manifestId && oldProtoManifest != null) { downloadManifest = oldProtoManifest; DebugLog.WriteLine("ContentDownloader", "Already have manifest " + depot.manifestId + " for depot " + depot.id + "."); } else { var newManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", depot.manifestId)); if (newManifestFileName != null) { downloadManifest = ProtoManifest.LoadFromFile(newManifestFileName); } if (downloadManifest != null) { DebugLog.WriteLine("ContentDownloader", "Already have manifest " + depot.manifestId + " for depot " + depot.id + "."); } else { DebugLog.WriteLine("ContentDownloader", "Downloading depot manifest..."); DepotManifest depotManifest = null; while (depotManifest == null) { CDNClient client = null; try { client = await cdnPool.GetConnectionForDepotAsync(appId, depot.id, depot.depotKey, CancellationToken.None).ConfigureAwait(false); //client = await cdnPool.GetConnectionForDepotAsync(appId, depot.id, depot.depotKey, CancellationToken.None); depotManifest = await client.DownloadManifestAsync(depot.id, depot.manifestId).ConfigureAwait(false); //depotManifest = await client.DownloadManifestAsync(depot.id, depot.manifestId); cdnPool.ReturnConnection(client); } catch (WebException e) { cdnPool.ReturnBrokenConnection(client); if (e.Status == WebExceptionStatus.ProtocolError) { var response = e.Response as HttpWebResponse; if (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden) { DebugLog.WriteLine("ContentDownloader", "Encountered 401 for depot manifest " + depot.id + " " + depot.manifestId + ". Aborting."); break; } else { DebugLog.WriteLine("ContentDownloader", "Encountered error downloading depot manifest " + depot.id + " " + depot.manifestId + ": " + response.StatusCode); } } else { DebugLog.WriteLine("ContentDownloader", "Encountered error downloading manifest for depot " + depot.id + " " + depot.manifestId + ": " + e.Status); } } catch (Exception e) { cdnPool.ReturnBrokenConnection(client); DebugLog.WriteLine("ContentDownloader", "Encountered error downloading manifest for depot " + depot.id + " " + depot.manifestId + ": " + e.Message); } } if (depotManifest == null) { DebugLog.WriteLine("ContentDownloader", "\nUnable to download manifest " + depot.manifestId + " for depot " + depot.id); return; } downloadManifest = new ProtoManifest(depotManifest, depot.manifestId); downloadManifest.SaveToFile(newManifestFileName); DebugLog.WriteLine("ContentDownloader", "Done!"); } } downloadManifest.Files.Sort((x, y) => { return(x.FileName.CompareTo(y.FileName)); }); if (downloadManifest != null) { onManifestReceived?.Invoke(appId, depot.id, depot.contentName, downloadManifest); } if (Config.DownloadManifestOnly) { continue; } complete_download_size = 0; size_downloaded = 0; string stagingDir = Path.Combine(depot.installDir, STAGING_DIR); var filesAfterExclusions = downloadManifest.Files.Where(f => TestIsFileIncluded(f.FileName)).ToList(); // Pre-process filesAfterExclusions.ForEach(file => { var fileFinalPath = Path.Combine(depot.installDir, file.FileName); var fileStagingPath = Path.Combine(stagingDir, file.FileName); if (file.Flags.HasFlag(EDepotFileFlag.Directory)) { Directory.CreateDirectory(fileFinalPath); Directory.CreateDirectory(fileStagingPath); } else { // Some manifests don't explicitly include all necessary directories Directory.CreateDirectory(Path.GetDirectoryName(fileFinalPath)); Directory.CreateDirectory(Path.GetDirectoryName(fileStagingPath)); complete_download_size += file.TotalSize; } }); var semaphore = new SemaphoreSlim(Config.MaxDownloads); var files = filesAfterExclusions.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory)).ToArray(); var tasks = new Task[files.Length]; for (var i = 0; i < files.Length; i++) { var file = files[i]; var task = Task.Run(async() => { cts.Token.ThrowIfCancellationRequested(); try { await semaphore.WaitAsync().ConfigureAwait(false); //await semaphore.WaitAsync(); cts.Token.ThrowIfCancellationRequested(); string fileFinalPath = Path.Combine(depot.installDir, file.FileName); string fileStagingPath = Path.Combine(stagingDir, file.FileName); // This may still exist if the previous run exited before cleanup DebugLog.WriteLine(DEBUG_NAME_FILES, "Checking if " + fileStagingPath + " exists"); if (File.Exists(fileStagingPath)) { DebugLog.WriteLine(DEBUG_NAME_FILES, fileStagingPath + " exists, deleting!"); File.Delete(fileStagingPath); } List <ProtoManifest.ChunkData> neededChunks; FileInfo fi = new FileInfo(fileFinalPath); DebugLog.WriteLine(DEBUG_NAME_FILES, "Checking if " + fileFinalPath + " exists"); if (!fi.Exists) { // create new file. need all chunks DebugLog.WriteLine(DEBUG_NAME_FILES, fileFinalPath + " does not exist, creating!"); using (FileStream fs = File.Create(fileFinalPath)) { fs.SetLength((long)file.TotalSize); neededChunks = new List <ProtoManifest.ChunkData>(file.Chunks); } } else { // open existing ProtoManifest.FileData oldManifestFile = null; if (oldProtoManifest != null) { oldManifestFile = oldProtoManifest.Files.SingleOrDefault(f => f.FileName == file.FileName); } if (oldManifestFile != null) { neededChunks = new List <ProtoManifest.ChunkData>(); if (Config.VerifyAll || !oldManifestFile.FileHash.SequenceEqual(file.FileHash)) { // we have a version of this file, but it doesn't fully match what we want var matchingChunks = new List <ChunkMatch>(); foreach (var chunk in file.Chunks) { var oldChunk = oldManifestFile.Chunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID)); if (oldChunk != null) { matchingChunks.Add(new ChunkMatch(oldChunk, chunk)); } else { neededChunks.Add(chunk); } } DebugLog.WriteLine(DEBUG_NAME_FILES, "Moving file " + fileFinalPath + " to " + fileStagingPath); File.Move(fileFinalPath, fileStagingPath); DebugLog.WriteLine(DEBUG_NAME_FILES, "Creating file " + fileFinalPath); using (FileStream fs = File.Open(fileFinalPath, FileMode.Create)) { fs.SetLength((long)file.TotalSize); DebugLog.WriteLine(DEBUG_NAME_FILES, "Opening file " + fileStagingPath); using (var fsOld = File.Open(fileStagingPath, FileMode.Open)) { foreach (var match in matchingChunks) { fsOld.Seek((long)match.OldChunk.Offset, SeekOrigin.Begin); byte[] tmp = new byte[match.OldChunk.UncompressedLength]; fsOld.Read(tmp, 0, tmp.Length); byte[] adler = Util.AdlerHash(tmp); if (!adler.SequenceEqual(match.OldChunk.Checksum)) { neededChunks.Add(match.NewChunk); } else { fs.Seek((long)match.NewChunk.Offset, SeekOrigin.Begin); fs.Write(tmp, 0, tmp.Length); } } } } DebugLog.WriteLine(DEBUG_NAME_FILES, "Deleting file " + fileStagingPath); File.Delete(fileStagingPath); } } else { // No old manifest or file not in old manifest. We must validate. DebugLog.WriteLine(DEBUG_NAME_FILES, "Opening file " + fileFinalPath); using (FileStream fs = File.Open(fileFinalPath, FileMode.Open)) { if ((ulong)fi.Length != file.TotalSize) { fs.SetLength((long)file.TotalSize); } neededChunks = Util.ValidateSteam3FileChecksums(fs, file.Chunks.OrderBy(x => x.Offset).ToArray()); } } if (neededChunks.Count() == 0) { size_downloaded += file.TotalSize; DebugLog.WriteLine("ContentDownloader", DownloadPercent * 100.0f + "% " + fileFinalPath); return; } else { size_downloaded += (file.TotalSize - ( ulong )neededChunks.Select(x => ( long )x.UncompressedLength).Sum()); } } DebugLog.WriteLine(DEBUG_NAME_FILES, "Opening file " + fileFinalPath); using (FileStream fs = File.Open(fileFinalPath, FileMode.Open)) { foreach (var chunk in neededChunks) { if (cts.IsCancellationRequested) { break; } string chunkID = Util.EncodeHexString(chunk.ChunkID); CDNClient.DepotChunk chunkData = null; while (!cts.IsCancellationRequested) { CDNClient client; try { client = await cdnPool.GetConnectionForDepotAsync(appId, depot.id, depot.depotKey, cts.Token).ConfigureAwait(false); //client = await cdnPool.GetConnectionForDepotAsync(appId, depot.id, depot.depotKey, cts.Token); } catch (OperationCanceledException) { break; } DepotManifest.ChunkData data = new DepotManifest.ChunkData(); data.ChunkID = chunk.ChunkID; data.Checksum = chunk.Checksum; data.Offset = chunk.Offset; data.CompressedLength = chunk.CompressedLength; data.UncompressedLength = chunk.UncompressedLength; try { //chunkData = await client.DownloadDepotChunkAsStreamAsync(depot.id, data).ConfigureAwait(false); chunkData = client.DownloadDepotChunkAsStreamAsync(depot.id, data).Result; //chunkData = await client.DownloadDepotChunkAsync(depot.id, data); cdnPool.ReturnConnection(client); break; } catch (WebException e) { cdnPool.ReturnBrokenConnection(client); if (e.Status == WebExceptionStatus.ProtocolError) { var response = e.Response as HttpWebResponse; if (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden) { DebugLog.WriteLine("ContentDownloader", "Encountered 401 for chunk " + chunkID + ". Aborting."); cts.Cancel(); break; } else { DebugLog.WriteLine("ContentDownloader", "Encountered error downloading chunk " + chunkID + ": " + response.StatusCode); } } else { DebugLog.WriteLine("ContentDownloader", "Encountered error downloading chunk " + chunkID + ": " + e.Status); } } catch (Exception e) { cdnPool.ReturnBrokenConnection(client); DebugLog.WriteLine("ContentDownloader", "Encountered unexpected error downloading chunk " + chunkID + ": " + e.Message); } } if (chunkData == null) { DebugLog.WriteLine("ContentDownloader", "Failed to find any server with chunk " + chunkID + " for depot " + depot.id + ". Aborting."); cts.Cancel(); return; } TotalBytesCompressed += chunk.CompressedLength; DepotBytesCompressed += chunk.CompressedLength; TotalBytesUncompressed += chunk.UncompressedLength; DepotBytesUncompressed += chunk.UncompressedLength; using (chunkData.DataStream) { fs.Seek((long)chunk.Offset, SeekOrigin.Begin); chunkData.DataStream.CopyTo(fs); //fs.Write(chunkData.Data, 0, chunkData.Data.Length); } size_downloaded += chunk.UncompressedLength; } } DebugLog.WriteLine("ContentDownloader", DownloadPercent * 100.0f + "% " + fileFinalPath); } finally { semaphore.Release(); } }); tasks[i] = task; } Task.WaitAll(tasks); ConfigStore.TheConfig.LastManifests[depot.id] = depot.manifestId; ConfigStore.Save(); DebugLog.WriteLine("ContentDownloader", "Depot " + depot.id + " - Downloaded " + DepotBytesCompressed + " bytes (" + DepotBytesUncompressed + " bytes uncompressed)"); } IsDownloading = false; downloadCompleteAction?.Invoke(); onDownloadCompleted?.Invoke(); DebugLog.WriteLine("ContentDownloader", "Total downloaded: " + TotalBytesCompressed + " bytes (" + TotalBytesUncompressed + " bytes uncompressed) from " + depots.Count + " depots"); }
private static async Task DownloadSteam3Async(uint appId, List <DepotDownloadInfo> depots) { ulong TotalBytesCompressed = 0; ulong TotalBytesUncompressed = 0; foreach (var depot in depots) { ulong DepotBytesCompressed = 0; ulong DepotBytesUncompressed = 0; Console.WriteLine("Downloading depot {0} - {1}", depot.id, depot.contentName); CancellationTokenSource cts = new CancellationTokenSource(); cdnPool.ExhaustedToken = cts; ProtoManifest oldProtoManifest = null; ProtoManifest newProtoManifest = null; string configDir = Path.Combine(depot.installDir, CONFIG_DIR); ulong lastManifestId = INVALID_MANIFEST_ID; DepotConfigStore.Instance.InstalledManifestIDs.TryGetValue(depot.id, out lastManifestId); // In case we have an early exit, this will force equiv of verifyall next run. DepotConfigStore.Instance.InstalledManifestIDs[depot.id] = INVALID_MANIFEST_ID; DepotConfigStore.Save(); if (lastManifestId != INVALID_MANIFEST_ID) { var oldManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", lastManifestId)); if (File.Exists(oldManifestFileName)) { byte[] expectedChecksum, currentChecksum; try { expectedChecksum = File.ReadAllBytes(oldManifestFileName + ".sha"); } catch (IOException) { expectedChecksum = null; } oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName, out currentChecksum); if (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum)) { // We only have to show this warning if the old manifest ID was different if (lastManifestId != depot.manifestId) { Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", lastManifestId); } oldProtoManifest = null; } } } if (lastManifestId == depot.manifestId && oldProtoManifest != null) { newProtoManifest = oldProtoManifest; Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id); } else { var newManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", depot.manifestId)); if (newManifestFileName != null) { byte[] expectedChecksum, currentChecksum; try { expectedChecksum = File.ReadAllBytes(newManifestFileName + ".sha"); } catch (IOException) { expectedChecksum = null; } newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName, out currentChecksum); if (newProtoManifest != null && (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum))) { Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", depot.manifestId); newProtoManifest = null; } } if (newProtoManifest != null) { Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id); } else { Console.Write("Downloading depot manifest..."); DepotManifest depotManifest = null; while (depotManifest == null) { Tuple <CDNClient.Server, string> connection = null; try { connection = await cdnPool.GetConnectionForDepot(appId, depot.id, CancellationToken.None); depotManifest = await cdnPool.CDNClient.DownloadManifestAsync(depot.id, depot.manifestId, connection.Item1, connection.Item2, depot.depotKey).ConfigureAwait(false); cdnPool.ReturnConnection(connection); } catch (SteamKitWebRequestException e) { cdnPool.ReturnBrokenConnection(connection); if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden) { Console.WriteLine("Encountered 401 for depot manifest {0} {1}. Aborting.", depot.id, depot.manifestId); break; } else { Console.WriteLine("Encountered error downloading depot manifest {0} {1}: {2}", depot.id, depot.manifestId, e.StatusCode); } } catch (Exception e) { cdnPool.ReturnBrokenConnection(connection); Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depot.id, depot.manifestId, e.Message); } } if (depotManifest == null) { Console.WriteLine("\nUnable to download manifest {0} for depot {1}", depot.manifestId, depot.id); return; } byte[] checksum; newProtoManifest = new ProtoManifest(depotManifest, depot.manifestId); newProtoManifest.SaveToFile(newManifestFileName, out checksum); File.WriteAllBytes(newManifestFileName + ".sha", checksum); Console.WriteLine(" Done!"); } } newProtoManifest.Files.Sort((x, y) => string.Compare(x.FileName, y.FileName, StringComparison.Ordinal)); if (Config.DownloadManifestOnly) { StringBuilder manifestBuilder = new StringBuilder(); string txtManifest = Path.Combine(depot.installDir, string.Format("manifest_{0}.txt", depot.id)); foreach (var file in newProtoManifest.Files) { if (file.Flags.HasFlag(EDepotFileFlag.Directory)) { continue; } manifestBuilder.Append(string.Format("{0}\n", file.FileName)); } File.WriteAllText(txtManifest, manifestBuilder.ToString()); continue; } ulong complete_download_size = 0; ulong size_downloaded = 0; string stagingDir = Path.Combine(depot.installDir, STAGING_DIR); var filesAfterExclusions = newProtoManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).ToList(); // Pre-process filesAfterExclusions.ForEach(file => { var fileFinalPath = Path.Combine(depot.installDir, file.FileName); var fileStagingPath = Path.Combine(stagingDir, file.FileName); if (file.Flags.HasFlag(EDepotFileFlag.Directory)) { Directory.CreateDirectory(fileFinalPath); Directory.CreateDirectory(fileStagingPath); } else { // Some manifests don't explicitly include all necessary directories Directory.CreateDirectory(Path.GetDirectoryName(fileFinalPath)); Directory.CreateDirectory(Path.GetDirectoryName(fileStagingPath)); complete_download_size += file.TotalSize; } }); var semaphore = new SemaphoreSlim(Config.MaxDownloads); var files = filesAfterExclusions.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory)).ToArray(); var tasks = new Task[files.Length]; for (var i = 0; i < files.Length; i++) { var file = files[i]; var task = Task.Run(async() => { cts.Token.ThrowIfCancellationRequested(); try { await semaphore.WaitAsync().ConfigureAwait(false); cts.Token.ThrowIfCancellationRequested(); string fileFinalPath = Path.Combine(depot.installDir, file.FileName); string fileStagingPath = Path.Combine(stagingDir, file.FileName); // This may still exist if the previous run exited before cleanup if (File.Exists(fileStagingPath)) { File.Delete(fileStagingPath); } FileStream fs = null; List <ProtoManifest.ChunkData> neededChunks; FileInfo fi = new FileInfo(fileFinalPath); if (!fi.Exists) { // create new file. need all chunks fs = File.Create(fileFinalPath); fs.SetLength(( long )file.TotalSize); neededChunks = new List <ProtoManifest.ChunkData>(file.Chunks); } else { // open existing ProtoManifest.FileData oldManifestFile = null; if (oldProtoManifest != null) { oldManifestFile = oldProtoManifest.Files.SingleOrDefault(f => f.FileName == file.FileName); } if (oldManifestFile != null) { neededChunks = new List <ProtoManifest.ChunkData>(); if (Config.VerifyAll || !oldManifestFile.FileHash.SequenceEqual(file.FileHash)) { // we have a version of this file, but it doesn't fully match what we want var matchingChunks = new List <ChunkMatch>(); foreach (var chunk in file.Chunks) { var oldChunk = oldManifestFile.Chunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID)); if (oldChunk != null) { matchingChunks.Add(new ChunkMatch(oldChunk, chunk)); } else { neededChunks.Add(chunk); } } File.Move(fileFinalPath, fileStagingPath); fs = File.Open(fileFinalPath, FileMode.Create); fs.SetLength(( long )file.TotalSize); using (var fsOld = File.Open(fileStagingPath, FileMode.Open)) { foreach (var match in matchingChunks) { fsOld.Seek(( long )match.OldChunk.Offset, SeekOrigin.Begin); byte[] tmp = new byte[match.OldChunk.UncompressedLength]; fsOld.Read(tmp, 0, tmp.Length); byte[] adler = Util.AdlerHash(tmp); if (!adler.SequenceEqual(match.OldChunk.Checksum)) { neededChunks.Add(match.NewChunk); } else { fs.Seek(( long )match.NewChunk.Offset, SeekOrigin.Begin); fs.Write(tmp, 0, tmp.Length); } } } File.Delete(fileStagingPath); } } else { // No old manifest or file not in old manifest. We must validate. fs = File.Open(fileFinalPath, FileMode.Open); if (( ulong )fi.Length != file.TotalSize) { fs.SetLength(( long )file.TotalSize); } neededChunks = Util.ValidateSteam3FileChecksums(fs, file.Chunks.OrderBy(x => x.Offset).ToArray()); } if (neededChunks.Count() == 0) { size_downloaded += file.TotalSize; Console.WriteLine("{0,6:#00.00}% {1}", (( float )size_downloaded / ( float )complete_download_size) * 100.0f, fileFinalPath); if (fs != null) { fs.Dispose(); } return; } else { size_downloaded += (file.TotalSize - ( ulong )neededChunks.Select(x => ( long )x.UncompressedLength).Sum()); } } foreach (var chunk in neededChunks) { if (cts.IsCancellationRequested) { break; } string chunkID = Util.EncodeHexString(chunk.ChunkID); CDNClient.DepotChunk chunkData = null; while (!cts.IsCancellationRequested) { Tuple <CDNClient.Server, string> connection; try { connection = await cdnPool.GetConnectionForDepot(appId, depot.id, cts.Token); } catch (OperationCanceledException) { break; } DepotManifest.ChunkData data = new DepotManifest.ChunkData(); data.ChunkID = chunk.ChunkID; data.Checksum = chunk.Checksum; data.Offset = chunk.Offset; data.CompressedLength = chunk.CompressedLength; data.UncompressedLength = chunk.UncompressedLength; try { chunkData = await cdnPool.CDNClient.DownloadDepotChunkAsync(depot.id, data, connection.Item1, connection.Item2, depot.depotKey).ConfigureAwait(false); cdnPool.ReturnConnection(connection); break; } catch (SteamKitWebRequestException e) { cdnPool.ReturnBrokenConnection(connection); if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden) { Console.WriteLine("Encountered 401 for chunk {0}. Aborting.", chunkID); cts.Cancel(); break; } else { Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, e.StatusCode); } } catch (Exception e) { cdnPool.ReturnBrokenConnection(connection); Console.WriteLine("Encountered unexpected error downloading chunk {0}: {1}", chunkID, e.Message); } } if (chunkData == null) { Console.WriteLine("Failed to find any server with chunk {0} for depot {1}. Aborting.", chunkID, depot.id); cts.Cancel(); } // Throw the cancellation exception if requested so that this task is marked failed cts.Token.ThrowIfCancellationRequested(); TotalBytesCompressed += chunk.CompressedLength; DepotBytesCompressed += chunk.CompressedLength; TotalBytesUncompressed += chunk.UncompressedLength; DepotBytesUncompressed += chunk.UncompressedLength; fs.Seek(( long )chunk.Offset, SeekOrigin.Begin); fs.Write(chunkData.Data, 0, chunkData.Data.Length); size_downloaded += chunk.UncompressedLength; } fs.Dispose(); Console.WriteLine("{0,6:#00.00}% {1}", (( float )size_downloaded / ( float )complete_download_size) * 100.0f, fileFinalPath); } finally { semaphore.Release(); } }); tasks[i] = task; } await Task.WhenAll(tasks).ConfigureAwait(false); DepotConfigStore.Instance.InstalledManifestIDs[depot.id] = depot.manifestId; DepotConfigStore.Save(); Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.id, DepotBytesCompressed, DepotBytesUncompressed); } Console.WriteLine("Total downloaded: {0} bytes ({1} bytes uncompressed) from {2} depots", TotalBytesCompressed, TotalBytesUncompressed, depots.Count); }
private static ulong DownloadSteam3(List<DepotDownloadInfo> depots) { ulong TotalBytesCompressed = 0; ulong TotalBytesUncompressed = 0; ulong result = 0; foreach (var depot in depots) { result ^= depot.manifestId; ulong DepotBytesCompressed = 0; ulong DepotBytesUncompressed = 0; Console.WriteLine("Downloading depot {0} - {1}", depot.id, depot.contentName); Console.Write("Finding content servers..."); List<CDNClient> cdnClients = null; Console.WriteLine(" Done!"); ProtoManifest oldProtoManifest = null; ProtoManifest newProtoManifest = null; string configDir = Path.Combine(depot.installDir, CONFIG_DIR); ulong lastManifestId = INVALID_MANIFEST_ID; ConfigStore.TheConfig.LastManifests.TryGetValue(depot.id, out lastManifestId); // In case we have an early exit, this will force equiv of verifyall next run. ConfigStore.TheConfig.LastManifests[depot.id] = INVALID_MANIFEST_ID; ConfigStore.Save(); if (lastManifestId != INVALID_MANIFEST_ID) { var oldManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", lastManifestId)); if (File.Exists(oldManifestFileName)) oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName); } if (lastManifestId == depot.manifestId && oldProtoManifest != null) { newProtoManifest = oldProtoManifest; Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id); } else { var newManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", depot.manifestId)); if (newManifestFileName != null) { newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName); } if (newProtoManifest != null) { Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id); } else { Console.Write("Downloading depot manifest..."); DepotManifest depotManifest = null; cdnClients = CollectCDNClientsForDepot(depot); foreach (var c in cdnClients) { try { depotManifest = c.DownloadManifest(depot.id, depot.manifestId); break; } catch (WebException) { } catch (SocketException) { } } if (depotManifest == null) { Console.WriteLine("\nUnable to download manifest {0} for depot {1}", depot.manifestId, depot.id); return 0; } newProtoManifest = new ProtoManifest(depotManifest, depot.manifestId); newProtoManifest.SaveToFile(newManifestFileName); Console.WriteLine(" Done!"); } } newProtoManifest.Files.Sort((x, y) => { return x.FileName.CompareTo(y.FileName); }); if (Config.DownloadManifestOnly) { StringBuilder manifestBuilder = new StringBuilder(); string txtManifest = Path.Combine(depot.installDir, string.Format("manifest_{0}.txt", depot.id)); foreach (var file in newProtoManifest.Files) { if (file.Flags.HasFlag(EDepotFileFlag.Directory)) continue; manifestBuilder.Append(string.Format("{0}\n", file.FileName)); } File.WriteAllText(txtManifest, manifestBuilder.ToString()); continue; } #region post-manifest ulong complete_download_size = 0; ulong size_downloaded = 0; string stagingDir = Path.Combine(depot.installDir, STAGING_DIR); var filesAfterExclusions = newProtoManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).ToList(); // Pre-process filesAfterExclusions.ForEach(file => { var fileFinalPath = Path.Combine(depot.installDir, file.FileName); var fileStagingPath = Path.Combine(stagingDir, file.FileName); if (file.Flags.HasFlag(EDepotFileFlag.Directory)) { Directory.CreateDirectory(fileFinalPath); Directory.CreateDirectory(fileStagingPath); } else { // Some manifests don't explicitly include all necessary directories Directory.CreateDirectory(Path.GetDirectoryName(fileFinalPath)); Directory.CreateDirectory(Path.GetDirectoryName(fileStagingPath)); complete_download_size += file.TotalSize; } }); var rand = new Random(); filesAfterExclusions.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory)) .AsParallel().WithDegreeOfParallelism(Config.MaxDownloads) .ForAll(file => { string fileFinalPath = Path.Combine(depot.installDir, file.FileName); string fileStagingPath = Path.Combine(stagingDir, file.FileName); // This may still exist if the previous run exited before cleanup if (File.Exists(fileStagingPath)) { File.Delete(fileStagingPath); } FileStream fs = null; List<ProtoManifest.ChunkData> neededChunks; FileInfo fi = new FileInfo(fileFinalPath); if (!fi.Exists) { // create new file. need all chunks fs = File.Create(fileFinalPath); fs.SetLength((long)file.TotalSize); neededChunks = new List<ProtoManifest.ChunkData>(file.Chunks); } else { // open existing ProtoManifest.FileData oldManifestFile = null; if (oldProtoManifest != null) { oldManifestFile = oldProtoManifest.Files.SingleOrDefault(f => f.FileName == file.FileName); } if (oldManifestFile != null) { neededChunks = new List<ProtoManifest.ChunkData>(); if (Config.VerifyAll || !oldManifestFile.FileHash.SequenceEqual(file.FileHash)) { // we have a version of this file, but it doesn't fully match what we want var matchingChunks = new List<ChunkMatch>(); foreach (var chunk in file.Chunks) { var oldChunk = oldManifestFile.Chunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID)); if (oldChunk != null) { matchingChunks.Add(new ChunkMatch(oldChunk, chunk)); } else { neededChunks.Add(chunk); } } File.Move(fileFinalPath, fileStagingPath); fs = File.Open(fileFinalPath, FileMode.Create); fs.SetLength((long)file.TotalSize); using (var fsOld = File.Open(fileStagingPath, FileMode.Open)) { foreach (var match in matchingChunks) { fs.Seek((long)match.NewChunk.Offset, SeekOrigin.Begin); fsOld.Seek((long)match.OldChunk.Offset, SeekOrigin.Begin); byte[] tmp = new byte[match.OldChunk.UncompressedLength]; fsOld.Read(tmp, 0, tmp.Length); fs.Write(tmp, 0, tmp.Length); } } File.Delete(fileStagingPath); } } else { // No old manifest or file not in old manifest. We must validate. fs = File.Open(fileFinalPath, FileMode.Open); if ((ulong)fi.Length != file.TotalSize) { fs.SetLength((long)file.TotalSize); } neededChunks = Util.ValidateSteam3FileChecksums(fs, file.Chunks.OrderBy(x => x.Offset).ToArray()); } if (neededChunks.Count() == 0) { size_downloaded += file.TotalSize; Console.WriteLine("{0,6:#00.00}% {1}", ((float)size_downloaded / (float)complete_download_size) * 100.0f, fileFinalPath); if (fs != null) fs.Close(); return; } else { size_downloaded += (file.TotalSize - (ulong)neededChunks.Select(x => (int)x.UncompressedLength).Sum()); } } int cdnClientIndex = 0; if (neededChunks.Count > 0 && cdnClients == null) { // If we didn't need to connect to get manifests, connect now. cdnClients = CollectCDNClientsForDepot(depot); cdnClientIndex = rand.Next(0, cdnClients.Count); } foreach (var chunk in neededChunks) { string chunkID = Util.EncodeHexString(chunk.ChunkID); CDNClient.DepotChunk chunkData = null; int idx = cdnClientIndex; while (true) { DepotManifest.ChunkData data = new DepotManifest.ChunkData(); data.ChunkID = chunk.ChunkID; data.Checksum = chunk.Checksum; data.Offset = chunk.Offset; data.CompressedLength = chunk.CompressedLength; data.UncompressedLength = chunk.UncompressedLength; try { chunkData = cdnClients[idx].DownloadDepotChunk(depot.id, data); break; } catch { if (++idx >= cdnClients.Count) idx = 0; if (idx == cdnClientIndex) break; } } if (chunkData == null) { Console.WriteLine("Failed to find any server with chunk {0} for depot {1}. Aborting.", chunkID, depot); return; } TotalBytesCompressed += chunk.CompressedLength; DepotBytesCompressed += chunk.CompressedLength; TotalBytesUncompressed += chunk.UncompressedLength; DepotBytesUncompressed += chunk.UncompressedLength; fs.Seek((long)chunk.Offset, SeekOrigin.Begin); fs.Write(chunkData.Data, 0, chunkData.Data.Length); size_downloaded += chunk.UncompressedLength; } fs.Close(); Console.WriteLine("{0,6:#00.00}% {1}", ((float)size_downloaded / (float)complete_download_size) * 100.0f, fileFinalPath); }); ConfigStore.TheConfig.LastManifests[depot.id] = depot.manifestId; ConfigStore.Save(); Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.id, DepotBytesCompressed, DepotBytesUncompressed); } #endregion Console.WriteLine("Total downloaded: {0} bytes ({1} bytes uncompressed) from {2} depots", TotalBytesCompressed, TotalBytesUncompressed, depots.Count); return result; }
public ChunkMatch(ProtoManifest.ChunkData oldChunk, ProtoManifest.ChunkData newChunk) { OldChunk = oldChunk; NewChunk = newChunk; }
// Validate a file against Steam3 Chunk data public static List<ProtoManifest.ChunkData> ValidateSteam3FileChecksums(FileStream fs, ProtoManifest.ChunkData[] chunkdata) { var neededChunks = new List<ProtoManifest.ChunkData>(); int read; foreach (var data in chunkdata) { byte[] chunk = new byte[data.UncompressedLength]; fs.Seek((long)data.Offset, SeekOrigin.Begin); read = fs.Read(chunk, 0, (int)data.UncompressedLength); byte[] tempchunk; if (read < data.UncompressedLength) { tempchunk = new byte[read]; Array.Copy(chunk, 0, tempchunk, 0, read); } else { tempchunk = chunk; } byte[] adler = AdlerHash(tempchunk); if (!adler.SequenceEqual(data.Checksum)) { neededChunks.Add(data); } } return neededChunks; }
private static async Task <DepotFilesData> ProcessDepotManifestAndFiles(CancellationTokenSource cts, uint appId, DepotDownloadInfo depot) { DepotDownloadCounter depotCounter = new DepotDownloadCounter(); Console.WriteLine("Processing depot {0} - {1}", depot.id, depot.contentName); ProtoManifest oldProtoManifest = null; ProtoManifest newProtoManifest = null; string configDir = Path.Combine(depot.installDir, CONFIG_DIR); ulong lastManifestId = INVALID_MANIFEST_ID; DepotConfigStore.Instance.InstalledManifestIDs.TryGetValue(depot.id, out lastManifestId); // In case we have an early exit, this will force equiv of verifyall next run. DepotConfigStore.Instance.InstalledManifestIDs[depot.id] = INVALID_MANIFEST_ID; DepotConfigStore.Save(); if (lastManifestId != INVALID_MANIFEST_ID) { var oldManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.id, lastManifestId)); if (File.Exists(oldManifestFileName)) { byte[] expectedChecksum, currentChecksum; try { expectedChecksum = File.ReadAllBytes(oldManifestFileName + ".sha"); } catch (IOException) { expectedChecksum = null; } oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName, out currentChecksum); if (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum)) { // We only have to show this warning if the old manifest ID was different if (lastManifestId != depot.manifestId) { Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", lastManifestId); } oldProtoManifest = null; } } } if (lastManifestId == depot.manifestId && oldProtoManifest != null) { newProtoManifest = oldProtoManifest; Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id); } else { var newManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.id, depot.manifestId)); if (newManifestFileName != null) { byte[] expectedChecksum, currentChecksum; try { expectedChecksum = File.ReadAllBytes(newManifestFileName + ".sha"); } catch (IOException) { expectedChecksum = null; } newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName, out currentChecksum); if (newProtoManifest != null && (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum))) { Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", depot.manifestId); newProtoManifest = null; } } if (newProtoManifest != null) { Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id); } else { Console.Write("Downloading depot manifest..."); DepotManifest depotManifest = null; do { cts.Token.ThrowIfCancellationRequested(); CDNClient.Server connection = null; try { connection = cdnPool.GetConnection(cts.Token); var cdnToken = await cdnPool.AuthenticateConnection(appId, depot.id, connection); depotManifest = await cdnPool.CDNClient.DownloadManifestAsync(depot.id, depot.manifestId, connection, cdnToken, depot.depotKey).ConfigureAwait(false); cdnPool.ReturnConnection(connection); } catch (TaskCanceledException) { Console.WriteLine("Connection timeout downloading depot manifest {0} {1}", depot.id, depot.manifestId); } catch (SteamKitWebRequestException e) { cdnPool.ReturnBrokenConnection(connection); if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden) { Console.WriteLine("Encountered 401 for depot manifest {0} {1}. Aborting.", depot.id, depot.manifestId); break; } else { Console.WriteLine("Encountered error downloading depot manifest {0} {1}: {2}", depot.id, depot.manifestId, e.StatusCode); } } catch (OperationCanceledException) { break; } catch (Exception e) { cdnPool.ReturnBrokenConnection(connection); Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depot.id, depot.manifestId, e.Message); } }while (depotManifest == null); if (depotManifest == null) { Console.WriteLine("\nUnable to download manifest {0} for depot {1}", depot.manifestId, depot.id); cts.Cancel(); } // Throw the cancellation exception if requested so that this task is marked failed cts.Token.ThrowIfCancellationRequested(); byte[] checksum; newProtoManifest = new ProtoManifest(depotManifest, depot.manifestId); newProtoManifest.SaveToFile(newManifestFileName, out checksum); File.WriteAllBytes(newManifestFileName + ".sha", checksum); Console.WriteLine(" Done!"); } } newProtoManifest.Files.Sort((x, y) => string.Compare(x.FileName, y.FileName, StringComparison.Ordinal)); Console.WriteLine("Manifest {0} ({1})", depot.manifestId, newProtoManifest.CreationTime); if (Config.DownloadManifestOnly) { StringBuilder manifestBuilder = new StringBuilder(); string txtManifest = Path.Combine(depot.installDir, string.Format("manifest_{0}_{1}.txt", depot.id, depot.manifestId)); manifestBuilder.Append(string.Format("{0}\n\n", newProtoManifest.CreationTime)); foreach (var file in newProtoManifest.Files) { if (file.Flags.HasFlag(EDepotFileFlag.Directory)) { continue; } manifestBuilder.Append(string.Format("{0}\n", file.FileName)); manifestBuilder.Append(string.Format("\t{0}\n", file.TotalSize)); manifestBuilder.Append(string.Format("\t{0}\n", BitConverter.ToString(file.FileHash).Replace("-", ""))); } File.WriteAllText(txtManifest, manifestBuilder.ToString()); return(null); } string stagingDir = Path.Combine(depot.installDir, STAGING_DIR); var filesAfterExclusions = newProtoManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).ToList(); var allFileNames = new HashSet <string>(filesAfterExclusions.Count); // Pre-process filesAfterExclusions.ForEach(file => { allFileNames.Add(file.FileName); var fileFinalPath = Path.Combine(depot.installDir, file.FileName); var fileStagingPath = Path.Combine(stagingDir, file.FileName); if (file.Flags.HasFlag(EDepotFileFlag.Directory)) { Directory.CreateDirectory(fileFinalPath); Directory.CreateDirectory(fileStagingPath); } else { // Some manifests don't explicitly include all necessary directories Directory.CreateDirectory(Path.GetDirectoryName(fileFinalPath)); Directory.CreateDirectory(Path.GetDirectoryName(fileStagingPath)); depotCounter.CompleteDownloadSize += file.TotalSize; } }); return(new DepotFilesData { depotDownloadInfo = depot, depotCounter = depotCounter, stagingDir = stagingDir, manifest = newProtoManifest, previousManifest = oldProtoManifest, filteredFiles = filesAfterExclusions, allFileNames = allFileNames }); }
private async void FileSelector_LoadAsync(object sender, EventArgs e) { string Title = this.Text; string Password = ""; if(ManifestID==ContentDownloader.INVALID_MANIFEST_ID) ManifestID = ContentDownloader.GetSteam3DepotManifestStatic(DepotID, AppID, Branch,ref Password); if (ManifestID == ContentDownloader.INVALID_MANIFEST_ID) { MessageBox.Show(Properties.Resources.NoManifestID, "FileSelector", MessageBoxButtons.OK, MessageBoxIcon.Exclamation); return; } this.labelManifestID.Text = "ManifestID:" + ManifestID.ToString(); System.IO.Directory.CreateDirectory(Program.CacheDir); var localProtoManifest = DepotDownloader.ProtoManifest.LoadFromFile(string.Format("{0}/{1}.bin",Program.CacheDir, ManifestID)); if (localProtoManifest!=null) { depotManifest = localProtoManifest; } else { this.Text = "Downloading File List..."; this.Refresh(); ContentDownloader.Steam3.RequestDepotKey(DepotID,AppID); if(!ContentDownloader.Steam3.DepotKeys.ContainsKey(DepotID)) { MessageBox.Show(Properties.Resources.NoDepotKey, "FileSelector",MessageBoxButtons.OK,MessageBoxIcon.Exclamation); //return; //User may still need to use FileRegex Close(); return; } byte[] depotKey = ContentDownloader.Steam3.DepotKeys[DepotID]; //ContentDownloader.Steam3.RequestAppTicket(AppID); ContentDownloader.Steam3.RequestAppTicket(DepotID); var client = await DepotDownloader.ContentDownloader.GlobalCDNPool.GetConnectionForDepotAsync(AppID, DepotID, depotKey, System.Threading.CancellationToken.None).ConfigureAwait(true); var SteamKitdepotManifest = await client.DownloadManifestAsync(DepotID, ManifestID).ConfigureAwait(true); if (SteamKitdepotManifest != null) { localProtoManifest = new ProtoManifest(SteamKitdepotManifest, ManifestID); localProtoManifest.SaveToFile(string.Format("{0}/{1}.bin", Program.CacheDir, ManifestID)); depotManifest = localProtoManifest; } } if(depotManifest!=null) { foreach (var file in localProtoManifest.Files) { //Process Dir var FilePathSplited = file.FileName.Split('\\'); List<string> FilePathList = new List<string>(FilePathSplited); TreeNode LastNode=null; TreeNodeCollection FileNodes = this.treeViewFileList.Nodes; while (FilePathList.Count != 0) { TreeNode TargetNode = null; foreach (TreeNode Tn in FileNodes) { if (Tn.Text == FilePathList[0]) { TargetNode = Tn; break; } } if (TargetNode == null) { LastNode = FileNodes.Add(FilePathList[0]); FileNodes = LastNode.Nodes; } else { FileNodes = TargetNode.Nodes; } FilePathList.RemoveAt(0); } if(file.Flags!=SteamKit2.EDepotFileFlag.Directory) { DepotMaxSize += file.TotalSize; //if(LastNode!=null) LastNode.Tag = file.TotalSize; } } float TargetSize = DepotMaxSize / 1024f; if(TargetSize<1024) { SizeDivisor = 1024f; UnitStr = "KB"; } else if(TargetSize/1024f<1024) { SizeDivisor = 1024 * 1024f; UnitStr = "MB"; } else { SizeDivisor = 1024 * 1024 * 1024f; UnitStr = "GB"; } this.labelSize.Text = string.Format("{0}{1}/{2}{1}", 0.ToString("#0.00"), UnitStr, (DepotMaxSize / SizeDivisor).ToString("#0.00")); } this.Text = Title; }