private static void DownloadSteam3AsyncDepotFile( CancellationTokenSource cts, DepotFilesData depotFilesData, ProtoManifest.FileData file, ConcurrentQueue <(FileStreamData, ProtoManifest.FileData, ProtoManifest.ChunkData)> networkChunkQueue)
private static async Task DownloadSteam3AsyncDepotFileChunk( CancellationTokenSource cts, uint appId, GlobalDownloadCounter downloadCounter, DepotFilesData depotFilesData, ProtoManifest.FileData file, FileStreamData fileStreamData, ProtoManifest.ChunkData chunk) { cts.Token.ThrowIfCancellationRequested(); var depot = depotFilesData.depotDownloadInfo; var depotDownloadCounter = depotFilesData.depotCounter; string chunkID = Util.EncodeHexString(chunk.ChunkID); DepotManifest.ChunkData data = new DepotManifest.ChunkData(); data.ChunkID = chunk.ChunkID; data.Checksum = chunk.Checksum; data.Offset = chunk.Offset; data.CompressedLength = chunk.CompressedLength; data.UncompressedLength = chunk.UncompressedLength; CDNClient.DepotChunk chunkData = null; do { cts.Token.ThrowIfCancellationRequested(); CDNClient.Server connection = null; try { connection = cdnPool.GetConnection(cts.Token); var cdnToken = await cdnPool.AuthenticateConnection(appId, depot.id, connection); chunkData = await cdnPool.CDNClient.DownloadDepotChunkAsync(depot.id, data, connection, cdnToken, depot.depotKey).ConfigureAwait(false); cdnPool.ReturnConnection(connection); } catch (TaskCanceledException) { Console.WriteLine("Connection timeout downloading chunk {0}", chunkID); } catch (SteamKitWebRequestException e) { cdnPool.ReturnBrokenConnection(connection); if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden) { Console.WriteLine("Encountered 401 for chunk {0}. Aborting.", chunkID); break; } else { Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, e.StatusCode); } } catch (OperationCanceledException) { break; } catch (Exception e) { cdnPool.ReturnBrokenConnection(connection); Console.WriteLine("Encountered unexpected error downloading chunk {0}: {1}", chunkID, e.Message); } }while (chunkData == null); if (chunkData == null) { Console.WriteLine("Failed to find any server with chunk {0} for depot {1}. Aborting.", chunkID, depot.id); cts.Cancel(); } // Throw the cancellation exception if requested so that this task is marked failed cts.Token.ThrowIfCancellationRequested(); try { await fileStreamData.fileLock.WaitAsync().ConfigureAwait(false); fileStreamData.fileStream.Seek((long)chunkData.ChunkInfo.Offset, SeekOrigin.Begin); await fileStreamData.fileStream.WriteAsync(chunkData.Data, 0, chunkData.Data.Length); } finally { fileStreamData.fileLock.Release(); } int remainingChunks = Interlocked.Decrement(ref fileStreamData.chunksToDownload); if (remainingChunks == 0) { fileStreamData.fileStream.Dispose(); fileStreamData.fileLock.Dispose(); } ulong sizeDownloaded = 0; lock (depotDownloadCounter) { sizeDownloaded = depotDownloadCounter.SizeDownloaded + (ulong)chunkData.Data.Length; depotDownloadCounter.SizeDownloaded = sizeDownloaded; depotDownloadCounter.DepotBytesCompressed += chunk.CompressedLength; depotDownloadCounter.DepotBytesUncompressed += chunk.UncompressedLength; } lock (downloadCounter) { downloadCounter.TotalBytesCompressed += chunk.CompressedLength; downloadCounter.TotalBytesUncompressed += chunk.UncompressedLength; } if (remainingChunks == 0) { var fileFinalPath = Path.Combine(depot.installDir, file.FileName); Console.WriteLine("{0,6:#00.00}% {1}", ((float)sizeDownloaded / (float)depotDownloadCounter.CompleteDownloadSize) * 100.0f, fileFinalPath); } }
private static async Task DownloadSteam3AsyncDepotFiles(CancellationTokenSource cts, UInt32 appId, GlobalDownloadCounter downloadCounter, DepotFilesData depotFilesData, HashSet <String> allFileNamesAllDepots, CDNClientPool cdnPool) { var depot = depotFilesData.depotDownloadInfo; var depotCounter = depotFilesData.depotCounter; FileLog.LogMessage("Downloading depot {0} - {1}", depot.id, depot.contentName); var files = depotFilesData.filteredFiles.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory)).ToArray(); var networkChunkQueue = new ConcurrentQueue <(FileStreamData fileStreamData, ProtoManifest.FileData fileData, ProtoManifest.ChunkData chunk)>(); await Util.InvokeAsync( files.Select(file => new Func <Task>(async() => await Task.Run(() => DownloadSteam3AsyncDepotFile(cts, depotFilesData, file, networkChunkQueue)))), maxDegreeOfParallelism : 8 ); await Util.InvokeAsync( networkChunkQueue.Select(q => new Func <Task>(async() => await Task.Run(() => DownloadSteam3AsyncDepotFileChunk(cts, appId, downloadCounter, depotFilesData, q.fileData, q.fileStreamData, q.chunk, cdnPool)))), maxDegreeOfParallelism : 8 ); // Check for deleted files if updating the depot. if (depotFilesData.previousManifest != null) { var previousFilteredFiles = depotFilesData.previousManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).Select(f => f.FileName).ToHashSet(); // Check if we are writing to a single output directory. If not, each depot folder is managed independently if (string.IsNullOrWhiteSpace(ContentDownloader.Config.InstallDirectory)) { // Of the list of files in the previous manifest, remove any file names that exist in the current set of all file names previousFilteredFiles.ExceptWith(depotFilesData.allFileNames); } else { // Of the list of files in the previous manifest, remove any file names that exist in the current set of all file names across all depots being downloaded previousFilteredFiles.ExceptWith(allFileNamesAllDepots); } foreach (var existingFileName in previousFilteredFiles) { string fileFinalPath = Path.Combine(depot.installDir, existingFileName); if (!File.Exists(fileFinalPath)) { continue; } File.Delete(fileFinalPath); FileLog.LogMessage("Deleted {0}", fileFinalPath); } } //DepotConfigStore.Instance.InstalledManifestIDs[depot.id] = depot.manifestId; //DepotConfigStore.Save(); FileLog.LogMessage("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.id, depotCounter.DepotBytesCompressed, depotCounter.DepotBytesUncompressed); }
private static void DownloadSteam3AsyncDepotFile( CancellationTokenSource cts, DepotFilesData depotFilesData, ProtoManifest.FileData file, ConcurrentQueue <Tuple <FileStreamData, ProtoManifest.FileData, ProtoManifest.ChunkData> > networkChunkQueue) { cts.Token.ThrowIfCancellationRequested(); var depot = depotFilesData.depotDownloadInfo; var stagingDir = depotFilesData.stagingDir; var depotDownloadCounter = depotFilesData.depotCounter; var oldProtoManifest = depotFilesData.previousManifest; string fileFinalPath = Path.Combine(depot.installDir, file.FileName); string fileStagingPath = Path.Combine(stagingDir, file.FileName); // This may still exist if the previous run exited before cleanup if (File.Exists(fileStagingPath)) { File.Delete(fileStagingPath); } FileStream fs = null; List <ProtoManifest.ChunkData> neededChunks; FileInfo fi = new FileInfo(fileFinalPath); if (!fi.Exists) { Console.WriteLine("Pre-allocating {0}", fileFinalPath); // create new file. need all chunks fs = File.Create(fileFinalPath); fs.SetLength((long)file.TotalSize); neededChunks = new List <ProtoManifest.ChunkData>(file.Chunks); } else { // open existing ProtoManifest.FileData oldManifestFile = null; if (oldProtoManifest != null) { oldManifestFile = oldProtoManifest.Files.SingleOrDefault(f => f.FileName == file.FileName); } if (oldManifestFile != null) { neededChunks = new List <ProtoManifest.ChunkData>(); if (Config.VerifyAll || !oldManifestFile.FileHash.SequenceEqual(file.FileHash)) { // we have a version of this file, but it doesn't fully match what we want if (Config.VerifyAll) { Console.WriteLine("Validating {0}", fileFinalPath); } var matchingChunks = new List <ChunkMatch>(); foreach (var chunk in file.Chunks) { var oldChunk = oldManifestFile.Chunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID)); if (oldChunk != null) { matchingChunks.Add(new ChunkMatch(oldChunk, chunk)); } else { neededChunks.Add(chunk); } } var orderedChunks = matchingChunks.OrderBy(x => x.OldChunk.Offset); File.Move(fileFinalPath, fileStagingPath); fs = File.Open(fileFinalPath, FileMode.Create); fs.SetLength((long)file.TotalSize); using (var fsOld = File.Open(fileStagingPath, FileMode.Open)) { foreach (var match in orderedChunks) { fsOld.Seek((long)match.OldChunk.Offset, SeekOrigin.Begin); byte[] tmp = new byte[match.OldChunk.UncompressedLength]; fsOld.Read(tmp, 0, tmp.Length); byte[] adler = Util.AdlerHash(tmp); if (!adler.SequenceEqual(match.OldChunk.Checksum)) { neededChunks.Add(match.NewChunk); } else { fs.Seek((long)match.NewChunk.Offset, SeekOrigin.Begin); fs.Write(tmp, 0, tmp.Length); } } } File.Delete(fileStagingPath); } } else { // No old manifest or file not in old manifest. We must validate. fs = File.Open(fileFinalPath, FileMode.Open); if ((ulong)fi.Length != file.TotalSize) { fs.SetLength((long)file.TotalSize); } Console.WriteLine("Validating {0}", fileFinalPath); neededChunks = Util.ValidateSteam3FileChecksums(fs, file.Chunks.OrderBy(x => x.Offset).ToArray()); } if (neededChunks.Count() == 0) { lock (depotDownloadCounter) { depotDownloadCounter.SizeDownloaded += (ulong)file.TotalSize; Console.WriteLine("{0,6:#00.00}% {1}", ((float)depotDownloadCounter.SizeDownloaded / (float)depotDownloadCounter.CompleteDownloadSize) * 100.0f, fileFinalPath); } if (fs != null) { fs.Dispose(); } return; } else { var sizeOnDisk = (file.TotalSize - (ulong)neededChunks.Select(x => (long)x.UncompressedLength).Sum()); lock (depotDownloadCounter) { depotDownloadCounter.SizeDownloaded += sizeOnDisk; } } } FileStreamData fileStreamData = new FileStreamData { fileStream = fs, fileLock = new SemaphoreSlim(1), chunksToDownload = neededChunks.Count }; foreach (var chunk in neededChunks) { networkChunkQueue.Enqueue(Tuple.Create(fileStreamData, file, chunk)); } }