private static async Task DownloadSteam3AsyncDepotFiles(CancellationTokenSource cts, UInt32 appId,
GlobalDownloadCounter downloadCounter, DepotFilesData depotFilesData, HashSet <String> allFileNamesAllDepots, CDNClientPool cdnPool)
{
var depot = depotFilesData.depotDownloadInfo;
var depotCounter = depotFilesData.depotCounter;
FileLog.LogMessage("Downloading depot {0} - {1}", depot.id, depot.contentName);
var files = depotFilesData.filteredFiles.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory)).ToArray();
var networkChunkQueue = new ConcurrentQueue <(FileStreamData fileStreamData, ProtoManifest.FileData fileData, ProtoManifest.ChunkData chunk)>();
await Util.InvokeAsync(
files.Select(file => new Func <Task>(async() =>
await Task.Run(() => DownloadSteam3AsyncDepotFile(cts, depotFilesData, file, networkChunkQueue)))),
maxDegreeOfParallelism : 8
);
await Util.InvokeAsync(
networkChunkQueue.Select(q => new Func <Task>(async() =>
await Task.Run(() => DownloadSteam3AsyncDepotFileChunk(cts, appId, downloadCounter, depotFilesData,
q.fileData, q.fileStreamData, q.chunk, cdnPool)))),
maxDegreeOfParallelism : 8
);
// Check for deleted files if updating the depot.
if (depotFilesData.previousManifest != null)
{
var previousFilteredFiles = depotFilesData.previousManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).Select(f => f.FileName).ToHashSet();
// Check if we are writing to a single output directory. If not, each depot folder is managed independently
if (string.IsNullOrWhiteSpace(ContentDownloader.Config.InstallDirectory))
{
// Of the list of files in the previous manifest, remove any file names that exist in the current set of all file names
previousFilteredFiles.ExceptWith(depotFilesData.allFileNames);
}
else
{
// Of the list of files in the previous manifest, remove any file names that exist in the current set of all file names across all depots being downloaded
previousFilteredFiles.ExceptWith(allFileNamesAllDepots);
}
foreach (var existingFileName in previousFilteredFiles)
{
string fileFinalPath = Path.Combine(depot.installDir, existingFileName);
if (!File.Exists(fileFinalPath))
{
continue;
}
File.Delete(fileFinalPath);
FileLog.LogMessage("Deleted {0}", fileFinalPath);
}
}
//DepotConfigStore.Instance.InstalledManifestIDs[depot.id] = depot.manifestId;
//DepotConfigStore.Save();
FileLog.LogMessage("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.id, depotCounter.DepotBytesCompressed, depotCounter.DepotBytesUncompressed);
}
private static async Task DownloadSteam3Async(uint appId, List <DepotDownloadInfo> depots)
{
CancellationTokenSource cts = new CancellationTokenSource();
cdnPool.ExhaustedToken = cts;
GlobalDownloadCounter downloadCounter = new GlobalDownloadCounter();
var depotsToDownload = new List <DepotFilesData>(depots.Count);
var allFileNamesAllDepots = new HashSet <String>();
// First, fetch all the manifests for each depot (including previous manifests) and perform the initial setup
foreach (var depot in depots)
{
var depotFileData = await ProcessDepotManifestAndFiles(cts, appId, depot);
if (depotFileData != null)
{
depotsToDownload.Add(depotFileData);
allFileNamesAllDepots.UnionWith(depotFileData.allFileNames);
}
cts.Token.ThrowIfCancellationRequested();
}
foreach (var depotFileData in depotsToDownload)
{
await DownloadSteam3AsyncDepotFiles(cts, appId, downloadCounter, depotFileData, allFileNamesAllDepots);
}
Console.WriteLine("Total downloaded: {0} bytes ({1} bytes uncompressed) from {2} depots",
downloadCounter.TotalBytesCompressed, downloadCounter.TotalBytesUncompressed, depots.Count);
}
private static async Task DownloadSteam3Async(uint appId, DepotDownloadInfo depot, CDNClientPool cdnPool)
{
CancellationTokenSource cts = new CancellationTokenSource();
cdnPool.ExhaustedToken = cts;
GlobalDownloadCounter downloadCounter = new GlobalDownloadCounter();
var allFileNamesAllDepots = new HashSet <String>();
// First, fetch all the manifests for each depot (including previous manifests) and perform the initial setup
var depotFileData = await ProcessDepotManifestAndFiles(cts, appId, depot, cdnPool);
if (depotFileData != null)
{
allFileNamesAllDepots.UnionWith(depotFileData.allFileNames);
}
cts.Token.ThrowIfCancellationRequested();
await DownloadSteam3AsyncDepotFiles(cts, appId, downloadCounter, depotFileData, allFileNamesAllDepots, cdnPool);
FileLog.LogMessage($"Total downloaded: {downloadCounter.TotalBytesCompressed} bytes ({downloadCounter.TotalBytesUncompressed} bytes uncompressed) from depot");
}
private static async Task DownloadSteam3AsyncDepotFileChunk(
CancellationTokenSource cts, uint appId,
GlobalDownloadCounter downloadCounter,
DepotFilesData depotFilesData,
ProtoManifest.FileData file,
FileStreamData fileStreamData,
ProtoManifest.ChunkData chunk)
{
cts.Token.ThrowIfCancellationRequested();
var depot = depotFilesData.depotDownloadInfo;
var depotDownloadCounter = depotFilesData.depotCounter;
string chunkID = Util.EncodeHexString(chunk.ChunkID);
DepotManifest.ChunkData data = new DepotManifest.ChunkData();
data.ChunkID = chunk.ChunkID;
data.Checksum = chunk.Checksum;
data.Offset = chunk.Offset;
data.CompressedLength = chunk.CompressedLength;
data.UncompressedLength = chunk.UncompressedLength;
CDNClient.DepotChunk chunkData = null;
do
{
cts.Token.ThrowIfCancellationRequested();
CDNClient.Server connection = null;
try
{
connection = cdnPool.GetConnection(cts.Token);
var cdnToken = await cdnPool.AuthenticateConnection(appId, depot.id, connection);
chunkData = await cdnPool.CDNClient.DownloadDepotChunkAsync(depot.id, data,
connection, cdnToken, depot.depotKey).ConfigureAwait(false);
cdnPool.ReturnConnection(connection);
}
catch (TaskCanceledException)
{
Console.WriteLine("Connection timeout downloading chunk {0}", chunkID);
}
catch (SteamKitWebRequestException e)
{
cdnPool.ReturnBrokenConnection(connection);
if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden)
{
Console.WriteLine("Encountered 401 for chunk {0}. Aborting.", chunkID);
break;
}
else
{
Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, e.StatusCode);
}
}
catch (OperationCanceledException)
{
break;
}
catch (Exception e)
{
cdnPool.ReturnBrokenConnection(connection);
Console.WriteLine("Encountered unexpected error downloading chunk {0}: {1}", chunkID, e.Message);
}
}while (chunkData == null);
if (chunkData == null)
{
Console.WriteLine("Failed to find any server with chunk {0} for depot {1}. Aborting.", chunkID, depot.id);
cts.Cancel();
}
// Throw the cancellation exception if requested so that this task is marked failed
cts.Token.ThrowIfCancellationRequested();
try
{
await fileStreamData.fileLock.WaitAsync().ConfigureAwait(false);
fileStreamData.fileStream.Seek((long)chunkData.ChunkInfo.Offset, SeekOrigin.Begin);
await fileStreamData.fileStream.WriteAsync(chunkData.Data, 0, chunkData.Data.Length);
}
finally
{
fileStreamData.fileLock.Release();
}
int remainingChunks = Interlocked.Decrement(ref fileStreamData.chunksToDownload);
if (remainingChunks == 0)
{
fileStreamData.fileStream.Dispose();
fileStreamData.fileLock.Dispose();
}
ulong sizeDownloaded = 0;
lock (depotDownloadCounter)
{
sizeDownloaded = depotDownloadCounter.SizeDownloaded + (ulong)chunkData.Data.Length;
depotDownloadCounter.SizeDownloaded = sizeDownloaded;
depotDownloadCounter.DepotBytesCompressed += chunk.CompressedLength;
depotDownloadCounter.DepotBytesUncompressed += chunk.UncompressedLength;
}
lock (downloadCounter)
{
downloadCounter.TotalBytesCompressed += chunk.CompressedLength;
downloadCounter.TotalBytesUncompressed += chunk.UncompressedLength;
}
if (remainingChunks == 0)
{
var fileFinalPath = Path.Combine(depot.installDir, file.FileName);
Console.WriteLine("{0,6:#00.00}% {1}", ((float)sizeDownloaded / (float)depotDownloadCounter.CompleteDownloadSize) * 100.0f, fileFinalPath);
}
}
