public ChunkData(DepotManifest.ChunkData sourceChunk)
{
ChunkID = sourceChunk.ChunkID;
Checksum = sourceChunk.Checksum;
Offset = sourceChunk.Offset;
CompressedLength = sourceChunk.CompressedLength;
UncompressedLength = sourceChunk.UncompressedLength;
}
public static ChunkData Create(DepotManifest.ChunkData sourceChunk)
{
return(new ChunkData
{
ChunkID = sourceChunk.ChunkID,
Checksum = sourceChunk.Checksum,
Offset = sourceChunk.Offset,
CompressedLength = sourceChunk.CompressedLength,
UncompressedLength = sourceChunk.UncompressedLength
});
}
/// <summary>
/// Initializes a new instance of the <see cref="DepotChunk"/> class.
/// </summary>
/// <param name="info">The manifest chunk information associated with this chunk.</param>
/// <param name="data">The underlying data for this chunk.</param>
public DepotChunk(DepotManifest.ChunkData info, byte[] data)
{
if (info is null)
{
throw new ArgumentNullException(nameof(info));
}
if (data is null)
{
throw new ArgumentNullException(nameof(data));
}
ChunkInfo = info;
Data = data;
}
public async Task ThrowsWhenNoChunkIDIsSet()
{
var configuration = SteamConfiguration.Create(x => x.WithHttpClientFactory(() => new HttpClient(new TeapotHttpMessageHandler())));
var steam = new SteamClient(configuration);
var client = new Client(steam);
var server = new Server
{
Protocol = Server.ConnectionProtocol.HTTP,
Host = "localhost",
VHost = "localhost",
Port = 80
};
var chunk = new DepotManifest.ChunkData();
var ex = await Assert.ThrowsAsync <ArgumentException>(() => client.DownloadDepotChunkAsync(depotId: 0, chunk, server));
Assert.Equal("chunk", ex.ParamName);
}
public async Task ThrowsSteamKitWebExceptionOnUnsuccessfulWebResponseForChunk()
{
var configuration = SteamConfiguration.Create(x => x.WithHttpClientFactory(() => new HttpClient(new TeapotHttpMessageHandler())));
var steam = new SteamClient(configuration);
var client = new Client(steam);
var server = new Server
{
Protocol = Server.ConnectionProtocol.HTTP,
Host = "localhost",
VHost = "localhost",
Port = 80
};
var chunk = new DepotManifest.ChunkData
{
ChunkID = new byte[] { 0xFF },
};
var ex = await Assert.ThrowsAsync <SteamKitWebRequestException>(() => client.DownloadDepotChunkAsync(depotId: 0, chunk, server));
Assert.Equal(( HttpStatusCode )418, ex.StatusCode);
}
/// <summary>
/// Downloads the specified depot chunk, and optionally processes the chunk and verifies the checksum if the depot decryption key has been provided.
/// </summary>
/// <remarks>
/// This function will also validate the length of the downloaded chunk with the value of <see cref="DepotManifest.ChunkData.CompressedLength"/>,
/// if it has been assigned a value.
/// </remarks>
/// <param name="depotId">The id of the depot being accessed.</param>
/// <param name="chunk">
/// A <see cref="DepotManifest.ChunkData"/> instance that represents the chunk to download.
/// This value should come from a manifest downloaded with <see cref="o:DownloadManifestAsync"/>.
/// </param>
/// <returns>A <see cref="DepotChunk"/> instance that contains the data for the given chunk.</returns>
/// <param name="server">The content server to connect to.</param>
/// <param name="depotKey">
/// The depot decryption key for the depot that will be downloaded.
/// This is used for decrypting filenames (if needed) in depot manifests, and processing depot chunks.
/// </param>
/// <param name="proxyServer">Optional content server marked as UseAsProxy which transforms the request.</param>
/// <exception cref="System.ArgumentNullException">chunk's <see cref="DepotManifest.ChunkData.ChunkID"/> was null.</exception>
/// <exception cref="System.IO.InvalidDataException">Thrown if the downloaded data does not match the expected length.</exception>
/// <exception cref="HttpRequestException">An network error occurred when performing the request.</exception>
/// <exception cref="SteamKitWebRequestException">A network error occurred when performing the request.</exception>
public async Task <DepotChunk> DownloadDepotChunkAsync(uint depotId, DepotManifest.ChunkData chunk, Server server, byte[]?depotKey = null, Server?proxyServer = null)
{
if (server == null)
{
throw new ArgumentNullException(nameof(server));
}
if (chunk == null)
{
throw new ArgumentNullException(nameof(chunk));
}
if (chunk.ChunkID == null)
{
throw new ArgumentException("Chunk must have a ChunkID.", nameof(chunk));
}
var chunkID = Utils.EncodeHexString(chunk.ChunkID);
var chunkData = await DoRawCommandAsync(server, string.Format("depot/{0}/chunk/{1}", depotId, chunkID), proxyServer).ConfigureAwait(false);
// assert that lengths match only if the chunk has a length assigned.
if (chunk.CompressedLength > 0 && chunkData.Length != chunk.CompressedLength)
{
throw new InvalidDataException($"Length mismatch after downloading depot chunk! (was {chunkData.Length}, but should be {chunk.CompressedLength})");
}
var depotChunk = new DepotChunk(chunk, chunkData);
if (depotKey != null)
{
// if we have the depot key, we can process the chunk immediately
depotChunk.Process(depotKey);
}
return(depotChunk);
}
private static void DownloadSteam3(uint appId, List <DepotDownloadInfo> depots)
{
ulong TotalBytesCompressed = 0;
ulong TotalBytesUncompressed = 0;
foreach (var depot in depots)
{
ulong DepotBytesCompressed = 0;
ulong DepotBytesUncompressed = 0;
Console.WriteLine("Downloading depot {0} - {1}", depot.id, depot.contentName);
CancellationTokenSource cts = new CancellationTokenSource();
ProtoManifest oldProtoManifest = null;
ProtoManifest newProtoManifest = null;
string configDir = Path.Combine(depot.installDir, CONFIG_DIR);
ulong lastManifestId = INVALID_MANIFEST_ID;
ConfigStore.TheConfig.LastManifests.TryGetValue(depot.id, out lastManifestId);
// In case we have an early exit, this will force equiv of verifyall next run.
ConfigStore.TheConfig.LastManifests[depot.id] = INVALID_MANIFEST_ID;
ConfigStore.Save();
if (lastManifestId != INVALID_MANIFEST_ID)
{
var oldManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", lastManifestId));
if (File.Exists(oldManifestFileName))
{
oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName);
}
}
if (lastManifestId == depot.manifestId && oldProtoManifest != null)
{
newProtoManifest = oldProtoManifest;
Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id);
}
else
{
var newManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", depot.manifestId));
if (newManifestFileName != null)
{
newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName);
}
if (newProtoManifest != null)
{
Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id);
}
else
{
Console.Write("Downloading depot manifest...");
DepotManifest depotManifest = null;
while (depotManifest == null)
{
CDNClient client = null;
try
{
client = cdnPool.GetConnectionForDepot(appId, depot.id, depot.depotKey, CancellationToken.None);
depotManifest = client.DownloadManifest(depot.id, depot.manifestId);
cdnPool.ReturnConnection(client);
}
catch (WebException e)
{
cdnPool.ReturnBrokenConnection(client);
if (e.Status == WebExceptionStatus.ProtocolError)
{
var response = e.Response as HttpWebResponse;
if (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden)
{
Console.WriteLine("Encountered 401 for depot manifest {0} {1}. Aborting.", depot.id, depot.manifestId);
break;
}
else
{
Console.WriteLine("Encountered error downloading depot manifest {0} {1}: {2}", depot.id, depot.manifestId, response.StatusCode);
}
}
else
{
Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depot.id, depot.manifestId, e.Status);
}
}
catch (Exception e)
{
cdnPool.ReturnBrokenConnection(client);
Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depot.id, depot.manifestId, e.Message);
}
}
if (depotManifest == null)
{
Console.WriteLine("\nUnable to download manifest {0} for depot {1}", depot.manifestId, depot.id);
return;
}
newProtoManifest = new ProtoManifest(depotManifest, depot.manifestId);
newProtoManifest.SaveToFile(newManifestFileName);
Console.WriteLine("Successfully downloaded manifest {0} for depot {1}", depot.manifestId, depot.id);
Console.WriteLine(" Done!");
}
}
newProtoManifest.Files.Sort((x, y) => { return(x.FileName.CompareTo(y.FileName)); });
if (Config.DownloadManifestOnly)
{
StringBuilder manifestBuilder = new StringBuilder();
string txtManifest = Path.Combine(depot.installDir, string.Format("manifest_{0}.txt", depot.id));
foreach (var file in newProtoManifest.Files)
{
if (file.Flags.HasFlag(EDepotFileFlag.Directory))
{
continue;
}
manifestBuilder.Append(string.Format("{0}\n", file.FileName));
}
File.WriteAllText(txtManifest, manifestBuilder.ToString());
continue;
}
ulong complete_download_size = 0;
ulong size_downloaded = 0;
string stagingDir = Path.Combine(depot.installDir, STAGING_DIR);
var filesAfterExclusions = newProtoManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).ToList();
// Pre-process
filesAfterExclusions.ForEach(file =>
{
var fileFinalPath = Path.Combine(depot.installDir, file.FileName);
var fileStagingPath = Path.Combine(stagingDir, file.FileName);
if (file.Flags.HasFlag(EDepotFileFlag.Directory))
{
Directory.CreateDirectory(fileFinalPath);
Directory.CreateDirectory(fileStagingPath);
}
else
{
// Some manifests don't explicitly include all necessary directories
Directory.CreateDirectory(Path.GetDirectoryName(fileFinalPath));
Directory.CreateDirectory(Path.GetDirectoryName(fileStagingPath));
complete_download_size += file.TotalSize;
}
});
filesAfterExclusions.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory))
.AsParallel().WithCancellation(cts.Token).WithDegreeOfParallelism(Config.MaxDownloads)
.ForAll(file =>
{
string fileFinalPath = Path.Combine(depot.installDir, file.FileName);
string fileStagingPath = Path.Combine(stagingDir, file.FileName);
// This may still exist if the previous run exited before cleanup
if (File.Exists(fileStagingPath))
{
File.Delete(fileStagingPath);
}
FileStream fs = null;
List <ProtoManifest.ChunkData> neededChunks;
FileInfo fi = new FileInfo(fileFinalPath);
if (!fi.Exists)
{
// create new file. need all chunks
fs = File.Create(fileFinalPath);
fs.SetLength(( long )file.TotalSize);
neededChunks = new List <ProtoManifest.ChunkData>(file.Chunks);
}
else
{
// open existing
ProtoManifest.FileData oldManifestFile = null;
if (oldProtoManifest != null)
{
oldManifestFile = oldProtoManifest.Files.SingleOrDefault(f => f.FileName == file.FileName);
}
if (oldManifestFile != null)
{
neededChunks = new List <ProtoManifest.ChunkData>();
if (Config.VerifyAll || !oldManifestFile.FileHash.SequenceEqual(file.FileHash))
{
// we have a version of this file, but it doesn't fully match what we want
var matchingChunks = new List <ChunkMatch>();
foreach (var chunk in file.Chunks)
{
var oldChunk = oldManifestFile.Chunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID));
if (oldChunk != null)
{
matchingChunks.Add(new ChunkMatch(oldChunk, chunk));
}
else
{
neededChunks.Add(chunk);
}
}
File.Move(fileFinalPath, fileStagingPath);
fs = File.Open(fileFinalPath, FileMode.Create);
fs.SetLength(( long )file.TotalSize);
using (var fsOld = File.Open(fileStagingPath, FileMode.Open))
{
foreach (var match in matchingChunks)
{
fsOld.Seek(( long )match.OldChunk.Offset, SeekOrigin.Begin);
byte[] tmp = new byte[match.OldChunk.UncompressedLength];
fsOld.Read(tmp, 0, tmp.Length);
byte[] adler = Util.AdlerHash(tmp);
if (!adler.SequenceEqual(match.OldChunk.Checksum))
{
neededChunks.Add(match.NewChunk);
}
else
{
fs.Seek(( long )match.NewChunk.Offset, SeekOrigin.Begin);
fs.Write(tmp, 0, tmp.Length);
}
}
}
File.Delete(fileStagingPath);
}
}
else
{
// No old manifest or file not in old manifest. We must validate.
fs = File.Open(fileFinalPath, FileMode.Open);
if (( ulong )fi.Length != file.TotalSize)
{
fs.SetLength(( long )file.TotalSize);
}
neededChunks = Util.ValidateSteam3FileChecksums(fs, file.Chunks.OrderBy(x => x.Offset).ToArray());
}
if (neededChunks.Count() == 0)
{
size_downloaded += file.TotalSize;
Console.WriteLine("{0,6:#00.00}% {1}", (( float )size_downloaded / ( float )complete_download_size) * 100.0f, fileFinalPath);
if (fs != null)
{
fs.Close();
}
return;
}
else
{
size_downloaded += (file.TotalSize - ( ulong )neededChunks.Select(x => ( long )x.UncompressedLength).Sum());
}
}
foreach (var chunk in neededChunks)
{
if (cts.IsCancellationRequested)
{
break;
}
string chunkID = Util.EncodeHexString(chunk.ChunkID);
CDNClient.DepotChunk chunkData = null;
while (!cts.IsCancellationRequested)
{
CDNClient client;
try
{
client = cdnPool.GetConnectionForDepot(appId, depot.id, depot.depotKey, cts.Token);
}
catch (OperationCanceledException)
{
break;
}
DepotManifest.ChunkData data = new DepotManifest.ChunkData();
data.ChunkID = chunk.ChunkID;
data.Checksum = chunk.Checksum;
data.Offset = chunk.Offset;
data.CompressedLength = chunk.CompressedLength;
data.UncompressedLength = chunk.UncompressedLength;
try
{
chunkData = client.DownloadDepotChunk(depot.id, data);
cdnPool.ReturnConnection(client);
break;
}
catch (WebException e)
{
cdnPool.ReturnBrokenConnection(client);
if (e.Status == WebExceptionStatus.ProtocolError)
{
var response = e.Response as HttpWebResponse;
if (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden)
{
Console.WriteLine("Encountered 401 for chunk {0}. Aborting.", chunkID);
cts.Cancel();
break;
}
else
{
Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, response.StatusCode);
}
}
else
{
Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, e.Status);
}
}
catch (Exception e)
{
cdnPool.ReturnBrokenConnection(client);
Console.WriteLine("Encountered unexpected error downloading chunk {0}: {1}", chunkID, e.Message);
}
}
if (chunkData == null)
{
Console.WriteLine("Failed to find any server with chunk {0} for depot {1}. Aborting.", chunkID, depot.id);
return;
}
TotalBytesCompressed += chunk.CompressedLength;
DepotBytesCompressed += chunk.CompressedLength;
TotalBytesUncompressed += chunk.UncompressedLength;
DepotBytesUncompressed += chunk.UncompressedLength;
fs.Seek(( long )chunk.Offset, SeekOrigin.Begin);
fs.Write(chunkData.Data, 0, chunkData.Data.Length);
size_downloaded += chunk.UncompressedLength;
}
fs.Close();
Console.WriteLine("{0,6:#00.00}% {1}", (( float )size_downloaded / ( float )complete_download_size) * 100.0f, fileFinalPath);
});
ConfigStore.TheConfig.LastManifests[depot.id] = depot.manifestId;
ConfigStore.Save();
Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.id, DepotBytesCompressed, DepotBytesUncompressed);
}
Console.WriteLine("Total downloaded: {0} bytes ({1} bytes uncompressed) from {2} depots", TotalBytesCompressed, TotalBytesUncompressed, depots.Count);
}
private static async Task DownloadSteam3Async(uint appId, List <DepotDownloadInfo> depots)
{
ulong TotalBytesCompressed = 0;
ulong TotalBytesUncompressed = 0;
foreach (var depot in depots)
{
ulong DepotBytesCompressed = 0;
ulong DepotBytesUncompressed = 0;
Console.WriteLine("Downloading depot {0} - {1}", depot.id, depot.contentName);
CancellationTokenSource cts = new CancellationTokenSource();
cdnPool.ExhaustedToken = cts;
ProtoManifest oldProtoManifest = null;
ProtoManifest newProtoManifest = null;
string configDir = Path.Combine(depot.installDir, CONFIG_DIR);
ulong lastManifestId = INVALID_MANIFEST_ID;
DepotConfigStore.Instance.InstalledManifestIDs.TryGetValue(depot.id, out lastManifestId);
// In case we have an early exit, this will force equiv of verifyall next run.
DepotConfigStore.Instance.InstalledManifestIDs[depot.id] = INVALID_MANIFEST_ID;
DepotConfigStore.Save();
if (lastManifestId != INVALID_MANIFEST_ID)
{
var oldManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", lastManifestId));
if (File.Exists(oldManifestFileName))
{
byte[] expectedChecksum, currentChecksum;
try
{
expectedChecksum = File.ReadAllBytes(oldManifestFileName + ".sha");
}
catch (IOException)
{
expectedChecksum = null;
}
oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName, out currentChecksum);
if (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum))
{
// We only have to show this warning if the old manifest ID was different
if (lastManifestId != depot.manifestId)
{
Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", lastManifestId);
}
oldProtoManifest = null;
}
}
}
if (lastManifestId == depot.manifestId && oldProtoManifest != null)
{
newProtoManifest = oldProtoManifest;
Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id);
}
else
{
var newManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", depot.manifestId));
if (newManifestFileName != null)
{
byte[] expectedChecksum, currentChecksum;
try
{
expectedChecksum = File.ReadAllBytes(newManifestFileName + ".sha");
}
catch (IOException)
{
expectedChecksum = null;
}
newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName, out currentChecksum);
if (newProtoManifest != null && (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum)))
{
Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", depot.manifestId);
newProtoManifest = null;
}
}
if (newProtoManifest != null)
{
Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id);
}
else
{
Console.Write("Downloading depot manifest...");
DepotManifest depotManifest = null;
while (depotManifest == null)
{
Tuple <CDNClient.Server, string> connection = null;
try
{
connection = await cdnPool.GetConnectionForDepot(appId, depot.id, CancellationToken.None);
depotManifest = await cdnPool.CDNClient.DownloadManifestAsync(depot.id, depot.manifestId,
connection.Item1, connection.Item2, depot.depotKey).ConfigureAwait(false);
cdnPool.ReturnConnection(connection);
}
catch (SteamKitWebRequestException e)
{
cdnPool.ReturnBrokenConnection(connection);
if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden)
{
Console.WriteLine("Encountered 401 for depot manifest {0} {1}. Aborting.", depot.id, depot.manifestId);
break;
}
else
{
Console.WriteLine("Encountered error downloading depot manifest {0} {1}: {2}", depot.id, depot.manifestId, e.StatusCode);
}
}
catch (Exception e)
{
cdnPool.ReturnBrokenConnection(connection);
Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depot.id, depot.manifestId, e.Message);
}
}
if (depotManifest == null)
{
Console.WriteLine("\nUnable to download manifest {0} for depot {1}", depot.manifestId, depot.id);
return;
}
byte[] checksum;
newProtoManifest = new ProtoManifest(depotManifest, depot.manifestId);
newProtoManifest.SaveToFile(newManifestFileName, out checksum);
File.WriteAllBytes(newManifestFileName + ".sha", checksum);
Console.WriteLine(" Done!");
}
}
newProtoManifest.Files.Sort((x, y) => string.Compare(x.FileName, y.FileName, StringComparison.Ordinal));
if (Config.DownloadManifestOnly)
{
StringBuilder manifestBuilder = new StringBuilder();
string txtManifest = Path.Combine(depot.installDir, string.Format("manifest_{0}.txt", depot.id));
foreach (var file in newProtoManifest.Files)
{
if (file.Flags.HasFlag(EDepotFileFlag.Directory))
{
continue;
}
manifestBuilder.Append(string.Format("{0}\n", file.FileName));
}
File.WriteAllText(txtManifest, manifestBuilder.ToString());
continue;
}
ulong complete_download_size = 0;
ulong size_downloaded = 0;
string stagingDir = Path.Combine(depot.installDir, STAGING_DIR);
var filesAfterExclusions = newProtoManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).ToList();
// Pre-process
filesAfterExclusions.ForEach(file =>
{
var fileFinalPath = Path.Combine(depot.installDir, file.FileName);
var fileStagingPath = Path.Combine(stagingDir, file.FileName);
if (file.Flags.HasFlag(EDepotFileFlag.Directory))
{
Directory.CreateDirectory(fileFinalPath);
Directory.CreateDirectory(fileStagingPath);
}
else
{
// Some manifests don't explicitly include all necessary directories
Directory.CreateDirectory(Path.GetDirectoryName(fileFinalPath));
Directory.CreateDirectory(Path.GetDirectoryName(fileStagingPath));
complete_download_size += file.TotalSize;
}
});
var semaphore = new SemaphoreSlim(Config.MaxDownloads);
var files = filesAfterExclusions.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory)).ToArray();
var tasks = new Task[files.Length];
for (var i = 0; i < files.Length; i++)
{
var file = files[i];
var task = Task.Run(async() =>
{
cts.Token.ThrowIfCancellationRequested();
try
{
await semaphore.WaitAsync().ConfigureAwait(false);
cts.Token.ThrowIfCancellationRequested();
string fileFinalPath = Path.Combine(depot.installDir, file.FileName);
string fileStagingPath = Path.Combine(stagingDir, file.FileName);
// This may still exist if the previous run exited before cleanup
if (File.Exists(fileStagingPath))
{
File.Delete(fileStagingPath);
}
FileStream fs = null;
List <ProtoManifest.ChunkData> neededChunks;
FileInfo fi = new FileInfo(fileFinalPath);
if (!fi.Exists)
{
// create new file. need all chunks
fs = File.Create(fileFinalPath);
fs.SetLength(( long )file.TotalSize);
neededChunks = new List <ProtoManifest.ChunkData>(file.Chunks);
}
else
{
// open existing
ProtoManifest.FileData oldManifestFile = null;
if (oldProtoManifest != null)
{
oldManifestFile = oldProtoManifest.Files.SingleOrDefault(f => f.FileName == file.FileName);
}
if (oldManifestFile != null)
{
neededChunks = new List <ProtoManifest.ChunkData>();
if (Config.VerifyAll || !oldManifestFile.FileHash.SequenceEqual(file.FileHash))
{
// we have a version of this file, but it doesn't fully match what we want
var matchingChunks = new List <ChunkMatch>();
foreach (var chunk in file.Chunks)
{
var oldChunk = oldManifestFile.Chunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID));
if (oldChunk != null)
{
matchingChunks.Add(new ChunkMatch(oldChunk, chunk));
}
else
{
neededChunks.Add(chunk);
}
}
File.Move(fileFinalPath, fileStagingPath);
fs = File.Open(fileFinalPath, FileMode.Create);
fs.SetLength(( long )file.TotalSize);
using (var fsOld = File.Open(fileStagingPath, FileMode.Open))
{
foreach (var match in matchingChunks)
{
fsOld.Seek(( long )match.OldChunk.Offset, SeekOrigin.Begin);
byte[] tmp = new byte[match.OldChunk.UncompressedLength];
fsOld.Read(tmp, 0, tmp.Length);
byte[] adler = Util.AdlerHash(tmp);
if (!adler.SequenceEqual(match.OldChunk.Checksum))
{
neededChunks.Add(match.NewChunk);
}
else
{
fs.Seek(( long )match.NewChunk.Offset, SeekOrigin.Begin);
fs.Write(tmp, 0, tmp.Length);
}
}
}
File.Delete(fileStagingPath);
}
}
else
{
// No old manifest or file not in old manifest. We must validate.
fs = File.Open(fileFinalPath, FileMode.Open);
if (( ulong )fi.Length != file.TotalSize)
{
fs.SetLength(( long )file.TotalSize);
}
neededChunks = Util.ValidateSteam3FileChecksums(fs, file.Chunks.OrderBy(x => x.Offset).ToArray());
}
if (neededChunks.Count() == 0)
{
size_downloaded += file.TotalSize;
Console.WriteLine("{0,6:#00.00}% {1}", (( float )size_downloaded / ( float )complete_download_size) * 100.0f, fileFinalPath);
if (fs != null)
{
fs.Dispose();
}
return;
}
else
{
size_downloaded += (file.TotalSize - ( ulong )neededChunks.Select(x => ( long )x.UncompressedLength).Sum());
}
}
foreach (var chunk in neededChunks)
{
if (cts.IsCancellationRequested)
{
break;
}
string chunkID = Util.EncodeHexString(chunk.ChunkID);
CDNClient.DepotChunk chunkData = null;
while (!cts.IsCancellationRequested)
{
Tuple <CDNClient.Server, string> connection;
try
{
connection = await cdnPool.GetConnectionForDepot(appId, depot.id, cts.Token);
}
catch (OperationCanceledException)
{
break;
}
DepotManifest.ChunkData data = new DepotManifest.ChunkData();
data.ChunkID = chunk.ChunkID;
data.Checksum = chunk.Checksum;
data.Offset = chunk.Offset;
data.CompressedLength = chunk.CompressedLength;
data.UncompressedLength = chunk.UncompressedLength;
try
{
chunkData = await cdnPool.CDNClient.DownloadDepotChunkAsync(depot.id, data,
connection.Item1, connection.Item2, depot.depotKey).ConfigureAwait(false);
cdnPool.ReturnConnection(connection);
break;
}
catch (SteamKitWebRequestException e)
{
cdnPool.ReturnBrokenConnection(connection);
if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden)
{
Console.WriteLine("Encountered 401 for chunk {0}. Aborting.", chunkID);
cts.Cancel();
break;
}
else
{
Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, e.StatusCode);
}
}
catch (Exception e)
{
cdnPool.ReturnBrokenConnection(connection);
Console.WriteLine("Encountered unexpected error downloading chunk {0}: {1}", chunkID, e.Message);
}
}
if (chunkData == null)
{
Console.WriteLine("Failed to find any server with chunk {0} for depot {1}. Aborting.", chunkID, depot.id);
cts.Cancel();
}
// Throw the cancellation exception if requested so that this task is marked failed
cts.Token.ThrowIfCancellationRequested();
TotalBytesCompressed += chunk.CompressedLength;
DepotBytesCompressed += chunk.CompressedLength;
TotalBytesUncompressed += chunk.UncompressedLength;
DepotBytesUncompressed += chunk.UncompressedLength;
fs.Seek(( long )chunk.Offset, SeekOrigin.Begin);
fs.Write(chunkData.Data, 0, chunkData.Data.Length);
size_downloaded += chunk.UncompressedLength;
}
fs.Dispose();
Console.WriteLine("{0,6:#00.00}% {1}", (( float )size_downloaded / ( float )complete_download_size) * 100.0f, fileFinalPath);
}
finally
{
semaphore.Release();
}
});
tasks[i] = task;
}
await Task.WhenAll(tasks).ConfigureAwait(false);
DepotConfigStore.Instance.InstalledManifestIDs[depot.id] = depot.manifestId;
DepotConfigStore.Save();
Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.id, DepotBytesCompressed, DepotBytesUncompressed);
}
Console.WriteLine("Total downloaded: {0} bytes ({1} bytes uncompressed) from {2} depots", TotalBytesCompressed, TotalBytesUncompressed, depots.Count);
}
private static async Task <bool> DownloadChunk(DepotProcessor.ManifestJob job, DepotManifest.ChunkData chunk, FileInfo downloadPath)
{
for (var i = 0; i <= 5; i++)
{
try
{
var chunkData = await CDNClient.DownloadDepotChunkAsync(job.DepotID, chunk, job.Server, job.CDNToken, job.DepotKey);
using (var fs = downloadPath.Open(FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite))
{
fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
await fs.WriteAsync(chunkData.Data, 0, chunkData.Data.Length);
}
return(true);
}
catch (Exception e)
{
Log.WriteWarn("FileDownloader", "{0} exception: {1}", job.DepotID, e.Message);
}
await Task.Delay(Utils.ExponentionalBackoff(i));
}
return(false);
}
private static async Task DownloadSteam3Async(uint appId, List <DepotDownloadInfo> depots, CancellationToken cancellationToken)
{
ulong TotalBytesCompressed = 0;
ulong TotalBytesUncompressed = 0;
foreach (var depot in depots)
{
ulong DepotBytesCompressed = 0;
ulong DepotBytesUncompressed = 0;
Console.WriteLine("Downloading depot {0} - {1}", depot.Id, depot.ContentName);
CancellationTokenSource cts = new CancellationTokenSource();
cdnPool.ExhaustedToken = cts;
ProtoManifest oldProtoManifest = null;
ProtoManifest newProtoManifest = null;
string configDir = Path.Combine(depot.InstallDir, CONFIG_DIR);
ulong lastManifestId = INVALID_MANIFEST_ID;
DepotConfigStore.Instance.InstalledManifestIDs.TryGetValue(depot.Id, out lastManifestId);
// In case we have an early exit, this will force equiv of verifyall next run.
DepotConfigStore.Instance.InstalledManifestIDs[depot.Id] = INVALID_MANIFEST_ID;
DepotConfigStore.Save();
if (lastManifestId != INVALID_MANIFEST_ID)
{
var oldManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", lastManifestId));
if (File.Exists(oldManifestFileName))
{
byte[] expectedChecksum, currentChecksum;
try
{
expectedChecksum = File.ReadAllBytes(oldManifestFileName + ".sha");
}
catch (IOException)
{
expectedChecksum = null;
}
oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName, out currentChecksum);
if (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum))
{
// We only have to show this warning if the old manifest ID was different
if (lastManifestId != depot.ManifestId)
{
Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", lastManifestId);
}
oldProtoManifest = null;
}
}
}
if (lastManifestId == depot.ManifestId && oldProtoManifest != null)
{
newProtoManifest = oldProtoManifest;
Console.WriteLine("Already have manifest {0} for depot {1}.", depot.ManifestId, depot.Id);
}
else
{
var newManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.Id, depot.ManifestId));
if (newManifestFileName != null)
{
byte[] expectedChecksum, currentChecksum;
try
{
expectedChecksum = File.ReadAllBytes(newManifestFileName + ".sha");
}
catch (IOException)
{
expectedChecksum = null;
}
newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName, out currentChecksum);
if (newProtoManifest != null && (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum)))
{
Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", depot.ManifestId);
newProtoManifest = null;
}
}
if (newProtoManifest != null)
{
Console.WriteLine("Already have manifest {0} for depot {1}.", depot.ManifestId, depot.Id);
}
else
{
Console.Write("Downloading depot manifest...");
DepotManifest depotManifest = await DownloadManifest(appId, depot.Id, depot.ManifestId, depot.DepotKey).ConfigureAwait(false);
byte[] checksum;
newProtoManifest = new ProtoManifest(depotManifest, depot.ManifestId);
newProtoManifest.SaveToFile(newManifestFileName, out checksum);
File.WriteAllBytes(newManifestFileName + ".sha", checksum);
Console.WriteLine(" Done!");
}
}
newProtoManifest.Files.Sort((x, y) => string.Compare(x.FileName, y.FileName, StringComparison.Ordinal));
Console.WriteLine("Manifest {0} ({1})", depot.ManifestId, newProtoManifest.CreationTime);
ulong complete_download_size = 0;
ulong size_downloaded = 0;
var downloadChanges = new ConcurrentQueue <(DateTime, ulong)>();
object downloadChangesLock = new object();
void reportProgress(string currentFilePath)
{
lock (downloadChangesLock)
{
while (downloadChanges.Count > 50) // Only average last 50 deltas
{
downloadChanges.TryDequeue(out _);
}
}
ulong bytesPerSecond = 0;
if (downloadChanges.Count > 0)
{
lock (downloadChangesLock)
{
TimeSpan timeSpan = downloadChanges.Last().Item1 - downloadChanges.First().Item1;
ulong totalBytes = downloadChanges.Aggregate(0ul, (a, b) => a + b.Item2);
double rate = totalBytes / timeSpan.TotalSeconds;
if (double.IsNaN(rate) || double.IsInfinity(rate))
{
rate = 0;
}
bytesPerSecond = (ulong)rate;
}
}
Globals.UiDispatcher.Invoke(() =>
{
Globals.AppState.DownloadState.DownloadPercentageComplete = ((double)size_downloaded / complete_download_size) * 100;
Globals.AppState.DownloadState.DownloadCurrentFile = currentFilePath;
Globals.AppState.DownloadState.DownloadedBytes = size_downloaded;
Globals.AppState.DownloadState.BytesPerSecond = bytesPerSecond;
});
}
string stagingDir = Path.Combine(depot.InstallDir, STAGING_DIR);
// Delete files that are removed in new manifest
if (oldProtoManifest != null)
{
var directoriesToCheck = new HashSet <string>(); // Directories to check if they're empty and delete
foreach (ProtoManifest.FileData oldFile in oldProtoManifest.Files)
{
if (!newProtoManifest.Files.Any(newFile => newFile.FileName == oldFile.FileName))
{
Console.WriteLine($"Deleting file: {oldFile.FileName}");
var fileFinalPath = Path.Combine(depot.InstallDir, oldFile.FileName);
if (File.Exists(fileFinalPath))
{
File.Delete(fileFinalPath);
directoriesToCheck.Add(Path.GetDirectoryName(fileFinalPath));
}
}
}
// Delete the now empty directories
foreach (string directoryPath in directoriesToCheck)
{
if (Directory.GetDirectories(directoryPath).Length == 0 && Directory.GetFiles(directoryPath).Length == 0)
{
Directory.Delete(directoryPath);
}
}
}
// Pre-process
newProtoManifest.Files.ForEach(file =>
{
var fileFinalPath = Path.Combine(depot.InstallDir, file.FileName);
var fileStagingPath = Path.Combine(stagingDir, file.FileName);
if (file.Flags.HasFlag(EDepotFileFlag.Directory))
{
Directory.CreateDirectory(fileFinalPath);
Directory.CreateDirectory(fileStagingPath);
}
else
{
// Some manifests don't explicitly include all necessary directories
Directory.CreateDirectory(Path.GetDirectoryName(fileFinalPath));
Directory.CreateDirectory(Path.GetDirectoryName(fileStagingPath));
complete_download_size += file.TotalSize;
}
});
Globals.UiDispatcher.Invoke(() =>
{
Globals.AppState.DownloadState.TotalBytes = complete_download_size;
});
cancellationToken.ThrowIfCancellationRequested();
var semaphore = new SemaphoreSlim(Config.MaxDownloads);
var files = newProtoManifest.Files.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory)).ToArray();
var tasks = new Task[files.Length];
for (var i = 0; i < files.Length; i++)
{
var file = files[i];
var task = Task.Run(async() =>
{
cts.Token.ThrowIfCancellationRequested();
try
{
await semaphore.WaitAsync().ConfigureAwait(false);
cts.Token.ThrowIfCancellationRequested();
cancellationToken.ThrowIfCancellationRequested();
string fileFinalPath = Path.Combine(depot.InstallDir, file.FileName);
string fileStagingPath = Path.Combine(stagingDir, file.FileName);
// This may still exist if the previous run exited before cleanup
if (File.Exists(fileStagingPath))
{
File.Delete(fileStagingPath);
}
FileStream fs = null;
List <ProtoManifest.ChunkData> neededChunks;
FileInfo fi = new FileInfo(fileFinalPath);
if (!fi.Exists)
{
// create new file. need all chunks
fs = File.Create(fileFinalPath);
fs.SetLength((long)file.TotalSize);
neededChunks = new List <ProtoManifest.ChunkData>(file.Chunks);
}
else
{
// open existing
ProtoManifest.FileData oldManifestFile = null;
if (oldProtoManifest != null)
{
oldManifestFile = oldProtoManifest.Files.SingleOrDefault(f => f.FileName == file.FileName);
}
if (oldManifestFile != null)
{
neededChunks = new List <ProtoManifest.ChunkData>();
if (Config.VerifyAll || !oldManifestFile.FileHash.SequenceEqual(file.FileHash))
{
// we have a version of this file, but it doesn't fully match what we want
var matchingChunks = new List <ChunkMatch>();
foreach (var chunk in file.Chunks)
{
var oldChunk = oldManifestFile.Chunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID));
if (oldChunk != null)
{
matchingChunks.Add(new ChunkMatch(oldChunk, chunk));
}
else
{
neededChunks.Add(chunk);
}
}
File.Move(fileFinalPath, fileStagingPath);
fs = File.Open(fileFinalPath, FileMode.Create);
fs.SetLength((long)file.TotalSize);
using (var fsOld = File.Open(fileStagingPath, FileMode.Open))
{
foreach (var match in matchingChunks)
{
fsOld.Seek((long)match.OldChunk.Offset, SeekOrigin.Begin);
byte[] tmp = new byte[match.OldChunk.UncompressedLength];
fsOld.Read(tmp, 0, tmp.Length);
byte[] adler = Util.AdlerHash(tmp);
if (!adler.SequenceEqual(match.OldChunk.Checksum))
{
neededChunks.Add(match.NewChunk);
}
else
{
fs.Seek((long)match.NewChunk.Offset, SeekOrigin.Begin);
fs.Write(tmp, 0, tmp.Length);
}
}
}
File.Delete(fileStagingPath);
}
}
else
{
// No old manifest or file not in old manifest. We must validate.
fs = File.Open(fileFinalPath, FileMode.Open);
if ((ulong)fi.Length != file.TotalSize)
{
fs.SetLength((long)file.TotalSize);
}
neededChunks = Util.ValidateSteam3FileChecksums(fs, file.Chunks.OrderBy(x => x.Offset).ToArray());
}
if (neededChunks.Count() == 0)
{
size_downloaded += file.TotalSize;
reportProgress(file.FileName);
if (fs != null)
{
fs.Dispose();
}
return;
}
else
{
ulong sizeDelta = (file.TotalSize - (ulong)neededChunks.Select(x => (long)x.UncompressedLength).Sum());
size_downloaded += sizeDelta;
reportProgress(file.FileName);
}
}
foreach (var chunk in neededChunks)
{
if (cts.IsCancellationRequested)
{
break;
}
string chunkID = Util.EncodeHexString(chunk.ChunkID);
CDNClient.DepotChunk chunkData = null;
while (!cts.IsCancellationRequested)
{
Tuple <CDNClient.Server, string> connection;
try
{
connection = await cdnPool.GetConnectionForDepot(appId, depot.Id, cts.Token);
}
catch (OperationCanceledException)
{
break;
}
DepotManifest.ChunkData data = new DepotManifest.ChunkData();
data.ChunkID = chunk.ChunkID;
data.Checksum = chunk.Checksum;
data.Offset = chunk.Offset;
data.CompressedLength = chunk.CompressedLength;
data.UncompressedLength = chunk.UncompressedLength;
try
{
chunkData = await cdnPool.CDNClient.DownloadDepotChunkAsync(depot.Id, data,
connection.Item1, connection.Item2, depot.DepotKey).ConfigureAwait(false);
cdnPool.ReturnConnection(connection);
break;
}
catch (SteamKitWebRequestException e)
{
cdnPool.ReturnBrokenConnection(connection);
if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden)
{
Console.WriteLine("Encountered 401 for chunk {0}. Aborting.", chunkID);
cts.Cancel();
break;
}
else
{
Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, e.StatusCode);
}
}
catch (TaskCanceledException)
{
Console.WriteLine("Connection timeout downloading chunk {0}", chunkID);
}
catch (Exception e)
{
cdnPool.ReturnBrokenConnection(connection);
Console.WriteLine("Encountered unexpected error downloading chunk {0}: {1}", chunkID, e.Message);
}
}
if (chunkData == null)
{
Console.WriteLine("Failed to find any server with chunk {0} for depot {1}. Aborting.", chunkID, depot.Id);
cts.Cancel();
}
// Throw the cancellation exception if requested so that this task is marked failed
cts.Token.ThrowIfCancellationRequested();
cancellationToken.ThrowIfCancellationRequested();
TotalBytesCompressed += chunk.CompressedLength;
DepotBytesCompressed += chunk.CompressedLength;
TotalBytesUncompressed += chunk.UncompressedLength;
DepotBytesUncompressed += chunk.UncompressedLength;
fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
fs.Write(chunkData.Data, 0, chunkData.Data.Length);
size_downloaded += chunk.UncompressedLength;
downloadChanges.Enqueue((DateTime.Now, chunk.CompressedLength));
reportProgress(file.FileName);
}
fs.Dispose();
reportProgress(file.FileName);
}
finally
{
semaphore.Release();
}
});
tasks[i] = task;
}
await Task.WhenAll(tasks).ConfigureAwait(false);
DepotConfigStore.Instance.InstalledManifestIDs[depot.Id] = depot.ManifestId;
DepotConfigStore.Save();
Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.Id, DepotBytesCompressed, DepotBytesUncompressed);
reportProgress(null);
}
Console.WriteLine("Total downloaded: {0} bytes ({1} bytes uncompressed) from {2} depots", TotalBytesCompressed, TotalBytesUncompressed, depots.Count);
}
private static async Task <bool> DownloadChunk(DepotProcessor.ManifestJob job, DepotManifest.ChunkData chunk, FileInfo downloadPath)
{
for (var i = 0; i <= 5; i++)
{
try
{
var chunkData = await CDNClient.DownloadDepotChunkAsync(job.DepotID, chunk, job.Server, string.Empty, job.DepotKey);
await using var fs = downloadPath.Open(FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite);
fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
await fs.WriteAsync(chunkData.Data, 0, chunkData.Data.Length);
return(true);
}
catch (Exception e)
{
Log.WriteWarn($"FileDownloader {job.DepotID}", $"Exception: {e}");
}
if (i < 5)
{
await Task.Delay(Utils.ExponentionalBackoff(i + 1));
}
}
return(false);
}
private static async Task DownloadSteam3Async(uint appId, List <DepotDownloadInfo> depots, Action downloadCompleteAction = null)
{
ulong TotalBytesCompressed = 0;
ulong TotalBytesUncompressed = 0;
foreach (var depot in depots)
{
ulong DepotBytesCompressed = 0;
ulong DepotBytesUncompressed = 0;
DebugLog.WriteLine("ContentDownloader", "Downloading depot " + depot.id + " - " + depot.contentName);
CancellationTokenSource cts = new CancellationTokenSource();
cdnPool.ExhaustedToken = cts;
ProtoManifest oldProtoManifest = null;
ProtoManifest downloadManifest = null;
string configDir = Path.Combine(depot.installDir, CONFIG_DIR);
ulong lastManifestId = INVALID_MANIFEST_ID;
ConfigStore.TheConfig.LastManifests.TryGetValue(depot.id, out lastManifestId);
// In case we have an early exit, this will force equiv of verifyall next run.
ConfigStore.TheConfig.LastManifests[depot.id] = INVALID_MANIFEST_ID;
ConfigStore.Save();
if (lastManifestId != INVALID_MANIFEST_ID)
{
var oldManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", lastManifestId));
DebugLog.WriteLine(DEBUG_NAME_FILES, "Checking if " + oldManifestFileName + " exists");
if (File.Exists(oldManifestFileName))
{
DebugLog.WriteLine(DEBUG_NAME_FILES, oldManifestFileName + " exists, reading!");
oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName);
}
}
if (lastManifestId == depot.manifestId && oldProtoManifest != null)
{
downloadManifest = oldProtoManifest;
DebugLog.WriteLine("ContentDownloader", "Already have manifest " + depot.manifestId + " for depot " + depot.id + ".");
}
else
{
var newManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", depot.manifestId));
if (newManifestFileName != null)
{
downloadManifest = ProtoManifest.LoadFromFile(newManifestFileName);
}
if (downloadManifest != null)
{
DebugLog.WriteLine("ContentDownloader", "Already have manifest " + depot.manifestId + " for depot " + depot.id + ".");
}
else
{
DebugLog.WriteLine("ContentDownloader", "Downloading depot manifest...");
DepotManifest depotManifest = null;
while (depotManifest == null)
{
CDNClient client = null;
try
{
client = await cdnPool.GetConnectionForDepotAsync(appId, depot.id, depot.depotKey, CancellationToken.None).ConfigureAwait(false);
//client = await cdnPool.GetConnectionForDepotAsync(appId, depot.id, depot.depotKey, CancellationToken.None);
depotManifest = await client.DownloadManifestAsync(depot.id, depot.manifestId).ConfigureAwait(false);
//depotManifest = await client.DownloadManifestAsync(depot.id, depot.manifestId);
cdnPool.ReturnConnection(client);
}
catch (WebException e)
{
cdnPool.ReturnBrokenConnection(client);
if (e.Status == WebExceptionStatus.ProtocolError)
{
var response = e.Response as HttpWebResponse;
if (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden)
{
DebugLog.WriteLine("ContentDownloader", "Encountered 401 for depot manifest " + depot.id + " " + depot.manifestId + ". Aborting.");
break;
}
else
{
DebugLog.WriteLine("ContentDownloader", "Encountered error downloading depot manifest " + depot.id + " " + depot.manifestId + ": " + response.StatusCode);
}
}
else
{
DebugLog.WriteLine("ContentDownloader", "Encountered error downloading manifest for depot " + depot.id + " " + depot.manifestId + ": " + e.Status);
}
}
catch (Exception e)
{
cdnPool.ReturnBrokenConnection(client);
DebugLog.WriteLine("ContentDownloader", "Encountered error downloading manifest for depot " + depot.id + " " + depot.manifestId + ": " + e.Message);
}
}
if (depotManifest == null)
{
DebugLog.WriteLine("ContentDownloader", "\nUnable to download manifest " + depot.manifestId + " for depot " + depot.id);
return;
}
downloadManifest = new ProtoManifest(depotManifest, depot.manifestId);
downloadManifest.SaveToFile(newManifestFileName);
DebugLog.WriteLine("ContentDownloader", "Done!");
}
}
downloadManifest.Files.Sort((x, y) => { return(x.FileName.CompareTo(y.FileName)); });
if (downloadManifest != null)
{
onManifestReceived?.Invoke(appId, depot.id, depot.contentName, downloadManifest);
}
if (Config.DownloadManifestOnly)
{
continue;
}
complete_download_size = 0;
size_downloaded = 0;
string stagingDir = Path.Combine(depot.installDir, STAGING_DIR);
var filesAfterExclusions = downloadManifest.Files.Where(f => TestIsFileIncluded(f.FileName)).ToList();
// Pre-process
filesAfterExclusions.ForEach(file =>
{
var fileFinalPath = Path.Combine(depot.installDir, file.FileName);
var fileStagingPath = Path.Combine(stagingDir, file.FileName);
if (file.Flags.HasFlag(EDepotFileFlag.Directory))
{
Directory.CreateDirectory(fileFinalPath);
Directory.CreateDirectory(fileStagingPath);
}
else
{
// Some manifests don't explicitly include all necessary directories
Directory.CreateDirectory(Path.GetDirectoryName(fileFinalPath));
Directory.CreateDirectory(Path.GetDirectoryName(fileStagingPath));
complete_download_size += file.TotalSize;
}
});
var semaphore = new SemaphoreSlim(Config.MaxDownloads);
var files = filesAfterExclusions.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory)).ToArray();
var tasks = new Task[files.Length];
for (var i = 0; i < files.Length; i++)
{
var file = files[i];
var task = Task.Run(async() =>
{
cts.Token.ThrowIfCancellationRequested();
try
{
await semaphore.WaitAsync().ConfigureAwait(false);
//await semaphore.WaitAsync();
cts.Token.ThrowIfCancellationRequested();
string fileFinalPath = Path.Combine(depot.installDir, file.FileName);
string fileStagingPath = Path.Combine(stagingDir, file.FileName);
// This may still exist if the previous run exited before cleanup
DebugLog.WriteLine(DEBUG_NAME_FILES, "Checking if " + fileStagingPath + " exists");
if (File.Exists(fileStagingPath))
{
DebugLog.WriteLine(DEBUG_NAME_FILES, fileStagingPath + " exists, deleting!");
File.Delete(fileStagingPath);
}
List <ProtoManifest.ChunkData> neededChunks;
FileInfo fi = new FileInfo(fileFinalPath);
DebugLog.WriteLine(DEBUG_NAME_FILES, "Checking if " + fileFinalPath + " exists");
if (!fi.Exists)
{
// create new file. need all chunks
DebugLog.WriteLine(DEBUG_NAME_FILES, fileFinalPath + " does not exist, creating!");
using (FileStream fs = File.Create(fileFinalPath))
{
fs.SetLength((long)file.TotalSize);
neededChunks = new List <ProtoManifest.ChunkData>(file.Chunks);
}
}
else
{
// open existing
ProtoManifest.FileData oldManifestFile = null;
if (oldProtoManifest != null)
{
oldManifestFile = oldProtoManifest.Files.SingleOrDefault(f => f.FileName == file.FileName);
}
if (oldManifestFile != null)
{
neededChunks = new List <ProtoManifest.ChunkData>();
if (Config.VerifyAll || !oldManifestFile.FileHash.SequenceEqual(file.FileHash))
{
// we have a version of this file, but it doesn't fully match what we want
var matchingChunks = new List <ChunkMatch>();
foreach (var chunk in file.Chunks)
{
var oldChunk = oldManifestFile.Chunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID));
if (oldChunk != null)
{
matchingChunks.Add(new ChunkMatch(oldChunk, chunk));
}
else
{
neededChunks.Add(chunk);
}
}
DebugLog.WriteLine(DEBUG_NAME_FILES, "Moving file " + fileFinalPath + " to " + fileStagingPath);
File.Move(fileFinalPath, fileStagingPath);
DebugLog.WriteLine(DEBUG_NAME_FILES, "Creating file " + fileFinalPath);
using (FileStream fs = File.Open(fileFinalPath, FileMode.Create))
{
fs.SetLength((long)file.TotalSize);
DebugLog.WriteLine(DEBUG_NAME_FILES, "Opening file " + fileStagingPath);
using (var fsOld = File.Open(fileStagingPath, FileMode.Open))
{
foreach (var match in matchingChunks)
{
fsOld.Seek((long)match.OldChunk.Offset, SeekOrigin.Begin);
byte[] tmp = new byte[match.OldChunk.UncompressedLength];
fsOld.Read(tmp, 0, tmp.Length);
byte[] adler = Util.AdlerHash(tmp);
if (!adler.SequenceEqual(match.OldChunk.Checksum))
{
neededChunks.Add(match.NewChunk);
}
else
{
fs.Seek((long)match.NewChunk.Offset, SeekOrigin.Begin);
fs.Write(tmp, 0, tmp.Length);
}
}
}
}
DebugLog.WriteLine(DEBUG_NAME_FILES, "Deleting file " + fileStagingPath);
File.Delete(fileStagingPath);
}
}
else
{
// No old manifest or file not in old manifest. We must validate.
DebugLog.WriteLine(DEBUG_NAME_FILES, "Opening file " + fileFinalPath);
using (FileStream fs = File.Open(fileFinalPath, FileMode.Open))
{
if ((ulong)fi.Length != file.TotalSize)
{
fs.SetLength((long)file.TotalSize);
}
neededChunks = Util.ValidateSteam3FileChecksums(fs, file.Chunks.OrderBy(x => x.Offset).ToArray());
}
}
if (neededChunks.Count() == 0)
{
size_downloaded += file.TotalSize;
DebugLog.WriteLine("ContentDownloader", DownloadPercent * 100.0f + "% " + fileFinalPath);
return;
}
else
{
size_downloaded += (file.TotalSize - ( ulong )neededChunks.Select(x => ( long )x.UncompressedLength).Sum());
}
}
DebugLog.WriteLine(DEBUG_NAME_FILES, "Opening file " + fileFinalPath);
using (FileStream fs = File.Open(fileFinalPath, FileMode.Open))
{
foreach (var chunk in neededChunks)
{
if (cts.IsCancellationRequested)
{
break;
}
string chunkID = Util.EncodeHexString(chunk.ChunkID);
CDNClient.DepotChunk chunkData = null;
while (!cts.IsCancellationRequested)
{
CDNClient client;
try
{
client = await cdnPool.GetConnectionForDepotAsync(appId, depot.id, depot.depotKey, cts.Token).ConfigureAwait(false);
//client = await cdnPool.GetConnectionForDepotAsync(appId, depot.id, depot.depotKey, cts.Token);
}
catch (OperationCanceledException)
{
break;
}
DepotManifest.ChunkData data = new DepotManifest.ChunkData();
data.ChunkID = chunk.ChunkID;
data.Checksum = chunk.Checksum;
data.Offset = chunk.Offset;
data.CompressedLength = chunk.CompressedLength;
data.UncompressedLength = chunk.UncompressedLength;
try
{
//chunkData = await client.DownloadDepotChunkAsStreamAsync(depot.id, data).ConfigureAwait(false);
chunkData = client.DownloadDepotChunkAsStreamAsync(depot.id, data).Result;
//chunkData = await client.DownloadDepotChunkAsync(depot.id, data);
cdnPool.ReturnConnection(client);
break;
}
catch (WebException e)
{
cdnPool.ReturnBrokenConnection(client);
if (e.Status == WebExceptionStatus.ProtocolError)
{
var response = e.Response as HttpWebResponse;
if (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden)
{
DebugLog.WriteLine("ContentDownloader", "Encountered 401 for chunk " + chunkID + ". Aborting.");
cts.Cancel();
break;
}
else
{
DebugLog.WriteLine("ContentDownloader", "Encountered error downloading chunk " + chunkID + ": " + response.StatusCode);
}
}
else
{
DebugLog.WriteLine("ContentDownloader", "Encountered error downloading chunk " + chunkID + ": " + e.Status);
}
}
catch (Exception e)
{
cdnPool.ReturnBrokenConnection(client);
DebugLog.WriteLine("ContentDownloader", "Encountered unexpected error downloading chunk " + chunkID + ": " + e.Message);
}
}
if (chunkData == null)
{
DebugLog.WriteLine("ContentDownloader", "Failed to find any server with chunk " + chunkID + " for depot " + depot.id + ". Aborting.");
cts.Cancel();
return;
}
TotalBytesCompressed += chunk.CompressedLength;
DepotBytesCompressed += chunk.CompressedLength;
TotalBytesUncompressed += chunk.UncompressedLength;
DepotBytesUncompressed += chunk.UncompressedLength;
using (chunkData.DataStream)
{
fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
chunkData.DataStream.CopyTo(fs);
//fs.Write(chunkData.Data, 0, chunkData.Data.Length);
}
size_downloaded += chunk.UncompressedLength;
}
}
DebugLog.WriteLine("ContentDownloader", DownloadPercent * 100.0f + "% " + fileFinalPath);
}
finally
{
semaphore.Release();
}
});
tasks[i] = task;
}
Task.WaitAll(tasks);
ConfigStore.TheConfig.LastManifests[depot.id] = depot.manifestId;
ConfigStore.Save();
DebugLog.WriteLine("ContentDownloader", "Depot " + depot.id + " - Downloaded " + DepotBytesCompressed + " bytes (" + DepotBytesUncompressed + " bytes uncompressed)");
}
IsDownloading = false;
downloadCompleteAction?.Invoke();
onDownloadCompleted?.Invoke();
DebugLog.WriteLine("ContentDownloader", "Total downloaded: " + TotalBytesCompressed + " bytes (" + TotalBytesUncompressed + " bytes uncompressed) from " + depots.Count + " depots");
}
private static async Task <bool> DownloadChunk(DepotProcessor.ManifestJob job, DepotManifest.ChunkData chunk, FileInfo downloadPath, CancellationTokenSource chunkCancellation)
{
const int TRIES = 3;
for (var i = 0; i <= TRIES; i++)
{
chunkCancellation.Token.ThrowIfCancellationRequested();
try
{
var chunkData = await CDNClient.DownloadDepotChunkAsync(job.DepotID, chunk, job.Server, string.Empty, job.DepotKey);
await using var fs = downloadPath.Open(FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite);
fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
await fs.WriteAsync(chunkData.Data);
return(true);
}
catch (Exception e)
{
Log.WriteWarn($"FileDownloader {job.DepotID}", $"Exception: {e}");
}
if (i < TRIES)
{
await Task.Delay(Utils.ExponentionalBackoff(i + 1));
}
}
return(false);
}
public async Task DownloadDepot(String path, uint appId, uint depotId, string branch,
Func <String, bool> includeFile)
{
Directory.CreateDirectory(path);
// retrieve app info
steam3.RequestAppInfo(appId);
var depotIDs = new List <uint>();
KeyValue depots = GetSteam3AppSection(appId, EAppInfoSection.Depots);
Console.WriteLine("Using app branch: '{0}'.", branch);
ulong manifestId = GetSteam3DepotManifest(depotId, appId, branch);
steam3.RequestDepotKey(depotId, appId);
byte[] depotKey = steam3.DepotKeys[depotId];
string contentName = GetAppOrDepotName(depotId, appId);
Console.WriteLine("Downloading depot {0} - {1}", depotId, contentName);
DepotManifest depotManifest = null;
CancellationTokenSource cts = new CancellationTokenSource();
CDNClientPool cdnPool = new CDNClientPool(steam3);
cdnPool.ExhaustedToken = cts;
while (depotManifest == null)
{
Tuple <CDNClient.Server, string> connection = null;
try
{
connection = await cdnPool.GetConnectionForDepot(appId, depotId, CancellationToken.None);
depotManifest = await cdnPool.CDNClient.DownloadManifestAsync(depotId, manifestId,
connection.Item1, connection.Item2, depotKey).ConfigureAwait(false);
cdnPool.ReturnConnection(connection);
}
catch (SteamKitWebRequestException e)
{
cdnPool.ReturnBrokenConnection(connection);
if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden)
{
Console.WriteLine("Encountered 401 for depot manifest {0} {1}. Aborting.", depotId, manifestId);
break;
}
else
{
Console.WriteLine("Encountered error downloading depot manifest {0} {1}: {2}", depotId,
manifestId, e.StatusCode);
}
}
catch (Exception e)
{
cdnPool.ReturnBrokenConnection(connection);
Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depotId,
manifestId, e.Message);
}
}
if (depotManifest == null)
{
Console.WriteLine("\nUnable to download manifest {0} for depot {1}", manifestId, depotId);
return;
}
ulong size_downloaded = 0;
foreach (var folder in depotManifest.Files.AsParallel().Where(f => f.Flags.HasFlag(EDepotFileFlag.Directory))
.ToList())
{
Directory.CreateDirectory(Path.Join(path, folder.FileName));
}
ulong TotalBytesCompressed = 0;
ulong TotalBytesUncompressed = 0;
ulong DepotBytesCompressed = 0;
ulong DepotBytesUncompressed = 0;
var semaphore = new SemaphoreSlim(10);
var files = depotManifest.Files.AsParallel().Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory) && includeFile(f.FileName))
.ToArray();
ulong complete_download_size = 0;
foreach (var file in files)
{
complete_download_size += file.TotalSize;
}
var tasks = new Task[files.Length];
for (var i = 0; i < files.Length; i++)
{
var file = files[i];
var task = Task.Run(async() =>
{
cts.Token.ThrowIfCancellationRequested();
try
{
await semaphore.WaitAsync().ConfigureAwait(false);
cts.Token.ThrowIfCancellationRequested();
string fileFinalPath = Path.Combine(path, file.FileName);
FileStream fs = null;
List <DepotManifest.ChunkData> neededChunks = new List <DepotManifest.ChunkData>();
FileInfo fi = new FileInfo(fileFinalPath);
if (!fi.Exists)
{
// create new file. need all chunks
fs = File.Create(fileFinalPath);
fs.SetLength((long)file.TotalSize);
neededChunks = new List <DepotManifest.ChunkData>(file.Chunks);
}
else
{
fs = File.Open(fileFinalPath, FileMode.Truncate);
fs.SetLength((long)file.TotalSize);
neededChunks = new List <DepotManifest.ChunkData>(file.Chunks);
}
foreach (var chunk in neededChunks)
{
if (cts.IsCancellationRequested)
{
break;
}
string chunkID = Util.EncodeHexString(chunk.ChunkID);
CDNClient.DepotChunk chunkData = null;
while (!cts.IsCancellationRequested)
{
Tuple <CDNClient.Server, string> connection;
try
{
connection = await cdnPool.GetConnectionForDepot(appId, depotId, cts.Token);
}
catch (OperationCanceledException)
{
break;
}
DepotManifest.ChunkData data = new DepotManifest.ChunkData();
data.ChunkID = chunk.ChunkID;
data.Checksum = chunk.Checksum;
data.Offset = chunk.Offset;
data.CompressedLength = chunk.CompressedLength;
data.UncompressedLength = chunk.UncompressedLength;
try
{
chunkData = await cdnPool.CDNClient.DownloadDepotChunkAsync(depotId, data,
connection.Item1, connection.Item2, depotKey).ConfigureAwait(false);
cdnPool.ReturnConnection(connection);
break;
}
catch (SteamKitWebRequestException e)
{
cdnPool.ReturnBrokenConnection(connection);
if (e.StatusCode == HttpStatusCode.Unauthorized ||
e.StatusCode == HttpStatusCode.Forbidden)
{
Console.WriteLine("Encountered 401 for chunk {0}. Aborting.", chunkID);
cts.Cancel();
break;
}
else
{
Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID,
e.StatusCode);
}
}
catch (Exception e)
{
cdnPool.ReturnBrokenConnection(connection);
Console.WriteLine("Encountered unexpected error downloading chunk {0}: {1}",
chunkID, e.Message);
}
}
if (chunkData == null)
{
Console.WriteLine("Failed to find any server with chunk {0} for depot {1}. Aborting.",
chunkID, depotId);
cts.Cancel();
}
// Throw the cancellation exception if requested so that this task is marked failed
cts.Token.ThrowIfCancellationRequested();
TotalBytesCompressed += chunk.CompressedLength;
DepotBytesCompressed += chunk.CompressedLength;
TotalBytesUncompressed += chunk.UncompressedLength;
DepotBytesUncompressed += chunk.UncompressedLength;
fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
fs.Write(chunkData.Data, 0, chunkData.Data.Length);
size_downloaded += chunk.UncompressedLength;
}
fs.Dispose();
Console.WriteLine("{0,6:#00.00}% {1}",
((float)size_downloaded / (float)complete_download_size) * 100.0f, fileFinalPath);
}
finally
{
semaphore.Release();
}
});
tasks[i] = task;
}
await Task.WhenAll(tasks).ConfigureAwait(false);
Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depotId,
DepotBytesCompressed, DepotBytesUncompressed);
}
private static async Task DownloadSteam3AsyncDepotFileChunk(
CancellationTokenSource cts, uint appId,
GlobalDownloadCounter downloadCounter,
DepotFilesData depotFilesData,
ProtoManifest.FileData file,
FileStreamData fileStreamData,
ProtoManifest.ChunkData chunk)
{
cts.Token.ThrowIfCancellationRequested();
var depot = depotFilesData.depotDownloadInfo;
var depotDownloadCounter = depotFilesData.depotCounter;
string chunkID = Util.EncodeHexString(chunk.ChunkID);
DepotManifest.ChunkData data = new DepotManifest.ChunkData();
data.ChunkID = chunk.ChunkID;
data.Checksum = chunk.Checksum;
data.Offset = chunk.Offset;
data.CompressedLength = chunk.CompressedLength;
data.UncompressedLength = chunk.UncompressedLength;
CDNClient.DepotChunk chunkData = null;
do
{
cts.Token.ThrowIfCancellationRequested();
CDNClient.Server connection = null;
try
{
connection = cdnPool.GetConnection(cts.Token);
var cdnToken = await cdnPool.AuthenticateConnection(appId, depot.id, connection);
chunkData = await cdnPool.CDNClient.DownloadDepotChunkAsync(depot.id, data,
connection, cdnToken, depot.depotKey).ConfigureAwait(false);
cdnPool.ReturnConnection(connection);
}
catch (TaskCanceledException)
{
Console.WriteLine("Connection timeout downloading chunk {0}", chunkID);
}
catch (SteamKitWebRequestException e)
{
cdnPool.ReturnBrokenConnection(connection);
if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden)
{
Console.WriteLine("Encountered 401 for chunk {0}. Aborting.", chunkID);
break;
}
else
{
Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, e.StatusCode);
}
}
catch (OperationCanceledException)
{
break;
}
catch (Exception e)
{
cdnPool.ReturnBrokenConnection(connection);
Console.WriteLine("Encountered unexpected error downloading chunk {0}: {1}", chunkID, e.Message);
}
}while (chunkData == null);
if (chunkData == null)
{
Console.WriteLine("Failed to find any server with chunk {0} for depot {1}. Aborting.", chunkID, depot.id);
cts.Cancel();
}
// Throw the cancellation exception if requested so that this task is marked failed
cts.Token.ThrowIfCancellationRequested();
try
{
await fileStreamData.fileLock.WaitAsync().ConfigureAwait(false);
fileStreamData.fileStream.Seek((long)chunkData.ChunkInfo.Offset, SeekOrigin.Begin);
await fileStreamData.fileStream.WriteAsync(chunkData.Data, 0, chunkData.Data.Length);
}
finally
{
fileStreamData.fileLock.Release();
}
int remainingChunks = Interlocked.Decrement(ref fileStreamData.chunksToDownload);
if (remainingChunks == 0)
{
fileStreamData.fileStream.Dispose();
fileStreamData.fileLock.Dispose();
}
ulong sizeDownloaded = 0;
lock (depotDownloadCounter)
{
sizeDownloaded = depotDownloadCounter.SizeDownloaded + (ulong)chunkData.Data.Length;
depotDownloadCounter.SizeDownloaded = sizeDownloaded;
depotDownloadCounter.DepotBytesCompressed += chunk.CompressedLength;
depotDownloadCounter.DepotBytesUncompressed += chunk.UncompressedLength;
}
lock (downloadCounter)
{
downloadCounter.TotalBytesCompressed += chunk.CompressedLength;
downloadCounter.TotalBytesUncompressed += chunk.UncompressedLength;
}
if (remainingChunks == 0)
{
var fileFinalPath = Path.Combine(depot.installDir, file.FileName);
Console.WriteLine("{0,6:#00.00}% {1}", ((float)sizeDownloaded / (float)depotDownloadCounter.CompleteDownloadSize) * 100.0f, fileFinalPath);
}
}
