public static async Task DownloadAppAsync(uint appId, uint depotId, ulong manifestId, CancellationToken cancellationToken)
{
// Load our configuration data containing the depots currently installed
string configPath = Config.InstallDirectory;
if (string.IsNullOrWhiteSpace(configPath))
{
configPath = DEFAULT_DOWNLOAD_DIR;
}
Directory.CreateDirectory(Path.Combine(configPath, CONFIG_DIR));
DepotConfigStore.LoadFromFile(Path.Combine(configPath, CONFIG_DIR, "depot.config"));
var depotIDs = new List <uint>()
{
depotId
};
KeyValue depots = await GetSteam3AppSectionAsync(appId, EAppInfoSection.Depots).ConfigureAwait(false);
var infos = new List <DepotDownloadInfo>();
foreach (var depot in depotIDs)
{
var info = await GetDepotInfoAsync(depot, appId, manifestId, "Public").ConfigureAwait(false);
if (info != null)
{
infos.Add(info);
}
cancellationToken.ThrowIfCancellationRequested();
}
cancellationToken.ThrowIfCancellationRequested();
try
{
await DownloadSteam3Async(appId, infos, cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
Console.WriteLine("App {0} was not completely downloaded.", appId);
throw;
}
finally
{
DepotConfigStore.Instance = null; // unload config store
}
}
public static void LoadFromFile(string filename)
{
if (Loaded)
{
throw new Exception("Config already loaded");
}
if (File.Exists(filename))
{
using (FileStream fs = File.Open(filename, FileMode.Open))
using (DeflateStream ds = new DeflateStream(fs, CompressionMode.Decompress))
Instance = Serializer.Deserialize <DepotConfigStore>(ds);
}
else
{
Instance = new DepotConfigStore();
}
Instance.FileName = filename;
}
private static async Task DownloadSteam3Async(uint appId, List <DepotDownloadInfo> depots, CancellationToken cancellationToken)
{
ulong TotalBytesCompressed = 0;
ulong TotalBytesUncompressed = 0;
foreach (var depot in depots)
{
ulong DepotBytesCompressed = 0;
ulong DepotBytesUncompressed = 0;
Console.WriteLine("Downloading depot {0} - {1}", depot.Id, depot.ContentName);
CancellationTokenSource cts = new CancellationTokenSource();
cdnPool.ExhaustedToken = cts;
ProtoManifest oldProtoManifest = null;
ProtoManifest newProtoManifest = null;
string configDir = Path.Combine(depot.InstallDir, CONFIG_DIR);
ulong lastManifestId = INVALID_MANIFEST_ID;
DepotConfigStore.Instance.InstalledManifestIDs.TryGetValue(depot.Id, out lastManifestId);
// In case we have an early exit, this will force equiv of verifyall next run.
DepotConfigStore.Instance.InstalledManifestIDs[depot.Id] = INVALID_MANIFEST_ID;
DepotConfigStore.Save();
if (lastManifestId != INVALID_MANIFEST_ID)
{
var oldManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", lastManifestId));
if (File.Exists(oldManifestFileName))
{
byte[] expectedChecksum, currentChecksum;
try
{
expectedChecksum = File.ReadAllBytes(oldManifestFileName + ".sha");
}
catch (IOException)
{
expectedChecksum = null;
}
oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName, out currentChecksum);
if (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum))
{
// We only have to show this warning if the old manifest ID was different
if (lastManifestId != depot.ManifestId)
{
Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", lastManifestId);
}
oldProtoManifest = null;
}
}
}
if (lastManifestId == depot.ManifestId && oldProtoManifest != null)
{
newProtoManifest = oldProtoManifest;
Console.WriteLine("Already have manifest {0} for depot {1}.", depot.ManifestId, depot.Id);
}
else
{
var newManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.Id, depot.ManifestId));
if (newManifestFileName != null)
{
byte[] expectedChecksum, currentChecksum;
try
{
expectedChecksum = File.ReadAllBytes(newManifestFileName + ".sha");
}
catch (IOException)
{
expectedChecksum = null;
}
newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName, out currentChecksum);
if (newProtoManifest != null && (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum)))
{
Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", depot.ManifestId);
newProtoManifest = null;
}
}
if (newProtoManifest != null)
{
Console.WriteLine("Already have manifest {0} for depot {1}.", depot.ManifestId, depot.Id);
}
else
{
Console.Write("Downloading depot manifest...");
DepotManifest depotManifest = await DownloadManifest(appId, depot.Id, depot.ManifestId, depot.DepotKey).ConfigureAwait(false);
byte[] checksum;
newProtoManifest = new ProtoManifest(depotManifest, depot.ManifestId);
newProtoManifest.SaveToFile(newManifestFileName, out checksum);
File.WriteAllBytes(newManifestFileName + ".sha", checksum);
Console.WriteLine(" Done!");
}
}
newProtoManifest.Files.Sort((x, y) => string.Compare(x.FileName, y.FileName, StringComparison.Ordinal));
Console.WriteLine("Manifest {0} ({1})", depot.ManifestId, newProtoManifest.CreationTime);
ulong complete_download_size = 0;
ulong size_downloaded = 0;
var downloadChanges = new ConcurrentQueue <(DateTime, ulong)>();
object downloadChangesLock = new object();
void reportProgress(string currentFilePath)
{
lock (downloadChangesLock)
{
while (downloadChanges.Count > 50) // Only average last 50 deltas
{
downloadChanges.TryDequeue(out _);
}
}
ulong bytesPerSecond = 0;
if (downloadChanges.Count > 0)
{
lock (downloadChangesLock)
{
TimeSpan timeSpan = downloadChanges.Last().Item1 - downloadChanges.First().Item1;
ulong totalBytes = downloadChanges.Aggregate(0ul, (a, b) => a + b.Item2);
double rate = totalBytes / timeSpan.TotalSeconds;
if (double.IsNaN(rate) || double.IsInfinity(rate))
{
rate = 0;
}
bytesPerSecond = (ulong)rate;
}
}
Globals.UiDispatcher.Invoke(() =>
{
Globals.AppState.DownloadState.DownloadPercentageComplete = ((double)size_downloaded / complete_download_size) * 100;
Globals.AppState.DownloadState.DownloadCurrentFile = currentFilePath;
Globals.AppState.DownloadState.DownloadedBytes = size_downloaded;
Globals.AppState.DownloadState.BytesPerSecond = bytesPerSecond;
});
}
string stagingDir = Path.Combine(depot.InstallDir, STAGING_DIR);
// Delete files that are removed in new manifest
if (oldProtoManifest != null)
{
var directoriesToCheck = new HashSet <string>(); // Directories to check if they're empty and delete
foreach (ProtoManifest.FileData oldFile in oldProtoManifest.Files)
{
if (!newProtoManifest.Files.Any(newFile => newFile.FileName == oldFile.FileName))
{
Console.WriteLine($"Deleting file: {oldFile.FileName}");
var fileFinalPath = Path.Combine(depot.InstallDir, oldFile.FileName);
if (File.Exists(fileFinalPath))
{
File.Delete(fileFinalPath);
directoriesToCheck.Add(Path.GetDirectoryName(fileFinalPath));
}
}
}
// Delete the now empty directories
foreach (string directoryPath in directoriesToCheck)
{
if (Directory.GetDirectories(directoryPath).Length == 0 && Directory.GetFiles(directoryPath).Length == 0)
{
Directory.Delete(directoryPath);
}
}
}
// Pre-process
newProtoManifest.Files.ForEach(file =>
{
var fileFinalPath = Path.Combine(depot.InstallDir, file.FileName);
var fileStagingPath = Path.Combine(stagingDir, file.FileName);
if (file.Flags.HasFlag(EDepotFileFlag.Directory))
{
Directory.CreateDirectory(fileFinalPath);
Directory.CreateDirectory(fileStagingPath);
}
else
{
// Some manifests don't explicitly include all necessary directories
Directory.CreateDirectory(Path.GetDirectoryName(fileFinalPath));
Directory.CreateDirectory(Path.GetDirectoryName(fileStagingPath));
complete_download_size += file.TotalSize;
}
});
Globals.UiDispatcher.Invoke(() =>
{
Globals.AppState.DownloadState.TotalBytes = complete_download_size;
});
cancellationToken.ThrowIfCancellationRequested();
var semaphore = new SemaphoreSlim(Config.MaxDownloads);
var files = newProtoManifest.Files.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory)).ToArray();
var tasks = new Task[files.Length];
for (var i = 0; i < files.Length; i++)
{
var file = files[i];
var task = Task.Run(async() =>
{
cts.Token.ThrowIfCancellationRequested();
try
{
await semaphore.WaitAsync().ConfigureAwait(false);
cts.Token.ThrowIfCancellationRequested();
cancellationToken.ThrowIfCancellationRequested();
string fileFinalPath = Path.Combine(depot.InstallDir, file.FileName);
string fileStagingPath = Path.Combine(stagingDir, file.FileName);
// This may still exist if the previous run exited before cleanup
if (File.Exists(fileStagingPath))
{
File.Delete(fileStagingPath);
}
FileStream fs = null;
List <ProtoManifest.ChunkData> neededChunks;
FileInfo fi = new FileInfo(fileFinalPath);
if (!fi.Exists)
{
// create new file. need all chunks
fs = File.Create(fileFinalPath);
fs.SetLength((long)file.TotalSize);
neededChunks = new List <ProtoManifest.ChunkData>(file.Chunks);
}
else
{
// open existing
ProtoManifest.FileData oldManifestFile = null;
if (oldProtoManifest != null)
{
oldManifestFile = oldProtoManifest.Files.SingleOrDefault(f => f.FileName == file.FileName);
}
if (oldManifestFile != null)
{
neededChunks = new List <ProtoManifest.ChunkData>();
if (Config.VerifyAll || !oldManifestFile.FileHash.SequenceEqual(file.FileHash))
{
// we have a version of this file, but it doesn't fully match what we want
var matchingChunks = new List <ChunkMatch>();
foreach (var chunk in file.Chunks)
{
var oldChunk = oldManifestFile.Chunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID));
if (oldChunk != null)
{
matchingChunks.Add(new ChunkMatch(oldChunk, chunk));
}
else
{
neededChunks.Add(chunk);
}
}
File.Move(fileFinalPath, fileStagingPath);
fs = File.Open(fileFinalPath, FileMode.Create);
fs.SetLength((long)file.TotalSize);
using (var fsOld = File.Open(fileStagingPath, FileMode.Open))
{
foreach (var match in matchingChunks)
{
fsOld.Seek((long)match.OldChunk.Offset, SeekOrigin.Begin);
byte[] tmp = new byte[match.OldChunk.UncompressedLength];
fsOld.Read(tmp, 0, tmp.Length);
byte[] adler = Util.AdlerHash(tmp);
if (!adler.SequenceEqual(match.OldChunk.Checksum))
{
neededChunks.Add(match.NewChunk);
}
else
{
fs.Seek((long)match.NewChunk.Offset, SeekOrigin.Begin);
fs.Write(tmp, 0, tmp.Length);
}
}
}
File.Delete(fileStagingPath);
}
}
else
{
// No old manifest or file not in old manifest. We must validate.
fs = File.Open(fileFinalPath, FileMode.Open);
if ((ulong)fi.Length != file.TotalSize)
{
fs.SetLength((long)file.TotalSize);
}
neededChunks = Util.ValidateSteam3FileChecksums(fs, file.Chunks.OrderBy(x => x.Offset).ToArray());
}
if (neededChunks.Count() == 0)
{
size_downloaded += file.TotalSize;
reportProgress(file.FileName);
if (fs != null)
{
fs.Dispose();
}
return;
}
else
{
ulong sizeDelta = (file.TotalSize - (ulong)neededChunks.Select(x => (long)x.UncompressedLength).Sum());
size_downloaded += sizeDelta;
reportProgress(file.FileName);
}
}
foreach (var chunk in neededChunks)
{
if (cts.IsCancellationRequested)
{
break;
}
string chunkID = Util.EncodeHexString(chunk.ChunkID);
CDNClient.DepotChunk chunkData = null;
while (!cts.IsCancellationRequested)
{
Tuple <CDNClient.Server, string> connection;
try
{
connection = await cdnPool.GetConnectionForDepot(appId, depot.Id, cts.Token);
}
catch (OperationCanceledException)
{
break;
}
DepotManifest.ChunkData data = new DepotManifest.ChunkData();
data.ChunkID = chunk.ChunkID;
data.Checksum = chunk.Checksum;
data.Offset = chunk.Offset;
data.CompressedLength = chunk.CompressedLength;
data.UncompressedLength = chunk.UncompressedLength;
try
{
chunkData = await cdnPool.CDNClient.DownloadDepotChunkAsync(depot.Id, data,
connection.Item1, connection.Item2, depot.DepotKey).ConfigureAwait(false);
cdnPool.ReturnConnection(connection);
break;
}
catch (SteamKitWebRequestException e)
{
cdnPool.ReturnBrokenConnection(connection);
if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden)
{
Console.WriteLine("Encountered 401 for chunk {0}. Aborting.", chunkID);
cts.Cancel();
break;
}
else
{
Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, e.StatusCode);
}
}
catch (TaskCanceledException)
{
Console.WriteLine("Connection timeout downloading chunk {0}", chunkID);
}
catch (Exception e)
{
cdnPool.ReturnBrokenConnection(connection);
Console.WriteLine("Encountered unexpected error downloading chunk {0}: {1}", chunkID, e.Message);
}
}
if (chunkData == null)
{
Console.WriteLine("Failed to find any server with chunk {0} for depot {1}. Aborting.", chunkID, depot.Id);
cts.Cancel();
}
// Throw the cancellation exception if requested so that this task is marked failed
cts.Token.ThrowIfCancellationRequested();
cancellationToken.ThrowIfCancellationRequested();
TotalBytesCompressed += chunk.CompressedLength;
DepotBytesCompressed += chunk.CompressedLength;
TotalBytesUncompressed += chunk.UncompressedLength;
DepotBytesUncompressed += chunk.UncompressedLength;
fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
fs.Write(chunkData.Data, 0, chunkData.Data.Length);
size_downloaded += chunk.UncompressedLength;
downloadChanges.Enqueue((DateTime.Now, chunk.CompressedLength));
reportProgress(file.FileName);
}
fs.Dispose();
reportProgress(file.FileName);
}
finally
{
semaphore.Release();
}
});
tasks[i] = task;
}
await Task.WhenAll(tasks).ConfigureAwait(false);
DepotConfigStore.Instance.InstalledManifestIDs[depot.Id] = depot.ManifestId;
DepotConfigStore.Save();
Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.Id, DepotBytesCompressed, DepotBytesUncompressed);
reportProgress(null);
}
Console.WriteLine("Total downloaded: {0} bytes ({1} bytes uncompressed) from {2} depots", TotalBytesCompressed, TotalBytesUncompressed, depots.Count);
}
