public static async Task DownloadAppAsync(UInt32 appId, UInt32 depotId, UInt64 manifestId, String downloadFolder)
{
await Program.SteamSession.RequestAppInfo(appId);
if (await AccountHasAccess(appId))
{
var cdnPool = new CDNClientPool(appId);
var info = await GetDepotInfo(depotId, appId, manifestId, "public", downloadFolder);
try
{
await DownloadSteam3Async(appId, info, cdnPool).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
FileLog.LogMessage($"App {appId} was not completely downloaded.");
throw;
}
finally
{
cdnPool.Shutdown();
}
}
else
{
FileLog.LogMessage("This account is not allowed to download Conan Exiles! Please purchase the product first!");
MessageBox.Show("This account is not allowed to download Conan Exiles! Please purchase the product first!", Program.AppName, MessageBoxButtons.OK, MessageBoxIcon.Error);
}
}
private static async Task DownloadSteam3AsyncDepotFiles(CancellationTokenSource cts, UInt32 appId,
GlobalDownloadCounter downloadCounter, DepotFilesData depotFilesData, HashSet <String> allFileNamesAllDepots, CDNClientPool cdnPool)
{
var depot = depotFilesData.depotDownloadInfo;
var depotCounter = depotFilesData.depotCounter;
FileLog.LogMessage("Downloading depot {0} - {1}", depot.id, depot.contentName);
var files = depotFilesData.filteredFiles.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory)).ToArray();
var networkChunkQueue = new ConcurrentQueue <(FileStreamData fileStreamData, ProtoManifest.FileData fileData, ProtoManifest.ChunkData chunk)>();
await Util.InvokeAsync(
files.Select(file => new Func <Task>(async() =>
await Task.Run(() => DownloadSteam3AsyncDepotFile(cts, depotFilesData, file, networkChunkQueue)))),
maxDegreeOfParallelism : 8
);
await Util.InvokeAsync(
networkChunkQueue.Select(q => new Func <Task>(async() =>
await Task.Run(() => DownloadSteam3AsyncDepotFileChunk(cts, appId, downloadCounter, depotFilesData,
q.fileData, q.fileStreamData, q.chunk, cdnPool)))),
maxDegreeOfParallelism : 8
);
// Check for deleted files if updating the depot.
if (depotFilesData.previousManifest != null)
{
var previousFilteredFiles = depotFilesData.previousManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).Select(f => f.FileName).ToHashSet();
// Check if we are writing to a single output directory. If not, each depot folder is managed independently
if (string.IsNullOrWhiteSpace(ContentDownloader.Config.InstallDirectory))
{
// Of the list of files in the previous manifest, remove any file names that exist in the current set of all file names
previousFilteredFiles.ExceptWith(depotFilesData.allFileNames);
}
else
{
// Of the list of files in the previous manifest, remove any file names that exist in the current set of all file names across all depots being downloaded
previousFilteredFiles.ExceptWith(allFileNamesAllDepots);
}
foreach (var existingFileName in previousFilteredFiles)
{
string fileFinalPath = Path.Combine(depot.installDir, existingFileName);
if (!File.Exists(fileFinalPath))
{
continue;
}
File.Delete(fileFinalPath);
FileLog.LogMessage("Deleted {0}", fileFinalPath);
}
}
//DepotConfigStore.Instance.InstalledManifestIDs[depot.id] = depot.manifestId;
//DepotConfigStore.Save();
FileLog.LogMessage("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.id, depotCounter.DepotBytesCompressed, depotCounter.DepotBytesUncompressed);
}
private static async Task <DepotFilesData> ProcessDepotManifestAndFiles(CancellationTokenSource cts,
UInt32 appId, DepotDownloadInfo depot, CDNClientPool cdnPool)
{
DepotDownloadCounter depotCounter = new DepotDownloadCounter();
FileLog.LogMessage("Processing depot {0} - {1}", depot.id, depot.contentName);
ProtoManifest oldProtoManifest = null;
ProtoManifest newProtoManifest = null;
string configDir = Path.Combine(depot.installDir, CONFIG_DIR);
ulong lastManifestId = INVALID_MANIFEST_ID;
//DepotConfigStore.Instance.InstalledManifestIDs.TryGetValue(depot.id, out lastManifestId);
//// In case we have an early exit, this will force equiv of verifyall next run.
//DepotConfigStore.Instance.InstalledManifestIDs[depot.id] = INVALID_MANIFEST_ID;
//DepotConfigStore.Save();
//if (lastManifestId != INVALID_MANIFEST_ID)
//{
// var oldManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.id, lastManifestId));
// if (File.Exists(oldManifestFileName))
// {
// byte[] expectedChecksum, currentChecksum;
// try
// {
// expectedChecksum = File.ReadAllBytes(oldManifestFileName + ".sha");
// }
// catch (IOException)
// {
// expectedChecksum = null;
// }
// oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName, out currentChecksum);
// if (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum))
// {
// // We only have to show this warning if the old manifest ID was different
// if (lastManifestId != depot.manifestId)
// FileLog.LogMessage("Manifest {0} on disk did not match the expected checksum.", lastManifestId);
// oldProtoManifest = null;
// }
// }
//}
if (lastManifestId == depot.manifestId && oldProtoManifest != null)
{
newProtoManifest = oldProtoManifest;
FileLog.LogMessage("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id);
}
else
{
var newManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.id, depot.manifestId));
if (newManifestFileName != null)
{
byte[] expectedChecksum, currentChecksum;
try
{
expectedChecksum = File.ReadAllBytes(newManifestFileName + ".sha");
}
catch (IOException)
{
expectedChecksum = null;
}
newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName, out currentChecksum);
if (newProtoManifest != null && (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum)))
{
FileLog.LogMessage("Manifest {0} on disk did not match the expected checksum.", depot.manifestId);
newProtoManifest = null;
}
}
if (newProtoManifest != null)
{
FileLog.LogMessage("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id);
}
else
{
Console.Write("Downloading depot manifest...");
DepotManifest depotManifest = null;
do
{
cts.Token.ThrowIfCancellationRequested();
CDNClient.Server connection = null;
try
{
connection = cdnPool.GetConnection(cts.Token);
var cdnToken = await cdnPool.AuthenticateConnection(appId, depot.id, connection);
#if STEAMKIT_UNRELEASED
depotManifest = await cdnPool.CDNClient.DownloadManifestAsync(depot.id, depot.manifestId,
connection, cdnToken, depot.depotKey, proxyServer : cdnPool.ProxyServer).ConfigureAwait(false);
#else
depotManifest = await cdnPool.CDNClient.DownloadManifestAsync(depot.id, depot.manifestId,
connection, cdnToken, depot.depotKey).ConfigureAwait(false);
#endif
cdnPool.ReturnConnection(connection);
}
catch (TaskCanceledException)
{
FileLog.LogMessage("Connection timeout downloading depot manifest {0} {1}", depot.id, depot.manifestId);
}
catch (SteamKitWebRequestException e)
{
cdnPool.ReturnBrokenConnection(connection);
if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden)
{
FileLog.LogMessage("Encountered 401 for depot manifest {0} {1}. Aborting.", depot.id, depot.manifestId);
break;
}
else if (e.StatusCode == HttpStatusCode.NotFound)
{
FileLog.LogMessage("Encountered 404 for depot manifest {0} {1}. Aborting.", depot.id, depot.manifestId);
break;
}
else
{
FileLog.LogMessage("Encountered error downloading depot manifest {0} {1}: {2}", depot.id, depot.manifestId, e.StatusCode);
}
}
catch (OperationCanceledException)
{
break;
}
catch (Exception e)
{
cdnPool.ReturnBrokenConnection(connection);
FileLog.LogMessage("Encountered error downloading manifest for depot {0} {1}: {2}", depot.id, depot.manifestId, e.Message);
}
}while (depotManifest == null);
if (depotManifest == null)
{
FileLog.LogMessage("\nUnable to download manifest {0} for depot {1}", depot.manifestId, depot.id);
cts.Cancel();
}
// Throw the cancellation exception if requested so that this task is marked failed
cts.Token.ThrowIfCancellationRequested();
byte[] checksum;
newProtoManifest = new ProtoManifest(depotManifest, depot.manifestId);
newProtoManifest.SaveToFile(newManifestFileName, out checksum);
File.WriteAllBytes(newManifestFileName + ".sha", checksum);
FileLog.LogMessage(" Done!");
}
}
newProtoManifest.Files.Sort((x, y) => string.Compare(x.FileName, y.FileName, StringComparison.Ordinal));
FileLog.LogMessage("Manifest {0} ({1})", depot.manifestId, newProtoManifest.CreationTime);
//if (Config.DownloadManifestOnly)
//{
// StringBuilder manifestBuilder = new StringBuilder();
// string txtManifest = Path.Combine(depot.installDir, string.Format("manifest_{0}_{1}.txt", depot.id, depot.manifestId));
// manifestBuilder.Append(string.Format("{0}\n\n", newProtoManifest.CreationTime));
// foreach (var file in newProtoManifest.Files)
// {
// if (file.Flags.HasFlag(EDepotFileFlag.Directory))
// continue;
// manifestBuilder.Append(string.Format("{0}\n", file.FileName));
// manifestBuilder.Append(string.Format("\t{0}\n", file.TotalSize));
// manifestBuilder.Append(string.Format("\t{0}\n", BitConverter.ToString(file.FileHash).Replace("-", "")));
// }
// File.WriteAllText(txtManifest, manifestBuilder.ToString());
// return null;
//}
string stagingDir = Path.Combine(depot.installDir, STAGING_DIR);
var filesAfterExclusions = newProtoManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).ToList();
var allFileNames = new HashSet <string>(filesAfterExclusions.Count);
// Pre-process
filesAfterExclusions.ForEach(file =>
{
allFileNames.Add(file.FileName);
var fileFinalPath = Path.Combine(depot.installDir, file.FileName);
var fileStagingPath = Path.Combine(stagingDir, file.FileName);
if (file.Flags.HasFlag(EDepotFileFlag.Directory))
{
Directory.CreateDirectory(fileFinalPath);
Directory.CreateDirectory(fileStagingPath);
}
else
{
// Some manifests don't explicitly include all necessary directories
Directory.CreateDirectory(Path.GetDirectoryName(fileFinalPath));
Directory.CreateDirectory(Path.GetDirectoryName(fileStagingPath));
depotCounter.CompleteDownloadSize += file.TotalSize;
}
});
return(new DepotFilesData
{
depotDownloadInfo = depot,
depotCounter = depotCounter,
stagingDir = stagingDir,
manifest = newProtoManifest,
previousManifest = oldProtoManifest,
filteredFiles = filesAfterExclusions,
allFileNames = allFileNames
});
}
private static async Task DownloadSteam3Async(uint appId, DepotDownloadInfo depot, CDNClientPool cdnPool)
{
CancellationTokenSource cts = new CancellationTokenSource();
cdnPool.ExhaustedToken = cts;
GlobalDownloadCounter downloadCounter = new GlobalDownloadCounter();
var allFileNamesAllDepots = new HashSet <String>();
// First, fetch all the manifests for each depot (including previous manifests) and perform the initial setup
var depotFileData = await ProcessDepotManifestAndFiles(cts, appId, depot, cdnPool);
if (depotFileData != null)
{
allFileNamesAllDepots.UnionWith(depotFileData.allFileNames);
}
cts.Token.ThrowIfCancellationRequested();
await DownloadSteam3AsyncDepotFiles(cts, appId, downloadCounter, depotFileData, allFileNamesAllDepots, cdnPool);
FileLog.LogMessage($"Total downloaded: {downloadCounter.TotalBytesCompressed} bytes ({downloadCounter.TotalBytesUncompressed} bytes uncompressed) from depot");
}