private static async Task <DepotDownloadInfo> GetDepotInfo(UInt32 depotId, UInt32 appId, UInt64 manifestId, String branch, String baseDir)
{
await Program.SteamSession.RequestAppInfo(appId);
string contentName = GetAppOrDepotName(depotId, appId);
if (false == (await AccountHasAccess(depotId)))
{
FileLog.LogMessage($"Depot {depotId} ({contentName}) is not available from this account.");
MessageBox.Show($"Depot {depotId} ({contentName}) is not available from this account.", Program.AppName);
return(null);
}
// Skip requesting an app ticket
Program.SteamSession.AppTickets[depotId] = null;
if (manifestId == INVALID_MANIFEST_ID)
{
manifestId = await GetSteam3DepotManifest(depotId, appId, branch);
if (manifestId == INVALID_MANIFEST_ID && branch != "public")
{
FileLog.LogMessage($"Warning: Depot {depotId} does not have branch named \"{branch}\". Trying public branch.");
branch = "public";
manifestId = await GetSteam3DepotManifest(depotId, appId, branch);
}
if (manifestId == INVALID_MANIFEST_ID)
{
FileLog.LogMessage($"Depot {depotId} ({contentName}) missing public subsection or manifest section.");
return(null);
}
}
string installDir;
if (!CreateDirectories(depotId, 0, baseDir, out installDir))
{
FileLog.LogMessage("Unable to download files to the target location! Missing IO permissions!");
return(null);
}
await Program.SteamSession.RequestDepotKey(depotId, appId);
if (false == Program.SteamSession.DepotKeys.ContainsKey(depotId))
{
FileLog.LogMessage($"No valid depot key for {depotId}, unable to download.");
return(null);
}
var depotKey = Program.SteamSession.DepotKeys[depotId];
var info = new DepotDownloadInfo(depotId, manifestId, installDir, contentName);
info.depotKey = depotKey;
return(info);
}
private static async Task DownloadSteam3Async(uint appId, DepotDownloadInfo depot, CDNClientPool cdnPool)
{
CancellationTokenSource cts = new CancellationTokenSource();
cdnPool.ExhaustedToken = cts;
GlobalDownloadCounter downloadCounter = new GlobalDownloadCounter();
var allFileNamesAllDepots = new HashSet <String>();
// First, fetch all the manifests for each depot (including previous manifests) and perform the initial setup
var depotFileData = await ProcessDepotManifestAndFiles(cts, appId, depot, cdnPool);
if (depotFileData != null)
{
allFileNamesAllDepots.UnionWith(depotFileData.allFileNames);
}
cts.Token.ThrowIfCancellationRequested();
await DownloadSteam3AsyncDepotFiles(cts, appId, downloadCounter, depotFileData, allFileNamesAllDepots, cdnPool);
FileLog.LogMessage($"Total downloaded: {downloadCounter.TotalBytesCompressed} bytes ({downloadCounter.TotalBytesUncompressed} bytes uncompressed) from depot");
}
private static async Task <DepotFilesData> ProcessDepotManifestAndFiles(CancellationTokenSource cts,
UInt32 appId, DepotDownloadInfo depot, CDNClientPool cdnPool)
{
DepotDownloadCounter depotCounter = new DepotDownloadCounter();
FileLog.LogMessage("Processing depot {0} - {1}", depot.id, depot.contentName);
ProtoManifest oldProtoManifest = null;
ProtoManifest newProtoManifest = null;
string configDir = Path.Combine(depot.installDir, CONFIG_DIR);
ulong lastManifestId = INVALID_MANIFEST_ID;
//DepotConfigStore.Instance.InstalledManifestIDs.TryGetValue(depot.id, out lastManifestId);
//// In case we have an early exit, this will force equiv of verifyall next run.
//DepotConfigStore.Instance.InstalledManifestIDs[depot.id] = INVALID_MANIFEST_ID;
//DepotConfigStore.Save();
//if (lastManifestId != INVALID_MANIFEST_ID)
//{
// var oldManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.id, lastManifestId));
// if (File.Exists(oldManifestFileName))
// {
// byte[] expectedChecksum, currentChecksum;
// try
// {
// expectedChecksum = File.ReadAllBytes(oldManifestFileName + ".sha");
// }
// catch (IOException)
// {
// expectedChecksum = null;
// }
// oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName, out currentChecksum);
// if (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum))
// {
// // We only have to show this warning if the old manifest ID was different
// if (lastManifestId != depot.manifestId)
// FileLog.LogMessage("Manifest {0} on disk did not match the expected checksum.", lastManifestId);
// oldProtoManifest = null;
// }
// }
//}
if (lastManifestId == depot.manifestId && oldProtoManifest != null)
{
newProtoManifest = oldProtoManifest;
FileLog.LogMessage("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id);
}
else
{
var newManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.id, depot.manifestId));
if (newManifestFileName != null)
{
byte[] expectedChecksum, currentChecksum;
try
{
expectedChecksum = File.ReadAllBytes(newManifestFileName + ".sha");
}
catch (IOException)
{
expectedChecksum = null;
}
newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName, out currentChecksum);
if (newProtoManifest != null && (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum)))
{
FileLog.LogMessage("Manifest {0} on disk did not match the expected checksum.", depot.manifestId);
newProtoManifest = null;
}
}
if (newProtoManifest != null)
{
FileLog.LogMessage("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id);
}
else
{
Console.Write("Downloading depot manifest...");
DepotManifest depotManifest = null;
do
{
cts.Token.ThrowIfCancellationRequested();
CDNClient.Server connection = null;
try
{
connection = cdnPool.GetConnection(cts.Token);
var cdnToken = await cdnPool.AuthenticateConnection(appId, depot.id, connection);
#if STEAMKIT_UNRELEASED
depotManifest = await cdnPool.CDNClient.DownloadManifestAsync(depot.id, depot.manifestId,
connection, cdnToken, depot.depotKey, proxyServer : cdnPool.ProxyServer).ConfigureAwait(false);
#else
depotManifest = await cdnPool.CDNClient.DownloadManifestAsync(depot.id, depot.manifestId,
connection, cdnToken, depot.depotKey).ConfigureAwait(false);
#endif
cdnPool.ReturnConnection(connection);
}
catch (TaskCanceledException)
{
FileLog.LogMessage("Connection timeout downloading depot manifest {0} {1}", depot.id, depot.manifestId);
}
catch (SteamKitWebRequestException e)
{
cdnPool.ReturnBrokenConnection(connection);
if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden)
{
FileLog.LogMessage("Encountered 401 for depot manifest {0} {1}. Aborting.", depot.id, depot.manifestId);
break;
}
else if (e.StatusCode == HttpStatusCode.NotFound)
{
FileLog.LogMessage("Encountered 404 for depot manifest {0} {1}. Aborting.", depot.id, depot.manifestId);
break;
}
else
{
FileLog.LogMessage("Encountered error downloading depot manifest {0} {1}: {2}", depot.id, depot.manifestId, e.StatusCode);
}
}
catch (OperationCanceledException)
{
break;
}
catch (Exception e)
{
cdnPool.ReturnBrokenConnection(connection);
FileLog.LogMessage("Encountered error downloading manifest for depot {0} {1}: {2}", depot.id, depot.manifestId, e.Message);
}
}while (depotManifest == null);
if (depotManifest == null)
{
FileLog.LogMessage("\nUnable to download manifest {0} for depot {1}", depot.manifestId, depot.id);
cts.Cancel();
}
// Throw the cancellation exception if requested so that this task is marked failed
cts.Token.ThrowIfCancellationRequested();
byte[] checksum;
newProtoManifest = new ProtoManifest(depotManifest, depot.manifestId);
newProtoManifest.SaveToFile(newManifestFileName, out checksum);
File.WriteAllBytes(newManifestFileName + ".sha", checksum);
FileLog.LogMessage(" Done!");
}
}
newProtoManifest.Files.Sort((x, y) => string.Compare(x.FileName, y.FileName, StringComparison.Ordinal));
FileLog.LogMessage("Manifest {0} ({1})", depot.manifestId, newProtoManifest.CreationTime);
//if (Config.DownloadManifestOnly)
//{
// StringBuilder manifestBuilder = new StringBuilder();
// string txtManifest = Path.Combine(depot.installDir, string.Format("manifest_{0}_{1}.txt", depot.id, depot.manifestId));
// manifestBuilder.Append(string.Format("{0}\n\n", newProtoManifest.CreationTime));
// foreach (var file in newProtoManifest.Files)
// {
// if (file.Flags.HasFlag(EDepotFileFlag.Directory))
// continue;
// manifestBuilder.Append(string.Format("{0}\n", file.FileName));
// manifestBuilder.Append(string.Format("\t{0}\n", file.TotalSize));
// manifestBuilder.Append(string.Format("\t{0}\n", BitConverter.ToString(file.FileHash).Replace("-", "")));
// }
// File.WriteAllText(txtManifest, manifestBuilder.ToString());
// return null;
//}
string stagingDir = Path.Combine(depot.installDir, STAGING_DIR);
var filesAfterExclusions = newProtoManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).ToList();
var allFileNames = new HashSet <string>(filesAfterExclusions.Count);
// Pre-process
filesAfterExclusions.ForEach(file =>
{
allFileNames.Add(file.FileName);
var fileFinalPath = Path.Combine(depot.installDir, file.FileName);
var fileStagingPath = Path.Combine(stagingDir, file.FileName);
if (file.Flags.HasFlag(EDepotFileFlag.Directory))
{
Directory.CreateDirectory(fileFinalPath);
Directory.CreateDirectory(fileStagingPath);
}
else
{
// Some manifests don't explicitly include all necessary directories
Directory.CreateDirectory(Path.GetDirectoryName(fileFinalPath));
Directory.CreateDirectory(Path.GetDirectoryName(fileStagingPath));
depotCounter.CompleteDownloadSize += file.TotalSize;
}
});
return(new DepotFilesData
{
depotDownloadInfo = depot,
depotCounter = depotCounter,
stagingDir = stagingDir,
manifest = newProtoManifest,
previousManifest = oldProtoManifest,
filteredFiles = filesAfterExclusions,
allFileNames = allFileNames
});
}
