public async Task ThrowsSteamKitWebExceptionOnUnsuccessfulWebResponse()
{
var configuration = SteamConfiguration.Create(x => x.WithHttpClientFactory(() => new HttpClient(new TeapotHttpMessageHandler())));
var steam = new SteamClient(configuration);
var client = new CDNClient(steam);
try
{
await client.DownloadManifestAsync(0, 0, "localhost", "12345");
throw new InvalidOperationException("This should be unreachable.");
}
catch (SteamKitWebRequestException ex)
{
Assert.Equal((HttpStatusCode)418, ex.StatusCode);
}
}
private async Task DownloadDepots(uint appID, List <ManifestJob> depots)
{
Log.WriteDebug("Depot Downloader", "Will process {0} depots ({1} depot locks left)", depots.Count(), DepotLocks.Count);
var processTasks = new List <Task <EResult> >();
bool anyFilesDownloaded = false;
foreach (var depot in depots)
{
var instance = depot.Anonymous ? Steam.Anonymous.Apps : Steam.Instance.Apps;
depot.DepotKey = await GetDepotDecryptionKey(instance, depot.DepotID, appID);
if (depot.DepotKey == null)
{
RemoveLock(depot.DepotID);
continue;
}
var cdnToken = await GetCDNAuthToken(instance, appID, depot.DepotID);
if (cdnToken == null)
{
RemoveLock(depot.DepotID);
Log.WriteDebug("Depot Downloader", "Got a depot key for depot {0} but no cdn auth token", depot.DepotID);
continue;
}
depot.CDNToken = cdnToken.Token;
depot.Server = GetContentServer();
DepotManifest depotManifest = null;
string lastError = string.Empty;
for (var i = 0; i <= 5; i++)
{
try
{
depotManifest = await CDNClient.DownloadManifestAsync(depot.DepotID, depot.ManifestID, depot.Server, depot.CDNToken, depot.DepotKey);
break;
}
catch (Exception e)
{
lastError = e.Message;
Log.WriteError("Depot Processor", "Failed to download depot manifest for app {0} depot {1} ({2}: {3}) (#{4})", appID, depot.DepotID, depot.Server, lastError, i);
}
// TODO: get new auth key if auth fails
depot.Server = GetContentServer();
if (depotManifest == null)
{
await Task.Delay(Utils.ExponentionalBackoff(i));
}
}
if (depotManifest == null)
{
LocalConfig.CDNAuthTokens.TryRemove(depot.DepotID, out _);
RemoveLock(depot.DepotID);
if (FileDownloader.IsImportantDepot(depot.DepotID))
{
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download manifest ({3})",
Colors.OLIVE, depot.DepotName, Colors.NORMAL, lastError);
}
if (!Settings.IsFullRun)
{
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
}
continue;
}
var task = ProcessDepotAfterDownload(depot, depotManifest);
processTasks.Add(task);
if (!FileDownloader.IsImportantDepot(depot.DepotID))
{
continue;
}
task = TaskManager.Run(async() =>
{
var result = EResult.Fail;
try
{
result = await FileDownloader.DownloadFilesFromDepot(depot, depotManifest);
if (result == EResult.OK)
{
anyFilesDownloaded = true;
}
}
catch (Exception e)
{
ErrorReporter.Notify($"Depot Processor {depot.DepotID}", e);
}
return(result);
}).Unwrap();
processTasks.Add(task);
}
if (SaveLocalConfig)
{
SaveLocalConfig = false;
LocalConfig.Save();
}
await Task.WhenAll(processTasks).ConfigureAwait(false);
Log.WriteDebug("Depot Downloader", "{0} depot downloads finished", depots.Count());
// TODO: use ContinueWith on tasks
if (!anyFilesDownloaded)
{
foreach (var depot in depots)
{
RemoveLock(depot.DepotID);
}
return;
}
if (!File.Exists(UpdateScript))
{
return;
}
lock (UpdateScriptLock)
{
foreach (var depot in depots)
{
if (depot.Result == EResult.OK)
{
RunUpdateScript(string.Format("{0} no-git", depot.DepotID));
}
else if (depot.Result != EResult.Ignored)
{
Log.WriteWarn("Depot Processor", "Dropping stored token for {0} due to download failures", depot.DepotID);
LocalConfig.CDNAuthTokens.TryRemove(depot.DepotID, out _);
using (var db = Database.Get())
{
// Mark this depot for redownload
db.Execute("UPDATE `Depots` SET `LastManifestID` = 0 WHERE `DepotID` = @DepotID", new { depot.DepotID });
}
}
RemoveLock(depot.DepotID);
}
// Only commit changes if all depots downloaded
if (processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored))
{
if (!RunUpdateScript(appID, depots.First().BuildID))
{
RunUpdateScript("0");
}
}
else
{
Log.WriteDebug("Depot Processor", "Reprocessing the app {0} because some files failed to download", appID);
IRC.Instance.SendOps("{0}[{1}]{2} Reprocessing the app due to download failures",
Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL
);
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
}
}
}
private static async Task DownloadSteam3Async(uint appId, List <DepotDownloadInfo> depots)
{
ulong TotalBytesCompressed = 0;
ulong TotalBytesUncompressed = 0;
foreach (var depot in depots)
{
ulong DepotBytesCompressed = 0;
ulong DepotBytesUncompressed = 0;
Console.WriteLine("Downloading depot {0} - {1}", depot.id, depot.contentName);
CancellationTokenSource cts = new CancellationTokenSource();
cdnPool.ExhaustedToken = cts;
ProtoManifest oldProtoManifest = null;
ProtoManifest newProtoManifest = null;
string configDir = Path.Combine(depot.installDir, CONFIG_DIR);
ulong lastManifestId = INVALID_MANIFEST_ID;
DepotConfigStore.Instance.InstalledManifestIDs.TryGetValue(depot.id, out lastManifestId);
// In case we have an early exit, this will force equiv of verifyall next run.
DepotConfigStore.Instance.InstalledManifestIDs[depot.id] = INVALID_MANIFEST_ID;
DepotConfigStore.Save();
if (lastManifestId != INVALID_MANIFEST_ID)
{
var oldManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", lastManifestId));
if (File.Exists(oldManifestFileName))
{
byte[] expectedChecksum, currentChecksum;
try
{
expectedChecksum = File.ReadAllBytes(oldManifestFileName + ".sha");
}
catch (IOException)
{
expectedChecksum = null;
}
oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName, out currentChecksum);
if (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum))
{
// We only have to show this warning if the old manifest ID was different
if (lastManifestId != depot.manifestId)
{
Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", lastManifestId);
}
oldProtoManifest = null;
}
}
}
if (lastManifestId == depot.manifestId && oldProtoManifest != null)
{
newProtoManifest = oldProtoManifest;
Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id);
}
else
{
var newManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", depot.manifestId));
if (newManifestFileName != null)
{
byte[] expectedChecksum, currentChecksum;
try
{
expectedChecksum = File.ReadAllBytes(newManifestFileName + ".sha");
}
catch (IOException)
{
expectedChecksum = null;
}
newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName, out currentChecksum);
if (newProtoManifest != null && (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum)))
{
Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", depot.manifestId);
newProtoManifest = null;
}
}
if (newProtoManifest != null)
{
Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id);
}
else
{
Console.Write("Downloading depot manifest...");
DepotManifest depotManifest = null;
while (depotManifest == null)
{
CDNClient client = null;
try
{
client = await cdnPool.GetConnectionForDepotAsync(appId, depot.id, depot.depotKey, CancellationToken.None).ConfigureAwait(false);
depotManifest = await client.DownloadManifestAsync(depot.id, depot.manifestId).ConfigureAwait(false);
cdnPool.ReturnConnection(client);
}
catch (WebException e)
{
cdnPool.ReturnBrokenConnection(client);
if (e.Status == WebExceptionStatus.ProtocolError)
{
var response = e.Response as HttpWebResponse;
if (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden)
{
Console.WriteLine("Encountered 401 for depot manifest {0} {1}. Aborting.", depot.id, depot.manifestId);
break;
}
else
{
Console.WriteLine("Encountered error downloading depot manifest {0} {1}: {2}", depot.id, depot.manifestId, response.StatusCode);
}
}
else
{
Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depot.id, depot.manifestId, e.Status);
}
}
catch (Exception e)
{
cdnPool.ReturnBrokenConnection(client);
Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depot.id, depot.manifestId, e.Message);
}
}
if (depotManifest == null)
{
Console.WriteLine("\nUnable to download manifest {0} for depot {1}", depot.manifestId, depot.id);
return;
}
byte[] checksum;
newProtoManifest = new ProtoManifest(depotManifest, depot.manifestId);
newProtoManifest.SaveToFile(newManifestFileName, out checksum);
File.WriteAllBytes(newManifestFileName + ".sha", checksum);
Console.WriteLine(" Done!");
}
}
newProtoManifest.Files.Sort((x, y) => { return(x.FileName.CompareTo(y.FileName)); });
if (Config.DownloadManifestOnly)
{
StringBuilder manifestBuilder = new StringBuilder();
string txtManifest = Path.Combine(depot.installDir, string.Format("manifest_{0}.txt", depot.id));
foreach (var file in newProtoManifest.Files)
{
if (file.Flags.HasFlag(EDepotFileFlag.Directory))
{
continue;
}
manifestBuilder.Append(string.Format("{0}\n", file.FileName));
}
File.WriteAllText(txtManifest, manifestBuilder.ToString());
continue;
}
ulong complete_download_size = 0;
ulong size_downloaded = 0;
string stagingDir = Path.Combine(depot.installDir, STAGING_DIR);
var filesAfterExclusions = newProtoManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).ToList();
// Pre-process
filesAfterExclusions.ForEach(file =>
{
var fileFinalPath = Path.Combine(depot.installDir, file.FileName);
var fileStagingPath = Path.Combine(stagingDir, file.FileName);
if (file.Flags.HasFlag(EDepotFileFlag.Directory))
{
Directory.CreateDirectory(fileFinalPath);
Directory.CreateDirectory(fileStagingPath);
}
else
{
// Some manifests don't explicitly include all necessary directories
Directory.CreateDirectory(Path.GetDirectoryName(fileFinalPath));
Directory.CreateDirectory(Path.GetDirectoryName(fileStagingPath));
complete_download_size += file.TotalSize;
}
});
var semaphore = new SemaphoreSlim(Config.MaxDownloads);
var files = filesAfterExclusions.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory)).ToArray();
var tasks = new Task[files.Length];
for (var i = 0; i < files.Length; i++)
{
var file = files[i];
var task = Task.Run(async() =>
{
cts.Token.ThrowIfCancellationRequested();
try
{
await semaphore.WaitAsync().ConfigureAwait(false);
cts.Token.ThrowIfCancellationRequested();
string fileFinalPath = Path.Combine(depot.installDir, file.FileName);
string fileStagingPath = Path.Combine(stagingDir, file.FileName);
// This may still exist if the previous run exited before cleanup
if (File.Exists(fileStagingPath))
{
File.Delete(fileStagingPath);
}
FileStream fs = null;
List <ProtoManifest.ChunkData> neededChunks;
FileInfo fi = new FileInfo(fileFinalPath);
if (!fi.Exists)
{
// create new file. need all chunks
fs = File.Create(fileFinalPath);
fs.SetLength(( long )file.TotalSize);
neededChunks = new List <ProtoManifest.ChunkData>(file.Chunks);
}
else
{
// open existing
ProtoManifest.FileData oldManifestFile = null;
if (oldProtoManifest != null)
{
oldManifestFile = oldProtoManifest.Files.SingleOrDefault(f => f.FileName == file.FileName);
}
if (oldManifestFile != null)
{
neededChunks = new List <ProtoManifest.ChunkData>();
if (Config.VerifyAll || !oldManifestFile.FileHash.SequenceEqual(file.FileHash))
{
// we have a version of this file, but it doesn't fully match what we want
var matchingChunks = new List <ChunkMatch>();
foreach (var chunk in file.Chunks)
{
var oldChunk = oldManifestFile.Chunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID));
if (oldChunk != null)
{
matchingChunks.Add(new ChunkMatch(oldChunk, chunk));
}
else
{
neededChunks.Add(chunk);
}
}
File.Move(fileFinalPath, fileStagingPath);
fs = File.Open(fileFinalPath, FileMode.Create);
fs.SetLength(( long )file.TotalSize);
using (var fsOld = File.Open(fileStagingPath, FileMode.Open))
{
foreach (var match in matchingChunks)
{
fsOld.Seek(( long )match.OldChunk.Offset, SeekOrigin.Begin);
byte[] tmp = new byte[match.OldChunk.UncompressedLength];
fsOld.Read(tmp, 0, tmp.Length);
byte[] adler = Util.AdlerHash(tmp);
if (!adler.SequenceEqual(match.OldChunk.Checksum))
{
neededChunks.Add(match.NewChunk);
}
else
{
fs.Seek(( long )match.NewChunk.Offset, SeekOrigin.Begin);
fs.Write(tmp, 0, tmp.Length);
}
}
}
File.Delete(fileStagingPath);
}
}
else
{
// No old manifest or file not in old manifest. We must validate.
fs = File.Open(fileFinalPath, FileMode.Open);
if (( ulong )fi.Length != file.TotalSize)
{
fs.SetLength(( long )file.TotalSize);
}
neededChunks = Util.ValidateSteam3FileChecksums(fs, file.Chunks.OrderBy(x => x.Offset).ToArray());
}
if (neededChunks.Count() == 0)
{
size_downloaded += file.TotalSize;
Console.WriteLine("{0,6:#00.00}% {1}", (( float )size_downloaded / ( float )complete_download_size) * 100.0f, fileFinalPath);
if (fs != null)
{
fs.Dispose();
}
return;
}
else
{
size_downloaded += (file.TotalSize - ( ulong )neededChunks.Select(x => ( long )x.UncompressedLength).Sum());
}
}
foreach (var chunk in neededChunks)
{
if (cts.IsCancellationRequested)
{
break;
}
string chunkID = Util.EncodeHexString(chunk.ChunkID);
CDNClient.DepotChunk chunkData = null;
while (!cts.IsCancellationRequested)
{
CDNClient client;
try
{
client = await cdnPool.GetConnectionForDepotAsync(appId, depot.id, depot.depotKey, cts.Token).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
break;
}
DepotManifest.ChunkData data = new DepotManifest.ChunkData();
data.ChunkID = chunk.ChunkID;
data.Checksum = chunk.Checksum;
data.Offset = chunk.Offset;
data.CompressedLength = chunk.CompressedLength;
data.UncompressedLength = chunk.UncompressedLength;
try
{
chunkData = await client.DownloadDepotChunkAsync(depot.id, data).ConfigureAwait(false);
cdnPool.ReturnConnection(client);
break;
}
catch (WebException e)
{
cdnPool.ReturnBrokenConnection(client);
if (e.Status == WebExceptionStatus.ProtocolError)
{
var response = e.Response as HttpWebResponse;
if (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden)
{
Console.WriteLine("Encountered 401 for chunk {0}. Aborting.", chunkID);
cts.Cancel();
break;
}
else
{
Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, response.StatusCode);
}
}
else
{
Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, e.Status);
}
}
catch (Exception e)
{
cdnPool.ReturnBrokenConnection(client);
Console.WriteLine("Encountered unexpected error downloading chunk {0}: {1}", chunkID, e.Message);
}
}
if (chunkData == null)
{
Console.WriteLine("Failed to find any server with chunk {0} for depot {1}. Aborting.", chunkID, depot.id);
cts.Cancel();
}
// Throw the cancellation exception if requested so that this task is marked failed
cts.Token.ThrowIfCancellationRequested();
TotalBytesCompressed += chunk.CompressedLength;
DepotBytesCompressed += chunk.CompressedLength;
TotalBytesUncompressed += chunk.UncompressedLength;
DepotBytesUncompressed += chunk.UncompressedLength;
fs.Seek(( long )chunk.Offset, SeekOrigin.Begin);
fs.Write(chunkData.Data, 0, chunkData.Data.Length);
size_downloaded += chunk.UncompressedLength;
}
fs.Dispose();
Console.WriteLine("{0,6:#00.00}% {1}", (( float )size_downloaded / ( float )complete_download_size) * 100.0f, fileFinalPath);
}
finally
{
semaphore.Release();
}
});
tasks[i] = task;
}
await Task.WhenAll(tasks).ConfigureAwait(false);
DepotConfigStore.Instance.InstalledManifestIDs[depot.id] = depot.manifestId;
DepotConfigStore.Save();
Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.id, DepotBytesCompressed, DepotBytesUncompressed);
}
Console.WriteLine("Total downloaded: {0} bytes ({1} bytes uncompressed) from {2} depots", TotalBytesCompressed, TotalBytesUncompressed, depots.Count);
}
private static async Task DownloadSteam3Async(uint appId, List <DepotDownloadInfo> depots, Action downloadCompleteAction = null)
{
ulong TotalBytesCompressed = 0;
ulong TotalBytesUncompressed = 0;
foreach (var depot in depots)
{
ulong DepotBytesCompressed = 0;
ulong DepotBytesUncompressed = 0;
DebugLog.WriteLine("ContentDownloader", "Downloading depot " + depot.id + " - " + depot.contentName);
CancellationTokenSource cts = new CancellationTokenSource();
cdnPool.ExhaustedToken = cts;
ProtoManifest oldProtoManifest = null;
ProtoManifest downloadManifest = null;
string configDir = Path.Combine(depot.installDir, CONFIG_DIR);
ulong lastManifestId = INVALID_MANIFEST_ID;
ConfigStore.TheConfig.LastManifests.TryGetValue(depot.id, out lastManifestId);
// In case we have an early exit, this will force equiv of verifyall next run.
ConfigStore.TheConfig.LastManifests[depot.id] = INVALID_MANIFEST_ID;
ConfigStore.Save();
if (lastManifestId != INVALID_MANIFEST_ID)
{
var oldManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", lastManifestId));
DebugLog.WriteLine(DEBUG_NAME_FILES, "Checking if " + oldManifestFileName + " exists");
if (File.Exists(oldManifestFileName))
{
DebugLog.WriteLine(DEBUG_NAME_FILES, oldManifestFileName + " exists, reading!");
oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName);
}
}
if (lastManifestId == depot.manifestId && oldProtoManifest != null)
{
downloadManifest = oldProtoManifest;
DebugLog.WriteLine("ContentDownloader", "Already have manifest " + depot.manifestId + " for depot " + depot.id + ".");
}
else
{
var newManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", depot.manifestId));
if (newManifestFileName != null)
{
downloadManifest = ProtoManifest.LoadFromFile(newManifestFileName);
}
if (downloadManifest != null)
{
DebugLog.WriteLine("ContentDownloader", "Already have manifest " + depot.manifestId + " for depot " + depot.id + ".");
}
else
{
DebugLog.WriteLine("ContentDownloader", "Downloading depot manifest...");
DepotManifest depotManifest = null;
while (depotManifest == null)
{
CDNClient client = null;
try
{
client = await cdnPool.GetConnectionForDepotAsync(appId, depot.id, depot.depotKey, CancellationToken.None).ConfigureAwait(false);
//client = await cdnPool.GetConnectionForDepotAsync(appId, depot.id, depot.depotKey, CancellationToken.None);
depotManifest = await client.DownloadManifestAsync(depot.id, depot.manifestId).ConfigureAwait(false);
//depotManifest = await client.DownloadManifestAsync(depot.id, depot.manifestId);
cdnPool.ReturnConnection(client);
}
catch (WebException e)
{
cdnPool.ReturnBrokenConnection(client);
if (e.Status == WebExceptionStatus.ProtocolError)
{
var response = e.Response as HttpWebResponse;
if (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden)
{
DebugLog.WriteLine("ContentDownloader", "Encountered 401 for depot manifest " + depot.id + " " + depot.manifestId + ". Aborting.");
break;
}
else
{
DebugLog.WriteLine("ContentDownloader", "Encountered error downloading depot manifest " + depot.id + " " + depot.manifestId + ": " + response.StatusCode);
}
}
else
{
DebugLog.WriteLine("ContentDownloader", "Encountered error downloading manifest for depot " + depot.id + " " + depot.manifestId + ": " + e.Status);
}
}
catch (Exception e)
{
cdnPool.ReturnBrokenConnection(client);
DebugLog.WriteLine("ContentDownloader", "Encountered error downloading manifest for depot " + depot.id + " " + depot.manifestId + ": " + e.Message);
}
}
if (depotManifest == null)
{
DebugLog.WriteLine("ContentDownloader", "\nUnable to download manifest " + depot.manifestId + " for depot " + depot.id);
return;
}
downloadManifest = new ProtoManifest(depotManifest, depot.manifestId);
downloadManifest.SaveToFile(newManifestFileName);
DebugLog.WriteLine("ContentDownloader", "Done!");
}
}
downloadManifest.Files.Sort((x, y) => { return(x.FileName.CompareTo(y.FileName)); });
if (downloadManifest != null)
{
onManifestReceived?.Invoke(appId, depot.id, depot.contentName, downloadManifest);
}
if (Config.DownloadManifestOnly)
{
continue;
}
complete_download_size = 0;
size_downloaded = 0;
string stagingDir = Path.Combine(depot.installDir, STAGING_DIR);
var filesAfterExclusions = downloadManifest.Files.Where(f => TestIsFileIncluded(f.FileName)).ToList();
// Pre-process
filesAfterExclusions.ForEach(file =>
{
var fileFinalPath = Path.Combine(depot.installDir, file.FileName);
var fileStagingPath = Path.Combine(stagingDir, file.FileName);
if (file.Flags.HasFlag(EDepotFileFlag.Directory))
{
Directory.CreateDirectory(fileFinalPath);
Directory.CreateDirectory(fileStagingPath);
}
else
{
// Some manifests don't explicitly include all necessary directories
Directory.CreateDirectory(Path.GetDirectoryName(fileFinalPath));
Directory.CreateDirectory(Path.GetDirectoryName(fileStagingPath));
complete_download_size += file.TotalSize;
}
});
var semaphore = new SemaphoreSlim(Config.MaxDownloads);
var files = filesAfterExclusions.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory)).ToArray();
var tasks = new Task[files.Length];
for (var i = 0; i < files.Length; i++)
{
var file = files[i];
var task = Task.Run(async() =>
{
cts.Token.ThrowIfCancellationRequested();
try
{
await semaphore.WaitAsync().ConfigureAwait(false);
//await semaphore.WaitAsync();
cts.Token.ThrowIfCancellationRequested();
string fileFinalPath = Path.Combine(depot.installDir, file.FileName);
string fileStagingPath = Path.Combine(stagingDir, file.FileName);
// This may still exist if the previous run exited before cleanup
DebugLog.WriteLine(DEBUG_NAME_FILES, "Checking if " + fileStagingPath + " exists");
if (File.Exists(fileStagingPath))
{
DebugLog.WriteLine(DEBUG_NAME_FILES, fileStagingPath + " exists, deleting!");
File.Delete(fileStagingPath);
}
List <ProtoManifest.ChunkData> neededChunks;
FileInfo fi = new FileInfo(fileFinalPath);
DebugLog.WriteLine(DEBUG_NAME_FILES, "Checking if " + fileFinalPath + " exists");
if (!fi.Exists)
{
// create new file. need all chunks
DebugLog.WriteLine(DEBUG_NAME_FILES, fileFinalPath + " does not exist, creating!");
using (FileStream fs = File.Create(fileFinalPath))
{
fs.SetLength((long)file.TotalSize);
neededChunks = new List <ProtoManifest.ChunkData>(file.Chunks);
}
}
else
{
// open existing
ProtoManifest.FileData oldManifestFile = null;
if (oldProtoManifest != null)
{
oldManifestFile = oldProtoManifest.Files.SingleOrDefault(f => f.FileName == file.FileName);
}
if (oldManifestFile != null)
{
neededChunks = new List <ProtoManifest.ChunkData>();
if (Config.VerifyAll || !oldManifestFile.FileHash.SequenceEqual(file.FileHash))
{
// we have a version of this file, but it doesn't fully match what we want
var matchingChunks = new List <ChunkMatch>();
foreach (var chunk in file.Chunks)
{
var oldChunk = oldManifestFile.Chunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID));
if (oldChunk != null)
{
matchingChunks.Add(new ChunkMatch(oldChunk, chunk));
}
else
{
neededChunks.Add(chunk);
}
}
DebugLog.WriteLine(DEBUG_NAME_FILES, "Moving file " + fileFinalPath + " to " + fileStagingPath);
File.Move(fileFinalPath, fileStagingPath);
DebugLog.WriteLine(DEBUG_NAME_FILES, "Creating file " + fileFinalPath);
using (FileStream fs = File.Open(fileFinalPath, FileMode.Create))
{
fs.SetLength((long)file.TotalSize);
DebugLog.WriteLine(DEBUG_NAME_FILES, "Opening file " + fileStagingPath);
using (var fsOld = File.Open(fileStagingPath, FileMode.Open))
{
foreach (var match in matchingChunks)
{
fsOld.Seek((long)match.OldChunk.Offset, SeekOrigin.Begin);
byte[] tmp = new byte[match.OldChunk.UncompressedLength];
fsOld.Read(tmp, 0, tmp.Length);
byte[] adler = Util.AdlerHash(tmp);
if (!adler.SequenceEqual(match.OldChunk.Checksum))
{
neededChunks.Add(match.NewChunk);
}
else
{
fs.Seek((long)match.NewChunk.Offset, SeekOrigin.Begin);
fs.Write(tmp, 0, tmp.Length);
}
}
}
}
DebugLog.WriteLine(DEBUG_NAME_FILES, "Deleting file " + fileStagingPath);
File.Delete(fileStagingPath);
}
}
else
{
// No old manifest or file not in old manifest. We must validate.
DebugLog.WriteLine(DEBUG_NAME_FILES, "Opening file " + fileFinalPath);
using (FileStream fs = File.Open(fileFinalPath, FileMode.Open))
{
if ((ulong)fi.Length != file.TotalSize)
{
fs.SetLength((long)file.TotalSize);
}
neededChunks = Util.ValidateSteam3FileChecksums(fs, file.Chunks.OrderBy(x => x.Offset).ToArray());
}
}
if (neededChunks.Count() == 0)
{
size_downloaded += file.TotalSize;
DebugLog.WriteLine("ContentDownloader", DownloadPercent * 100.0f + "% " + fileFinalPath);
return;
}
else
{
size_downloaded += (file.TotalSize - ( ulong )neededChunks.Select(x => ( long )x.UncompressedLength).Sum());
}
}
DebugLog.WriteLine(DEBUG_NAME_FILES, "Opening file " + fileFinalPath);
using (FileStream fs = File.Open(fileFinalPath, FileMode.Open))
{
foreach (var chunk in neededChunks)
{
if (cts.IsCancellationRequested)
{
break;
}
string chunkID = Util.EncodeHexString(chunk.ChunkID);
CDNClient.DepotChunk chunkData = null;
while (!cts.IsCancellationRequested)
{
CDNClient client;
try
{
client = await cdnPool.GetConnectionForDepotAsync(appId, depot.id, depot.depotKey, cts.Token).ConfigureAwait(false);
//client = await cdnPool.GetConnectionForDepotAsync(appId, depot.id, depot.depotKey, cts.Token);
}
catch (OperationCanceledException)
{
break;
}
DepotManifest.ChunkData data = new DepotManifest.ChunkData();
data.ChunkID = chunk.ChunkID;
data.Checksum = chunk.Checksum;
data.Offset = chunk.Offset;
data.CompressedLength = chunk.CompressedLength;
data.UncompressedLength = chunk.UncompressedLength;
try
{
//chunkData = await client.DownloadDepotChunkAsStreamAsync(depot.id, data).ConfigureAwait(false);
chunkData = client.DownloadDepotChunkAsStreamAsync(depot.id, data).Result;
//chunkData = await client.DownloadDepotChunkAsync(depot.id, data);
cdnPool.ReturnConnection(client);
break;
}
catch (WebException e)
{
cdnPool.ReturnBrokenConnection(client);
if (e.Status == WebExceptionStatus.ProtocolError)
{
var response = e.Response as HttpWebResponse;
if (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden)
{
DebugLog.WriteLine("ContentDownloader", "Encountered 401 for chunk " + chunkID + ". Aborting.");
cts.Cancel();
break;
}
else
{
DebugLog.WriteLine("ContentDownloader", "Encountered error downloading chunk " + chunkID + ": " + response.StatusCode);
}
}
else
{
DebugLog.WriteLine("ContentDownloader", "Encountered error downloading chunk " + chunkID + ": " + e.Status);
}
}
catch (Exception e)
{
cdnPool.ReturnBrokenConnection(client);
DebugLog.WriteLine("ContentDownloader", "Encountered unexpected error downloading chunk " + chunkID + ": " + e.Message);
}
}
if (chunkData == null)
{
DebugLog.WriteLine("ContentDownloader", "Failed to find any server with chunk " + chunkID + " for depot " + depot.id + ". Aborting.");
cts.Cancel();
return;
}
TotalBytesCompressed += chunk.CompressedLength;
DepotBytesCompressed += chunk.CompressedLength;
TotalBytesUncompressed += chunk.UncompressedLength;
DepotBytesUncompressed += chunk.UncompressedLength;
using (chunkData.DataStream)
{
fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
chunkData.DataStream.CopyTo(fs);
//fs.Write(chunkData.Data, 0, chunkData.Data.Length);
}
size_downloaded += chunk.UncompressedLength;
}
}
DebugLog.WriteLine("ContentDownloader", DownloadPercent * 100.0f + "% " + fileFinalPath);
}
finally
{
semaphore.Release();
}
});
tasks[i] = task;
}
Task.WaitAll(tasks);
ConfigStore.TheConfig.LastManifests[depot.id] = depot.manifestId;
ConfigStore.Save();
DebugLog.WriteLine("ContentDownloader", "Depot " + depot.id + " - Downloaded " + DepotBytesCompressed + " bytes (" + DepotBytesUncompressed + " bytes uncompressed)");
}
IsDownloading = false;
downloadCompleteAction?.Invoke();
onDownloadCompleted?.Invoke();
DebugLog.WriteLine("ContentDownloader", "Total downloaded: " + TotalBytesCompressed + " bytes (" + TotalBytesUncompressed + " bytes uncompressed) from " + depots.Count + " depots");
}
private async Task DownloadDepots(uint appID, List <ManifestJob> depots)
{
Log.WriteDebug(nameof(DepotProcessor), $"Will process {depots.Count} depots from app {appID} ({DepotLocks.Count} depot locks left)");
var processTasks = new List <Task <(uint DepotID, EResult Result)> >();
var anyFilesDownloaded = false;
var willDownloadFiles = false;
foreach (var depot in depots)
{
if (depot.DepotKey == null)
{
await GetDepotDecryptionKey(Steam.Instance.Apps, depot, appID);
if (depot.DepotKey == null &&
depot.LastManifestID == depot.ManifestID &&
Settings.FullRun != FullRunState.WithForcedDepots)
{
RemoveLock(depot.DepotID);
continue;
}
}
depot.Server = GetContentServer();
DepotManifest depotManifest = null;
var lastError = string.Empty;
for (var i = 0; i <= 5; i++)
{
try
{
await ManifestDownloadSemaphore.WaitAsync(TaskManager.TaskCancellationToken.Token).ConfigureAwait(false);
depotManifest = await CDNClient.DownloadManifestAsync(depot.DepotID, depot.ManifestID, depot.Server, string.Empty, depot.DepotKey);
break;
}
catch (Exception e)
{
lastError = e.Message;
Log.WriteError(nameof(DepotProcessor), $"Failed to download depot manifest for app {appID} depot {depot.DepotID} ({depot.Server}: {lastError}) (#{i})");
}
finally
{
ManifestDownloadSemaphore.Release();
}
if (depot.DepotKey != null)
{
RemoveErroredServer(depot.Server);
}
if (depotManifest == null && i < 5)
{
await Task.Delay(Utils.ExponentionalBackoff(i + 1));
depot.Server = GetContentServer();
}
}
if (depotManifest == null)
{
RemoveLock(depot.DepotID);
if (FileDownloader.IsImportantDepot(depot.DepotID))
{
IRC.Instance.SendOps($"{Colors.OLIVE}[{depot.DepotName}]{Colors.NORMAL} Failed to download manifest ({lastError})");
}
if (!Settings.IsFullRun && depot.DepotKey != null)
{
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
}
continue;
}
var task = ProcessDepotAfterDownload(depot, depotManifest);
processTasks.Add(task);
if (!FileDownloader.IsImportantDepot(depot.DepotID) || depot.DepotKey == null)
{
depot.Result = EResult.Ignored;
continue;
}
willDownloadFiles = true;
task = TaskManager.Run(async() =>
{
var result = EResult.Fail;
try
{
result = await FileDownloader.DownloadFilesFromDepot(depot, depotManifest);
if (result == EResult.OK)
{
anyFilesDownloaded = true;
}
}
catch (Exception e)
{
ErrorReporter.Notify($"Depot Processor {depot.DepotID}", e);
}
return(depot.DepotID, result);
});
processTasks.Add(task);
}
if (!anyFilesDownloaded && !willDownloadFiles)
{
foreach (var task in processTasks)
{
_ = task.ContinueWith(result =>
{
RemoveLock(result.Result.DepotID);
}, TaskManager.TaskCancellationToken.Token);
}
await Task.WhenAll(processTasks).ConfigureAwait(false);
return;
}
await Task.WhenAll(processTasks).ConfigureAwait(false);
Log.WriteDebug(nameof(DepotProcessor), $"{depots.Count} depot downloads finished for app {appID}");
lock (UpdateScriptLock)
{
foreach (var depot in depots)
{
if (depot.Result == EResult.OK)
{
RunUpdateScript(UpdateScript, $"{depot.DepotID} no-git");
}
else if (depot.Result != EResult.Ignored)
{
Log.WriteWarn(nameof(DepotProcessor), $"Download failed for {depot.DepotID}: {depot.Result}");
RemoveErroredServer(depot.Server);
// Mark this depot for redownload
using var db = Database.Get();
db.Execute("UPDATE `Depots` SET `LastManifestID` = 0 WHERE `DepotID` = @DepotID", new { depot.DepotID });
}
RemoveLock(depot.DepotID);
}
// Only commit changes if all depots downloaded
if (processTasks.All(x => x.Result.Result == EResult.OK || x.Result.Result == EResult.Ignored))
{
if (!RunUpdateScriptForApp(appID, depots[0].BuildID))
{
RunUpdateScript(UpdateScript, "0");
}
}
else
{
Log.WriteDebug(nameof(DepotProcessor), $"Reprocessing the app {appID} because some files failed to download");
IRC.Instance.SendOps($"{Colors.OLIVE}[{Steam.GetAppName(appID)}]{Colors.NORMAL} Reprocessing the app due to download failures");
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
}
}
}
private async Task DownloadDepots(uint appID, List <ManifestJob> depots)
{
Log.WriteDebug("Depot Downloader", "Will process {0} depots ({1} depot locks left)", depots.Count, DepotLocks.Count);
var processTasks = new List <Task <EResult> >();
var anyFilesDownloaded = false;
var willDownloadFiles = false;
foreach (var depot in depots)
{
if (depot.DepotKey == null)
{
await GetDepotDecryptionKey(Steam.Instance.Apps, depot, appID);
if (depot.DepotKey == null && depot.LastManifestID == depot.ManifestID)
{
RemoveLock(depot.DepotID);
continue;
}
}
depot.Server = GetContentServer();
DepotManifest depotManifest = null;
var lastError = string.Empty;
for (var i = 0; i <= 5; i++)
{
try
{
depotManifest = await CDNClient.DownloadManifestAsync(depot.DepotID, depot.ManifestID, depot.Server, string.Empty, depot.DepotKey);
break;
}
catch (Exception e)
{
lastError = e.Message;
Log.WriteError("Depot Processor", "Failed to download depot manifest for app {0} depot {1} ({2}: {3}) (#{4})", appID, depot.DepotID, depot.Server, lastError, i);
}
if (depot.DepotKey != null)
{
RemoveErroredServer(depot.Server);
}
depot.Server = GetContentServer();
if (depotManifest == null)
{
await Task.Delay(Utils.ExponentionalBackoff(i));
}
}
if (depotManifest == null)
{
RemoveLock(depot.DepotID);
if (FileDownloader.IsImportantDepot(depot.DepotID))
{
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download manifest ({3})",
Colors.OLIVE, depot.DepotName, Colors.NORMAL, lastError);
}
if (!Settings.IsFullRun && depot.DepotKey != null)
{
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
}
continue;
}
var task = ProcessDepotAfterDownload(depot, depotManifest);
processTasks.Add(task);
if (!FileDownloader.IsImportantDepot(depot.DepotID) || depot.DepotKey == null)
{
continue;
}
willDownloadFiles = true;
task = TaskManager.Run(async() =>
{
var result = EResult.Fail;
try
{
result = await FileDownloader.DownloadFilesFromDepot(depot, depotManifest);
if (result == EResult.OK)
{
anyFilesDownloaded = true;
}
}
catch (Exception e)
{
ErrorReporter.Notify($"Depot Processor {depot.DepotID}", e);
}
return(result);
}).Unwrap();
processTasks.Add(task);
}
if (SaveLocalConfig)
{
SaveLocalConfig = false;
LocalConfig.Save();
}
await Task.WhenAll(processTasks).ConfigureAwait(false);
Log.WriteDebug("Depot Downloader", $"{depots.Count} depot downloads finished for app {appID}");
// TODO: use ContinueWith on tasks
if (!anyFilesDownloaded && !willDownloadFiles)
{
foreach (var depot in depots)
{
RemoveLock(depot.DepotID);
}
return;
}
lock (UpdateScriptLock)
{
foreach (var depot in depots)
{
if (depot.Result == EResult.OK)
{
RunUpdateScript(UpdateScript, string.Format("{0} no-git", depot.DepotID));
}
else if (depot.Result != EResult.Ignored)
{
Log.WriteWarn("Depot Processor", $"Download failed for {depot.DepotID}");
// Mark this depot for redownload
var db = Database.Get();
db.Execute("UPDATE `Depots` SET `LastManifestID` = 0 WHERE `DepotID` = @DepotID", new { depot.DepotID });
}
RemoveLock(depot.DepotID);
}
// Only commit changes if all depots downloaded
if (processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored))
{
if (!RunUpdateScriptForApp(appID, depots[0].BuildID))
{
RunUpdateScript(UpdateScript, "0");
}
}
else
{
Log.WriteDebug("Depot Processor", "Reprocessing the app {0} because some files failed to download", appID);
IRC.Instance.SendOps("{0}[{1}]{2} Reprocessing the app due to download failures",
Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL
);
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
}
}
}
async private void dataGridView1_CellDoubleClick(object sender, DataGridViewCellEventArgs e)
{
if (e.RowIndex < 0)
{
return;
}
try
{
ConfigItem item = datagridConfigs.Rows[e.RowIndex].DataBoundItem as ConfigItem;
if (item != null)
{
if (item.URL == "")
{
var callback = await steamWorkshop.RequestInfo(241100, item.Details.publishedfileid);
var itemInfo = callback.Items.FirstOrDefault();
var ticket = await steamClient.GetHandler <SteamApps>().GetAppOwnershipTicket(241100);
var decryptKey = await steamClient.GetHandler <SteamApps>().GetDepotDecryptionKey(241100, 241100);
var cdn = new CDNClient(steamClient, ticket.Ticket);
// var servers = cdn.FetchServerList();
var servers = await cdn.FetchServerListAsync();
int i = 0;
foreach (CDNClient.Server server in servers)
{
Log.w($"Server{i}: {server}");
++i;
}
await cdn.ConnectAsync(servers.First());
await cdn.AuthenticateDepotAsync(241100, decryptKey.DepotKey);
var manifest = await cdn.DownloadManifestAsync(241100, itemInfo.ManifestID);
manifest.DecryptFilenames(decryptKey.DepotKey);
if (manifest.Files.Count == 0)
{
MessageBox.Show("Steam Refused Download Request");
return;
}
var chunk = cdn.DownloadDepotChunkAsync(241100, manifest.Files.First().Chunks.First());
if (saveFileDialog1.ShowDialog() == DialogResult.OK)
{
using (var wc = new WebClient())
{
using (var io = saveFileDialog1.OpenFile())
{
io.Write(chunk.Result.Data, 0, chunk.Result.Data.Length);
MessageBox.Show($"Download Done!\nSaved {saveFileDialog1.FileName}");
Log.w($"Downloaded {saveFileDialog1.FileName}");
}
}
}
}
else
{
if (saveFileDialog1.ShowDialog() == DialogResult.OK)
{
using (var wc = new WebClient())
{
wc.DownloadFile(new Uri(item.URL), saveFileDialog1.FileName);
MessageBox.Show("Download Done!");
}
}
}
}
}
catch (TaskCanceledException)
{
MessageBox.Show("Timeout! This can happen if you're using anonymous account and trying to download.");
}
catch (Exception ex)
{
MessageBox.Show($"Error Downloading Config: {ex.ToString()}");
}
}