/// <summary>
/// Downloads the depot manifest specified by the given manifest ID, and optionally decrypts the manifest's filenames if the depot decryption key has been provided.
/// </summary>
/// <param name="depotId">The id of the depot being accessed.</param>
/// <param name="manifestId">The unique identifier of the manifest to be downloaded.</param>
/// <param name="manifestRequestCode">The manifest request code for the manifest that is being downloaded.</param>
/// <param name="server">The content server to connect to.</param>
/// <param name="depotKey">
/// The depot decryption key for the depot that will be downloaded.
/// This is used for decrypting filenames (if needed) in depot manifests, and processing depot chunks.
/// </param>
/// <param name="proxyServer">Optional content server marked as UseAsProxy which transforms the request.</param>
/// <returns>A <see cref="DepotManifest"/> instance that contains information about the files present within a depot.</returns>
/// <exception cref="System.ArgumentNullException"><see ref="server"/> was null.</exception>
/// <exception cref="HttpRequestException">An network error occurred when performing the request.</exception>
/// <exception cref="SteamKitWebRequestException">A network error occurred when performing the request.</exception>
public async Task <DepotManifest> DownloadManifestAsync(uint depotId, ulong manifestId, ulong manifestRequestCode, Server server, byte[]?depotKey = null, Server?proxyServer = null)
{
if (server == null)
{
throw new ArgumentNullException(nameof(server));
}
const uint MANIFEST_VERSION = 5;
string url;
if (manifestRequestCode > 0)
{
url = $"depot/{depotId}/manifest/{manifestId}/{MANIFEST_VERSION}/{manifestRequestCode}";
}
else
{
url = $"depot/{depotId}/manifest/{manifestId}/{MANIFEST_VERSION}";
}
var manifestData = await DoRawCommandAsync(server, url, proxyServer).ConfigureAwait(false);
manifestData = ZipUtil.Decompress(manifestData);
var depotManifest = new DepotManifest(manifestData);
if (depotKey != null)
{
// if we have the depot key, decrypt the manifest filenames
depotManifest.DecryptFilenames(depotKey);
}
return(depotManifest);
}
private async Task <EResult> ProcessDepotAfterDownload(ManifestJob request, DepotManifest depotManifest)
{
using (var db = await Database.GetConnectionAsync())
using (var transaction = await db.BeginTransactionAsync())
{
var result = await ProcessDepotAfterDownload(db, transaction, request, depotManifest);
await transaction.CommitAsync();
return(result);
}
}
private static async Task <DepotManifest> DownloadManifest(uint appId, uint depotId, ulong manifestId, byte[] depotKey)
{
DepotManifest depotManifest = null;
while (depotManifest == null)
{
Tuple <CDNClient.Server, string> connection = null;
try
{
connection = await cdnPool.GetConnectionForDepot(appId, depotId, CancellationToken.None);
depotManifest = await cdnPool.CDNClient.DownloadManifestAsync(depotId, manifestId,
connection.Item1, connection.Item2, depotKey).ConfigureAwait(false);
cdnPool.ReturnConnection(connection);
}
catch (SteamKitWebRequestException e)
{
cdnPool.ReturnBrokenConnection(connection);
if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden)
{
Console.WriteLine("Encountered 401 for depot manifest {0} {1}. Aborting.", depotId, manifestId);
break;
}
else
{
Console.WriteLine("Encountered error downloading depot manifest {0} {1}: {2}", depotId, manifestId, e.StatusCode);
}
}
catch (Exception e)
{
cdnPool.ReturnBrokenConnection(connection);
Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depotId, manifestId, e.Message);
}
}
if (depotManifest == null)
{
Console.WriteLine("\nUnable to download manifest {0} for depot {1}", manifestId, depotId);
return(null);
}
return(depotManifest);
}
private static void HandleManifest()
{
if (!string.IsNullOrEmpty(inputFile) && File.Exists(inputFile))
{
byte[] fileBytes = File.ReadAllBytes(inputFile);
BindingFlags flags = BindingFlags.NonPublic | BindingFlags.Instance;
DepotManifest depotManifest = null;
try
{
// DepotManifest constructor is internal, so use Activator to call it
depotManifest = (DepotManifest)Activator.CreateInstance(typeof(DepotManifest), flags, null, new object[] { fileBytes }, System.Globalization.CultureInfo.InvariantCulture);
}
catch (Exception ex)
{
Console.WriteLine("Something bad happened! Exception message:\r\n{0}", ex.Message);
}
if (depotManifest != null)
{
FCIVExport exporter = new FCIVExport(outputFile);
foreach (DepotManifest.FileData filedata in depotManifest.Files)
{
bool isDir = filedata.Flags == EDepotFileFlag.Directory;
string strHash = filedata.FileHash.Aggregate(new StringBuilder(), (sb, v) => sb.Append(v.ToString("x2"))).ToString().ToUpper();
if (string.IsNullOrEmpty(outputFile))
{
Console.WriteLine("{0} {1}", filedata.FileName, isDir ? "" : strHash);
}
if (!isDir)
{
exporter.AddEntry(filedata.FileName, strHash);
}
}
exporter.Finalize();
}
}
else
{
Console.WriteLine("File {0} doesn't exists!", inputFile);
}
}
private static void DownloadSteam3(uint appId, List <DepotDownloadInfo> depots)
{
ulong TotalBytesCompressed = 0;
ulong TotalBytesUncompressed = 0;
foreach (var depot in depots)
{
ulong DepotBytesCompressed = 0;
ulong DepotBytesUncompressed = 0;
Console.WriteLine("Downloading depot {0} - {1}", depot.id, depot.contentName);
CancellationTokenSource cts = new CancellationTokenSource();
ProtoManifest oldProtoManifest = null;
ProtoManifest newProtoManifest = null;
string configDir = Path.Combine(depot.installDir, CONFIG_DIR);
ulong lastManifestId = INVALID_MANIFEST_ID;
ConfigStore.TheConfig.LastManifests.TryGetValue(depot.id, out lastManifestId);
// In case we have an early exit, this will force equiv of verifyall next run.
ConfigStore.TheConfig.LastManifests[depot.id] = INVALID_MANIFEST_ID;
ConfigStore.Save();
if (lastManifestId != INVALID_MANIFEST_ID)
{
var oldManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", lastManifestId));
if (File.Exists(oldManifestFileName))
{
oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName);
}
}
if (lastManifestId == depot.manifestId && oldProtoManifest != null)
{
newProtoManifest = oldProtoManifest;
Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id);
}
else
{
var newManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", depot.manifestId));
if (newManifestFileName != null)
{
newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName);
}
if (newProtoManifest != null)
{
Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id);
}
else
{
Console.Write("Downloading depot manifest...");
DepotManifest depotManifest = null;
while (depotManifest == null)
{
CDNClient client = null;
try
{
client = cdnPool.GetConnectionForDepot(appId, depot.id, depot.depotKey, CancellationToken.None);
depotManifest = client.DownloadManifest(depot.id, depot.manifestId);
cdnPool.ReturnConnection(client);
}
catch (WebException e)
{
cdnPool.ReturnBrokenConnection(client);
if (e.Status == WebExceptionStatus.ProtocolError)
{
var response = e.Response as HttpWebResponse;
if (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden)
{
Console.WriteLine("Encountered 401 for depot manifest {0} {1}. Aborting.", depot.id, depot.manifestId);
break;
}
else
{
Console.WriteLine("Encountered error downloading depot manifest {0} {1}: {2}", depot.id, depot.manifestId, response.StatusCode);
}
}
else
{
Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depot.id, depot.manifestId, e.Status);
}
}
catch (Exception e)
{
cdnPool.ReturnBrokenConnection(client);
Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depot.id, depot.manifestId, e.Message);
}
}
if (depotManifest == null)
{
Console.WriteLine("\nUnable to download manifest {0} for depot {1}", depot.manifestId, depot.id);
return;
}
newProtoManifest = new ProtoManifest(depotManifest, depot.manifestId);
newProtoManifest.SaveToFile(newManifestFileName);
Console.WriteLine("Successfully downloaded manifest {0} for depot {1}", depot.manifestId, depot.id);
Console.WriteLine(" Done!");
}
}
newProtoManifest.Files.Sort((x, y) => { return(x.FileName.CompareTo(y.FileName)); });
if (Config.DownloadManifestOnly)
{
StringBuilder manifestBuilder = new StringBuilder();
string txtManifest = Path.Combine(depot.installDir, string.Format("manifest_{0}.txt", depot.id));
foreach (var file in newProtoManifest.Files)
{
if (file.Flags.HasFlag(EDepotFileFlag.Directory))
{
continue;
}
manifestBuilder.Append(string.Format("{0}\n", file.FileName));
}
File.WriteAllText(txtManifest, manifestBuilder.ToString());
continue;
}
ulong complete_download_size = 0;
ulong size_downloaded = 0;
string stagingDir = Path.Combine(depot.installDir, STAGING_DIR);
var filesAfterExclusions = newProtoManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).ToList();
// Pre-process
filesAfterExclusions.ForEach(file =>
{
var fileFinalPath = Path.Combine(depot.installDir, file.FileName);
var fileStagingPath = Path.Combine(stagingDir, file.FileName);
if (file.Flags.HasFlag(EDepotFileFlag.Directory))
{
Directory.CreateDirectory(fileFinalPath);
Directory.CreateDirectory(fileStagingPath);
}
else
{
// Some manifests don't explicitly include all necessary directories
Directory.CreateDirectory(Path.GetDirectoryName(fileFinalPath));
Directory.CreateDirectory(Path.GetDirectoryName(fileStagingPath));
complete_download_size += file.TotalSize;
}
});
filesAfterExclusions.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory))
.AsParallel().WithCancellation(cts.Token).WithDegreeOfParallelism(Config.MaxDownloads)
.ForAll(file =>
{
string fileFinalPath = Path.Combine(depot.installDir, file.FileName);
string fileStagingPath = Path.Combine(stagingDir, file.FileName);
// This may still exist if the previous run exited before cleanup
if (File.Exists(fileStagingPath))
{
File.Delete(fileStagingPath);
}
FileStream fs = null;
List <ProtoManifest.ChunkData> neededChunks;
FileInfo fi = new FileInfo(fileFinalPath);
if (!fi.Exists)
{
// create new file. need all chunks
fs = File.Create(fileFinalPath);
fs.SetLength(( long )file.TotalSize);
neededChunks = new List <ProtoManifest.ChunkData>(file.Chunks);
}
else
{
// open existing
ProtoManifest.FileData oldManifestFile = null;
if (oldProtoManifest != null)
{
oldManifestFile = oldProtoManifest.Files.SingleOrDefault(f => f.FileName == file.FileName);
}
if (oldManifestFile != null)
{
neededChunks = new List <ProtoManifest.ChunkData>();
if (Config.VerifyAll || !oldManifestFile.FileHash.SequenceEqual(file.FileHash))
{
// we have a version of this file, but it doesn't fully match what we want
var matchingChunks = new List <ChunkMatch>();
foreach (var chunk in file.Chunks)
{
var oldChunk = oldManifestFile.Chunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID));
if (oldChunk != null)
{
matchingChunks.Add(new ChunkMatch(oldChunk, chunk));
}
else
{
neededChunks.Add(chunk);
}
}
File.Move(fileFinalPath, fileStagingPath);
fs = File.Open(fileFinalPath, FileMode.Create);
fs.SetLength(( long )file.TotalSize);
using (var fsOld = File.Open(fileStagingPath, FileMode.Open))
{
foreach (var match in matchingChunks)
{
fsOld.Seek(( long )match.OldChunk.Offset, SeekOrigin.Begin);
byte[] tmp = new byte[match.OldChunk.UncompressedLength];
fsOld.Read(tmp, 0, tmp.Length);
byte[] adler = Util.AdlerHash(tmp);
if (!adler.SequenceEqual(match.OldChunk.Checksum))
{
neededChunks.Add(match.NewChunk);
}
else
{
fs.Seek(( long )match.NewChunk.Offset, SeekOrigin.Begin);
fs.Write(tmp, 0, tmp.Length);
}
}
}
File.Delete(fileStagingPath);
}
}
else
{
// No old manifest or file not in old manifest. We must validate.
fs = File.Open(fileFinalPath, FileMode.Open);
if (( ulong )fi.Length != file.TotalSize)
{
fs.SetLength(( long )file.TotalSize);
}
neededChunks = Util.ValidateSteam3FileChecksums(fs, file.Chunks.OrderBy(x => x.Offset).ToArray());
}
if (neededChunks.Count() == 0)
{
size_downloaded += file.TotalSize;
Console.WriteLine("{0,6:#00.00}% {1}", (( float )size_downloaded / ( float )complete_download_size) * 100.0f, fileFinalPath);
if (fs != null)
{
fs.Close();
}
return;
}
else
{
size_downloaded += (file.TotalSize - ( ulong )neededChunks.Select(x => ( long )x.UncompressedLength).Sum());
}
}
foreach (var chunk in neededChunks)
{
if (cts.IsCancellationRequested)
{
break;
}
string chunkID = Util.EncodeHexString(chunk.ChunkID);
CDNClient.DepotChunk chunkData = null;
while (!cts.IsCancellationRequested)
{
CDNClient client;
try
{
client = cdnPool.GetConnectionForDepot(appId, depot.id, depot.depotKey, cts.Token);
}
catch (OperationCanceledException)
{
break;
}
DepotManifest.ChunkData data = new DepotManifest.ChunkData();
data.ChunkID = chunk.ChunkID;
data.Checksum = chunk.Checksum;
data.Offset = chunk.Offset;
data.CompressedLength = chunk.CompressedLength;
data.UncompressedLength = chunk.UncompressedLength;
try
{
chunkData = client.DownloadDepotChunk(depot.id, data);
cdnPool.ReturnConnection(client);
break;
}
catch (WebException e)
{
cdnPool.ReturnBrokenConnection(client);
if (e.Status == WebExceptionStatus.ProtocolError)
{
var response = e.Response as HttpWebResponse;
if (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden)
{
Console.WriteLine("Encountered 401 for chunk {0}. Aborting.", chunkID);
cts.Cancel();
break;
}
else
{
Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, response.StatusCode);
}
}
else
{
Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, e.Status);
}
}
catch (Exception e)
{
cdnPool.ReturnBrokenConnection(client);
Console.WriteLine("Encountered unexpected error downloading chunk {0}: {1}", chunkID, e.Message);
}
}
if (chunkData == null)
{
Console.WriteLine("Failed to find any server with chunk {0} for depot {1}. Aborting.", chunkID, depot.id);
return;
}
TotalBytesCompressed += chunk.CompressedLength;
DepotBytesCompressed += chunk.CompressedLength;
TotalBytesUncompressed += chunk.UncompressedLength;
DepotBytesUncompressed += chunk.UncompressedLength;
fs.Seek(( long )chunk.Offset, SeekOrigin.Begin);
fs.Write(chunkData.Data, 0, chunkData.Data.Length);
size_downloaded += chunk.UncompressedLength;
}
fs.Close();
Console.WriteLine("{0,6:#00.00}% {1}", (( float )size_downloaded / ( float )complete_download_size) * 100.0f, fileFinalPath);
});
ConfigStore.TheConfig.LastManifests[depot.id] = depot.manifestId;
ConfigStore.Save();
Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.id, DepotBytesCompressed, DepotBytesUncompressed);
}
Console.WriteLine("Total downloaded: {0} bytes ({1} bytes uncompressed) from {2} depots", TotalBytesCompressed, TotalBytesUncompressed, depots.Count);
}
/*
* Here be dragons.
*/
public static async Task <EResult> DownloadFilesFromDepot(DepotProcessor.ManifestJob job, DepotManifest depotManifest)
{
var files = depotManifest.Files.Where(x => IsFileNameMatching(job.DepotID, x.FileName)).ToList();
var downloadState = EResult.Fail;
var hashesFile = Path.Combine(Application.Path, "files", ".support", "hashes", string.Format("{0}.json", job.DepotID));
ConcurrentDictionary <string, byte[]> hashes;
if (File.Exists(hashesFile))
{
hashes = JsonConvert.DeserializeObject <ConcurrentDictionary <string, byte[]> >(File.ReadAllText(hashesFile));
}
else
{
hashes = new ConcurrentDictionary <string, byte[]>();
}
foreach (var file in hashes.Keys.Except(files.Select(x => x.FileName)))
{
Log.WriteWarn(nameof(FileDownloader), $"\"{file}\" no longer exists in manifest");
}
Log.WriteInfo("FileDownloader", "Will download {0} files from depot {1}", files.Count, job.DepotID);
var downloadedFiles = 0;
var fileTasks = new Task[files.Count];
for (var i = 0; i < fileTasks.Length; i++)
{
var file = files[i];
fileTasks[i] = TaskManager.Run(async() =>
{
hashes.TryGetValue(file.FileName, out var hash);
var fileState = await DownloadFile(job, file, hash);
if (fileState == EResult.OK || fileState == EResult.SameAsPreviousValue)
{
hashes[file.FileName] = file.FileHash;
downloadedFiles++;
}
if (fileState != EResult.SameAsPreviousValue)
{
// Do not write progress info to log file
Console.WriteLine("{1} [{0,6:#00.00}%] {2} files left to download", downloadedFiles / (float)files.Count * 100.0f, job.DepotName, files.Count - downloadedFiles);
}
if (downloadState == EResult.DataCorruption)
{
return;
}
if (fileState == EResult.OK || fileState == EResult.DataCorruption)
{
downloadState = fileState;
}
}).Unwrap();
// Register error handler on inner task
TaskManager.RegisterErrorHandler(fileTasks[i]);
}
await Task.WhenAll(fileTasks).ConfigureAwait(false);
if (downloadState == EResult.OK)
{
File.WriteAllText(hashesFile, JsonConvert.SerializeObject(hashes));
job.Result = EResult.OK;
}
else
{
job.Result = EResult.Ignored;
}
return(job.Result);
}
private async Task DownloadDepots(uint appID, List <ManifestJob> depots)
{
Log.WriteDebug("Depot Downloader", "Will process {0} depots ({1} depot locks left)", depots.Count(), DepotLocks.Count);
var processTasks = new List <Task <EResult> >();
bool anyFilesDownloaded = false;
foreach (var depot in depots)
{
var instance = depot.Anonymous ? Steam.Anonymous.Apps : Steam.Instance.Apps;
depot.DepotKey = await GetDepotDecryptionKey(instance, depot.DepotID, appID);
if (depot.DepotKey == null)
{
RemoveLock(depot.DepotID);
continue;
}
var cdnToken = await GetCDNAuthToken(instance, appID, depot.DepotID);
if (cdnToken == null)
{
RemoveLock(depot.DepotID);
Log.WriteDebug("Depot Downloader", "Got a depot key for depot {0} but no cdn auth token", depot.DepotID);
continue;
}
depot.CDNToken = cdnToken.Token;
depot.Server = cdnToken.Server;
DepotManifest depotManifest = null;
string lastError = string.Empty;
for (var i = 0; i <= 5; i++)
{
try
{
depotManifest = CDNClient.DownloadManifest(depot.DepotID, depot.ManifestID, depot.Server, depot.CDNToken, depot.DepotKey);
break;
}
catch (Exception e)
{
Log.WriteWarn("Depot Downloader", "[{0}] Manifest download failed: {1} - {2}", depot.DepotID, e.GetType(), e.Message);
lastError = e.Message;
}
// TODO: get new auth key if auth fails
if (depotManifest == null)
{
await Task.Delay(Utils.ExponentionalBackoff(i));
}
}
if (depotManifest == null)
{
RemoveLock(depot.DepotID);
Log.WriteError("Depot Processor", "Failed to download depot manifest for app {0} depot {1} ({2}: {3})", appID, depot.DepotID, depot.Server, lastError);
if (FileDownloader.IsImportantDepot(depot.DepotID))
{
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download app {3} depot {4} manifest ({5}: {6})",
Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL, appID, depot.DepotID, depot.Server, lastError);
}
continue;
}
var task = TaskManager.Run(() =>
{
using (var db = Database.GetConnection())
{
using (var transaction = db.BeginTransaction())
{
var result = ProcessDepotAfterDownload(db, depot, depotManifest);
transaction.Commit();
return(result);
}
}
});
processTasks.Add(task);
if (FileDownloader.IsImportantDepot(depot.DepotID))
{
task = TaskManager.Run(() =>
{
var result = FileDownloader.DownloadFilesFromDepot(appID, depot, depotManifest);
if (result == EResult.OK)
{
anyFilesDownloaded = true;
}
return(result);
}, TaskCreationOptions.LongRunning);
TaskManager.RegisterErrorHandler(task);
processTasks.Add(task);
}
}
if (SaveLocalConfig)
{
SaveLocalConfig = false;
LocalConfig.Save();
}
await Task.WhenAll(processTasks);
Log.WriteDebug("Depot Downloader", "{0} depot downloads finished", depots.Count());
// TODO: use ContinueWith on tasks
if (!anyFilesDownloaded)
{
foreach (var depot in depots)
{
RemoveLock(depot.DepotID);
}
return;
}
if (!File.Exists(UpdateScript))
{
return;
}
bool lockTaken = false;
try
{
UpdateScriptLock.Enter(ref lockTaken);
foreach (var depot in depots)
{
if (depot.Result == EResult.OK)
{
RunUpdateScript(string.Format("{0} no-git", depot.DepotID));
}
RemoveLock(depot.DepotID);
}
// Only commit changes if all depots downloaded
if (processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored))
{
if (!RunUpdateScript(appID))
{
RunUpdateScript("0");
}
}
else
{
Log.WriteDebug("Depot Processor", "Reprocessing the app {0} because some files failed to download", appID);
IRC.Instance.SendOps("{0}[{1}]{2} Reprocessing the app due to download failures",
Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL
);
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
}
}
finally
{
if (lockTaken)
{
UpdateScriptLock.Exit();
}
}
}
private async Task DownloadDepots(uint appID, List <ManifestJob> depots)
{
Log.WriteDebug(nameof(DepotProcessor), $"Will process {depots.Count} depots from app {appID} ({DepotLocks.Count} depot locks left)");
var processTasks = new List <Task <(uint DepotID, EResult Result)> >();
var anyFilesDownloaded = false;
var willDownloadFiles = false;
foreach (var depot in depots)
{
if (depot.DepotKey == null)
{
await GetDepotDecryptionKey(Steam.Instance.Apps, depot, appID);
if (depot.DepotKey == null &&
depot.LastManifestID == depot.ManifestID &&
Settings.FullRun != FullRunState.WithForcedDepots)
{
RemoveLock(depot.DepotID);
continue;
}
}
depot.Server = GetContentServer();
DepotManifest depotManifest = null;
var lastError = string.Empty;
for (var i = 0; i <= 5; i++)
{
try
{
await ManifestDownloadSemaphore.WaitAsync(TaskManager.TaskCancellationToken.Token).ConfigureAwait(false);
depotManifest = await CDNClient.DownloadManifestAsync(depot.DepotID, depot.ManifestID, depot.Server, string.Empty, depot.DepotKey);
break;
}
catch (Exception e)
{
lastError = e.Message;
Log.WriteError(nameof(DepotProcessor), $"Failed to download depot manifest for app {appID} depot {depot.DepotID} ({depot.Server}: {lastError}) (#{i})");
}
finally
{
ManifestDownloadSemaphore.Release();
}
if (depot.DepotKey != null)
{
RemoveErroredServer(depot.Server);
}
if (depotManifest == null && i < 5)
{
await Task.Delay(Utils.ExponentionalBackoff(i + 1));
depot.Server = GetContentServer();
}
}
if (depotManifest == null)
{
RemoveLock(depot.DepotID);
if (FileDownloader.IsImportantDepot(depot.DepotID))
{
IRC.Instance.SendOps($"{Colors.OLIVE}[{depot.DepotName}]{Colors.NORMAL} Failed to download manifest ({lastError})");
}
if (!Settings.IsFullRun && depot.DepotKey != null)
{
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
}
continue;
}
var task = ProcessDepotAfterDownload(depot, depotManifest);
processTasks.Add(task);
if (!FileDownloader.IsImportantDepot(depot.DepotID) || depot.DepotKey == null)
{
depot.Result = EResult.Ignored;
continue;
}
willDownloadFiles = true;
task = TaskManager.Run(async() =>
{
var result = EResult.Fail;
try
{
result = await FileDownloader.DownloadFilesFromDepot(depot, depotManifest);
if (result == EResult.OK)
{
anyFilesDownloaded = true;
}
}
catch (Exception e)
{
ErrorReporter.Notify($"Depot Processor {depot.DepotID}", e);
}
return(depot.DepotID, result);
});
processTasks.Add(task);
}
if (!anyFilesDownloaded && !willDownloadFiles)
{
foreach (var task in processTasks)
{
_ = task.ContinueWith(result =>
{
RemoveLock(result.Result.DepotID);
}, TaskManager.TaskCancellationToken.Token);
}
await Task.WhenAll(processTasks).ConfigureAwait(false);
return;
}
await Task.WhenAll(processTasks).ConfigureAwait(false);
Log.WriteDebug(nameof(DepotProcessor), $"{depots.Count} depot downloads finished for app {appID}");
lock (UpdateScriptLock)
{
foreach (var depot in depots)
{
if (depot.Result == EResult.OK)
{
RunUpdateScript(UpdateScript, $"{depot.DepotID} no-git");
}
else if (depot.Result != EResult.Ignored)
{
Log.WriteWarn(nameof(DepotProcessor), $"Download failed for {depot.DepotID}: {depot.Result}");
RemoveErroredServer(depot.Server);
// Mark this depot for redownload
using var db = Database.Get();
db.Execute("UPDATE `Depots` SET `LastManifestID` = 0 WHERE `DepotID` = @DepotID", new { depot.DepotID });
}
RemoveLock(depot.DepotID);
}
// Only commit changes if all depots downloaded
if (processTasks.All(x => x.Result.Result == EResult.OK || x.Result.Result == EResult.Ignored))
{
if (!RunUpdateScriptForApp(appID, depots[0].BuildID))
{
RunUpdateScript(UpdateScript, "0");
}
}
else
{
Log.WriteDebug(nameof(DepotProcessor), $"Reprocessing the app {appID} because some files failed to download");
IRC.Instance.SendOps($"{Colors.OLIVE}[{Steam.GetAppName(appID)}]{Colors.NORMAL} Reprocessing the app due to download failures");
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
}
}
}
private static async Task <(uint, EResult)> ProcessDepotAfterDownload(ManifestJob request, DepotManifest depotManifest)
{
if (depotManifest.FilenamesEncrypted && request.DepotKey != null)
{
Log.WriteError(nameof(DepotProcessor), $"Depot key for depot {request.DepotID} is invalid?");
IRC.Instance.SendOps($"[Tokens] Looks like the depot key for depot {request.DepotID} is invalid");
}
await using var db = await Database.GetConnectionAsync();
await using var transaction = await db.BeginTransactionAsync();
var result = await ProcessDepotAfterDownload(db, transaction, request, depotManifest);
await transaction.CommitAsync();
return(request.DepotID, result);
}
private static void DownloadManifest(ManifestJob request)
{
DepotManifest depotManifest = null;
string lastError = string.Empty;
// CDN is very random, just keep trying
for (var i = 0; i <= 5; i++)
{
try
{
depotManifest = CDNClient.DownloadManifest(request.DepotID, request.ManifestID, request.Server, request.DepotKey, request.CDNToken);
break;
}
catch (Exception e)
{
lastError = e.Message;
}
}
if (depotManifest == null)
{
Log.WriteError("Depot Processor", "Failed to download depot manifest for depot {0} (jobs still in queue: {1}) ({2}: {3})", request.DepotID, ManifestJobs.Count - 1, request.Server, lastError);
if (SteamProxy.Instance.ImportantApps.Contains(request.ParentAppID))
{
IRC.SendMain("Important depot update: {0}{1}{2} -{3} failed to download depot manifest", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.RED);
}
return;
}
if (SteamProxy.Instance.ImportantApps.Contains(request.ParentAppID))
{
IRC.SendMain("Important depot update: {0}{1}{2} -{3} {4}", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.DARK_BLUE, SteamDB.GetDepotURL(request.DepotID, "history"));
}
var sortedFiles = depotManifest.Files.OrderBy(f => f.FileName, StringComparer.OrdinalIgnoreCase);
bool shouldHistorize = false;
var filesNew = new List <DepotFile>();
var filesOld = new Dictionary <string, DepotFile>();
foreach (var file in sortedFiles)
{
System.Text.Encoding.UTF8.GetString(file.FileHash);
var depotFile = new DepotFile
{
Name = file.FileName.Replace('\\', '/'),
Size = file.TotalSize,
Chunks = file.Chunks.Count,
Flags = (int)file.Flags
};
// TODO: Ideally we would check if filehash is not empty
if (!file.Flags.HasFlag(EDepotFileFlag.Directory))
{
depotFile.Hash = string.Concat(Array.ConvertAll(file.FileHash, x => x.ToString("X2")));
}
filesNew.Add(depotFile);
}
using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `Files` FROM `Depots` WHERE `DepotID` = @DepotID LIMIT 1", new MySqlParameter("DepotID", request.DepotID)))
{
if (Reader.Read())
{
string files = Reader.GetString("Files");
if (!string.IsNullOrEmpty(files))
{
shouldHistorize = true;
var _filesOld = JsonConvert.DeserializeObject <List <DepotFile> >(files);
filesOld = _filesOld.ToDictionary(x => x.Name);
}
}
}
DbWorker.ExecuteNonQuery("UPDATE `Depots` SET `Files` = @Files WHERE `DepotID` = @DepotID",
new MySqlParameter("@DepotID", request.DepotID),
new MySqlParameter("@Files", JsonConvert.SerializeObject(filesNew, new JsonSerializerSettings {
DefaultValueHandling = DefaultValueHandling.Ignore
}))
);
if (shouldHistorize)
{
var filesAdded = new List <string>();
foreach (var file in filesNew)
{
if (filesOld.ContainsKey(file.Name))
{
var oldFile = filesOld[file.Name];
if (oldFile.Size != file.Size)
{
MakeHistory(request, file.Name, "modified", oldFile.Size, file.Size);
}
else if (file.Hash != null && oldFile.Hash != null && !file.Hash.Equals(oldFile.Hash))
{
MakeHistory(request, file.Name, "modified", oldFile.Size, file.Size);
}
filesOld.Remove(file.Name);
}
else
{
// We want to historize modifications first, and only then deletions and additions
filesAdded.Add(file.Name);
}
}
foreach (var file in filesOld)
{
MakeHistory(request, file.Value.Name, "removed");
}
foreach (string file in filesAdded)
{
MakeHistory(request, file, "added");
}
}
lock (ManifestJobs)
{
Log.WriteDebug("Depot Processor", "DepotID: Processed {0} (jobs still in queue: {1})", request.DepotID, ManifestJobs.Count - 1);
}
}
/*
* Here be dragons.
*/
public static EResult DownloadFilesFromDepot(uint appID, DepotProcessor.ManifestJob job, DepotManifest depotManifest)
{
var files = depotManifest.Files.Where(x => IsFileNameMatching(job.DepotID, x.FileName)).ToList();
var filesUpdated = false;
var filesAnyFailed = false;
var hashesFile = Path.Combine(Application.Path, "files", ".support", "hashes", string.Format("{0}.json", job.DepotID));
var hashes = new Dictionary <string, byte[]>();
if (File.Exists(hashesFile))
{
hashes = JsonConvert.DeserializeObject <Dictionary <string, byte[]> >(File.ReadAllText(hashesFile));
}
Log.WriteDebug("FileDownloader", "Will download {0} files from depot {1}", files.Count, job.DepotID);
Parallel.ForEach(files, new ParallelOptions {
MaxDegreeOfParallelism = 2
}, (file, state2) =>
{
string directory = Path.Combine(Application.Path, "files", DownloadFolders[job.DepotID], Path.GetDirectoryName(file.FileName));
string finalPath = Path.Combine(directory, Path.GetFileName(file.FileName));
string downloadPath = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".steamdb_tmp"));
if (!Directory.Exists(directory))
{
Directory.CreateDirectory(directory);
}
else if (file.TotalSize == 0)
{
if (File.Exists(finalPath))
{
var f = new FileInfo(finalPath);
if (f.Length == 0)
{
#if DEBUG
Log.WriteDebug("FileDownloader", "{0} is already empty", file.FileName);
#endif
return;
}
}
else
{
File.Create(finalPath);
Log.WriteInfo("FileDownloader", "{0} created an empty file", file.FileName);
return;
}
}
else if (hashes.ContainsKey(file.FileName) && file.FileHash.SequenceEqual(hashes[file.FileName]))
{
#if DEBUG
Log.WriteDebug("FileDownloader", "{0} already matches the file we have", file.FileName);
#endif
return;
}
var chunks = file.Chunks.OrderBy(x => x.Offset).ToList();
Log.WriteInfo("FileDownloader", "Downloading {0} ({1} bytes, {2} chunks)", file.FileName, file.TotalSize, chunks.Count);
uint count = 0;
byte[] checksum;
string lastError = "or checksum failed";
string oldChunksFile;
using (var sha = new SHA1Managed())
{
oldChunksFile = Path.Combine(Application.Path, "files", ".support", "chunks",
string.Format("{0}-{1}.json", job.DepotID, BitConverter.ToString(sha.ComputeHash(Encoding.UTF8.GetBytes(file.FileName))))
);
}
using (var fs = File.Open(downloadPath, FileMode.OpenOrCreate, FileAccess.ReadWrite))
{
fs.SetLength((long)file.TotalSize);
var lockObject = new object();
var neededChunks = new List <DepotManifest.ChunkData>();
if (File.Exists(oldChunksFile) && File.Exists(finalPath))
{
var oldChunks = JsonConvert.DeserializeObject <List <DepotManifest.ChunkData> >(
File.ReadAllText(oldChunksFile),
new JsonSerializerSettings {
PreserveReferencesHandling = PreserveReferencesHandling.All
}
);
using (var fsOld = File.Open(finalPath, FileMode.Open, FileAccess.Read))
{
foreach (var chunk in chunks)
{
var oldChunk = oldChunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID));
if (oldChunk != null)
{
var oldData = new byte[oldChunk.UncompressedLength];
fsOld.Seek((long)oldChunk.Offset, SeekOrigin.Begin);
fsOld.Read(oldData, 0, oldData.Length);
var existingChecksum = Utils.AdlerHash(oldData);
if (existingChecksum.SequenceEqual(chunk.Checksum))
{
fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
fs.Write(oldData, 0, oldData.Length);
#if DEBUG
Log.WriteDebug("FileDownloader", "{0} Found chunk ({1}), not downloading ({2}/{3})", file.FileName, chunk.Offset, ++count, chunks.Count);
#endif
}
else
{
neededChunks.Add(chunk);
#if DEBUG
Log.WriteDebug("FileDownloader", "{0} Found chunk ({1}), but checksum differs", file.FileName, chunk.Offset);
#endif
}
}
else
{
neededChunks.Add(chunk);
}
}
}
}
else
{
neededChunks = chunks;
}
Parallel.ForEach(neededChunks, new ParallelOptions {
MaxDegreeOfParallelism = 3
}, (chunk, state) =>
{
var downloaded = false;
for (var i = 0; i <= 5; i++)
{
try
{
var chunkData = CDNClient.DownloadDepotChunk(job.DepotID, chunk, job.Server, job.CDNToken, job.DepotKey);
lock (lockObject)
{
fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
fs.Write(chunkData.Data, 0, chunkData.Data.Length);
Log.WriteDebug("FileDownloader", "Downloaded {0} ({1}/{2})", file.FileName, ++count, chunks.Count);
}
downloaded = true;
break;
}
catch (Exception e)
{
lastError = e.Message;
}
Task.Delay(Utils.ExponentionalBackoff(i)).Wait();
}
if (!downloaded)
{
state.Stop();
}
});
fs.Seek(0, SeekOrigin.Begin);
using (var sha = new SHA1Managed())
{
checksum = sha.ComputeHash(fs);
}
}
if (file.FileHash.SequenceEqual(checksum))
{
Log.WriteInfo("FileDownloader", "Downloaded {0} from {1}", file.FileName, Steam.GetAppName(appID));
hashes[file.FileName] = checksum;
if (File.Exists(finalPath))
{
File.Delete(finalPath);
}
File.Move(downloadPath, finalPath);
if (chunks.Count > 1)
{
File.WriteAllText(oldChunksFile,
JsonConvert.SerializeObject(
chunks,
Formatting.None,
new JsonSerializerSettings {
PreserveReferencesHandling = PreserveReferencesHandling.All
}
)
);
}
else if (File.Exists(oldChunksFile))
{
File.Delete(oldChunksFile);
}
filesUpdated = true;
}
else
{
filesAnyFailed = true;
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download {3}: Only {4} out of {5} chunks downloaded ({6})",
Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL, file.FileName, count, chunks.Count, lastError);
Log.WriteError("FileDownloader", "Failed to download {0}: Only {1} out of {2} chunks downloaded from {3} ({4})",
file.FileName, count, chunks.Count, job.Server, lastError);
File.Delete(downloadPath);
}
});
if (filesAnyFailed)
{
using (var db = Database.GetConnection())
{
// Mark this depot for redownload
db.Execute("UPDATE `Depots` SET `LastManifestID` = 0 WHERE `DepotID` = @DepotID", new { job.DepotID });
}
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download some files, not running update script to prevent broken diffs.",
Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL);
}
else if (filesUpdated)
{
File.WriteAllText(hashesFile, JsonConvert.SerializeObject(hashes));
job.Result = EResult.OK;
}
else
{
job.Result = EResult.Ignored;
}
return(job.Result);
}
private static async Task <DepotFilesData> ProcessDepotManifestAndFiles(CancellationTokenSource cts,
UInt32 appId, DepotDownloadInfo depot, CDNClientPool cdnPool)
{
DepotDownloadCounter depotCounter = new DepotDownloadCounter();
FileLog.LogMessage("Processing depot {0} - {1}", depot.id, depot.contentName);
ProtoManifest oldProtoManifest = null;
ProtoManifest newProtoManifest = null;
string configDir = Path.Combine(depot.installDir, CONFIG_DIR);
ulong lastManifestId = INVALID_MANIFEST_ID;
//DepotConfigStore.Instance.InstalledManifestIDs.TryGetValue(depot.id, out lastManifestId);
//// In case we have an early exit, this will force equiv of verifyall next run.
//DepotConfigStore.Instance.InstalledManifestIDs[depot.id] = INVALID_MANIFEST_ID;
//DepotConfigStore.Save();
//if (lastManifestId != INVALID_MANIFEST_ID)
//{
// var oldManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.id, lastManifestId));
// if (File.Exists(oldManifestFileName))
// {
// byte[] expectedChecksum, currentChecksum;
// try
// {
// expectedChecksum = File.ReadAllBytes(oldManifestFileName + ".sha");
// }
// catch (IOException)
// {
// expectedChecksum = null;
// }
// oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName, out currentChecksum);
// if (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum))
// {
// // We only have to show this warning if the old manifest ID was different
// if (lastManifestId != depot.manifestId)
// FileLog.LogMessage("Manifest {0} on disk did not match the expected checksum.", lastManifestId);
// oldProtoManifest = null;
// }
// }
//}
if (lastManifestId == depot.manifestId && oldProtoManifest != null)
{
newProtoManifest = oldProtoManifest;
FileLog.LogMessage("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id);
}
else
{
var newManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.id, depot.manifestId));
if (newManifestFileName != null)
{
byte[] expectedChecksum, currentChecksum;
try
{
expectedChecksum = File.ReadAllBytes(newManifestFileName + ".sha");
}
catch (IOException)
{
expectedChecksum = null;
}
newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName, out currentChecksum);
if (newProtoManifest != null && (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum)))
{
FileLog.LogMessage("Manifest {0} on disk did not match the expected checksum.", depot.manifestId);
newProtoManifest = null;
}
}
if (newProtoManifest != null)
{
FileLog.LogMessage("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id);
}
else
{
Console.Write("Downloading depot manifest...");
DepotManifest depotManifest = null;
do
{
cts.Token.ThrowIfCancellationRequested();
CDNClient.Server connection = null;
try
{
connection = cdnPool.GetConnection(cts.Token);
var cdnToken = await cdnPool.AuthenticateConnection(appId, depot.id, connection);
#if STEAMKIT_UNRELEASED
depotManifest = await cdnPool.CDNClient.DownloadManifestAsync(depot.id, depot.manifestId,
connection, cdnToken, depot.depotKey, proxyServer : cdnPool.ProxyServer).ConfigureAwait(false);
#else
depotManifest = await cdnPool.CDNClient.DownloadManifestAsync(depot.id, depot.manifestId,
connection, cdnToken, depot.depotKey).ConfigureAwait(false);
#endif
cdnPool.ReturnConnection(connection);
}
catch (TaskCanceledException)
{
FileLog.LogMessage("Connection timeout downloading depot manifest {0} {1}", depot.id, depot.manifestId);
}
catch (SteamKitWebRequestException e)
{
cdnPool.ReturnBrokenConnection(connection);
if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden)
{
FileLog.LogMessage("Encountered 401 for depot manifest {0} {1}. Aborting.", depot.id, depot.manifestId);
break;
}
else if (e.StatusCode == HttpStatusCode.NotFound)
{
FileLog.LogMessage("Encountered 404 for depot manifest {0} {1}. Aborting.", depot.id, depot.manifestId);
break;
}
else
{
FileLog.LogMessage("Encountered error downloading depot manifest {0} {1}: {2}", depot.id, depot.manifestId, e.StatusCode);
}
}
catch (OperationCanceledException)
{
break;
}
catch (Exception e)
{
cdnPool.ReturnBrokenConnection(connection);
FileLog.LogMessage("Encountered error downloading manifest for depot {0} {1}: {2}", depot.id, depot.manifestId, e.Message);
}
}while (depotManifest == null);
if (depotManifest == null)
{
FileLog.LogMessage("\nUnable to download manifest {0} for depot {1}", depot.manifestId, depot.id);
cts.Cancel();
}
// Throw the cancellation exception if requested so that this task is marked failed
cts.Token.ThrowIfCancellationRequested();
byte[] checksum;
newProtoManifest = new ProtoManifest(depotManifest, depot.manifestId);
newProtoManifest.SaveToFile(newManifestFileName, out checksum);
File.WriteAllBytes(newManifestFileName + ".sha", checksum);
FileLog.LogMessage(" Done!");
}
}
newProtoManifest.Files.Sort((x, y) => string.Compare(x.FileName, y.FileName, StringComparison.Ordinal));
FileLog.LogMessage("Manifest {0} ({1})", depot.manifestId, newProtoManifest.CreationTime);
//if (Config.DownloadManifestOnly)
//{
// StringBuilder manifestBuilder = new StringBuilder();
// string txtManifest = Path.Combine(depot.installDir, string.Format("manifest_{0}_{1}.txt", depot.id, depot.manifestId));
// manifestBuilder.Append(string.Format("{0}\n\n", newProtoManifest.CreationTime));
// foreach (var file in newProtoManifest.Files)
// {
// if (file.Flags.HasFlag(EDepotFileFlag.Directory))
// continue;
// manifestBuilder.Append(string.Format("{0}\n", file.FileName));
// manifestBuilder.Append(string.Format("\t{0}\n", file.TotalSize));
// manifestBuilder.Append(string.Format("\t{0}\n", BitConverter.ToString(file.FileHash).Replace("-", "")));
// }
// File.WriteAllText(txtManifest, manifestBuilder.ToString());
// return null;
//}
string stagingDir = Path.Combine(depot.installDir, STAGING_DIR);
var filesAfterExclusions = newProtoManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).ToList();
var allFileNames = new HashSet <string>(filesAfterExclusions.Count);
// Pre-process
filesAfterExclusions.ForEach(file =>
{
allFileNames.Add(file.FileName);
var fileFinalPath = Path.Combine(depot.installDir, file.FileName);
var fileStagingPath = Path.Combine(stagingDir, file.FileName);
if (file.Flags.HasFlag(EDepotFileFlag.Directory))
{
Directory.CreateDirectory(fileFinalPath);
Directory.CreateDirectory(fileStagingPath);
}
else
{
// Some manifests don't explicitly include all necessary directories
Directory.CreateDirectory(Path.GetDirectoryName(fileFinalPath));
Directory.CreateDirectory(Path.GetDirectoryName(fileStagingPath));
depotCounter.CompleteDownloadSize += file.TotalSize;
}
});
return(new DepotFilesData
{
depotDownloadInfo = depot,
depotCounter = depotCounter,
stagingDir = stagingDir,
manifest = newProtoManifest,
previousManifest = oldProtoManifest,
filteredFiles = filesAfterExclusions,
allFileNames = allFileNames
});
}
private async Task DownloadDepots(uint appID, List <ManifestJob> depots)
{
Log.WriteDebug("Depot Downloader", "Will process {0} depots ({1} depot locks left)", depots.Count, DepotLocks.Count);
var processTasks = new List <Task <EResult> >();
var anyFilesDownloaded = false;
var willDownloadFiles = false;
foreach (var depot in depots)
{
if (depot.DepotKey == null)
{
await GetDepotDecryptionKey(Steam.Instance.Apps, depot, appID);
if (depot.DepotKey == null && depot.LastManifestID == depot.ManifestID)
{
RemoveLock(depot.DepotID);
continue;
}
}
depot.Server = GetContentServer();
DepotManifest depotManifest = null;
var lastError = string.Empty;
for (var i = 0; i <= 5; i++)
{
try
{
depotManifest = await CDNClient.DownloadManifestAsync(depot.DepotID, depot.ManifestID, depot.Server, string.Empty, depot.DepotKey);
break;
}
catch (Exception e)
{
lastError = e.Message;
Log.WriteError("Depot Processor", "Failed to download depot manifest for app {0} depot {1} ({2}: {3}) (#{4})", appID, depot.DepotID, depot.Server, lastError, i);
}
if (depot.DepotKey != null)
{
RemoveErroredServer(depot.Server);
}
depot.Server = GetContentServer();
if (depotManifest == null)
{
await Task.Delay(Utils.ExponentionalBackoff(i));
}
}
if (depotManifest == null)
{
RemoveLock(depot.DepotID);
if (FileDownloader.IsImportantDepot(depot.DepotID))
{
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download manifest ({3})",
Colors.OLIVE, depot.DepotName, Colors.NORMAL, lastError);
}
if (!Settings.IsFullRun && depot.DepotKey != null)
{
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
}
continue;
}
var task = ProcessDepotAfterDownload(depot, depotManifest);
processTasks.Add(task);
if (!FileDownloader.IsImportantDepot(depot.DepotID) || depot.DepotKey == null)
{
continue;
}
willDownloadFiles = true;
task = TaskManager.Run(async() =>
{
var result = EResult.Fail;
try
{
result = await FileDownloader.DownloadFilesFromDepot(depot, depotManifest);
if (result == EResult.OK)
{
anyFilesDownloaded = true;
}
}
catch (Exception e)
{
ErrorReporter.Notify($"Depot Processor {depot.DepotID}", e);
}
return(result);
}).Unwrap();
processTasks.Add(task);
}
if (SaveLocalConfig)
{
SaveLocalConfig = false;
LocalConfig.Save();
}
await Task.WhenAll(processTasks).ConfigureAwait(false);
Log.WriteDebug("Depot Downloader", $"{depots.Count} depot downloads finished for app {appID}");
// TODO: use ContinueWith on tasks
if (!anyFilesDownloaded && !willDownloadFiles)
{
foreach (var depot in depots)
{
RemoveLock(depot.DepotID);
}
return;
}
lock (UpdateScriptLock)
{
foreach (var depot in depots)
{
if (depot.Result == EResult.OK)
{
RunUpdateScript(UpdateScript, string.Format("{0} no-git", depot.DepotID));
}
else if (depot.Result != EResult.Ignored)
{
Log.WriteWarn("Depot Processor", $"Download failed for {depot.DepotID}");
// Mark this depot for redownload
var db = Database.Get();
db.Execute("UPDATE `Depots` SET `LastManifestID` = 0 WHERE `DepotID` = @DepotID", new { depot.DepotID });
}
RemoveLock(depot.DepotID);
}
// Only commit changes if all depots downloaded
if (processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored))
{
if (!RunUpdateScriptForApp(appID, depots[0].BuildID))
{
RunUpdateScript(UpdateScript, "0");
}
}
else
{
Log.WriteDebug("Depot Processor", "Reprocessing the app {0} because some files failed to download", appID);
IRC.Instance.SendOps("{0}[{1}]{2} Reprocessing the app due to download failures",
Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL
);
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
}
}
}
public async Task DownloadDepot(String path, uint appId, uint depotId, string branch,
Func <String, bool> includeFile)
{
Directory.CreateDirectory(path);
// retrieve app info
steam3.RequestAppInfo(appId);
var depotIDs = new List <uint>();
KeyValue depots = GetSteam3AppSection(appId, EAppInfoSection.Depots);
Console.WriteLine("Using app branch: '{0}'.", branch);
ulong manifestId = GetSteam3DepotManifest(depotId, appId, branch);
steam3.RequestDepotKey(depotId, appId);
byte[] depotKey = steam3.DepotKeys[depotId];
string contentName = GetAppOrDepotName(depotId, appId);
Console.WriteLine("Downloading depot {0} - {1}", depotId, contentName);
DepotManifest depotManifest = null;
CancellationTokenSource cts = new CancellationTokenSource();
CDNClientPool cdnPool = new CDNClientPool(steam3);
cdnPool.ExhaustedToken = cts;
while (depotManifest == null)
{
Tuple <CDNClient.Server, string> connection = null;
try
{
connection = await cdnPool.GetConnectionForDepot(appId, depotId, CancellationToken.None);
depotManifest = await cdnPool.CDNClient.DownloadManifestAsync(depotId, manifestId,
connection.Item1, connection.Item2, depotKey).ConfigureAwait(false);
cdnPool.ReturnConnection(connection);
}
catch (SteamKitWebRequestException e)
{
cdnPool.ReturnBrokenConnection(connection);
if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden)
{
Console.WriteLine("Encountered 401 for depot manifest {0} {1}. Aborting.", depotId, manifestId);
break;
}
else
{
Console.WriteLine("Encountered error downloading depot manifest {0} {1}: {2}", depotId,
manifestId, e.StatusCode);
}
}
catch (Exception e)
{
cdnPool.ReturnBrokenConnection(connection);
Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depotId,
manifestId, e.Message);
}
}
if (depotManifest == null)
{
Console.WriteLine("\nUnable to download manifest {0} for depot {1}", manifestId, depotId);
return;
}
ulong size_downloaded = 0;
foreach (var folder in depotManifest.Files.AsParallel().Where(f => f.Flags.HasFlag(EDepotFileFlag.Directory))
.ToList())
{
Directory.CreateDirectory(Path.Join(path, folder.FileName));
}
ulong TotalBytesCompressed = 0;
ulong TotalBytesUncompressed = 0;
ulong DepotBytesCompressed = 0;
ulong DepotBytesUncompressed = 0;
var semaphore = new SemaphoreSlim(10);
var files = depotManifest.Files.AsParallel().Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory) && includeFile(f.FileName))
.ToArray();
ulong complete_download_size = 0;
foreach (var file in files)
{
complete_download_size += file.TotalSize;
}
var tasks = new Task[files.Length];
for (var i = 0; i < files.Length; i++)
{
var file = files[i];
var task = Task.Run(async() =>
{
cts.Token.ThrowIfCancellationRequested();
try
{
await semaphore.WaitAsync().ConfigureAwait(false);
cts.Token.ThrowIfCancellationRequested();
string fileFinalPath = Path.Combine(path, file.FileName);
FileStream fs = null;
List <DepotManifest.ChunkData> neededChunks = new List <DepotManifest.ChunkData>();
FileInfo fi = new FileInfo(fileFinalPath);
if (!fi.Exists)
{
// create new file. need all chunks
fs = File.Create(fileFinalPath);
fs.SetLength((long)file.TotalSize);
neededChunks = new List <DepotManifest.ChunkData>(file.Chunks);
}
else
{
fs = File.Open(fileFinalPath, FileMode.Truncate);
fs.SetLength((long)file.TotalSize);
neededChunks = new List <DepotManifest.ChunkData>(file.Chunks);
}
foreach (var chunk in neededChunks)
{
if (cts.IsCancellationRequested)
{
break;
}
string chunkID = Util.EncodeHexString(chunk.ChunkID);
CDNClient.DepotChunk chunkData = null;
while (!cts.IsCancellationRequested)
{
Tuple <CDNClient.Server, string> connection;
try
{
connection = await cdnPool.GetConnectionForDepot(appId, depotId, cts.Token);
}
catch (OperationCanceledException)
{
break;
}
DepotManifest.ChunkData data = new DepotManifest.ChunkData();
data.ChunkID = chunk.ChunkID;
data.Checksum = chunk.Checksum;
data.Offset = chunk.Offset;
data.CompressedLength = chunk.CompressedLength;
data.UncompressedLength = chunk.UncompressedLength;
try
{
chunkData = await cdnPool.CDNClient.DownloadDepotChunkAsync(depotId, data,
connection.Item1, connection.Item2, depotKey).ConfigureAwait(false);
cdnPool.ReturnConnection(connection);
break;
}
catch (SteamKitWebRequestException e)
{
cdnPool.ReturnBrokenConnection(connection);
if (e.StatusCode == HttpStatusCode.Unauthorized ||
e.StatusCode == HttpStatusCode.Forbidden)
{
Console.WriteLine("Encountered 401 for chunk {0}. Aborting.", chunkID);
cts.Cancel();
break;
}
else
{
Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID,
e.StatusCode);
}
}
catch (Exception e)
{
cdnPool.ReturnBrokenConnection(connection);
Console.WriteLine("Encountered unexpected error downloading chunk {0}: {1}",
chunkID, e.Message);
}
}
if (chunkData == null)
{
Console.WriteLine("Failed to find any server with chunk {0} for depot {1}. Aborting.",
chunkID, depotId);
cts.Cancel();
}
// Throw the cancellation exception if requested so that this task is marked failed
cts.Token.ThrowIfCancellationRequested();
TotalBytesCompressed += chunk.CompressedLength;
DepotBytesCompressed += chunk.CompressedLength;
TotalBytesUncompressed += chunk.UncompressedLength;
DepotBytesUncompressed += chunk.UncompressedLength;
fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
fs.Write(chunkData.Data, 0, chunkData.Data.Length);
size_downloaded += chunk.UncompressedLength;
}
fs.Dispose();
Console.WriteLine("{0,6:#00.00}% {1}",
((float)size_downloaded / (float)complete_download_size) * 100.0f, fileFinalPath);
}
finally
{
semaphore.Release();
}
});
tasks[i] = task;
}
await Task.WhenAll(tasks).ConfigureAwait(false);
Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depotId,
DepotBytesCompressed, DepotBytesUncompressed);
}
public static void DownloadFilesFromDepot(DepotProcessor.ManifestJob job, DepotManifest depotManifest)
{
var files = depotManifest.Files.Where(x => IsFileNameMatching(job.DepotID, x.FileName)).ToList();
var filesUpdated = false;
Log.WriteDebug("FileDownloader", "Will download {0} files from depot {1}", files.Count(), job.DepotID);
foreach (var file in files)
{
string directory = Path.Combine(Application.Path, FILES_DIRECTORY, job.DepotID.ToString(), Path.GetDirectoryName(file.FileName));
string finalPath = Path.Combine(directory, Path.GetFileName(file.FileName));
string downloadPath = Path.GetTempFileName();
if (!Directory.Exists(directory))
{
Directory.CreateDirectory(directory);
}
else if (File.Exists(finalPath))
{
using (var fs = File.Open(finalPath, FileMode.Open))
{
using (var sha = new SHA1Managed())
{
if (file.FileHash.SequenceEqual(sha.ComputeHash(fs)))
{
Log.WriteDebug("FileDownloader", "{0} already matches the file we have", file.FileName);
continue;
}
}
}
}
Log.WriteInfo("FileDownloader", "Downloading {0} ({1} bytes, {2} chunks)", file.FileName, file.TotalSize, file.Chunks.Count);
uint count = 0;
byte[] checksum;
string lastError = "or checksum failed";
using (var fs = File.Open(downloadPath, FileMode.OpenOrCreate))
{
fs.SetLength((long)file.TotalSize);
var lockObject = new object();
// TODO: We *could* verify each chunk and only download needed ones
Parallel.ForEach(file.Chunks, (chunk, state) =>
{
var downloaded = false;
for (var i = 0; i <= 5; i++)
{
try
{
var chunkData = CDNClient.DownloadDepotChunk(job.DepotID, chunk, job.Server, job.CDNToken, job.DepotKey);
lock (lockObject)
{
fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
fs.Write(chunkData.Data, 0, chunkData.Data.Length);
Log.WriteDebug("FileDownloader", "Downloaded {0} ({1}/{2})", file.FileName, ++count, file.Chunks.Count);
}
downloaded = true;
break;
}
catch (Exception e)
{
lastError = e.Message;
}
}
if (!downloaded)
{
state.Stop();
}
});
fs.Seek(0, SeekOrigin.Begin);
using (var sha = new SHA1Managed())
{
checksum = sha.ComputeHash(fs);
}
}
if (file.Chunks.Count == 0 || file.FileHash.SequenceEqual(checksum))
{
Log.WriteInfo("FileDownloader", "Downloaded {0} from {1}", file.FileName, Steam.GetAppName(job.ParentAppID));
if (File.Exists(finalPath))
{
File.Delete(finalPath);
}
File.Move(downloadPath, finalPath);
filesUpdated = true;
}
else
{
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download {3}: Only {4} out of {5} chunks downloaded ({6})",
Colors.OLIVE, Steam.GetAppName(job.ParentAppID), Colors.NORMAL, file.FileName, count, file.Chunks.Count, lastError);
Log.WriteError("FileDownloader", "Failed to download {0}: Only {1} out of {2} chunks downloaded from {3} ({4})",
file.FileName, count, file.Chunks.Count, job.Server, lastError);
File.Delete(downloadPath);
}
}
if (filesUpdated)
{
var updateScript = Path.Combine(Application.Path, "files", "update.sh");
if (File.Exists(updateScript))
{
// YOLO
Process.Start(updateScript, job.DepotID.ToString());
}
}
}
private static async Task <EResult> ProcessDepotAfterDownload(IDbConnection db, IDbTransaction transaction, ManifestJob request, DepotManifest depotManifest)
{
var filesOld = (await db.QueryAsync <DepotFile>("SELECT `File`, `Hash`, `Size`, `Flags` FROM `DepotsFiles` WHERE `DepotID` = @DepotID", new { request.DepotID }, transaction)).ToDictionary(x => x.File, x => x);
var filesAdded = new List <DepotFile>();
var shouldHistorize = filesOld.Count > 0 && !depotManifest.FilenamesEncrypted; // Don't historize file additions if we didn't have any data before
if (request.StoredFilenamesEncrypted && !depotManifest.FilenamesEncrypted)
{
Log.WriteInfo(nameof(DepotProcessor), $"Depot {request.DepotID} will decrypt stored filenames");
var decryptedFilesOld = new Dictionary <string, DepotFile>();
foreach (var file in filesOld.Values)
{
var oldFile = file.File;
file.File = DecryptFilename(oldFile, request.DepotKey);
decryptedFilesOld.Add(file.File, file);
await db.ExecuteAsync("UPDATE `DepotsFiles` SET `File` = @File WHERE `DepotID` = @DepotID AND `File` = @OldFile", new
{
request.DepotID,
file.File,
OldFile = oldFile
}, transaction);
}
await MakeHistory(db, transaction, request, string.Empty, "files_decrypted");
filesOld = decryptedFilesOld;
}
foreach (var file in depotManifest.Files.OrderByDescending(x => x.FileName))
{
var name = depotManifest.FilenamesEncrypted ? file.FileName.Replace("\n", "") : file.FileName.Replace('\\', '/');
byte[] hash = null;
// Store empty hashes as NULL (e.g. an empty file)
if ((file.Flags & EDepotFileFlag.Directory) == 0 && file.FileHash.Length > 0 && file.FileHash.Any(c => c != 0))
{
hash = file.FileHash;
}
// Limit path names to 260 characters (default windows max length)
// File column is varchar(260) and not higher to prevent reducing performance
// See https://stackoverflow.com/questions/1962310/importance-of-varchar-length-in-mysql-table/1962329#1962329
// Until 2019 there hasn't been a single file that went over this limit, so far there has been only one
// game with a big node_modules path, so we're safeguarding by limiting it.
if (name.Length > 260)
{
if (depotManifest.FilenamesEncrypted)
{
continue;
}
using var sha = SHA1.Create();
var nameHash = Utils.ByteArrayToString(sha.ComputeHash(Encoding.UTF8.GetBytes(name)));
name = $"{{SteamDB file name is too long}}/{nameHash}/...{name.Substring(name.Length - 150)}";
}
if (filesOld.ContainsKey(name))
{
var oldFile = filesOld[name];
var updateFile = false;
if (oldFile.Size != file.TotalSize || !Utils.ByteArrayEquals(hash, oldFile.Hash))
{
await MakeHistory(db, transaction, request, name, "modified", oldFile.Size, file.TotalSize);
updateFile = true;
}
if (oldFile.Flags != file.Flags)
{
await MakeHistory(db, transaction, request, name, "modified_flags", (ulong)oldFile.Flags, (ulong)file.Flags);
updateFile = true;
}
if (updateFile)
{
await db.ExecuteAsync("UPDATE `DepotsFiles` SET `Hash` = @Hash, `Size` = @Size, `Flags` = @Flags WHERE `DepotID` = @DepotID AND `File` = @File", new DepotFile
{
DepotID = request.DepotID,
File = name,
Hash = hash,
Size = file.TotalSize,
Flags = file.Flags
}, transaction);
}
filesOld.Remove(name);
}
else
{
// We want to historize modifications first, and only then deletions and additions
filesAdded.Add(new DepotFile
{
DepotID = request.DepotID,
Hash = hash,
File = name,
Size = file.TotalSize,
Flags = file.Flags
});
}
}
if (filesOld.Count > 0)
{
// Chunk file deletion queries so it doesn't go over max_allowed_packet
var filesOldChunks = filesOld.Select(x => x.Value.File).Split(1000);
foreach (var filesOldChunk in filesOldChunks)
{
await db.ExecuteAsync("DELETE FROM `DepotsFiles` WHERE `DepotID` = @DepotID AND `File` IN @Files",
new
{
request.DepotID,
Files = filesOldChunk,
}, transaction);
}
if (shouldHistorize)
{
await db.ExecuteAsync(HistoryQuery, filesOld.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ManifestID = request.ManifestID,
ChangeID = request.ChangeNumber,
Action = "removed",
File = x.Value.File,
OldValue = x.Value.Size
}), transaction);
}
}
if (filesAdded.Count > 0)
{
await db.ExecuteAsync("INSERT INTO `DepotsFiles` (`DepotID`, `File`, `Hash`, `Size`, `Flags`) VALUES (@DepotID, @File, @Hash, @Size, @Flags)", filesAdded, transaction);
if (shouldHistorize)
{
await db.ExecuteAsync(HistoryQuery, filesAdded.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ManifestID = request.ManifestID,
ChangeID = request.ChangeNumber,
Action = "added",
File = x.File,
NewValue = x.Size
}), transaction);
}
}
await db.ExecuteAsync(
request.LastManifestID == request.ManifestID?
"UPDATE `Depots` SET `LastManifestID` = @ManifestID, `ManifestDate` = @ManifestDate, `FilenamesEncrypted` = @FilenamesEncrypted, `SizeOriginal` = @SizeOriginal, `SizeCompressed` = @SizeCompressed WHERE `DepotID` = @DepotID" :
"UPDATE `Depots` SET `LastManifestID` = @ManifestID, `ManifestDate` = @ManifestDate, `FilenamesEncrypted` = @FilenamesEncrypted, `SizeOriginal` = @SizeOriginal, `SizeCompressed` = @SizeCompressed, `LastUpdated` = CURRENT_TIMESTAMP() WHERE `DepotID` = @DepotID",
new
{
request.DepotID,
request.ManifestID,
depotManifest.FilenamesEncrypted,
ManifestDate = depotManifest.CreationTime,
SizeOriginal = depotManifest.TotalUncompressedSize,
SizeCompressed = depotManifest.TotalCompressedSize,
}, transaction);
return(EResult.OK);
}
private async Task DownloadDepots(IEnumerable <ManifestJob> depots)
{
Log.WriteDebug("Depot Downloader", "Will process {0} depots ({1} depot locks left)", depots.Count(), DepotLocks.Count);
var processTasks = new List <Task <EResult> >();
bool anyFilesDownloaded = false;
foreach (var depot in depots)
{
depot.DepotKey = await GetDepotDecryptionKey(depot.DepotID, depot.ParentAppID);
if (depot.DepotKey == null)
{
RemoveLock(depot.DepotID);
continue;
}
var cdnToken = await GetCDNAuthToken(depot.DepotID);
if (cdnToken == null)
{
RemoveLock(depot.DepotID);
Log.WriteDebug("Depot Downloader", "Got a depot key for depot {0} but no cdn auth token", depot.DepotID);
continue;
}
depot.CDNToken = cdnToken.Token;
depot.Server = cdnToken.Server;
DepotManifest depotManifest = null;
string lastError = string.Empty;
for (var i = 0; i <= 5; i++)
{
try
{
depotManifest = CDNClient.DownloadManifest(depot.DepotID, depot.ManifestID, depot.Server, depot.CDNToken, depot.DepotKey);
break;
}
catch (Exception e)
{
Log.WriteWarn("Depot Downloader", "{0} Manifest download failed: {1} - {2}", depot.DepotID, e.GetType(), e.Message);
lastError = e.Message;
}
// TODO: get new auth key if auth fails
if (depotManifest == null)
{
await Task.Delay(Utils.ExponentionalBackoff(i));
}
}
if (depotManifest == null)
{
RemoveLock(depot.DepotID);
Log.WriteError("Depot Processor", "Failed to download depot manifest for depot {0} ({1}: {2})", depot.DepotID, depot.Server, lastError);
if (FileDownloader.IsImportantDepot(depot.DepotID))
{
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download depot {3} manifest ({4}: {5})",
Colors.OLIVE, Steam.GetAppName(depot.ParentAppID), Colors.NORMAL, depot.DepotID, depot.Server, lastError);
}
continue;
}
var task = TaskManager.Run(() =>
{
using (var db = Database.GetConnection())
{
return(ProcessDepotAfterDownload(db, depot, depotManifest));
}
});
processTasks.Add(task);
if (FileDownloader.IsImportantDepot(depot.DepotID))
{
task = TaskManager.Run(() =>
{
var result = FileDownloader.DownloadFilesFromDepot(depot, depotManifest);
if (result == EResult.OK)
{
anyFilesDownloaded = true;
}
return(result);
}, TaskCreationOptions.LongRunning);
TaskManager.RegisterErrorHandler(task);
processTasks.Add(task);
}
}
if (SaveLocalConfig)
{
SaveLocalConfig = false;
LocalConfig.Save();
}
await Task.WhenAll(processTasks);
// TODO: use ContinueWith on tasks
if (!anyFilesDownloaded)
{
Log.WriteDebug("Depot Downloader", "Tasks awaited for {0} depot downloads", depots.Count());
foreach (var depot in depots)
{
RemoveLock(depot.DepotID);
}
return;
}
var canUpdate = processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored) && File.Exists(UpdateScript);
#if true
Log.WriteDebug("Depot Downloader", "Tasks awaited for {0} depot downloads (will run script: {1})", depots.Count(), canUpdate);
#endif
bool lockTaken = false;
try
{
UpdateScriptLock.Enter(ref lockTaken);
foreach (var depot in depots)
{
// TODO: this only needs to run if any downloaded files changed
if (canUpdate && FileDownloader.IsImportantDepot(depot.DepotID))
{
RunUpdateScript(string.Format("{0} no-git", depot.DepotID));
}
RemoveLock(depot.DepotID);
}
if (canUpdate)
{
RunUpdateScript("0");
}
}
finally
{
if (lockTaken)
{
UpdateScriptLock.Exit();
}
}
}
private void DownloadManifest(ManifestJob request)
{
Log.WriteInfo("Depot Processor", "DepotID: {0}", request.DepotID);
DepotManifest depotManifest = null;
string lastError = string.Empty;
// CDN is very random, just keep trying
for (var i = 0; i <= 5; i++)
{
try
{
depotManifest = CDNClient.DownloadManifest(request.DepotID, request.ManifestID, request.Server, request.CDNToken, request.DepotKey);
break;
}
catch (Exception e)
{
lastError = e.Message;
}
}
if (depotManifest == null)
{
Log.WriteError("Depot Processor", "Failed to download depot manifest for depot {0} ({1}: {2}) (#{3})", request.DepotID, request.Server, lastError, request.Tries);
if (--request.Tries >= 0)
{
request.Server = GetContentServer(request.Tries);
JobManager.AddJob(() => Steam.Instance.Apps.GetCDNAuthToken(request.DepotID, request.Server), request);
return;
}
RemoveLock(request.DepotID); // TODO: Remove this once task in OnCDNAuthTokenCallback is used
if (FileDownloader.IsImportantDepot(request.DepotID))
{
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download depot {3} manifest ({4}: {5})",
Colors.OLIVE, Steam.GetAppName(request.ParentAppID), Colors.NORMAL, request.DepotID, request.Server, lastError);
}
return;
}
if (FileDownloader.IsImportantDepot(request.DepotID))
{
TaskManager.Run(() => FileDownloader.DownloadFilesFromDepot(request, depotManifest));
}
// TODO: Task here instead of in OnCDNAuthTokenCallback due to mono's silly threadpool
TaskManager.Run(() =>
{
using (var db = Database.GetConnection())
{
ProcessDepotAfterDownload(db, request, depotManifest);
}
}).ContinueWith(task =>
{
RemoveLock(request.DepotID);
Log.WriteDebug("Depot Processor", "Processed depot {0} ({1} depot locks left)", request.DepotID, DepotLocks.Count);
});
}
private EResult ProcessDepotAfterDownload(IDbConnection db, ManifestJob request, DepotManifest depotManifest)
{
var filesOld = db.Query <DepotFile>("SELECT `ID`, `File`, `Hash`, `Size`, `Flags` FROM `DepotsFiles` WHERE `DepotID` = @DepotID", new { request.DepotID }).ToDictionary(x => x.File, x => x);
var filesNew = new List <DepotFile>();
var filesAdded = new List <DepotFile>();
var shouldHistorize = filesOld.Any(); // Don't historize file additions if we didn't have any data before
foreach (var file in depotManifest.Files)
{
var name = file.FileName.Replace('\\', '/');
// safe guard
if (name.Length > 255)
{
ErrorReporter.Notify(new OverflowException(string.Format("File \"{0}\" in depot {1} is too long", name, request.DepotID)));
continue;
}
var depotFile = new DepotFile
{
DepotID = request.DepotID,
File = name,
Size = file.TotalSize,
Flags = file.Flags
};
if (file.FileHash.Length > 0 && !file.Flags.HasFlag(EDepotFileFlag.Directory))
{
depotFile.Hash = Utils.ByteArrayToString(file.FileHash);
}
else
{
depotFile.Hash = "0000000000000000000000000000000000000000";
}
filesNew.Add(depotFile);
}
foreach (var file in filesNew)
{
if (filesOld.ContainsKey(file.File))
{
var oldFile = filesOld[file.File];
var updateFile = false;
if (oldFile.Size != file.Size || !file.Hash.Equals(oldFile.Hash))
{
MakeHistory(db, request, file.File, "modified", oldFile.Size, file.Size);
updateFile = true;
}
if (oldFile.Flags != file.Flags)
{
MakeHistory(db, request, file.File, "modified_flags", (ulong)oldFile.Flags, (ulong)file.Flags);
updateFile = true;
}
if (updateFile)
{
file.ID = oldFile.ID;
db.Execute("UPDATE `DepotsFiles` SET `Hash` = @Hash, `Size` = @Size, `Flags` = @Flags WHERE `DepotID` = @DepotID AND `ID` = @ID", file);
}
filesOld.Remove(file.File);
}
else
{
// We want to historize modifications first, and only then deletions and additions
filesAdded.Add(file);
}
}
if (filesOld.Any())
{
db.Execute("DELETE FROM `DepotsFiles` WHERE `DepotID` = @DepotID AND `ID` IN @Files", new { request.DepotID, Files = filesOld.Select(x => x.Value.ID) });
db.Execute(GetHistoryQuery(), filesOld.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ChangeID = request.ChangeNumber,
Action = "removed",
File = x.Value.File
}));
}
if (filesAdded.Any())
{
db.Execute("INSERT INTO `DepotsFiles` (`DepotID`, `File`, `Hash`, `Size`, `Flags`) VALUES (@DepotID, @File, @Hash, @Size, @Flags)", filesAdded);
if (shouldHistorize)
{
db.Execute(GetHistoryQuery(), filesAdded.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ChangeID = request.ChangeNumber,
Action = "added",
File = x.File
}));
}
}
db.Execute("UPDATE `Depots` SET `LastManifestID` = @ManifestID WHERE `DepotID` = @DepotID", new { request.DepotID, request.ManifestID });
return(EResult.OK);
}
public List <DLC> GetInstalledDLCFiles()
{
List <DLC> dlcList = new List <DLC>();
if (AppManifestPath != null && File.Exists(AppManifestPath))
{
KeyValue appManifest = KeyValue.LoadAsText(AppManifestPath);
Dictionary <string, string> depotManifests = new Dictionary <string, string>();
foreach (KeyValue mountedDepot in appManifest["MountedDepots"].Children)
{
depotManifests[mountedDepot.Name] = mountedDepot.Value;
}
foreach (KeyValue mountedDepot in appManifest["InstalledDepots"].Children)
{
depotManifests[mountedDepot.Name] = mountedDepot["manifest"].Value;
}
foreach (KeyValuePair <string, string> depotManifest in depotManifests)
{
uint dlcappid = Convert.ToUInt32(depotManifest.Key);
string manifestPath = Path.Combine(SteamPath, "depotcache", $"{dlcappid}_{depotManifest.Value}.manifest");
if (File.Exists(manifestPath))
{
DLC dlc = new DLC(dlcappid);
DepotManifest manifest = DepotManifest.Deserialize(File.ReadAllBytes(manifestPath));
foreach (DepotManifest.FileData file in manifest.Files)
{
if (file.Flags.HasFlag(EDepotFileFlag.Directory))
{
continue;
}
string fileName = file.FileName.ToLower();
string extension = Path.GetExtension(fileName).ToLower();
if (fileName.Contains("assets"))
{
if (extension.Contains("xml") || extension.Contains("bin"))
{
dlc.IncludedFiles.Add(NormalizePath(GetRelativePath(Path.Combine(RWPath, "Assets"), Path.Combine(RWPath, fileName))));
}
if (extension == ".ap")
{
string absoluteFileName = Path.Combine(RWPath, fileName);
try
{
ZipArchive zipFile = ZipFile.OpenRead(absoluteFileName);
dlc.IncludedFiles.AddRange(from x in zipFile.Entries where (x.FullName.Contains(".xml") || x.FullName.Contains(".bin")) select NormalizePath(GetRelativePath(Path.Combine(RWPath, "Assets"), Path.Combine(Path.GetDirectoryName(absoluteFileName), x.FullName))));
}
catch { }
}
}
}
dlcList.Add(dlc);
}
}
}
return(dlcList);
}
private static async Task DownloadSteam3Async(uint appId, List <DepotDownloadInfo> depots, CancellationToken cancellationToken)
{
ulong TotalBytesCompressed = 0;
ulong TotalBytesUncompressed = 0;
foreach (var depot in depots)
{
ulong DepotBytesCompressed = 0;
ulong DepotBytesUncompressed = 0;
Console.WriteLine("Downloading depot {0} - {1}", depot.Id, depot.ContentName);
CancellationTokenSource cts = new CancellationTokenSource();
cdnPool.ExhaustedToken = cts;
ProtoManifest oldProtoManifest = null;
ProtoManifest newProtoManifest = null;
string configDir = Path.Combine(depot.InstallDir, CONFIG_DIR);
ulong lastManifestId = INVALID_MANIFEST_ID;
DepotConfigStore.Instance.InstalledManifestIDs.TryGetValue(depot.Id, out lastManifestId);
// In case we have an early exit, this will force equiv of verifyall next run.
DepotConfigStore.Instance.InstalledManifestIDs[depot.Id] = INVALID_MANIFEST_ID;
DepotConfigStore.Save();
if (lastManifestId != INVALID_MANIFEST_ID)
{
var oldManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", lastManifestId));
if (File.Exists(oldManifestFileName))
{
byte[] expectedChecksum, currentChecksum;
try
{
expectedChecksum = File.ReadAllBytes(oldManifestFileName + ".sha");
}
catch (IOException)
{
expectedChecksum = null;
}
oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName, out currentChecksum);
if (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum))
{
// We only have to show this warning if the old manifest ID was different
if (lastManifestId != depot.ManifestId)
{
Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", lastManifestId);
}
oldProtoManifest = null;
}
}
}
if (lastManifestId == depot.ManifestId && oldProtoManifest != null)
{
newProtoManifest = oldProtoManifest;
Console.WriteLine("Already have manifest {0} for depot {1}.", depot.ManifestId, depot.Id);
}
else
{
var newManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.Id, depot.ManifestId));
if (newManifestFileName != null)
{
byte[] expectedChecksum, currentChecksum;
try
{
expectedChecksum = File.ReadAllBytes(newManifestFileName + ".sha");
}
catch (IOException)
{
expectedChecksum = null;
}
newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName, out currentChecksum);
if (newProtoManifest != null && (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum)))
{
Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", depot.ManifestId);
newProtoManifest = null;
}
}
if (newProtoManifest != null)
{
Console.WriteLine("Already have manifest {0} for depot {1}.", depot.ManifestId, depot.Id);
}
else
{
Console.Write("Downloading depot manifest...");
DepotManifest depotManifest = await DownloadManifest(appId, depot.Id, depot.ManifestId, depot.DepotKey).ConfigureAwait(false);
byte[] checksum;
newProtoManifest = new ProtoManifest(depotManifest, depot.ManifestId);
newProtoManifest.SaveToFile(newManifestFileName, out checksum);
File.WriteAllBytes(newManifestFileName + ".sha", checksum);
Console.WriteLine(" Done!");
}
}
newProtoManifest.Files.Sort((x, y) => string.Compare(x.FileName, y.FileName, StringComparison.Ordinal));
Console.WriteLine("Manifest {0} ({1})", depot.ManifestId, newProtoManifest.CreationTime);
ulong complete_download_size = 0;
ulong size_downloaded = 0;
var downloadChanges = new ConcurrentQueue <(DateTime, ulong)>();
object downloadChangesLock = new object();
void reportProgress(string currentFilePath)
{
lock (downloadChangesLock)
{
while (downloadChanges.Count > 50) // Only average last 50 deltas
{
downloadChanges.TryDequeue(out _);
}
}
ulong bytesPerSecond = 0;
if (downloadChanges.Count > 0)
{
lock (downloadChangesLock)
{
TimeSpan timeSpan = downloadChanges.Last().Item1 - downloadChanges.First().Item1;
ulong totalBytes = downloadChanges.Aggregate(0ul, (a, b) => a + b.Item2);
double rate = totalBytes / timeSpan.TotalSeconds;
if (double.IsNaN(rate) || double.IsInfinity(rate))
{
rate = 0;
}
bytesPerSecond = (ulong)rate;
}
}
Globals.UiDispatcher.Invoke(() =>
{
Globals.AppState.DownloadState.DownloadPercentageComplete = ((double)size_downloaded / complete_download_size) * 100;
Globals.AppState.DownloadState.DownloadCurrentFile = currentFilePath;
Globals.AppState.DownloadState.DownloadedBytes = size_downloaded;
Globals.AppState.DownloadState.BytesPerSecond = bytesPerSecond;
});
}
string stagingDir = Path.Combine(depot.InstallDir, STAGING_DIR);
// Delete files that are removed in new manifest
if (oldProtoManifest != null)
{
var directoriesToCheck = new HashSet <string>(); // Directories to check if they're empty and delete
foreach (ProtoManifest.FileData oldFile in oldProtoManifest.Files)
{
if (!newProtoManifest.Files.Any(newFile => newFile.FileName == oldFile.FileName))
{
Console.WriteLine($"Deleting file: {oldFile.FileName}");
var fileFinalPath = Path.Combine(depot.InstallDir, oldFile.FileName);
if (File.Exists(fileFinalPath))
{
File.Delete(fileFinalPath);
directoriesToCheck.Add(Path.GetDirectoryName(fileFinalPath));
}
}
}
// Delete the now empty directories
foreach (string directoryPath in directoriesToCheck)
{
if (Directory.GetDirectories(directoryPath).Length == 0 && Directory.GetFiles(directoryPath).Length == 0)
{
Directory.Delete(directoryPath);
}
}
}
// Pre-process
newProtoManifest.Files.ForEach(file =>
{
var fileFinalPath = Path.Combine(depot.InstallDir, file.FileName);
var fileStagingPath = Path.Combine(stagingDir, file.FileName);
if (file.Flags.HasFlag(EDepotFileFlag.Directory))
{
Directory.CreateDirectory(fileFinalPath);
Directory.CreateDirectory(fileStagingPath);
}
else
{
// Some manifests don't explicitly include all necessary directories
Directory.CreateDirectory(Path.GetDirectoryName(fileFinalPath));
Directory.CreateDirectory(Path.GetDirectoryName(fileStagingPath));
complete_download_size += file.TotalSize;
}
});
Globals.UiDispatcher.Invoke(() =>
{
Globals.AppState.DownloadState.TotalBytes = complete_download_size;
});
cancellationToken.ThrowIfCancellationRequested();
var semaphore = new SemaphoreSlim(Config.MaxDownloads);
var files = newProtoManifest.Files.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory)).ToArray();
var tasks = new Task[files.Length];
for (var i = 0; i < files.Length; i++)
{
var file = files[i];
var task = Task.Run(async() =>
{
cts.Token.ThrowIfCancellationRequested();
try
{
await semaphore.WaitAsync().ConfigureAwait(false);
cts.Token.ThrowIfCancellationRequested();
cancellationToken.ThrowIfCancellationRequested();
string fileFinalPath = Path.Combine(depot.InstallDir, file.FileName);
string fileStagingPath = Path.Combine(stagingDir, file.FileName);
// This may still exist if the previous run exited before cleanup
if (File.Exists(fileStagingPath))
{
File.Delete(fileStagingPath);
}
FileStream fs = null;
List <ProtoManifest.ChunkData> neededChunks;
FileInfo fi = new FileInfo(fileFinalPath);
if (!fi.Exists)
{
// create new file. need all chunks
fs = File.Create(fileFinalPath);
fs.SetLength((long)file.TotalSize);
neededChunks = new List <ProtoManifest.ChunkData>(file.Chunks);
}
else
{
// open existing
ProtoManifest.FileData oldManifestFile = null;
if (oldProtoManifest != null)
{
oldManifestFile = oldProtoManifest.Files.SingleOrDefault(f => f.FileName == file.FileName);
}
if (oldManifestFile != null)
{
neededChunks = new List <ProtoManifest.ChunkData>();
if (Config.VerifyAll || !oldManifestFile.FileHash.SequenceEqual(file.FileHash))
{
// we have a version of this file, but it doesn't fully match what we want
var matchingChunks = new List <ChunkMatch>();
foreach (var chunk in file.Chunks)
{
var oldChunk = oldManifestFile.Chunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID));
if (oldChunk != null)
{
matchingChunks.Add(new ChunkMatch(oldChunk, chunk));
}
else
{
neededChunks.Add(chunk);
}
}
File.Move(fileFinalPath, fileStagingPath);
fs = File.Open(fileFinalPath, FileMode.Create);
fs.SetLength((long)file.TotalSize);
using (var fsOld = File.Open(fileStagingPath, FileMode.Open))
{
foreach (var match in matchingChunks)
{
fsOld.Seek((long)match.OldChunk.Offset, SeekOrigin.Begin);
byte[] tmp = new byte[match.OldChunk.UncompressedLength];
fsOld.Read(tmp, 0, tmp.Length);
byte[] adler = Util.AdlerHash(tmp);
if (!adler.SequenceEqual(match.OldChunk.Checksum))
{
neededChunks.Add(match.NewChunk);
}
else
{
fs.Seek((long)match.NewChunk.Offset, SeekOrigin.Begin);
fs.Write(tmp, 0, tmp.Length);
}
}
}
File.Delete(fileStagingPath);
}
}
else
{
// No old manifest or file not in old manifest. We must validate.
fs = File.Open(fileFinalPath, FileMode.Open);
if ((ulong)fi.Length != file.TotalSize)
{
fs.SetLength((long)file.TotalSize);
}
neededChunks = Util.ValidateSteam3FileChecksums(fs, file.Chunks.OrderBy(x => x.Offset).ToArray());
}
if (neededChunks.Count() == 0)
{
size_downloaded += file.TotalSize;
reportProgress(file.FileName);
if (fs != null)
{
fs.Dispose();
}
return;
}
else
{
ulong sizeDelta = (file.TotalSize - (ulong)neededChunks.Select(x => (long)x.UncompressedLength).Sum());
size_downloaded += sizeDelta;
reportProgress(file.FileName);
}
}
foreach (var chunk in neededChunks)
{
if (cts.IsCancellationRequested)
{
break;
}
string chunkID = Util.EncodeHexString(chunk.ChunkID);
CDNClient.DepotChunk chunkData = null;
while (!cts.IsCancellationRequested)
{
Tuple <CDNClient.Server, string> connection;
try
{
connection = await cdnPool.GetConnectionForDepot(appId, depot.Id, cts.Token);
}
catch (OperationCanceledException)
{
break;
}
DepotManifest.ChunkData data = new DepotManifest.ChunkData();
data.ChunkID = chunk.ChunkID;
data.Checksum = chunk.Checksum;
data.Offset = chunk.Offset;
data.CompressedLength = chunk.CompressedLength;
data.UncompressedLength = chunk.UncompressedLength;
try
{
chunkData = await cdnPool.CDNClient.DownloadDepotChunkAsync(depot.Id, data,
connection.Item1, connection.Item2, depot.DepotKey).ConfigureAwait(false);
cdnPool.ReturnConnection(connection);
break;
}
catch (SteamKitWebRequestException e)
{
cdnPool.ReturnBrokenConnection(connection);
if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden)
{
Console.WriteLine("Encountered 401 for chunk {0}. Aborting.", chunkID);
cts.Cancel();
break;
}
else
{
Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, e.StatusCode);
}
}
catch (TaskCanceledException)
{
Console.WriteLine("Connection timeout downloading chunk {0}", chunkID);
}
catch (Exception e)
{
cdnPool.ReturnBrokenConnection(connection);
Console.WriteLine("Encountered unexpected error downloading chunk {0}: {1}", chunkID, e.Message);
}
}
if (chunkData == null)
{
Console.WriteLine("Failed to find any server with chunk {0} for depot {1}. Aborting.", chunkID, depot.Id);
cts.Cancel();
}
// Throw the cancellation exception if requested so that this task is marked failed
cts.Token.ThrowIfCancellationRequested();
cancellationToken.ThrowIfCancellationRequested();
TotalBytesCompressed += chunk.CompressedLength;
DepotBytesCompressed += chunk.CompressedLength;
TotalBytesUncompressed += chunk.UncompressedLength;
DepotBytesUncompressed += chunk.UncompressedLength;
fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
fs.Write(chunkData.Data, 0, chunkData.Data.Length);
size_downloaded += chunk.UncompressedLength;
downloadChanges.Enqueue((DateTime.Now, chunk.CompressedLength));
reportProgress(file.FileName);
}
fs.Dispose();
reportProgress(file.FileName);
}
finally
{
semaphore.Release();
}
});
tasks[i] = task;
}
await Task.WhenAll(tasks).ConfigureAwait(false);
DepotConfigStore.Instance.InstalledManifestIDs[depot.Id] = depot.ManifestId;
DepotConfigStore.Save();
Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.Id, DepotBytesCompressed, DepotBytesUncompressed);
reportProgress(null);
}
Console.WriteLine("Total downloaded: {0} bytes ({1} bytes uncompressed) from {2} depots", TotalBytesCompressed, TotalBytesUncompressed, depots.Count);
}
private static void VerifyApp(SteamClientUtils steamClient, string appManifestPath, bool deleteUnknownFiles)
{
Console.WriteLine($"Parsing {appManifestPath}");
var kv = KeyValue.LoadAsText(appManifestPath);
var mountedDepots = kv["MountedDepots"];
var gamePath = Path.Join(Path.GetDirectoryName(appManifestPath), "common", kv["installdir"].Value);
if (!Directory.Exists(gamePath))
{
throw new DirectoryNotFoundException($"Game not found: {gamePath}");
}
Console.WriteLine($"Scanning {gamePath}");
var allKnownDepotFiles = new Dictionary <string, ulong>();
foreach (var mountedDepot in mountedDepots.Children)
{
var manifestPath = Path.Join(steamClient.SteamPath, "depotcache", $"{mountedDepot.Name}_{mountedDepot.Value}.manifest");
if (!File.Exists(manifestPath))
{
Console.Error.WriteLine($"Manifest does not exist: {manifestPath}");
continue;
}
var manifest = DepotManifest.Deserialize(File.ReadAllBytes(manifestPath));
foreach (var file in manifest.Files)
{
if (file.Flags.HasFlag(EDepotFileFlag.Directory))
{
continue;
}
allKnownDepotFiles[file.FileName] = file.TotalSize;
}
Console.WriteLine($"{manifestPath} - {manifest.Files.Count} files");
}
Console.WriteLine($"{allKnownDepotFiles.Count} files in depot manifests");
var filesOnDisk = Directory.GetFiles(gamePath, "*", SearchOption.AllDirectories).ToList();
filesOnDisk.Sort();
Console.WriteLine($"{filesOnDisk.Count} files on disk");
Console.WriteLine();
Console.ForegroundColor = ConsoleColor.Red;
var filenamesOnDisk = new HashSet <string>();
foreach (var file in filesOnDisk)
{
var unprefixedPath = file.Substring(gamePath.Length + 1);
filenamesOnDisk.Add(unprefixedPath);
if (unprefixedPath == "installscript.vdf")
{
continue;
}
if (!allKnownDepotFiles.ContainsKey(unprefixedPath))
{
Console.Write($"Unknown file: {unprefixedPath}");
if (deleteUnknownFiles && !InsideWhitelist(file))
{
try
{
File.Delete(file);
Console.Write(" -> [Deleted]");
}
catch (Exception ex)
{
Console.WriteLine("Unable to delete!");
Console.WriteLine(ex);
}
}
Console.WriteLine();
continue;
}
var length = new FileInfo(file).Length;
if (allKnownDepotFiles[unprefixedPath] != (ulong)length)
{
Console.WriteLine($"Mismatching file size: {unprefixedPath} (is {length} bytes, should be {allKnownDepotFiles[unprefixedPath]})");
continue;
}
}
foreach (var file in allKnownDepotFiles.Keys.Where(file => !filenamesOnDisk.Contains(file)))
{
Console.WriteLine($"Missing file: {file}");
}
Console.ResetColor();
}
public ProtoManifest(DepotManifest sourceManifest, ulong id) : this()
{
sourceManifest.Files.ForEach(f => Files.Add(new FileData(f)));
ID = id;
}
/*
* Here be dragons.
*/
public static async Task <EResult> DownloadFilesFromDepot(DepotProcessor.ManifestJob job, DepotManifest depotManifest)
{
var filesRegex = Files[job.DepotID];
var files = depotManifest.Files.Where(x => filesRegex.IsMatch(x.FileName.Replace('\\', '/'))).ToList();
var downloadState = EResult.Fail;
ConcurrentDictionary <string, ExistingFileData> existingFileData;
await using (var db = await Database.GetConnectionAsync())
{
var data = db.ExecuteScalar <string>("SELECT `Value` FROM `LocalConfig` WHERE `ConfigKey` = @Key", new { Key = $"depot.{job.DepotID}" });
if (data != null)
{
existingFileData = JsonConvert.DeserializeObject <ConcurrentDictionary <string, ExistingFileData> >(data);
}
else
{
existingFileData = new ConcurrentDictionary <string, ExistingFileData>();
}
}
foreach (var file in existingFileData.Keys.Except(files.Select(x => x.FileName)))
{
Log.WriteWarn(nameof(FileDownloader), $"\"{file}\" no longer exists in manifest");
}
Log.WriteInfo($"FileDownloader {job.DepotID}", $"Will download {files.Count} files");
var downloadedFiles = 0;
var fileTasks = new Task[files.Count];
for (var i = 0; i < fileTasks.Length; i++)
{
var file = files[i];
fileTasks[i] = TaskManager.Run(async() =>
{
var existingFile = existingFileData.GetOrAdd(file.FileName, _ => new ExistingFileData());
EResult fileState;
try
{
await ChunkDownloadingSemaphore.WaitAsync().ConfigureAwait(false);
fileState = await DownloadFile(job, file, existingFile);
}
finally
{
ChunkDownloadingSemaphore.Release();
}
if (fileState == EResult.OK || fileState == EResult.SameAsPreviousValue)
{
existingFile.FileHash = file.FileHash;
downloadedFiles++;
}
if (fileState != EResult.SameAsPreviousValue)
{
// Do not write progress info to log file
Console.WriteLine($"{job.DepotName} [{downloadedFiles / (float) files.Count * 100.0f,6:#00.00}%] {files.Count - downloadedFiles} files left to download");
}
if (downloadState == EResult.DataCorruption)
{
return;
}
if (fileState == EResult.OK || fileState == EResult.DataCorruption)
{
downloadState = fileState;
}
});
}
await Task.WhenAll(fileTasks).ConfigureAwait(false);
await LocalConfig.Update($"depot.{job.DepotID}", JsonConvert.SerializeObject(existingFileData));
job.Result = downloadState switch
{
EResult.OK => EResult.OK,
EResult.DataCorruption => EResult.DataCorruption,
_ => EResult.Ignored
};
return(job.Result);
}
private static async Task DownloadSteam3Async(uint appId, List <DepotDownloadInfo> depots)
{
ulong TotalBytesCompressed = 0;
ulong TotalBytesUncompressed = 0;
foreach (var depot in depots)
{
ulong DepotBytesCompressed = 0;
ulong DepotBytesUncompressed = 0;
Console.WriteLine("Downloading depot {0} - {1}", depot.id, depot.contentName);
CancellationTokenSource cts = new CancellationTokenSource();
cdnPool.ExhaustedToken = cts;
ProtoManifest oldProtoManifest = null;
ProtoManifest newProtoManifest = null;
string configDir = Path.Combine(depot.installDir, CONFIG_DIR);
ulong lastManifestId = INVALID_MANIFEST_ID;
DepotConfigStore.Instance.InstalledManifestIDs.TryGetValue(depot.id, out lastManifestId);
// In case we have an early exit, this will force equiv of verifyall next run.
DepotConfigStore.Instance.InstalledManifestIDs[depot.id] = INVALID_MANIFEST_ID;
DepotConfigStore.Save();
if (lastManifestId != INVALID_MANIFEST_ID)
{
var oldManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", lastManifestId));
if (File.Exists(oldManifestFileName))
{
byte[] expectedChecksum, currentChecksum;
try
{
expectedChecksum = File.ReadAllBytes(oldManifestFileName + ".sha");
}
catch (IOException)
{
expectedChecksum = null;
}
oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName, out currentChecksum);
if (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum))
{
// We only have to show this warning if the old manifest ID was different
if (lastManifestId != depot.manifestId)
{
Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", lastManifestId);
}
oldProtoManifest = null;
}
}
}
if (lastManifestId == depot.manifestId && oldProtoManifest != null)
{
newProtoManifest = oldProtoManifest;
Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id);
}
else
{
var newManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", depot.manifestId));
if (newManifestFileName != null)
{
byte[] expectedChecksum, currentChecksum;
try
{
expectedChecksum = File.ReadAllBytes(newManifestFileName + ".sha");
}
catch (IOException)
{
expectedChecksum = null;
}
newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName, out currentChecksum);
if (newProtoManifest != null && (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum)))
{
Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", depot.manifestId);
newProtoManifest = null;
}
}
if (newProtoManifest != null)
{
Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id);
}
else
{
Console.Write("Downloading depot manifest...");
DepotManifest depotManifest = null;
while (depotManifest == null)
{
Tuple <CDNClient.Server, string> connection = null;
try
{
connection = await cdnPool.GetConnectionForDepot(appId, depot.id, CancellationToken.None);
depotManifest = await cdnPool.CDNClient.DownloadManifestAsync(depot.id, depot.manifestId,
connection.Item1, connection.Item2, depot.depotKey).ConfigureAwait(false);
cdnPool.ReturnConnection(connection);
}
catch (SteamKitWebRequestException e)
{
cdnPool.ReturnBrokenConnection(connection);
if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden)
{
Console.WriteLine("Encountered 401 for depot manifest {0} {1}. Aborting.", depot.id, depot.manifestId);
break;
}
else
{
Console.WriteLine("Encountered error downloading depot manifest {0} {1}: {2}", depot.id, depot.manifestId, e.StatusCode);
}
}
catch (Exception e)
{
cdnPool.ReturnBrokenConnection(connection);
Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depot.id, depot.manifestId, e.Message);
}
}
if (depotManifest == null)
{
Console.WriteLine("\nUnable to download manifest {0} for depot {1}", depot.manifestId, depot.id);
return;
}
byte[] checksum;
newProtoManifest = new ProtoManifest(depotManifest, depot.manifestId);
newProtoManifest.SaveToFile(newManifestFileName, out checksum);
File.WriteAllBytes(newManifestFileName + ".sha", checksum);
Console.WriteLine(" Done!");
}
}
newProtoManifest.Files.Sort((x, y) => string.Compare(x.FileName, y.FileName, StringComparison.Ordinal));
if (Config.DownloadManifestOnly)
{
StringBuilder manifestBuilder = new StringBuilder();
string txtManifest = Path.Combine(depot.installDir, string.Format("manifest_{0}.txt", depot.id));
foreach (var file in newProtoManifest.Files)
{
if (file.Flags.HasFlag(EDepotFileFlag.Directory))
{
continue;
}
manifestBuilder.Append(string.Format("{0}\n", file.FileName));
}
File.WriteAllText(txtManifest, manifestBuilder.ToString());
continue;
}
ulong complete_download_size = 0;
ulong size_downloaded = 0;
string stagingDir = Path.Combine(depot.installDir, STAGING_DIR);
var filesAfterExclusions = newProtoManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).ToList();
// Pre-process
filesAfterExclusions.ForEach(file =>
{
var fileFinalPath = Path.Combine(depot.installDir, file.FileName);
var fileStagingPath = Path.Combine(stagingDir, file.FileName);
if (file.Flags.HasFlag(EDepotFileFlag.Directory))
{
Directory.CreateDirectory(fileFinalPath);
Directory.CreateDirectory(fileStagingPath);
}
else
{
// Some manifests don't explicitly include all necessary directories
Directory.CreateDirectory(Path.GetDirectoryName(fileFinalPath));
Directory.CreateDirectory(Path.GetDirectoryName(fileStagingPath));
complete_download_size += file.TotalSize;
}
});
var semaphore = new SemaphoreSlim(Config.MaxDownloads);
var files = filesAfterExclusions.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory)).ToArray();
var tasks = new Task[files.Length];
for (var i = 0; i < files.Length; i++)
{
var file = files[i];
var task = Task.Run(async() =>
{
cts.Token.ThrowIfCancellationRequested();
try
{
await semaphore.WaitAsync().ConfigureAwait(false);
cts.Token.ThrowIfCancellationRequested();
string fileFinalPath = Path.Combine(depot.installDir, file.FileName);
string fileStagingPath = Path.Combine(stagingDir, file.FileName);
// This may still exist if the previous run exited before cleanup
if (File.Exists(fileStagingPath))
{
File.Delete(fileStagingPath);
}
FileStream fs = null;
List <ProtoManifest.ChunkData> neededChunks;
FileInfo fi = new FileInfo(fileFinalPath);
if (!fi.Exists)
{
// create new file. need all chunks
fs = File.Create(fileFinalPath);
fs.SetLength(( long )file.TotalSize);
neededChunks = new List <ProtoManifest.ChunkData>(file.Chunks);
}
else
{
// open existing
ProtoManifest.FileData oldManifestFile = null;
if (oldProtoManifest != null)
{
oldManifestFile = oldProtoManifest.Files.SingleOrDefault(f => f.FileName == file.FileName);
}
if (oldManifestFile != null)
{
neededChunks = new List <ProtoManifest.ChunkData>();
if (Config.VerifyAll || !oldManifestFile.FileHash.SequenceEqual(file.FileHash))
{
// we have a version of this file, but it doesn't fully match what we want
var matchingChunks = new List <ChunkMatch>();
foreach (var chunk in file.Chunks)
{
var oldChunk = oldManifestFile.Chunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID));
if (oldChunk != null)
{
matchingChunks.Add(new ChunkMatch(oldChunk, chunk));
}
else
{
neededChunks.Add(chunk);
}
}
File.Move(fileFinalPath, fileStagingPath);
fs = File.Open(fileFinalPath, FileMode.Create);
fs.SetLength(( long )file.TotalSize);
using (var fsOld = File.Open(fileStagingPath, FileMode.Open))
{
foreach (var match in matchingChunks)
{
fsOld.Seek(( long )match.OldChunk.Offset, SeekOrigin.Begin);
byte[] tmp = new byte[match.OldChunk.UncompressedLength];
fsOld.Read(tmp, 0, tmp.Length);
byte[] adler = Util.AdlerHash(tmp);
if (!adler.SequenceEqual(match.OldChunk.Checksum))
{
neededChunks.Add(match.NewChunk);
}
else
{
fs.Seek(( long )match.NewChunk.Offset, SeekOrigin.Begin);
fs.Write(tmp, 0, tmp.Length);
}
}
}
File.Delete(fileStagingPath);
}
}
else
{
// No old manifest or file not in old manifest. We must validate.
fs = File.Open(fileFinalPath, FileMode.Open);
if (( ulong )fi.Length != file.TotalSize)
{
fs.SetLength(( long )file.TotalSize);
}
neededChunks = Util.ValidateSteam3FileChecksums(fs, file.Chunks.OrderBy(x => x.Offset).ToArray());
}
if (neededChunks.Count() == 0)
{
size_downloaded += file.TotalSize;
Console.WriteLine("{0,6:#00.00}% {1}", (( float )size_downloaded / ( float )complete_download_size) * 100.0f, fileFinalPath);
if (fs != null)
{
fs.Dispose();
}
return;
}
else
{
size_downloaded += (file.TotalSize - ( ulong )neededChunks.Select(x => ( long )x.UncompressedLength).Sum());
}
}
foreach (var chunk in neededChunks)
{
if (cts.IsCancellationRequested)
{
break;
}
string chunkID = Util.EncodeHexString(chunk.ChunkID);
CDNClient.DepotChunk chunkData = null;
while (!cts.IsCancellationRequested)
{
Tuple <CDNClient.Server, string> connection;
try
{
connection = await cdnPool.GetConnectionForDepot(appId, depot.id, cts.Token);
}
catch (OperationCanceledException)
{
break;
}
DepotManifest.ChunkData data = new DepotManifest.ChunkData();
data.ChunkID = chunk.ChunkID;
data.Checksum = chunk.Checksum;
data.Offset = chunk.Offset;
data.CompressedLength = chunk.CompressedLength;
data.UncompressedLength = chunk.UncompressedLength;
try
{
chunkData = await cdnPool.CDNClient.DownloadDepotChunkAsync(depot.id, data,
connection.Item1, connection.Item2, depot.depotKey).ConfigureAwait(false);
cdnPool.ReturnConnection(connection);
break;
}
catch (SteamKitWebRequestException e)
{
cdnPool.ReturnBrokenConnection(connection);
if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden)
{
Console.WriteLine("Encountered 401 for chunk {0}. Aborting.", chunkID);
cts.Cancel();
break;
}
else
{
Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, e.StatusCode);
}
}
catch (Exception e)
{
cdnPool.ReturnBrokenConnection(connection);
Console.WriteLine("Encountered unexpected error downloading chunk {0}: {1}", chunkID, e.Message);
}
}
if (chunkData == null)
{
Console.WriteLine("Failed to find any server with chunk {0} for depot {1}. Aborting.", chunkID, depot.id);
cts.Cancel();
}
// Throw the cancellation exception if requested so that this task is marked failed
cts.Token.ThrowIfCancellationRequested();
TotalBytesCompressed += chunk.CompressedLength;
DepotBytesCompressed += chunk.CompressedLength;
TotalBytesUncompressed += chunk.UncompressedLength;
DepotBytesUncompressed += chunk.UncompressedLength;
fs.Seek(( long )chunk.Offset, SeekOrigin.Begin);
fs.Write(chunkData.Data, 0, chunkData.Data.Length);
size_downloaded += chunk.UncompressedLength;
}
fs.Dispose();
Console.WriteLine("{0,6:#00.00}% {1}", (( float )size_downloaded / ( float )complete_download_size) * 100.0f, fileFinalPath);
}
finally
{
semaphore.Release();
}
});
tasks[i] = task;
}
await Task.WhenAll(tasks).ConfigureAwait(false);
DepotConfigStore.Instance.InstalledManifestIDs[depot.id] = depot.manifestId;
DepotConfigStore.Save();
Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.id, DepotBytesCompressed, DepotBytesUncompressed);
}
Console.WriteLine("Total downloaded: {0} bytes ({1} bytes uncompressed) from {2} depots", TotalBytesCompressed, TotalBytesUncompressed, depots.Count);
}
private static void DownloadSteam3(List <DepotDownloadInfo3> depots)
{
foreach (var depot in depots)
{
int depotId = depot.id;
ulong depot_manifest = depot.manifestId;
byte[] depotKey = depot.depotKey;
string installDir = depot.installDir;
Console.WriteLine("Downloading depot {0} - {1}", depot.id, depot.contentName);
Console.Write("Finding content servers...");
List <IPEndPoint> serverList = steam3.steamClient.GetServersOfType(EServerType.CS);
List <CDNClient.ClientEndPoint> cdnServers = null;
int counterDeferred = 0;
for (int i = 0; ; i++)
{
IPEndPoint endpoint = serverList[i % serverList.Count];
cdnServers = CDNClient.FetchServerList(new CDNClient.ClientEndPoint(endpoint.Address.ToString(), endpoint.Port), Config.CellID);
if (cdnServers == null)
{
counterDeferred++;
}
if (cdnServers != null && cdnServers.Count((ep) => { return(ep.Type == "CS"); }) > 0)
{
break;
}
if (((i + 1) % serverList.Count) == 0)
{
Console.WriteLine("Unable to find any Steam3 content servers");
return;
}
}
Console.WriteLine(" Done!");
Console.Write("Downloading depot manifest...");
List <CDNClient.ClientEndPoint> cdnEndpoints = cdnServers.Where((ep) => { return(ep.Type == "CDN"); }).ToList();
List <CDNClient.ClientEndPoint> csEndpoints = cdnServers.Where((ep) => { return(ep.Type == "CS"); }).ToList();
List <CDNClient> cdnClients = new List <CDNClient>();
byte[] appTicket = steam3.AppTickets[(uint)depotId];
foreach (var server in csEndpoints)
{
CDNClient client;
if (appTicket == null)
{
client = new CDNClient(server, (uint)depotId, steam3.steamUser.SteamID);
}
else
{
client = new CDNClient(server, appTicket);
}
if (client.Connect())
{
cdnClients.Add(client);
if (cdnClients.Count >= NUM_STEAM3_CONNECTIONS)
{
break;
}
}
}
if (cdnClients.Count == 0)
{
Console.WriteLine("\nCould not initialize connection with CDN.");
return;
}
DepotManifest depotManifest = cdnClients[0].DownloadDepotManifest(depotId, depot_manifest);
if (depotManifest == null)
{
// TODO: check for 401s
for (int i = 1; i < cdnClients.Count && depotManifest == null; i++)
{
depotManifest = cdnClients[i].DownloadDepotManifest(depotId, depot_manifest);
}
if (depotManifest == null)
{
Console.WriteLine("\nUnable to download manifest {0} for depot {1}", depot_manifest, depotId);
return;
}
}
if (!depotManifest.DecryptFilenames(depotKey))
{
Console.WriteLine("\nUnable to decrypt manifest for depot {0}", depotId);
return;
}
Console.WriteLine(" Done!");
ulong complete_download_size = 0;
ulong size_downloaded = 0;
depotManifest.Files.Sort((x, y) => { return(x.FileName.CompareTo(y.FileName)); });
if (Config.DownloadManifestOnly)
{
StringBuilder manifestBuilder = new StringBuilder();
string txtManifest = Path.Combine(depot.installDir, string.Format("manifest_{0}.txt", depot.id));
foreach (var file in depotManifest.Files)
{
if (file.Flags.HasFlag(EDepotFileFlag.Directory))
{
continue;
}
manifestBuilder.Append(string.Format("{0}\n", file.FileName));
}
File.WriteAllText(txtManifest, manifestBuilder.ToString());
continue;
}
depotManifest.Files.RemoveAll((x) => !TestIsFileIncluded(x.FileName));
foreach (var file in depotManifest.Files)
{
complete_download_size += file.TotalSize;
}
foreach (var file in depotManifest.Files)
{
string download_path = Path.Combine(installDir, file.FileName);
if (file.Flags.HasFlag(EDepotFileFlag.Directory))
{
if (!Directory.Exists(download_path))
{
Directory.CreateDirectory(download_path);
}
continue;
}
string dir_path = Path.GetDirectoryName(download_path);
if (!Directory.Exists(dir_path))
{
Directory.CreateDirectory(dir_path);
}
FileStream fs;
DepotManifest.ChunkData[] neededChunks;
FileInfo fi = new FileInfo(download_path);
if (!fi.Exists)
{
// create new file. need all chunks
fs = File.Create(download_path);
neededChunks = file.Chunks.ToArray();
}
else
{
// open existing
fs = File.Open(download_path, FileMode.Open);
if ((ulong)fi.Length != file.TotalSize)
{
fs.SetLength((long)file.TotalSize);
}
// find which chunks we need, in order so that we aren't seeking every which way
neededChunks = Util.ValidateSteam3FileChecksums(fs, file.Chunks.OrderBy(x => x.Offset).ToArray());
if (neededChunks.Count() == 0)
{
size_downloaded += file.TotalSize;
Console.WriteLine("{0,6:#00.00}% {1}", ((float)size_downloaded / (float)complete_download_size) * 100.0f, download_path);
fs.Close();
continue;
}
else
{
size_downloaded += (file.TotalSize - (ulong)neededChunks.Select(x => (int)x.UncompressedLength).Sum());
}
}
Console.Write("{0,6:#00.00}% {1}", ((float)size_downloaded / (float)complete_download_size) * 100.0f, download_path);
foreach (var chunk in neededChunks)
{
string chunkID = EncodeHexString(chunk.ChunkID);
byte[] encrypted_chunk = cdnClients[0].DownloadDepotChunk(depotId, chunkID);
if (encrypted_chunk == null)
{
for (int i = 1; i < cdnClients.Count && encrypted_chunk == null; i++)
{
encrypted_chunk = cdnClients[i].DownloadDepotChunk(depotId, chunkID);
}
if (encrypted_chunk == null)
{
Console.WriteLine("Unable to download chunk id {0} for depot {1}", chunkID, depotId);
fs.Close();
return;
}
}
byte[] chunk_data = CDNClient.ProcessChunk(encrypted_chunk, depotKey);
fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
fs.Write(chunk_data, 0, chunk_data.Length);
size_downloaded += chunk.UncompressedLength;
Console.CursorLeft = 0;
Console.Write("{0,6:#00.00}%", ((float)size_downloaded / (float)complete_download_size) * 100.0f);
}
fs.Close();
Console.WriteLine();
}
}
}
private async Task DownloadDepots(uint appID, List <ManifestJob> depots)
{
Log.WriteDebug("Depot Downloader", "Will process {0} depots ({1} depot locks left)", depots.Count(), DepotLocks.Count);
var processTasks = new List <Task <EResult> >();
bool anyFilesDownloaded = false;
foreach (var depot in depots)
{
var instance = depot.Anonymous ? Steam.Anonymous.Apps : Steam.Instance.Apps;
depot.DepotKey = await GetDepotDecryptionKey(instance, depot.DepotID, appID);
if (depot.DepotKey == null)
{
RemoveLock(depot.DepotID);
continue;
}
var cdnToken = await GetCDNAuthToken(instance, appID, depot.DepotID);
if (cdnToken == null)
{
RemoveLock(depot.DepotID);
Log.WriteDebug("Depot Downloader", "Got a depot key for depot {0} but no cdn auth token", depot.DepotID);
continue;
}
depot.CDNToken = cdnToken.Token;
depot.Server = GetContentServer();
DepotManifest depotManifest = null;
string lastError = string.Empty;
for (var i = 0; i <= 5; i++)
{
try
{
depotManifest = await CDNClient.DownloadManifestAsync(depot.DepotID, depot.ManifestID, depot.Server, depot.CDNToken, depot.DepotKey);
break;
}
catch (Exception e)
{
lastError = e.Message;
Log.WriteError("Depot Processor", "Failed to download depot manifest for app {0} depot {1} ({2}: {3}) (#{4})", appID, depot.DepotID, depot.Server, lastError, i);
}
// TODO: get new auth key if auth fails
depot.Server = GetContentServer();
if (depotManifest == null)
{
await Task.Delay(Utils.ExponentionalBackoff(i));
}
}
if (depotManifest == null)
{
LocalConfig.CDNAuthTokens.TryRemove(depot.DepotID, out _);
RemoveLock(depot.DepotID);
if (FileDownloader.IsImportantDepot(depot.DepotID))
{
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download manifest ({3})",
Colors.OLIVE, depot.DepotName, Colors.NORMAL, lastError);
}
if (!Settings.IsFullRun)
{
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
}
continue;
}
var task = ProcessDepotAfterDownload(depot, depotManifest);
processTasks.Add(task);
if (!FileDownloader.IsImportantDepot(depot.DepotID))
{
continue;
}
task = TaskManager.Run(async() =>
{
var result = EResult.Fail;
try
{
result = await FileDownloader.DownloadFilesFromDepot(depot, depotManifest);
if (result == EResult.OK)
{
anyFilesDownloaded = true;
}
}
catch (Exception e)
{
ErrorReporter.Notify($"Depot Processor {depot.DepotID}", e);
}
return(result);
}).Unwrap();
processTasks.Add(task);
}
if (SaveLocalConfig)
{
SaveLocalConfig = false;
LocalConfig.Save();
}
await Task.WhenAll(processTasks).ConfigureAwait(false);
Log.WriteDebug("Depot Downloader", "{0} depot downloads finished", depots.Count());
// TODO: use ContinueWith on tasks
if (!anyFilesDownloaded)
{
foreach (var depot in depots)
{
RemoveLock(depot.DepotID);
}
return;
}
if (!File.Exists(UpdateScript))
{
return;
}
lock (UpdateScriptLock)
{
foreach (var depot in depots)
{
if (depot.Result == EResult.OK)
{
RunUpdateScript(string.Format("{0} no-git", depot.DepotID));
}
else if (depot.Result != EResult.Ignored)
{
Log.WriteWarn("Depot Processor", "Dropping stored token for {0} due to download failures", depot.DepotID);
LocalConfig.CDNAuthTokens.TryRemove(depot.DepotID, out _);
using (var db = Database.Get())
{
// Mark this depot for redownload
db.Execute("UPDATE `Depots` SET `LastManifestID` = 0 WHERE `DepotID` = @DepotID", new { depot.DepotID });
}
}
RemoveLock(depot.DepotID);
}
// Only commit changes if all depots downloaded
if (processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored))
{
if (!RunUpdateScript(appID, depots.First().BuildID))
{
RunUpdateScript("0");
}
}
else
{
Log.WriteDebug("Depot Processor", "Reprocessing the app {0} because some files failed to download", appID);
IRC.Instance.SendOps("{0}[{1}]{2} Reprocessing the app due to download failures",
Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL
);
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
}
}
}
private static async Task DownloadSteam3Async(uint appId, List <DepotDownloadInfo> depots, Action downloadCompleteAction = null)
{
ulong TotalBytesCompressed = 0;
ulong TotalBytesUncompressed = 0;
foreach (var depot in depots)
{
ulong DepotBytesCompressed = 0;
ulong DepotBytesUncompressed = 0;
DebugLog.WriteLine("ContentDownloader", "Downloading depot " + depot.id + " - " + depot.contentName);
CancellationTokenSource cts = new CancellationTokenSource();
cdnPool.ExhaustedToken = cts;
ProtoManifest oldProtoManifest = null;
ProtoManifest downloadManifest = null;
string configDir = Path.Combine(depot.installDir, CONFIG_DIR);
ulong lastManifestId = INVALID_MANIFEST_ID;
ConfigStore.TheConfig.LastManifests.TryGetValue(depot.id, out lastManifestId);
// In case we have an early exit, this will force equiv of verifyall next run.
ConfigStore.TheConfig.LastManifests[depot.id] = INVALID_MANIFEST_ID;
ConfigStore.Save();
if (lastManifestId != INVALID_MANIFEST_ID)
{
var oldManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", lastManifestId));
DebugLog.WriteLine(DEBUG_NAME_FILES, "Checking if " + oldManifestFileName + " exists");
if (File.Exists(oldManifestFileName))
{
DebugLog.WriteLine(DEBUG_NAME_FILES, oldManifestFileName + " exists, reading!");
oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName);
}
}
if (lastManifestId == depot.manifestId && oldProtoManifest != null)
{
downloadManifest = oldProtoManifest;
DebugLog.WriteLine("ContentDownloader", "Already have manifest " + depot.manifestId + " for depot " + depot.id + ".");
}
else
{
var newManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", depot.manifestId));
if (newManifestFileName != null)
{
downloadManifest = ProtoManifest.LoadFromFile(newManifestFileName);
}
if (downloadManifest != null)
{
DebugLog.WriteLine("ContentDownloader", "Already have manifest " + depot.manifestId + " for depot " + depot.id + ".");
}
else
{
DebugLog.WriteLine("ContentDownloader", "Downloading depot manifest...");
DepotManifest depotManifest = null;
while (depotManifest == null)
{
CDNClient client = null;
try
{
client = await cdnPool.GetConnectionForDepotAsync(appId, depot.id, depot.depotKey, CancellationToken.None).ConfigureAwait(false);
//client = await cdnPool.GetConnectionForDepotAsync(appId, depot.id, depot.depotKey, CancellationToken.None);
depotManifest = await client.DownloadManifestAsync(depot.id, depot.manifestId).ConfigureAwait(false);
//depotManifest = await client.DownloadManifestAsync(depot.id, depot.manifestId);
cdnPool.ReturnConnection(client);
}
catch (WebException e)
{
cdnPool.ReturnBrokenConnection(client);
if (e.Status == WebExceptionStatus.ProtocolError)
{
var response = e.Response as HttpWebResponse;
if (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden)
{
DebugLog.WriteLine("ContentDownloader", "Encountered 401 for depot manifest " + depot.id + " " + depot.manifestId + ". Aborting.");
break;
}
else
{
DebugLog.WriteLine("ContentDownloader", "Encountered error downloading depot manifest " + depot.id + " " + depot.manifestId + ": " + response.StatusCode);
}
}
else
{
DebugLog.WriteLine("ContentDownloader", "Encountered error downloading manifest for depot " + depot.id + " " + depot.manifestId + ": " + e.Status);
}
}
catch (Exception e)
{
cdnPool.ReturnBrokenConnection(client);
DebugLog.WriteLine("ContentDownloader", "Encountered error downloading manifest for depot " + depot.id + " " + depot.manifestId + ": " + e.Message);
}
}
if (depotManifest == null)
{
DebugLog.WriteLine("ContentDownloader", "\nUnable to download manifest " + depot.manifestId + " for depot " + depot.id);
return;
}
downloadManifest = new ProtoManifest(depotManifest, depot.manifestId);
downloadManifest.SaveToFile(newManifestFileName);
DebugLog.WriteLine("ContentDownloader", "Done!");
}
}
downloadManifest.Files.Sort((x, y) => { return(x.FileName.CompareTo(y.FileName)); });
if (downloadManifest != null)
{
onManifestReceived?.Invoke(appId, depot.id, depot.contentName, downloadManifest);
}
if (Config.DownloadManifestOnly)
{
continue;
}
complete_download_size = 0;
size_downloaded = 0;
string stagingDir = Path.Combine(depot.installDir, STAGING_DIR);
var filesAfterExclusions = downloadManifest.Files.Where(f => TestIsFileIncluded(f.FileName)).ToList();
// Pre-process
filesAfterExclusions.ForEach(file =>
{
var fileFinalPath = Path.Combine(depot.installDir, file.FileName);
var fileStagingPath = Path.Combine(stagingDir, file.FileName);
if (file.Flags.HasFlag(EDepotFileFlag.Directory))
{
Directory.CreateDirectory(fileFinalPath);
Directory.CreateDirectory(fileStagingPath);
}
else
{
// Some manifests don't explicitly include all necessary directories
Directory.CreateDirectory(Path.GetDirectoryName(fileFinalPath));
Directory.CreateDirectory(Path.GetDirectoryName(fileStagingPath));
complete_download_size += file.TotalSize;
}
});
var semaphore = new SemaphoreSlim(Config.MaxDownloads);
var files = filesAfterExclusions.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory)).ToArray();
var tasks = new Task[files.Length];
for (var i = 0; i < files.Length; i++)
{
var file = files[i];
var task = Task.Run(async() =>
{
cts.Token.ThrowIfCancellationRequested();
try
{
await semaphore.WaitAsync().ConfigureAwait(false);
//await semaphore.WaitAsync();
cts.Token.ThrowIfCancellationRequested();
string fileFinalPath = Path.Combine(depot.installDir, file.FileName);
string fileStagingPath = Path.Combine(stagingDir, file.FileName);
// This may still exist if the previous run exited before cleanup
DebugLog.WriteLine(DEBUG_NAME_FILES, "Checking if " + fileStagingPath + " exists");
if (File.Exists(fileStagingPath))
{
DebugLog.WriteLine(DEBUG_NAME_FILES, fileStagingPath + " exists, deleting!");
File.Delete(fileStagingPath);
}
List <ProtoManifest.ChunkData> neededChunks;
FileInfo fi = new FileInfo(fileFinalPath);
DebugLog.WriteLine(DEBUG_NAME_FILES, "Checking if " + fileFinalPath + " exists");
if (!fi.Exists)
{
// create new file. need all chunks
DebugLog.WriteLine(DEBUG_NAME_FILES, fileFinalPath + " does not exist, creating!");
using (FileStream fs = File.Create(fileFinalPath))
{
fs.SetLength((long)file.TotalSize);
neededChunks = new List <ProtoManifest.ChunkData>(file.Chunks);
}
}
else
{
// open existing
ProtoManifest.FileData oldManifestFile = null;
if (oldProtoManifest != null)
{
oldManifestFile = oldProtoManifest.Files.SingleOrDefault(f => f.FileName == file.FileName);
}
if (oldManifestFile != null)
{
neededChunks = new List <ProtoManifest.ChunkData>();
if (Config.VerifyAll || !oldManifestFile.FileHash.SequenceEqual(file.FileHash))
{
// we have a version of this file, but it doesn't fully match what we want
var matchingChunks = new List <ChunkMatch>();
foreach (var chunk in file.Chunks)
{
var oldChunk = oldManifestFile.Chunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID));
if (oldChunk != null)
{
matchingChunks.Add(new ChunkMatch(oldChunk, chunk));
}
else
{
neededChunks.Add(chunk);
}
}
DebugLog.WriteLine(DEBUG_NAME_FILES, "Moving file " + fileFinalPath + " to " + fileStagingPath);
File.Move(fileFinalPath, fileStagingPath);
DebugLog.WriteLine(DEBUG_NAME_FILES, "Creating file " + fileFinalPath);
using (FileStream fs = File.Open(fileFinalPath, FileMode.Create))
{
fs.SetLength((long)file.TotalSize);
DebugLog.WriteLine(DEBUG_NAME_FILES, "Opening file " + fileStagingPath);
using (var fsOld = File.Open(fileStagingPath, FileMode.Open))
{
foreach (var match in matchingChunks)
{
fsOld.Seek((long)match.OldChunk.Offset, SeekOrigin.Begin);
byte[] tmp = new byte[match.OldChunk.UncompressedLength];
fsOld.Read(tmp, 0, tmp.Length);
byte[] adler = Util.AdlerHash(tmp);
if (!adler.SequenceEqual(match.OldChunk.Checksum))
{
neededChunks.Add(match.NewChunk);
}
else
{
fs.Seek((long)match.NewChunk.Offset, SeekOrigin.Begin);
fs.Write(tmp, 0, tmp.Length);
}
}
}
}
DebugLog.WriteLine(DEBUG_NAME_FILES, "Deleting file " + fileStagingPath);
File.Delete(fileStagingPath);
}
}
else
{
// No old manifest or file not in old manifest. We must validate.
DebugLog.WriteLine(DEBUG_NAME_FILES, "Opening file " + fileFinalPath);
using (FileStream fs = File.Open(fileFinalPath, FileMode.Open))
{
if ((ulong)fi.Length != file.TotalSize)
{
fs.SetLength((long)file.TotalSize);
}
neededChunks = Util.ValidateSteam3FileChecksums(fs, file.Chunks.OrderBy(x => x.Offset).ToArray());
}
}
if (neededChunks.Count() == 0)
{
size_downloaded += file.TotalSize;
DebugLog.WriteLine("ContentDownloader", DownloadPercent * 100.0f + "% " + fileFinalPath);
return;
}
else
{
size_downloaded += (file.TotalSize - ( ulong )neededChunks.Select(x => ( long )x.UncompressedLength).Sum());
}
}
DebugLog.WriteLine(DEBUG_NAME_FILES, "Opening file " + fileFinalPath);
using (FileStream fs = File.Open(fileFinalPath, FileMode.Open))
{
foreach (var chunk in neededChunks)
{
if (cts.IsCancellationRequested)
{
break;
}
string chunkID = Util.EncodeHexString(chunk.ChunkID);
CDNClient.DepotChunk chunkData = null;
while (!cts.IsCancellationRequested)
{
CDNClient client;
try
{
client = await cdnPool.GetConnectionForDepotAsync(appId, depot.id, depot.depotKey, cts.Token).ConfigureAwait(false);
//client = await cdnPool.GetConnectionForDepotAsync(appId, depot.id, depot.depotKey, cts.Token);
}
catch (OperationCanceledException)
{
break;
}
DepotManifest.ChunkData data = new DepotManifest.ChunkData();
data.ChunkID = chunk.ChunkID;
data.Checksum = chunk.Checksum;
data.Offset = chunk.Offset;
data.CompressedLength = chunk.CompressedLength;
data.UncompressedLength = chunk.UncompressedLength;
try
{
//chunkData = await client.DownloadDepotChunkAsStreamAsync(depot.id, data).ConfigureAwait(false);
chunkData = client.DownloadDepotChunkAsStreamAsync(depot.id, data).Result;
//chunkData = await client.DownloadDepotChunkAsync(depot.id, data);
cdnPool.ReturnConnection(client);
break;
}
catch (WebException e)
{
cdnPool.ReturnBrokenConnection(client);
if (e.Status == WebExceptionStatus.ProtocolError)
{
var response = e.Response as HttpWebResponse;
if (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden)
{
DebugLog.WriteLine("ContentDownloader", "Encountered 401 for chunk " + chunkID + ". Aborting.");
cts.Cancel();
break;
}
else
{
DebugLog.WriteLine("ContentDownloader", "Encountered error downloading chunk " + chunkID + ": " + response.StatusCode);
}
}
else
{
DebugLog.WriteLine("ContentDownloader", "Encountered error downloading chunk " + chunkID + ": " + e.Status);
}
}
catch (Exception e)
{
cdnPool.ReturnBrokenConnection(client);
DebugLog.WriteLine("ContentDownloader", "Encountered unexpected error downloading chunk " + chunkID + ": " + e.Message);
}
}
if (chunkData == null)
{
DebugLog.WriteLine("ContentDownloader", "Failed to find any server with chunk " + chunkID + " for depot " + depot.id + ". Aborting.");
cts.Cancel();
return;
}
TotalBytesCompressed += chunk.CompressedLength;
DepotBytesCompressed += chunk.CompressedLength;
TotalBytesUncompressed += chunk.UncompressedLength;
DepotBytesUncompressed += chunk.UncompressedLength;
using (chunkData.DataStream)
{
fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
chunkData.DataStream.CopyTo(fs);
//fs.Write(chunkData.Data, 0, chunkData.Data.Length);
}
size_downloaded += chunk.UncompressedLength;
}
}
DebugLog.WriteLine("ContentDownloader", DownloadPercent * 100.0f + "% " + fileFinalPath);
}
finally
{
semaphore.Release();
}
});
tasks[i] = task;
}
Task.WaitAll(tasks);
ConfigStore.TheConfig.LastManifests[depot.id] = depot.manifestId;
ConfigStore.Save();
DebugLog.WriteLine("ContentDownloader", "Depot " + depot.id + " - Downloaded " + DepotBytesCompressed + " bytes (" + DepotBytesUncompressed + " bytes uncompressed)");
}
IsDownloading = false;
downloadCompleteAction?.Invoke();
onDownloadCompleted?.Invoke();
DebugLog.WriteLine("ContentDownloader", "Total downloaded: " + TotalBytesCompressed + " bytes (" + TotalBytesUncompressed + " bytes uncompressed) from " + depots.Count + " depots");
}
private async Task <EResult> ProcessDepotAfterDownload(IDbConnection db, IDbTransaction transaction, ManifestJob request, DepotManifest depotManifest)
{
var filesOld = (await db.QueryAsync <DepotFile>("SELECT `ID`, `File`, `Hash`, `Size`, `Flags` FROM `DepotsFiles` WHERE `DepotID` = @DepotID", new { request.DepotID }, transaction: transaction)).ToDictionary(x => x.File, x => x);
var filesAdded = new List <DepotFile>();
var shouldHistorize = filesOld.Any(); // Don't historize file additions if we didn't have any data before
foreach (var file in depotManifest.Files)
{
var name = file.FileName.Replace('\\', '/');
byte[] hash = null;
// Store empty hashes as NULL (e.g. an empty file)
if (file.FileHash.Length > 0 && !file.Flags.HasFlag(EDepotFileFlag.Directory))
{
for (int i = 0; i < file.FileHash.Length; ++i)
{
if (file.FileHash[i] != 0)
{
hash = file.FileHash;
break;
}
}
}
// safe guard
if (name.Length > 255)
{
ErrorReporter.Notify("Depot Processor", new OverflowException(string.Format("File \"{0}\" in depot {1} is too long", name, request.DepotID)));
continue;
}
if (filesOld.ContainsKey(name))
{
var oldFile = filesOld[name];
var updateFile = false;
if (oldFile.Size != file.TotalSize || !Utils.IsEqualSHA1(hash, oldFile.Hash))
{
await MakeHistory(db, transaction, request, name, "modified", oldFile.Size, file.TotalSize);
updateFile = true;
}
if (oldFile.Flags != file.Flags)
{
await MakeHistory(db, transaction, request, name, "modified_flags", (ulong)oldFile.Flags, (ulong)file.Flags);
updateFile = true;
}
if (updateFile)
{
await db.ExecuteAsync("UPDATE `DepotsFiles` SET `Hash` = @Hash, `Size` = @Size, `Flags` = @Flags WHERE `DepotID` = @DepotID AND `ID` = @ID", new DepotFile
{
ID = oldFile.ID,
DepotID = request.DepotID,
Hash = hash,
Size = file.TotalSize,
Flags = file.Flags
}, transaction : transaction);
}
filesOld.Remove(name);
}
else
{
// We want to historize modifications first, and only then deletions and additions
filesAdded.Add(new DepotFile
{
DepotID = request.DepotID,
Hash = hash,
File = name,
Size = file.TotalSize,
Flags = file.Flags
});
}
}
if (filesOld.Any())
{
await db.ExecuteAsync("DELETE FROM `DepotsFiles` WHERE `DepotID` = @DepotID AND `ID` IN @Files", new { request.DepotID, Files = filesOld.Select(x => x.Value.ID) }, transaction : transaction);
await db.ExecuteAsync(HistoryQuery, filesOld.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ChangeID = request.ChangeNumber,
Action = "removed",
File = x.Value.File
}), transaction : transaction);
}
if (filesAdded.Any())
{
await db.ExecuteAsync("INSERT INTO `DepotsFiles` (`DepotID`, `File`, `Hash`, `Size`, `Flags`) VALUES (@DepotID, @File, @Hash, @Size, @Flags)", filesAdded, transaction : transaction);
if (shouldHistorize)
{
await db.ExecuteAsync(HistoryQuery, filesAdded.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ChangeID = request.ChangeNumber,
Action = "added",
File = x.File
}), transaction : transaction);
}
}
await db.ExecuteAsync("UPDATE `Depots` SET `LastManifestID` = @ManifestID, `LastUpdated` = CURRENT_TIMESTAMP() WHERE `DepotID` = @DepotID", new { request.DepotID, request.ManifestID }, transaction : transaction);
return(EResult.OK);
}
private static async Task <EResult> ProcessDepotAfterDownload(IDbConnection db, IDbTransaction transaction, ManifestJob request, DepotManifest depotManifest)
{
var filesOld = (await db.QueryAsync <DepotFile>("SELECT `ID`, `File`, `Hash`, `Size`, `Flags` FROM `DepotsFiles` WHERE `DepotID` = @DepotID", new { request.DepotID }, transaction: transaction)).ToDictionary(x => x.File, x => x);
var filesAdded = new List <DepotFile>();
var shouldHistorize = filesOld.Count > 0; // Don't historize file additions if we didn't have any data before
foreach (var file in depotManifest.Files.OrderByDescending(x => x.FileName))
{
var name = file.FileName.Replace('\\', '/');
byte[] hash = null;
// Store empty hashes as NULL (e.g. an empty file)
if (file.FileHash.Length > 0 && (file.Flags & EDepotFileFlag.Directory) == 0)
{
for (var i = 0; i < file.FileHash.Length; ++i)
{
if (file.FileHash[i] != 0)
{
hash = file.FileHash;
break;
}
}
}
// Limit path names to 260 characters (default windows max length)
// File column is varchar(260) and not higher to prevent reducing performance
// See https://stackoverflow.com/questions/1962310/importance-of-varchar-length-in-mysql-table/1962329#1962329
// Until 2019 there hasn't been a single file that went over this limit, so far there has been only one
// game with a big node_modules path, so we're safeguarding by limiting it.
if (name.Length > 260)
{
using var sha = new System.Security.Cryptography.SHA1Managed();
var nameHash = Utils.ByteArrayToString(sha.ComputeHash(System.Text.Encoding.UTF8.GetBytes(name)));
name = $"{{SteamDB file name is too long}}/{nameHash}/...{name.Substring(name.Length - 150)}";
}
if (filesOld.ContainsKey(name))
{
var oldFile = filesOld[name];
var updateFile = false;
if (oldFile.Size != file.TotalSize || !Utils.IsEqualSHA1(hash, oldFile.Hash))
{
await MakeHistory(db, transaction, request, name, "modified", oldFile.Size, file.TotalSize);
updateFile = true;
}
if (oldFile.Flags != file.Flags)
{
await MakeHistory(db, transaction, request, name, "modified_flags", (ulong)oldFile.Flags, (ulong)file.Flags);
updateFile = true;
}
if (updateFile)
{
await db.ExecuteAsync("UPDATE `DepotsFiles` SET `Hash` = @Hash, `Size` = @Size, `Flags` = @Flags WHERE `DepotID` = @DepotID AND `ID` = @ID", new DepotFile
{
ID = oldFile.ID,
DepotID = request.DepotID,
Hash = hash,
Size = file.TotalSize,
Flags = file.Flags
}, transaction : transaction);
}
filesOld.Remove(name);
}
else
{
// We want to historize modifications first, and only then deletions and additions
filesAdded.Add(new DepotFile
{
DepotID = request.DepotID,
Hash = hash,
File = name,
Size = file.TotalSize,
Flags = file.Flags
});
}
}
if (filesOld.Count > 0)
{
await db.ExecuteAsync("DELETE FROM `DepotsFiles` WHERE `DepotID` = @DepotID AND `ID` IN @Files", new { request.DepotID, Files = filesOld.Select(x => x.Value.ID) }, transaction : transaction);
await db.ExecuteAsync(HistoryQuery, filesOld.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ManifestID = request.ManifestID,
ChangeID = request.ChangeNumber,
Action = "removed",
File = x.Value.File,
OldValue = x.Value.Size
}), transaction : transaction);
}
if (filesAdded.Count > 0)
{
await db.ExecuteAsync("INSERT INTO `DepotsFiles` (`DepotID`, `File`, `Hash`, `Size`, `Flags`) VALUES (@DepotID, @File, @Hash, @Size, @Flags)", filesAdded, transaction : transaction);
if (shouldHistorize)
{
await db.ExecuteAsync(HistoryQuery, filesAdded.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ManifestID = request.ManifestID,
ChangeID = request.ChangeNumber,
Action = "added",
File = x.File,
NewValue = x.Size
}), transaction : transaction);
}
}
await db.ExecuteAsync("UPDATE `Depots` SET `LastManifestID` = @ManifestID, `LastUpdated` = CURRENT_TIMESTAMP() WHERE `DepotID` = @DepotID", new { request.DepotID, request.ManifestID }, transaction : transaction);
return(EResult.OK);
}
