public static void LoadServers()
{
Log.WriteInfo("Steam", "Loading Steam servers...");
var loadServersTask = SteamDirectory.Initialize(CellID);
TaskManager.RegisterErrorHandler(loadServersTask);
loadServersTask.Wait();
if (loadServersTask.IsFaulted)
{
Environment.Exit(1);
}
}
/*
* Here be dragons.
*/
public static async Task <EResult> DownloadFilesFromDepot(DepotProcessor.ManifestJob job, DepotManifest depotManifest)
{
var files = depotManifest.Files.Where(x => IsFileNameMatching(job.DepotID, x.FileName)).ToList();
var downloadState = EResult.Fail;
var hashesFile = Path.Combine(Application.Path, "files", ".support", "hashes", string.Format("{0}.json", job.DepotID));
ConcurrentDictionary <string, byte[]> hashes;
if (File.Exists(hashesFile))
{
hashes = JsonConvert.DeserializeObject <ConcurrentDictionary <string, byte[]> >(File.ReadAllText(hashesFile));
}
else
{
hashes = new ConcurrentDictionary <string, byte[]>();
}
foreach (var file in hashes.Keys.Except(files.Select(x => x.FileName)))
{
Log.WriteWarn(nameof(FileDownloader), $"\"{file}\" no longer exists in manifest");
}
Log.WriteInfo("FileDownloader", "Will download {0} files from depot {1}", files.Count, job.DepotID);
var downloadedFiles = 0;
var fileTasks = new Task[files.Count];
for (var i = 0; i < fileTasks.Length; i++)
{
var file = files[i];
fileTasks[i] = TaskManager.Run(async() =>
{
hashes.TryGetValue(file.FileName, out var hash);
var fileState = await DownloadFile(job, file, hash);
if (fileState == EResult.OK || fileState == EResult.SameAsPreviousValue)
{
hashes[file.FileName] = file.FileHash;
downloadedFiles++;
}
if (fileState != EResult.SameAsPreviousValue)
{
// Do not write progress info to log file
Console.WriteLine("{1} [{0,6:#00.00}%] {2} files left to download", downloadedFiles / (float)files.Count * 100.0f, job.DepotName, files.Count - downloadedFiles);
}
if (downloadState == EResult.DataCorruption)
{
return;
}
if (fileState == EResult.OK || fileState == EResult.DataCorruption)
{
downloadState = fileState;
}
}).Unwrap();
// Register error handler on inner task
TaskManager.RegisterErrorHandler(fileTasks[i]);
}
await Task.WhenAll(fileTasks).ConfigureAwait(false);
if (downloadState == EResult.OK)
{
File.WriteAllText(hashesFile, JsonConvert.SerializeObject(hashes));
job.Result = EResult.OK;
}
else
{
job.Result = EResult.Ignored;
}
return(job.Result);
}
private static async Task <EResult> DownloadFile(DepotProcessor.ManifestJob job, DepotManifest.FileData file, byte[] hash)
{
var directory = Path.Combine(Application.Path, "files", DownloadFolders[job.DepotID], Path.GetDirectoryName(file.FileName));
var finalPath = new FileInfo(Path.Combine(directory, Path.GetFileName(file.FileName)));
var downloadPath = new FileInfo(Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".steamdb_tmp")));
if (!Directory.Exists(directory))
{
Directory.CreateDirectory(directory);
}
else if (file.TotalSize == 0)
{
if (!finalPath.Exists)
{
using (var _ = finalPath.Create())
{
// FileInfo.Create returns a stream but we don't need it
}
Log.WriteInfo("FileDownloader", "{0} created an empty file", file.FileName);
return(EResult.SameAsPreviousValue);
}
else if (finalPath.Length == 0)
{
#if DEBUG
Log.WriteDebug("FileDownloader", "{0} is already empty", file.FileName);
#endif
return(EResult.SameAsPreviousValue);
}
}
else if (hash != null && file.FileHash.SequenceEqual(hash))
{
#if DEBUG
Log.WriteDebug("FileDownloader", "{0} already matches the file we have", file.FileName);
#endif
return(EResult.SameAsPreviousValue);
}
byte[] checksum;
using (var sha = SHA1.Create())
{
checksum = sha.ComputeHash(Encoding.UTF8.GetBytes(file.FileName));
}
var neededChunks = new List <DepotManifest.ChunkData>();
var chunks = file.Chunks.OrderBy(x => x.Offset).ToList();
var oldChunksFile = Path.Combine(Application.Path, "files", ".support", "chunks", string.Format("{0}-{1}.json", job.DepotID, BitConverter.ToString(checksum)));
using (var fs = downloadPath.Open(FileMode.OpenOrCreate, FileAccess.ReadWrite))
{
fs.SetLength((long)file.TotalSize);
if (finalPath.Exists && File.Exists(oldChunksFile))
{
var oldChunks = JsonConvert.DeserializeObject <List <DepotManifest.ChunkData> >(File.ReadAllText(oldChunksFile), JsonHandleAllReferences);
using (var fsOld = finalPath.Open(FileMode.Open, FileAccess.Read))
{
foreach (var chunk in chunks)
{
var oldChunk = oldChunks.Find(c => c.ChunkID.SequenceEqual(chunk.ChunkID));
if (oldChunk != null)
{
var oldData = new byte[oldChunk.UncompressedLength];
fsOld.Seek((long)oldChunk.Offset, SeekOrigin.Begin);
fsOld.Read(oldData, 0, oldData.Length);
var existingChecksum = Utils.AdlerHash(oldData);
if (existingChecksum.SequenceEqual(chunk.Checksum))
{
fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
fs.Write(oldData, 0, oldData.Length);
#if DEBUG
Log.WriteDebug("FileDownloader", "{0} Found chunk ({1}), not downloading", file.FileName, chunk.Offset);
#endif
}
else
{
neededChunks.Add(chunk);
#if DEBUG
Log.WriteDebug("FileDownloader", "{0} Found chunk ({1}), but checksum differs", file.FileName, chunk.Offset);
#endif
}
}
else
{
neededChunks.Add(chunk);
}
}
}
}
else
{
neededChunks = chunks;
}
}
var downloadedSize = file.TotalSize - (ulong)neededChunks.Sum(x => x.UncompressedLength);
var chunkCancellation = new CancellationTokenSource();
var chunkTasks = new Task[neededChunks.Count];
Log.WriteInfo("FileDownloader", "Downloading {0} ({1} bytes, {2} out of {3} chunks)", file.FileName, downloadedSize, neededChunks.Count, chunks.Count);
for (var i = 0; i < chunkTasks.Length; i++)
{
var chunk = neededChunks[i];
chunkTasks[i] = TaskManager.Run(async() =>
{
try
{
chunkCancellation.Token.ThrowIfCancellationRequested();
await ChunkDownloadingSemaphore.WaitAsync(chunkCancellation.Token).ConfigureAwait(false);
var result = await DownloadChunk(job, chunk, downloadPath);
if (!result)
{
Log.WriteWarn("FileDownloader", "Failed to download chunk for {0}", file.FileName);
chunkCancellation.Cancel();
}
else
{
downloadedSize += chunk.UncompressedLength;
// Do not write progress info to log file
Console.WriteLine("{2} [{0,6:#00.00}%] {1}", downloadedSize / (float)file.TotalSize * 100.0f, file.FileName, job.DepotName);
}
}
finally
{
ChunkDownloadingSemaphore.Release();
}
}).Unwrap();
// Register error handler on inner task
TaskManager.RegisterErrorHandler(chunkTasks[i]);
}
await Task.WhenAll(chunkTasks).ConfigureAwait(false);
using (var fs = downloadPath.Open(FileMode.Open, FileAccess.ReadWrite))
{
fs.Seek(0, SeekOrigin.Begin);
using (var sha = SHA1.Create())
{
checksum = sha.ComputeHash(fs);
}
}
if (!file.FileHash.SequenceEqual(checksum))
{
IRC.Instance.SendOps("{0}[{1}]{2} Failed to correctly download {3}{4}",
Colors.OLIVE, job.DepotName, Colors.NORMAL, Colors.BLUE, file.FileName);
Log.WriteWarn("FileDownloader", "Failed to download file {0} ({1})", file.FileName, job.Server);
downloadPath.Delete();
return(EResult.DataCorruption);
}
Log.WriteInfo("FileDownloader", "Downloaded {0} from {1}", file.FileName, job.DepotName);
finalPath.Delete();
downloadPath.MoveTo(finalPath.FullName);
if (chunks.Count > 1)
{
File.WriteAllText(oldChunksFile, JsonConvert.SerializeObject(chunks, Formatting.None, JsonHandleAllReferences));
}
else if (File.Exists(oldChunksFile))
{
File.Delete(oldChunksFile);
}
return(EResult.OK);
}
private static void OnPICSProductInfo(SteamApps.PICSProductInfoCallback callback)
{
JobManager.TryRemoveJob(callback.JobID);
var processors = new List <BaseProcessor>(
callback.Apps.Count +
callback.Packages.Count +
callback.UnknownApps.Count +
callback.UnknownPackages.Count
);
processors.AddRange(callback.Apps.Select(app => new AppProcessor(app.Key, app.Value)));
processors.AddRange(callback.Packages.Select(package => new SubProcessor(package.Key, package.Value)));
processors.AddRange(callback.UnknownApps.Select(app => new AppProcessor(app, null)));
processors.AddRange(callback.UnknownPackages.Select(package => new SubProcessor(package, null)));
foreach (var workaround in processors)
{
var processor = workaround;
Task mostRecentItem;
lock (CurrentlyProcessing)
{
CurrentlyProcessing.TryGetValue(processor.Id, out mostRecentItem);
}
var workerItem = TaskManager.Run(async() =>
{
try
{
await Semaphore.WaitAsync(TaskManager.TaskCancellationToken.Token).ConfigureAwait(false);
if (mostRecentItem?.IsCompleted == false)
{
Log.WriteDebug(processor.ToString(), $"Waiting for previous task to finish processing ({CurrentlyProcessing.Count})");
await mostRecentItem.ConfigureAwait(false);
#if DEBUG
Log.WriteDebug(processor.ToString(), "Previous task lock ended");
#endif
}
await processor.Process().ConfigureAwait(false);
}
catch (Exception e)
{
ErrorReporter.Notify(processor.ToString(), e);
}
finally
{
Semaphore.Release();
processor.Dispose();
}
return(processor);
}).Unwrap();
lock (CurrentlyProcessing)
{
CurrentlyProcessing[processor.Id] = workerItem;
}
// Register error handler on inner task and the continuation
TaskManager.RegisterErrorHandler(workerItem);
TaskManager.RegisterErrorHandler(workerItem.ContinueWith(RemoveProcessorLock, TaskManager.TaskCancellationToken.Token));
}
}
private async Task DownloadDepots(uint appID, List <ManifestJob> depots)
{
Log.WriteDebug("Depot Downloader", "Will process {0} depots ({1} depot locks left)", depots.Count(), DepotLocks.Count);
var processTasks = new List <Task <EResult> >();
bool anyFilesDownloaded = false;
foreach (var depot in depots)
{
var instance = depot.Anonymous ? Steam.Anonymous.Apps : Steam.Instance.Apps;
depot.DepotKey = await GetDepotDecryptionKey(instance, depot.DepotID, appID);
if (depot.DepotKey == null)
{
RemoveLock(depot.DepotID);
continue;
}
var cdnToken = await GetCDNAuthToken(instance, appID, depot.DepotID);
if (cdnToken == null)
{
RemoveLock(depot.DepotID);
Log.WriteDebug("Depot Downloader", "Got a depot key for depot {0} but no cdn auth token", depot.DepotID);
continue;
}
depot.CDNToken = cdnToken.Token;
depot.Server = cdnToken.Server;
DepotManifest depotManifest = null;
string lastError = string.Empty;
for (var i = 0; i <= 5; i++)
{
try
{
depotManifest = CDNClient.DownloadManifest(depot.DepotID, depot.ManifestID, depot.Server, depot.CDNToken, depot.DepotKey);
break;
}
catch (Exception e)
{
Log.WriteWarn("Depot Downloader", "[{0}] Manifest download failed: {1} - {2}", depot.DepotID, e.GetType(), e.Message);
lastError = e.Message;
}
// TODO: get new auth key if auth fails
if (depotManifest == null)
{
await Task.Delay(Utils.ExponentionalBackoff(i));
}
}
if (depotManifest == null)
{
RemoveLock(depot.DepotID);
Log.WriteError("Depot Processor", "Failed to download depot manifest for app {0} depot {1} ({2}: {3})", appID, depot.DepotID, depot.Server, lastError);
if (FileDownloader.IsImportantDepot(depot.DepotID))
{
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download app {3} depot {4} manifest ({5}: {6})",
Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL, appID, depot.DepotID, depot.Server, lastError);
}
continue;
}
var task = TaskManager.Run(() =>
{
using (var db = Database.GetConnection())
{
using (var transaction = db.BeginTransaction())
{
var result = ProcessDepotAfterDownload(db, depot, depotManifest);
transaction.Commit();
return(result);
}
}
});
processTasks.Add(task);
if (FileDownloader.IsImportantDepot(depot.DepotID))
{
task = TaskManager.Run(() =>
{
var result = FileDownloader.DownloadFilesFromDepot(appID, depot, depotManifest);
if (result == EResult.OK)
{
anyFilesDownloaded = true;
}
return(result);
}, TaskCreationOptions.LongRunning);
TaskManager.RegisterErrorHandler(task);
processTasks.Add(task);
}
}
if (SaveLocalConfig)
{
SaveLocalConfig = false;
LocalConfig.Save();
}
await Task.WhenAll(processTasks);
Log.WriteDebug("Depot Downloader", "{0} depot downloads finished", depots.Count());
// TODO: use ContinueWith on tasks
if (!anyFilesDownloaded)
{
foreach (var depot in depots)
{
RemoveLock(depot.DepotID);
}
return;
}
if (!File.Exists(UpdateScript))
{
return;
}
bool lockTaken = false;
try
{
UpdateScriptLock.Enter(ref lockTaken);
foreach (var depot in depots)
{
if (depot.Result == EResult.OK)
{
RunUpdateScript(string.Format("{0} no-git", depot.DepotID));
}
RemoveLock(depot.DepotID);
}
// Only commit changes if all depots downloaded
if (processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored))
{
if (!RunUpdateScript(appID))
{
RunUpdateScript("0");
}
}
else
{
Log.WriteDebug("Depot Processor", "Reprocessing the app {0} because some files failed to download", appID);
IRC.Instance.SendOps("{0}[{1}]{2} Reprocessing the app due to download failures",
Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL
);
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
}
}
finally
{
if (lockTaken)
{
UpdateScriptLock.Exit();
}
}
}
private async Task DownloadDepots(IEnumerable <ManifestJob> depots)
{
Log.WriteDebug("Depot Downloader", "Will process {0} depots ({1} depot locks left)", depots.Count(), DepotLocks.Count);
var processTasks = new List <Task <EResult> >();
bool anyFilesDownloaded = false;
foreach (var depot in depots)
{
depot.DepotKey = await GetDepotDecryptionKey(depot.DepotID, depot.ParentAppID);
if (depot.DepotKey == null)
{
RemoveLock(depot.DepotID);
continue;
}
var cdnToken = await GetCDNAuthToken(depot.DepotID);
if (cdnToken == null)
{
RemoveLock(depot.DepotID);
Log.WriteDebug("Depot Downloader", "Got a depot key for depot {0} but no cdn auth token", depot.DepotID);
continue;
}
depot.CDNToken = cdnToken.Token;
depot.Server = cdnToken.Server;
DepotManifest depotManifest = null;
string lastError = string.Empty;
for (var i = 0; i <= 5; i++)
{
try
{
depotManifest = CDNClient.DownloadManifest(depot.DepotID, depot.ManifestID, depot.Server, depot.CDNToken, depot.DepotKey);
break;
}
catch (Exception e)
{
Log.WriteWarn("Depot Downloader", "{0} Manifest download failed: {1} - {2}", depot.DepotID, e.GetType(), e.Message);
lastError = e.Message;
}
// TODO: get new auth key if auth fails
if (depotManifest == null)
{
await Task.Delay(Utils.ExponentionalBackoff(i));
}
}
if (depotManifest == null)
{
RemoveLock(depot.DepotID);
Log.WriteError("Depot Processor", "Failed to download depot manifest for depot {0} ({1}: {2})", depot.DepotID, depot.Server, lastError);
if (FileDownloader.IsImportantDepot(depot.DepotID))
{
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download depot {3} manifest ({4}: {5})",
Colors.OLIVE, Steam.GetAppName(depot.ParentAppID), Colors.NORMAL, depot.DepotID, depot.Server, lastError);
}
continue;
}
var task = TaskManager.Run(() =>
{
using (var db = Database.GetConnection())
{
return(ProcessDepotAfterDownload(db, depot, depotManifest));
}
});
processTasks.Add(task);
if (FileDownloader.IsImportantDepot(depot.DepotID))
{
task = TaskManager.Run(() =>
{
var result = FileDownloader.DownloadFilesFromDepot(depot, depotManifest);
if (result == EResult.OK)
{
anyFilesDownloaded = true;
}
return(result);
}, TaskCreationOptions.LongRunning);
TaskManager.RegisterErrorHandler(task);
processTasks.Add(task);
}
}
if (SaveLocalConfig)
{
SaveLocalConfig = false;
LocalConfig.Save();
}
await Task.WhenAll(processTasks);
// TODO: use ContinueWith on tasks
if (!anyFilesDownloaded)
{
Log.WriteDebug("Depot Downloader", "Tasks awaited for {0} depot downloads", depots.Count());
foreach (var depot in depots)
{
RemoveLock(depot.DepotID);
}
return;
}
var canUpdate = processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored) && File.Exists(UpdateScript);
#if true
Log.WriteDebug("Depot Downloader", "Tasks awaited for {0} depot downloads (will run script: {1})", depots.Count(), canUpdate);
#endif
bool lockTaken = false;
try
{
UpdateScriptLock.Enter(ref lockTaken);
foreach (var depot in depots)
{
// TODO: this only needs to run if any downloaded files changed
if (canUpdate && FileDownloader.IsImportantDepot(depot.DepotID))
{
RunUpdateScript(string.Format("{0} no-git", depot.DepotID));
}
RemoveLock(depot.DepotID);
}
if (canUpdate)
{
RunUpdateScript("0");
}
}
finally
{
if (lockTaken)
{
UpdateScriptLock.Exit();
}
}
}