Exemple #1
0
        public static void LoadServers()
        {
            Log.WriteInfo("Steam", "Loading Steam servers...");

            var loadServersTask = SteamDirectory.Initialize(CellID);

            TaskManager.RegisterErrorHandler(loadServersTask);

            loadServersTask.Wait();

            if (loadServersTask.IsFaulted)
            {
                Environment.Exit(1);
            }
        }
Exemple #2
0
        /*
         * Here be dragons.
         */
        public static async Task <EResult> DownloadFilesFromDepot(DepotProcessor.ManifestJob job, DepotManifest depotManifest)
        {
            var files         = depotManifest.Files.Where(x => IsFileNameMatching(job.DepotID, x.FileName)).ToList();
            var downloadState = EResult.Fail;

            var hashesFile = Path.Combine(Application.Path, "files", ".support", "hashes", string.Format("{0}.json", job.DepotID));
            ConcurrentDictionary <string, byte[]> hashes;

            if (File.Exists(hashesFile))
            {
                hashes = JsonConvert.DeserializeObject <ConcurrentDictionary <string, byte[]> >(File.ReadAllText(hashesFile));
            }
            else
            {
                hashes = new ConcurrentDictionary <string, byte[]>();
            }

            foreach (var file in hashes.Keys.Except(files.Select(x => x.FileName)))
            {
                Log.WriteWarn(nameof(FileDownloader), $"\"{file}\" no longer exists in manifest");
            }

            Log.WriteInfo("FileDownloader", "Will download {0} files from depot {1}", files.Count, job.DepotID);

            var downloadedFiles = 0;
            var fileTasks       = new Task[files.Count];

            for (var i = 0; i < fileTasks.Length; i++)
            {
                var file = files[i];
                fileTasks[i] = TaskManager.Run(async() =>
                {
                    hashes.TryGetValue(file.FileName, out var hash);

                    var fileState = await DownloadFile(job, file, hash);

                    if (fileState == EResult.OK || fileState == EResult.SameAsPreviousValue)
                    {
                        hashes[file.FileName] = file.FileHash;

                        downloadedFiles++;
                    }

                    if (fileState != EResult.SameAsPreviousValue)
                    {
                        // Do not write progress info to log file
                        Console.WriteLine("{1} [{0,6:#00.00}%] {2} files left to download", downloadedFiles / (float)files.Count * 100.0f, job.DepotName, files.Count - downloadedFiles);
                    }

                    if (downloadState == EResult.DataCorruption)
                    {
                        return;
                    }

                    if (fileState == EResult.OK || fileState == EResult.DataCorruption)
                    {
                        downloadState = fileState;
                    }
                }).Unwrap();

                // Register error handler on inner task
                TaskManager.RegisterErrorHandler(fileTasks[i]);
            }

            await Task.WhenAll(fileTasks).ConfigureAwait(false);

            if (downloadState == EResult.OK)
            {
                File.WriteAllText(hashesFile, JsonConvert.SerializeObject(hashes));

                job.Result = EResult.OK;
            }
            else
            {
                job.Result = EResult.Ignored;
            }

            return(job.Result);
        }
Exemple #3
0
        private static async Task <EResult> DownloadFile(DepotProcessor.ManifestJob job, DepotManifest.FileData file, byte[] hash)
        {
            var directory    = Path.Combine(Application.Path, "files", DownloadFolders[job.DepotID], Path.GetDirectoryName(file.FileName));
            var finalPath    = new FileInfo(Path.Combine(directory, Path.GetFileName(file.FileName)));
            var downloadPath = new FileInfo(Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".steamdb_tmp")));

            if (!Directory.Exists(directory))
            {
                Directory.CreateDirectory(directory);
            }
            else if (file.TotalSize == 0)
            {
                if (!finalPath.Exists)
                {
                    using (var _ = finalPath.Create())
                    {
                        // FileInfo.Create returns a stream but we don't need it
                    }

                    Log.WriteInfo("FileDownloader", "{0} created an empty file", file.FileName);

                    return(EResult.SameAsPreviousValue);
                }
                else if (finalPath.Length == 0)
                {
#if DEBUG
                    Log.WriteDebug("FileDownloader", "{0} is already empty", file.FileName);
#endif

                    return(EResult.SameAsPreviousValue);
                }
            }
            else if (hash != null && file.FileHash.SequenceEqual(hash))
            {
#if DEBUG
                Log.WriteDebug("FileDownloader", "{0} already matches the file we have", file.FileName);
#endif

                return(EResult.SameAsPreviousValue);
            }

            byte[] checksum;

            using (var sha = SHA1.Create())
            {
                checksum = sha.ComputeHash(Encoding.UTF8.GetBytes(file.FileName));
            }

            var neededChunks  = new List <DepotManifest.ChunkData>();
            var chunks        = file.Chunks.OrderBy(x => x.Offset).ToList();
            var oldChunksFile = Path.Combine(Application.Path, "files", ".support", "chunks", string.Format("{0}-{1}.json", job.DepotID, BitConverter.ToString(checksum)));

            using (var fs = downloadPath.Open(FileMode.OpenOrCreate, FileAccess.ReadWrite))
            {
                fs.SetLength((long)file.TotalSize);

                if (finalPath.Exists && File.Exists(oldChunksFile))
                {
                    var oldChunks = JsonConvert.DeserializeObject <List <DepotManifest.ChunkData> >(File.ReadAllText(oldChunksFile), JsonHandleAllReferences);

                    using (var fsOld = finalPath.Open(FileMode.Open, FileAccess.Read))
                    {
                        foreach (var chunk in chunks)
                        {
                            var oldChunk = oldChunks.Find(c => c.ChunkID.SequenceEqual(chunk.ChunkID));

                            if (oldChunk != null)
                            {
                                var oldData = new byte[oldChunk.UncompressedLength];
                                fsOld.Seek((long)oldChunk.Offset, SeekOrigin.Begin);
                                fsOld.Read(oldData, 0, oldData.Length);

                                var existingChecksum = Utils.AdlerHash(oldData);

                                if (existingChecksum.SequenceEqual(chunk.Checksum))
                                {
                                    fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
                                    fs.Write(oldData, 0, oldData.Length);

#if DEBUG
                                    Log.WriteDebug("FileDownloader", "{0} Found chunk ({1}), not downloading", file.FileName, chunk.Offset);
#endif
                                }
                                else
                                {
                                    neededChunks.Add(chunk);

#if DEBUG
                                    Log.WriteDebug("FileDownloader", "{0} Found chunk ({1}), but checksum differs", file.FileName, chunk.Offset);
#endif
                                }
                            }
                            else
                            {
                                neededChunks.Add(chunk);
                            }
                        }
                    }
                }
                else
                {
                    neededChunks = chunks;
                }
            }

            var downloadedSize    = file.TotalSize - (ulong)neededChunks.Sum(x => x.UncompressedLength);
            var chunkCancellation = new CancellationTokenSource();
            var chunkTasks        = new Task[neededChunks.Count];

            Log.WriteInfo("FileDownloader", "Downloading {0} ({1} bytes, {2} out of {3} chunks)", file.FileName, downloadedSize, neededChunks.Count, chunks.Count);

            for (var i = 0; i < chunkTasks.Length; i++)
            {
                var chunk = neededChunks[i];
                chunkTasks[i] = TaskManager.Run(async() =>
                {
                    try
                    {
                        chunkCancellation.Token.ThrowIfCancellationRequested();

                        await ChunkDownloadingSemaphore.WaitAsync(chunkCancellation.Token).ConfigureAwait(false);

                        var result = await DownloadChunk(job, chunk, downloadPath);

                        if (!result)
                        {
                            Log.WriteWarn("FileDownloader", "Failed to download chunk for {0}", file.FileName);

                            chunkCancellation.Cancel();
                        }
                        else
                        {
                            downloadedSize += chunk.UncompressedLength;

                            // Do not write progress info to log file
                            Console.WriteLine("{2} [{0,6:#00.00}%] {1}", downloadedSize / (float)file.TotalSize * 100.0f, file.FileName, job.DepotName);
                        }
                    }
                    finally
                    {
                        ChunkDownloadingSemaphore.Release();
                    }
                }).Unwrap();

                // Register error handler on inner task
                TaskManager.RegisterErrorHandler(chunkTasks[i]);
            }

            await Task.WhenAll(chunkTasks).ConfigureAwait(false);

            using (var fs = downloadPath.Open(FileMode.Open, FileAccess.ReadWrite))
            {
                fs.Seek(0, SeekOrigin.Begin);

                using (var sha = SHA1.Create())
                {
                    checksum = sha.ComputeHash(fs);
                }
            }

            if (!file.FileHash.SequenceEqual(checksum))
            {
                IRC.Instance.SendOps("{0}[{1}]{2} Failed to correctly download {3}{4}",
                                     Colors.OLIVE, job.DepotName, Colors.NORMAL, Colors.BLUE, file.FileName);

                Log.WriteWarn("FileDownloader", "Failed to download file {0} ({1})", file.FileName, job.Server);

                downloadPath.Delete();

                return(EResult.DataCorruption);
            }

            Log.WriteInfo("FileDownloader", "Downloaded {0} from {1}", file.FileName, job.DepotName);

            finalPath.Delete();

            downloadPath.MoveTo(finalPath.FullName);

            if (chunks.Count > 1)
            {
                File.WriteAllText(oldChunksFile, JsonConvert.SerializeObject(chunks, Formatting.None, JsonHandleAllReferences));
            }
            else if (File.Exists(oldChunksFile))
            {
                File.Delete(oldChunksFile);
            }

            return(EResult.OK);
        }
Exemple #4
0
        private static void OnPICSProductInfo(SteamApps.PICSProductInfoCallback callback)
        {
            JobManager.TryRemoveJob(callback.JobID);

            var processors = new List <BaseProcessor>(
                callback.Apps.Count +
                callback.Packages.Count +
                callback.UnknownApps.Count +
                callback.UnknownPackages.Count
                );

            processors.AddRange(callback.Apps.Select(app => new AppProcessor(app.Key, app.Value)));
            processors.AddRange(callback.Packages.Select(package => new SubProcessor(package.Key, package.Value)));
            processors.AddRange(callback.UnknownApps.Select(app => new AppProcessor(app, null)));
            processors.AddRange(callback.UnknownPackages.Select(package => new SubProcessor(package, null)));

            foreach (var workaround in processors)
            {
                var processor = workaround;

                Task mostRecentItem;

                lock (CurrentlyProcessing)
                {
                    CurrentlyProcessing.TryGetValue(processor.Id, out mostRecentItem);
                }

                var workerItem = TaskManager.Run(async() =>
                {
                    try
                    {
                        await Semaphore.WaitAsync(TaskManager.TaskCancellationToken.Token).ConfigureAwait(false);

                        if (mostRecentItem?.IsCompleted == false)
                        {
                            Log.WriteDebug(processor.ToString(), $"Waiting for previous task to finish processing ({CurrentlyProcessing.Count})");

                            await mostRecentItem.ConfigureAwait(false);

#if DEBUG
                            Log.WriteDebug(processor.ToString(), "Previous task lock ended");
#endif
                        }

                        await processor.Process().ConfigureAwait(false);
                    }
                    catch (Exception e)
                    {
                        ErrorReporter.Notify(processor.ToString(), e);
                    }
                    finally
                    {
                        Semaphore.Release();

                        processor.Dispose();
                    }

                    return(processor);
                }).Unwrap();

                lock (CurrentlyProcessing)
                {
                    CurrentlyProcessing[processor.Id] = workerItem;
                }

                // Register error handler on inner task and the continuation
                TaskManager.RegisterErrorHandler(workerItem);
                TaskManager.RegisterErrorHandler(workerItem.ContinueWith(RemoveProcessorLock, TaskManager.TaskCancellationToken.Token));
            }
        }
        private async Task DownloadDepots(uint appID, List <ManifestJob> depots)
        {
            Log.WriteDebug("Depot Downloader", "Will process {0} depots ({1} depot locks left)", depots.Count(), DepotLocks.Count);

            var  processTasks       = new List <Task <EResult> >();
            bool anyFilesDownloaded = false;

            foreach (var depot in depots)
            {
                var instance = depot.Anonymous ? Steam.Anonymous.Apps : Steam.Instance.Apps;

                depot.DepotKey = await GetDepotDecryptionKey(instance, depot.DepotID, appID);

                if (depot.DepotKey == null)
                {
                    RemoveLock(depot.DepotID);

                    continue;
                }

                var cdnToken = await GetCDNAuthToken(instance, appID, depot.DepotID);

                if (cdnToken == null)
                {
                    RemoveLock(depot.DepotID);

                    Log.WriteDebug("Depot Downloader", "Got a depot key for depot {0} but no cdn auth token", depot.DepotID);

                    continue;
                }

                depot.CDNToken = cdnToken.Token;
                depot.Server   = cdnToken.Server;

                DepotManifest depotManifest = null;
                string        lastError     = string.Empty;

                for (var i = 0; i <= 5; i++)
                {
                    try
                    {
                        depotManifest = CDNClient.DownloadManifest(depot.DepotID, depot.ManifestID, depot.Server, depot.CDNToken, depot.DepotKey);

                        break;
                    }
                    catch (Exception e)
                    {
                        Log.WriteWarn("Depot Downloader", "[{0}] Manifest download failed: {1} - {2}", depot.DepotID, e.GetType(), e.Message);

                        lastError = e.Message;
                    }

                    // TODO: get new auth key if auth fails

                    if (depotManifest == null)
                    {
                        await Task.Delay(Utils.ExponentionalBackoff(i));
                    }
                }

                if (depotManifest == null)
                {
                    RemoveLock(depot.DepotID);

                    Log.WriteError("Depot Processor", "Failed to download depot manifest for app {0} depot {1} ({2}: {3})", appID, depot.DepotID, depot.Server, lastError);

                    if (FileDownloader.IsImportantDepot(depot.DepotID))
                    {
                        IRC.Instance.SendOps("{0}[{1}]{2} Failed to download app {3} depot {4} manifest ({5}: {6})",
                                             Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL, appID, depot.DepotID, depot.Server, lastError);
                    }

                    continue;
                }

                var task = TaskManager.Run(() =>
                {
                    using (var db = Database.GetConnection())
                    {
                        using (var transaction = db.BeginTransaction())
                        {
                            var result = ProcessDepotAfterDownload(db, depot, depotManifest);
                            transaction.Commit();
                            return(result);
                        }
                    }
                });

                processTasks.Add(task);

                if (FileDownloader.IsImportantDepot(depot.DepotID))
                {
                    task = TaskManager.Run(() =>
                    {
                        var result = FileDownloader.DownloadFilesFromDepot(appID, depot, depotManifest);

                        if (result == EResult.OK)
                        {
                            anyFilesDownloaded = true;
                        }

                        return(result);
                    }, TaskCreationOptions.LongRunning);

                    TaskManager.RegisterErrorHandler(task);

                    processTasks.Add(task);
                }
            }

            if (SaveLocalConfig)
            {
                SaveLocalConfig = false;

                LocalConfig.Save();
            }

            await Task.WhenAll(processTasks);

            Log.WriteDebug("Depot Downloader", "{0} depot downloads finished", depots.Count());

            // TODO: use ContinueWith on tasks
            if (!anyFilesDownloaded)
            {
                foreach (var depot in depots)
                {
                    RemoveLock(depot.DepotID);
                }

                return;
            }

            if (!File.Exists(UpdateScript))
            {
                return;
            }

            bool lockTaken = false;

            try
            {
                UpdateScriptLock.Enter(ref lockTaken);

                foreach (var depot in depots)
                {
                    if (depot.Result == EResult.OK)
                    {
                        RunUpdateScript(string.Format("{0} no-git", depot.DepotID));
                    }

                    RemoveLock(depot.DepotID);
                }

                // Only commit changes if all depots downloaded
                if (processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored))
                {
                    if (!RunUpdateScript(appID))
                    {
                        RunUpdateScript("0");
                    }
                }
                else
                {
                    Log.WriteDebug("Depot Processor", "Reprocessing the app {0} because some files failed to download", appID);

                    IRC.Instance.SendOps("{0}[{1}]{2} Reprocessing the app due to download failures",
                                         Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL
                                         );

                    JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
                }
            }
            finally
            {
                if (lockTaken)
                {
                    UpdateScriptLock.Exit();
                }
            }
        }
Exemple #6
0
        private async Task DownloadDepots(IEnumerable <ManifestJob> depots)
        {
            Log.WriteDebug("Depot Downloader", "Will process {0} depots ({1} depot locks left)", depots.Count(), DepotLocks.Count);

            var  processTasks       = new List <Task <EResult> >();
            bool anyFilesDownloaded = false;

            foreach (var depot in depots)
            {
                depot.DepotKey = await GetDepotDecryptionKey(depot.DepotID, depot.ParentAppID);

                if (depot.DepotKey == null)
                {
                    RemoveLock(depot.DepotID);

                    continue;
                }

                var cdnToken = await GetCDNAuthToken(depot.DepotID);

                if (cdnToken == null)
                {
                    RemoveLock(depot.DepotID);

                    Log.WriteDebug("Depot Downloader", "Got a depot key for depot {0} but no cdn auth token", depot.DepotID);

                    continue;
                }

                depot.CDNToken = cdnToken.Token;
                depot.Server   = cdnToken.Server;

                DepotManifest depotManifest = null;
                string        lastError     = string.Empty;

                for (var i = 0; i <= 5; i++)
                {
                    try
                    {
                        depotManifest = CDNClient.DownloadManifest(depot.DepotID, depot.ManifestID, depot.Server, depot.CDNToken, depot.DepotKey);

                        break;
                    }
                    catch (Exception e)
                    {
                        Log.WriteWarn("Depot Downloader", "{0} Manifest download failed: {1} - {2}", depot.DepotID, e.GetType(), e.Message);

                        lastError = e.Message;
                    }

                    // TODO: get new auth key if auth fails

                    if (depotManifest == null)
                    {
                        await Task.Delay(Utils.ExponentionalBackoff(i));
                    }
                }

                if (depotManifest == null)
                {
                    RemoveLock(depot.DepotID);

                    Log.WriteError("Depot Processor", "Failed to download depot manifest for depot {0} ({1}: {2})", depot.DepotID, depot.Server, lastError);

                    if (FileDownloader.IsImportantDepot(depot.DepotID))
                    {
                        IRC.Instance.SendOps("{0}[{1}]{2} Failed to download depot {3} manifest ({4}: {5})",
                                             Colors.OLIVE, Steam.GetAppName(depot.ParentAppID), Colors.NORMAL, depot.DepotID, depot.Server, lastError);
                    }

                    continue;
                }

                var task = TaskManager.Run(() =>
                {
                    using (var db = Database.GetConnection())
                    {
                        return(ProcessDepotAfterDownload(db, depot, depotManifest));
                    }
                });

                processTasks.Add(task);

                if (FileDownloader.IsImportantDepot(depot.DepotID))
                {
                    task = TaskManager.Run(() =>
                    {
                        var result = FileDownloader.DownloadFilesFromDepot(depot, depotManifest);

                        if (result == EResult.OK)
                        {
                            anyFilesDownloaded = true;
                        }

                        return(result);
                    }, TaskCreationOptions.LongRunning);

                    TaskManager.RegisterErrorHandler(task);

                    processTasks.Add(task);
                }
            }

            if (SaveLocalConfig)
            {
                SaveLocalConfig = false;

                LocalConfig.Save();
            }

            await Task.WhenAll(processTasks);

            // TODO: use ContinueWith on tasks
            if (!anyFilesDownloaded)
            {
                Log.WriteDebug("Depot Downloader", "Tasks awaited for {0} depot downloads", depots.Count());

                foreach (var depot in depots)
                {
                    RemoveLock(depot.DepotID);
                }

                return;
            }

            var canUpdate = processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored) && File.Exists(UpdateScript);

#if true
            Log.WriteDebug("Depot Downloader", "Tasks awaited for {0} depot downloads (will run script: {1})", depots.Count(), canUpdate);
#endif
            bool lockTaken = false;

            try
            {
                UpdateScriptLock.Enter(ref lockTaken);

                foreach (var depot in depots)
                {
                    // TODO: this only needs to run if any downloaded files changed
                    if (canUpdate && FileDownloader.IsImportantDepot(depot.DepotID))
                    {
                        RunUpdateScript(string.Format("{0} no-git", depot.DepotID));
                    }

                    RemoveLock(depot.DepotID);
                }

                if (canUpdate)
                {
                    RunUpdateScript("0");
                }
            }
            finally
            {
                if (lockTaken)
                {
                    UpdateScriptLock.Exit();
                }
            }
        }