Beispiel #1
0
        public async Task DownloadMissingArchives(List <Archive> missing, bool download = true)
        {
            if (download)
            {
                foreach (var a in missing.Where(a => a.State.GetType() == typeof(ManualDownloader.State)))
                {
                    var outputPath = DownloadFolder.Combine(a.Name);
                    await a.State.Download(a, outputPath);
                }
            }

            DesiredThreads.OnNext(DownloadThreads);
            await missing.Where(a => a.State.GetType() != typeof(ManualDownloader.State))
            .PMap(Queue, async archive =>
            {
                Info($"Downloading {archive.Name}");
                var outputPath = DownloadFolder.Combine(archive.Name);

                if (download)
                {
                    if (outputPath.Exists)
                    {
                        var origName  = Path.GetFileNameWithoutExtension(archive.Name);
                        var ext       = Path.GetExtension(archive.Name);
                        var uniqueKey = archive.State.PrimaryKeyString.StringSha256Hex();
                        outputPath    = DownloadFolder.Combine(origName + "_" + uniqueKey + "_" + ext);
                        await outputPath.DeleteAsync();
                    }
                }

                return(await DownloadArchive(archive, download, outputPath));
            });

            DesiredThreads.OnNext(DiskThreads);
        }
Beispiel #2
0
        protected override async Task <bool> _Begin(CancellationToken cancel)
        {
            await Metrics.Send("begin_compiling", ModListName ?? "unknown");

            if (cancel.IsCancellationRequested)
            {
                return(false);
            }

            DesiredThreads.OnNext(DiskThreads);
            FileExtractor2.FavorPerfOverRAM = FavorPerfOverRam;

            UpdateTracker.Reset();
            UpdateTracker.NextStep("Gathering information");

            Utils.Log($"Compiling Game: {CompilingGame.Game}");
            Utils.Log("Games from setting files:");
            foreach (var game in Settings.IncludedGames)
            {
                Utils.Log($"- {game}");
            }

            Utils.Log($"VFS File Location: {VFSCacheName}");
            Utils.Log($"MO2 Folder: {SourcePath}");
            Utils.Log($"Downloads Folder: {DownloadsPath}");
            Utils.Log($"Game Folder: {GamePath}");

            var watcher = new DiskSpaceWatcher(cancel,
                                               new[] { SourcePath, DownloadsPath, GamePath, AbsolutePath.EntryPoint }, (long)2 << 31,
                                               drive =>
                    {
                    Utils.Log($"Aborting due to low space on {drive.Name}");
                    Abort();
                });
            var watcherTask = watcher.Start();

            if (cancel.IsCancellationRequested)
            {
                return(false);
            }

            List <AbsolutePath> roots = new List <AbsolutePath> {
                SourcePath, GamePath, DownloadsPath
            };

            roots.AddRange(Settings.IncludedGames.Select(g => g.MetaData().GameLocation()));

            UpdateTracker.NextStep("Indexing folders");

            if (cancel.IsCancellationRequested)
            {
                return(false);
            }

            await VFS.AddRoots(roots);

            UpdateTracker.NextStep("Cleaning output folder");
            await ModListOutputFolder.DeleteDirectory();

            if (cancel.IsCancellationRequested)
            {
                return(false);
            }

            UpdateTracker.NextStep("Inferring metas for game file downloads");
            await InferMetas();

            if (cancel.IsCancellationRequested)
            {
                return(false);
            }

            UpdateTracker.NextStep("Reindexing downloads after meta inferring");
            await VFS.AddRoot(DownloadsPath);

            if (cancel.IsCancellationRequested)
            {
                return(false);
            }

            UpdateTracker.NextStep("Pre-validating Archives");


            // Find all Downloads
            IndexedArchives = (await DownloadsPath.EnumerateFiles()
                               .Where(f => f.WithExtension(Consts.MetaFileExtension).Exists)
                               .PMap(Queue,
                                     async f => new IndexedArchive(VFS.Index.ByRootPath[f])
            {
                Name = (string)f.FileName,
                IniData = f.WithExtension(Consts.MetaFileExtension).LoadIniFile(),
                Meta = await f.WithExtension(Consts.MetaFileExtension).ReadAllTextAsync()
            })).ToList();


            await IndexGameFileHashes();

            IndexedArchives = IndexedArchives.DistinctBy(a => a.File.AbsoluteName).ToList();

            await CleanInvalidArchivesAndFillState();

            UpdateTracker.NextStep("Finding Install Files");
            ModListOutputFolder.CreateDirectory();

            var mo2Files = SourcePath.EnumerateFiles()
                           .Where(p => p.IsFile)
                           .Select(p =>
            {
                if (!VFS.Index.ByRootPath.ContainsKey(p))
                {
                    Utils.Log($"WELL THERE'S YOUR PROBLEM: {p} {VFS.Index.ByRootPath.Count}");
                }

                return(new RawSourceFile(VFS.Index.ByRootPath[p], p.RelativeTo(SourcePath)));
            });

            // If Game Folder Files exists, ignore the game folder
            IndexedFiles = IndexedArchives.SelectMany(f => f.File.ThisAndAllChildren)
                           .OrderBy(f => f.NestingFactor)
                           .GroupBy(f => f.Hash)
                           .ToDictionary(f => f.Key, f => f.AsEnumerable());

            AllFiles.SetTo(mo2Files
                           .DistinctBy(f => f.Path));

            Info($"Found {AllFiles.Count} files to build into mod list");

            if (cancel.IsCancellationRequested)
            {
                return(false);
            }

            UpdateTracker.NextStep("Verifying destinations");

            var dups = AllFiles.GroupBy(f => f.Path)
                       .Where(fs => fs.Count() > 1)
                       .Select(fs =>
            {
                Utils.Log(
                    $"Duplicate files installed to {fs.Key} from : {String.Join(", ", fs.Select(f => f.AbsolutePath))}");
                return(fs);
            }).ToList();

            if (dups.Count > 0)
            {
                Error($"Found {dups.Count} duplicates, exiting");
            }

            if (cancel.IsCancellationRequested)
            {
                return(false);
            }

            UpdateTracker.NextStep("Loading INIs");

            ArchivesByFullPath = IndexedArchives.ToDictionary(a => a.File.AbsoluteName);

            if (cancel.IsCancellationRequested)
            {
                return(false);
            }

            var stack = MakeStack();

            UpdateTracker.NextStep("Running Compilation Stack");
            var results = await AllFiles.PMap(Queue, UpdateTracker, f => RunStack(stack, f));

            // Add the extra files that were generated by the stack
            if (cancel.IsCancellationRequested)
            {
                return(false);
            }
            var noMatch = results.OfType <NoMatch>().ToArray();

            PrintNoMatches(noMatch);
            if (CheckForNoMatchExit(noMatch))
            {
                return(false);
            }

            foreach (var ignored in results.OfType <IgnoredDirectly>())
            {
                Utils.Log($"Ignored {ignored.To} because {ignored.Reason}");
            }

            InstallDirectives.SetTo(results.Where(i => !(i is IgnoredDirectly)));

            Info("Getting Nexus api_key, please click authorize if a browser window appears");

            UpdateTracker.NextStep("Building Patches");
            await BuildPatches();

            UpdateTracker.NextStep("Gathering Archives");
            await GatherArchives();

            UpdateTracker.NextStep("Gathering Metadata");
            await GatherMetaData();

            ModList = new ModList
            {
                GameType         = CompilingGame.Game,
                WabbajackVersion = Consts.CurrentMinimumWabbajackVersion,
                Archives         = SelectedArchives.ToList(),
                ModManager       = ModManager.MO2,
                Directives       = InstallDirectives,
                Name             = ModListName ?? "untitled",
                Author           = ModListAuthor ?? "",
                Description      = ModListDescription ?? "",
                Readme           = ModlistReadme ?? "",
                Image            = ModListImage != default ? ModListImage.FileName : default,
 public TestInstaller(AbsolutePath archive, ModList modList, AbsolutePath outputFolder, AbsolutePath downloadFolder, SystemParameters parameters)
     : base(archive, modList, outputFolder, downloadFolder, parameters, steps: 1, modList.GameType)
 {
     DesiredThreads.OnNext(1);
 }
Beispiel #4
0
        protected override async Task <bool> _Begin(CancellationToken cancel)
        {
            await Metrics.Send("begin_compiling", MO2Profile ?? "unknown");

            if (cancel.IsCancellationRequested)
            {
                return(false);
            }

            DesiredThreads.OnNext(DiskThreads);
            FileExtractor2.FavorPerfOverRAM = FavorPerfOverRam;

            UpdateTracker.Reset();
            UpdateTracker.NextStep("Gathering information");

            Utils.Log("Loading compiler Settings");
            Settings = await CompilerSettings.Load(MO2ProfileDir);

            Settings.IncludedGames = Settings.IncludedGames.Add(CompilingGame.Game);

            Info("Looking for other profiles");
            var otherProfilesPath = MO2ProfileDir.Combine("otherprofiles.txt");

            SelectedProfiles = new HashSet <string>();
            if (otherProfilesPath.Exists)
            {
                SelectedProfiles = (await otherProfilesPath.ReadAllLinesAsync()).ToHashSet();
            }

            SelectedProfiles.Add(MO2Profile !);

            Info("Using Profiles: " + string.Join(", ", SelectedProfiles.OrderBy(p => p)));

            Utils.Log($"Compiling Game: {CompilingGame.Game}");
            Utils.Log("Games from setting files:");
            foreach (var game in Settings.IncludedGames)
            {
                Utils.Log($"- {game}");
            }

            Utils.Log($"VFS File Location: {VFSCacheName}");
            Utils.Log($"MO2 Folder: {SourcePath}");
            Utils.Log($"Downloads Folder: {DownloadsPath}");
            Utils.Log($"Game Folder: {GamePath}");

            var watcher = new DiskSpaceWatcher(cancel,
                                               new[] { SourcePath, DownloadsPath, GamePath, AbsolutePath.EntryPoint }, (long)2 << 31,
                                               drive =>
                    {
                    Utils.Log($"Aborting due to low space on {drive.Name}");
                    Abort();
                });
            var watcherTask = watcher.Start();

            if (cancel.IsCancellationRequested)
            {
                return(false);
            }

            List <AbsolutePath> roots;

            if (UseGamePaths)
            {
                roots = new List <AbsolutePath> {
                    SourcePath, GamePath, DownloadsPath
                };
                roots.AddRange(Settings.IncludedGames.Select(g => g.MetaData().GameLocation()));
            }
            else
            {
                roots = new List <AbsolutePath> {
                    SourcePath, DownloadsPath
                };
            }

            // TODO: make this generic so we can add more paths

            var lootPath = (AbsolutePath)Path.Combine(
                Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),
                "LOOT");
            IEnumerable <RawSourceFile> lootFiles = new List <RawSourceFile>();

            if (lootPath.Exists)
            {
                roots.Add(lootPath);
            }

            UpdateTracker.NextStep("Indexing folders");

            if (cancel.IsCancellationRequested)
            {
                return(false);
            }

            await VFS.AddRoots(roots);

            if (lootPath.Exists)
            {
                if (CompilingGame.MO2Name == null)
                {
                    throw new ArgumentException("Compiling game had no MO2 name specified.");
                }

                var lootGameDirs = new[]
                {
                    CompilingGame.MO2Name,                 // most of the games use the MO2 name
                    CompilingGame.MO2Name.Replace(" ", "") //eg: Fallout 4 -> Fallout4
                };

                var lootGameDir = lootGameDirs.Select(x => lootPath.Combine(x))
                                  .FirstOrDefault(p => p.IsDirectory);

                if (lootGameDir != default)
                {
                    Utils.Log($"Found LOOT game folder at {lootGameDir}");
                    lootFiles = lootGameDir.EnumerateFiles(false)
                                .Where(p => p.FileName == (RelativePath)"userlist.yaml")
                                .Where(p => p.IsFile)
                                .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p],
                                                               Consts.LOOTFolderFilesDir.Combine(p.RelativeTo(lootPath))));

                    if (!lootFiles.Any())
                    {
                        Utils.Log(
                            $"Found no LOOT user data for {CompilingGame.HumanFriendlyGameName} at {lootGameDir}!");
                    }
                }
            }

            if (cancel.IsCancellationRequested)
            {
                return(false);
            }

            UpdateTracker.NextStep("Cleaning output folder");
            await ModListOutputFolder.DeleteDirectory();

            if (cancel.IsCancellationRequested)
            {
                return(false);
            }

            UpdateTracker.NextStep("Inferring metas for game file downloads");
            await InferMetas();

            if (cancel.IsCancellationRequested)
            {
                return(false);
            }

            UpdateTracker.NextStep("Reindexing downloads after meta inferring");
            await VFS.AddRoot(DownloadsPath);

            if (cancel.IsCancellationRequested)
            {
                return(false);
            }

            UpdateTracker.NextStep("Pre-validating Archives");


            // Find all Downloads
            IndexedArchives = (await DownloadsPath.EnumerateFiles()
                               .Where(f => f.WithExtension(Consts.MetaFileExtension).Exists)
                               .PMap(Queue, UpdateTracker,
                                     async f => new IndexedArchive(VFS.Index.ByRootPath[f])
            {
                Name = (string)f.FileName,
                IniData = f.WithExtension(Consts.MetaFileExtension).LoadIniFile(),
                Meta = await f.WithExtension(Consts.MetaFileExtension).ReadAllTextAsync()
            })).ToList();


            await IndexGameFileHashes();

            IndexedArchives = IndexedArchives.DistinctBy(a => a.File.AbsoluteName).ToList();

            await CleanInvalidArchivesAndFillState();

            UpdateTracker.NextStep("Finding Install Files");
            ModListOutputFolder.CreateDirectory();

            var mo2Files = SourcePath.EnumerateFiles()
                           .Where(p => p.IsFile)
                           .Select(p =>
            {
                if (!VFS.Index.ByRootPath.ContainsKey(p))
                {
                    Utils.Log($"WELL THERE'S YOUR PROBLEM: {p} {VFS.Index.ByRootPath.Count}");
                }

                return(new RawSourceFile(VFS.Index.ByRootPath[p], p.RelativeTo(SourcePath)));
            });

            // If Game Folder Files exists, ignore the game folder
            IndexedFiles = IndexedArchives.SelectMany(f => f.File.ThisAndAllChildren)
                           .OrderBy(f => f.NestingFactor)
                           .GroupBy(f => f.Hash)
                           .ToDictionary(f => f.Key, f => f.AsEnumerable());

            AllFiles.SetTo(mo2Files
                           .Concat(lootFiles)
                           .DistinctBy(f => f.Path));

            Info($"Found {AllFiles.Count} files to build into mod list");

            if (cancel.IsCancellationRequested)
            {
                return(false);
            }

            UpdateTracker.NextStep("Verifying destinations");

            var dups = AllFiles.GroupBy(f => f.Path)
                       .Where(fs => fs.Count() > 1)
                       .Select(fs =>
            {
                Utils.Log(
                    $"Duplicate files installed to {fs.Key} from : {String.Join(", ", fs.Select(f => f.AbsolutePath))}");
                return(fs);
            }).ToList();

            if (dups.Count > 0)
            {
                Error($"Found {dups.Count} duplicates, exiting");
            }

            if (cancel.IsCancellationRequested)
            {
                return(false);
            }

            UpdateTracker.NextStep("Loading INIs");

            ModInis.SetTo(SourcePath.Combine(Consts.MO2ModFolderName)
                          .EnumerateDirectories()
                          .Select(f =>
            {
                var modName  = f.FileName;
                var metaPath = f.Combine("meta.ini");
                return(metaPath.Exists ? (mod_name: f, metaPath.LoadIniFile()) : default);
Beispiel #5
0
        protected override async Task <bool> _Begin(CancellationToken cancel)
        {
            if (cancel.IsCancellationRequested)
            {
                return(false);
            }
            await Metrics.Send(Metrics.BeginInstall, ModList.Name);

            Utils.Log("Configuring Processor");

            FileExtractor2.FavorPerfOverRAM = FavorPerfOverRam;

            if (GameFolder == null)
            {
                GameFolder = Game.TryGetGameLocation();
            }

            if (GameFolder == null)
            {
                var otherGame = Game.CommonlyConfusedWith.Where(g => g.MetaData().IsInstalled).Select(g => g.MetaData()).FirstOrDefault();
                if (otherGame != null)
                {
                    await Utils.Log(new CriticalFailureIntervention(
                                        $"In order to do a proper install Wabbajack needs to know where your {Game.HumanFriendlyGameName} folder resides. However this game doesn't seem to be installed, we did however find a installed " +
                                        $"copy of {otherGame.HumanFriendlyGameName}, did you install the wrong game?",
                                        $"Could not locate {Game.HumanFriendlyGameName}"))
                    .Task;
                }
                else
                {
                    await Utils.Log(new CriticalFailureIntervention(
                                        $"In order to do a proper install Wabbajack needs to know where your {Game.HumanFriendlyGameName} folder resides. However this game doesn't seem to be installed",
                                        $"Could not locate {Game.HumanFriendlyGameName}"))
                    .Task;
                }

                Utils.Log("Exiting because we couldn't find the game folder.");
                return(false);
            }

            Utils.Log($"Install Folder: {OutputFolder}");
            Utils.Log($"Downloads Folder: {DownloadFolder}");
            Utils.Log($"Game Folder: {GameFolder.Value}");
            Utils.Log($"Wabbajack Folder: {AbsolutePath.EntryPoint}");


            var watcher = new DiskSpaceWatcher(cancel, new[] { OutputFolder, DownloadFolder, GameFolder.Value, AbsolutePath.EntryPoint }, (long)2 << 31,
                                               drive =>
                    {
                    Utils.Log($"Aborting due to low space on {drive.Name}");
                    Abort();
                });
            var watcherTask = watcher.Start();

            if (cancel.IsCancellationRequested)
            {
                return(false);
            }
            UpdateTracker.NextStep("Validating Game ESMs");
            await ValidateGameESMs();

            if (cancel.IsCancellationRequested)
            {
                return(false);
            }
            UpdateTracker.NextStep("Creating Output Folders");

            OutputFolder.CreateDirectory();
            DownloadFolder.CreateDirectory();

            if (OutputFolder.Combine(Consts.MO2ModFolderName).IsDirectory&& WarnOnOverwrite)
            {
                if ((await Utils.Log(new ConfirmUpdateOfExistingInstall {
                    ModListName = ModList.Name, OutputFolder = OutputFolder
                }).Task) == ConfirmUpdateOfExistingInstall.Choice.Abort)
                {
                    Utils.Log("Exiting installation at the request of the user, existing mods folder found.");
                    return(false);
                }
            }

            // Reduce to one thread if downloads on HDD, else use specified. Hashing on HDD has no benefit with more threads.
            if (new PhysicalDisk(DownloadFolder.DriveInfo().Name).MediaType == PhysicalDisk.MediaTypes.HDD && ReduceHDDThreads)
            {
                DesiredThreads.OnNext(1);
            }
            else
            {
                DesiredThreads.OnNext(DiskThreads);
            }

            if (cancel.IsCancellationRequested)
            {
                return(false);
            }
            UpdateTracker.NextStep("Optimizing ModList");
            await OptimizeModlist();

            if (cancel.IsCancellationRequested)
            {
                return(false);
            }
            UpdateTracker.NextStep("Hashing Archives");
            await HashArchives();

            // Set to download thread count.
            DesiredThreads.OnNext(DownloadThreads);

            if (cancel.IsCancellationRequested)
            {
                return(false);
            }
            UpdateTracker.NextStep("Downloading Missing Archives");
            await DownloadArchives();

            // Reduce to one thread if downloads on HDD, else use specified. Hashing on HDD has no benefit with more threads.
            if (new PhysicalDisk(DownloadFolder.DriveInfo().Name).MediaType == PhysicalDisk.MediaTypes.HDD && ReduceHDDThreads)
            {
                DesiredThreads.OnNext(1);
            }
            else
            {
                DesiredThreads.OnNext(DiskThreads);
            }

            if (cancel.IsCancellationRequested)
            {
                return(false);
            }
            UpdateTracker.NextStep("Hashing Remaining Archives");
            await HashArchives();

            var missing = ModList.Archives.Where(a => !HashedArchives.ContainsKey(a.Hash)).ToList();

            if (missing.Count > 0)
            {
                foreach (var a in missing)
                {
                    Info($"Unable to download {a.Name} ({a.State.PrimaryKeyString})");
                }
                if (IgnoreMissingFiles)
                {
                    Info("Missing some archives, but continuing anyways at the request of the user");
                }
                else
                {
                    Error("Cannot continue, was unable to download one or more archives");
                }
            }

            // Reduce to two threads if output on HDD, else use specified. Installing files seems to have a slight benefit with two threads.
            if (new PhysicalDisk(OutputFolder.DriveInfo().Name).MediaType == PhysicalDisk.MediaTypes.HDD && ReduceHDDThreads)
            {
                DesiredThreads.OnNext(2);
            }
            else
            {
                DesiredThreads.OnNext(DiskThreads);
            }

            if (cancel.IsCancellationRequested)
            {
                return(false);
            }
            UpdateTracker.NextStep("Extracting Modlist contents");
            await ExtractModlist();

            if (cancel.IsCancellationRequested)
            {
                return(false);
            }
            UpdateTracker.NextStep("Priming VFS");
            await PrimeVFS();

            if (cancel.IsCancellationRequested)
            {
                return(false);
            }
            UpdateTracker.NextStep("Building Folder Structure");
            BuildFolderStructure();

            if (cancel.IsCancellationRequested)
            {
                return(false);
            }
            UpdateTracker.NextStep("Installing Archives");
            await InstallArchives();

            if (cancel.IsCancellationRequested)
            {
                return(false);
            }
            UpdateTracker.NextStep("Installing Included files");
            await InstallIncludedFiles();

            if (cancel.IsCancellationRequested)
            {
                return(false);
            }
            UpdateTracker.NextStep("Installing Archive Metas");
            await InstallIncludedDownloadMetas();

            if (cancel.IsCancellationRequested)
            {
                return(false);
            }
            UpdateTracker.NextStep("Building BSAs");
            await BuildBSAs();

            if (cancel.IsCancellationRequested)
            {
                return(false);
            }
            UpdateTracker.NextStep("Generating Merges");
            await zEditIntegration.GenerateMerges(this);

            UpdateTracker.NextStep("Set MO2 into portable");
            await ForcePortable();

            UpdateTracker.NextStep("Create Empty Output Mods");
            CreateOutputMods();

            UpdateTracker.NextStep("Updating System-specific ini settings");
            SetScreenSizeInPrefs();

            UpdateTracker.NextStep("Compacting files");
            await CompactFiles();

            UpdateTracker.NextStep("Installation complete! You may exit the program.");
            await ExtractedModlistFolder !.DisposeAsync();
            await Metrics.Send(Metrics.FinishInstall, ModList.Name);

            return(true);
        }