protected override async Task <bool> _Begin(CancellationToken cancel) { await Metrics.Send("begin_compiling", MO2Profile ?? "unknown"); if (cancel.IsCancellationRequested) { return(false); } DesiredThreads.OnNext(DiskThreads); FileExtractor2.FavorPerfOverRAM = FavorPerfOverRam; UpdateTracker.Reset(); UpdateTracker.NextStep("Gathering information"); Utils.Log("Loading compiler Settings"); Settings = await CompilerSettings.Load(MO2ProfileDir); Settings.IncludedGames = Settings.IncludedGames.Add(CompilingGame.Game); Info("Looking for other profiles"); var otherProfilesPath = MO2ProfileDir.Combine("otherprofiles.txt"); SelectedProfiles = new HashSet <string>(); if (otherProfilesPath.Exists) { SelectedProfiles = (await otherProfilesPath.ReadAllLinesAsync()).ToHashSet(); } SelectedProfiles.Add(MO2Profile !); Info("Using Profiles: " + string.Join(", ", SelectedProfiles.OrderBy(p => p))); Utils.Log($"Compiling Game: {CompilingGame.Game}"); Utils.Log("Games from setting files:"); foreach (var game in Settings.IncludedGames) { Utils.Log($"- {game}"); } Utils.Log($"VFS File Location: {VFSCacheName}"); Utils.Log($"MO2 Folder: {SourcePath}"); Utils.Log($"Downloads Folder: {DownloadsPath}"); Utils.Log($"Game Folder: {GamePath}"); var watcher = new DiskSpaceWatcher(cancel, new[] { SourcePath, DownloadsPath, GamePath, AbsolutePath.EntryPoint }, (long)2 << 31, drive => { Utils.Log($"Aborting due to low space on {drive.Name}"); Abort(); }); var watcherTask = watcher.Start(); if (cancel.IsCancellationRequested) { return(false); } List <AbsolutePath> roots; if (UseGamePaths) { roots = new List <AbsolutePath> { SourcePath, GamePath, DownloadsPath }; roots.AddRange(Settings.IncludedGames.Select(g => g.MetaData().GameLocation())); } else { roots = new List <AbsolutePath> { SourcePath, DownloadsPath }; } // TODO: make this generic so we can add more paths var lootPath = (AbsolutePath)Path.Combine( Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "LOOT"); IEnumerable <RawSourceFile> lootFiles = new List <RawSourceFile>(); if (lootPath.Exists) { roots.Add(lootPath); } UpdateTracker.NextStep("Indexing folders"); if (cancel.IsCancellationRequested) { return(false); } await VFS.AddRoots(roots); if (lootPath.Exists) { if (CompilingGame.MO2Name == null) { throw new ArgumentException("Compiling game had no MO2 name specified."); } var lootGameDirs = new[] { CompilingGame.MO2Name, // most of the games use the MO2 name CompilingGame.MO2Name.Replace(" ", "") //eg: Fallout 4 -> Fallout4 }; var lootGameDir = lootGameDirs.Select(x => lootPath.Combine(x)) .FirstOrDefault(p => p.IsDirectory); if (lootGameDir != default) { Utils.Log($"Found LOOT game folder at {lootGameDir}"); lootFiles = lootGameDir.EnumerateFiles(false) .Where(p => p.FileName == (RelativePath)"userlist.yaml") .Where(p => p.IsFile) .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p], Consts.LOOTFolderFilesDir.Combine(p.RelativeTo(lootPath)))); if (!lootFiles.Any()) { Utils.Log( $"Found no LOOT user data for {CompilingGame.HumanFriendlyGameName} at {lootGameDir}!"); } } } if (cancel.IsCancellationRequested) { return(false); } UpdateTracker.NextStep("Cleaning output folder"); await ModListOutputFolder.DeleteDirectory(); if (cancel.IsCancellationRequested) { return(false); } UpdateTracker.NextStep("Inferring metas for game file downloads"); await InferMetas(); if (cancel.IsCancellationRequested) { return(false); } UpdateTracker.NextStep("Reindexing downloads after meta inferring"); await VFS.AddRoot(DownloadsPath); if (cancel.IsCancellationRequested) { return(false); } UpdateTracker.NextStep("Pre-validating Archives"); // Find all Downloads IndexedArchives = (await DownloadsPath.EnumerateFiles() .Where(f => f.WithExtension(Consts.MetaFileExtension).Exists) .PMap(Queue, UpdateTracker, async f => new IndexedArchive(VFS.Index.ByRootPath[f]) { Name = (string)f.FileName, IniData = f.WithExtension(Consts.MetaFileExtension).LoadIniFile(), Meta = await f.WithExtension(Consts.MetaFileExtension).ReadAllTextAsync() })).ToList(); await IndexGameFileHashes(); IndexedArchives = IndexedArchives.DistinctBy(a => a.File.AbsoluteName).ToList(); await CleanInvalidArchivesAndFillState(); UpdateTracker.NextStep("Finding Install Files"); ModListOutputFolder.CreateDirectory(); var mo2Files = SourcePath.EnumerateFiles() .Where(p => p.IsFile) .Select(p => { if (!VFS.Index.ByRootPath.ContainsKey(p)) { Utils.Log($"WELL THERE'S YOUR PROBLEM: {p} {VFS.Index.ByRootPath.Count}"); } return(new RawSourceFile(VFS.Index.ByRootPath[p], p.RelativeTo(SourcePath))); }); // If Game Folder Files exists, ignore the game folder IndexedFiles = IndexedArchives.SelectMany(f => f.File.ThisAndAllChildren) .OrderBy(f => f.NestingFactor) .GroupBy(f => f.Hash) .ToDictionary(f => f.Key, f => f.AsEnumerable()); AllFiles.SetTo(mo2Files .Concat(lootFiles) .DistinctBy(f => f.Path)); Info($"Found {AllFiles.Count} files to build into mod list"); if (cancel.IsCancellationRequested) { return(false); } UpdateTracker.NextStep("Verifying destinations"); var dups = AllFiles.GroupBy(f => f.Path) .Where(fs => fs.Count() > 1) .Select(fs => { Utils.Log( $"Duplicate files installed to {fs.Key} from : {String.Join(", ", fs.Select(f => f.AbsolutePath))}"); return(fs); }).ToList(); if (dups.Count > 0) { Error($"Found {dups.Count} duplicates, exiting"); } if (cancel.IsCancellationRequested) { return(false); } UpdateTracker.NextStep("Loading INIs"); ModInis.SetTo(SourcePath.Combine(Consts.MO2ModFolderName) .EnumerateDirectories() .Select(f => { var modName = f.FileName; var metaPath = f.Combine("meta.ini"); return(metaPath.Exists ? (mod_name: f, metaPath.LoadIniFile()) : default);
protected override async Task <bool> _Begin(CancellationToken cancel) { await Metrics.Send("begin_compiling", ModListName ?? "unknown"); if (cancel.IsCancellationRequested) { return(false); } DesiredThreads.OnNext(DiskThreads); FileExtractor2.FavorPerfOverRAM = FavorPerfOverRam; UpdateTracker.Reset(); UpdateTracker.NextStep("Gathering information"); Utils.Log($"Compiling Game: {CompilingGame.Game}"); Utils.Log("Games from setting files:"); foreach (var game in Settings.IncludedGames) { Utils.Log($"- {game}"); } Utils.Log($"VFS File Location: {VFSCacheName}"); Utils.Log($"MO2 Folder: {SourcePath}"); Utils.Log($"Downloads Folder: {DownloadsPath}"); Utils.Log($"Game Folder: {GamePath}"); var watcher = new DiskSpaceWatcher(cancel, new[] { SourcePath, DownloadsPath, GamePath, AbsolutePath.EntryPoint }, (long)2 << 31, drive => { Utils.Log($"Aborting due to low space on {drive.Name}"); Abort(); }); var watcherTask = watcher.Start(); if (cancel.IsCancellationRequested) { return(false); } List <AbsolutePath> roots = new List <AbsolutePath> { SourcePath, GamePath, DownloadsPath }; roots.AddRange(Settings.IncludedGames.Select(g => g.MetaData().GameLocation())); UpdateTracker.NextStep("Indexing folders"); if (cancel.IsCancellationRequested) { return(false); } await VFS.AddRoots(roots); UpdateTracker.NextStep("Cleaning output folder"); await ModListOutputFolder.DeleteDirectory(); if (cancel.IsCancellationRequested) { return(false); } UpdateTracker.NextStep("Inferring metas for game file downloads"); await InferMetas(); if (cancel.IsCancellationRequested) { return(false); } UpdateTracker.NextStep("Reindexing downloads after meta inferring"); await VFS.AddRoot(DownloadsPath); if (cancel.IsCancellationRequested) { return(false); } UpdateTracker.NextStep("Pre-validating Archives"); // Find all Downloads IndexedArchives = (await DownloadsPath.EnumerateFiles() .Where(f => f.WithExtension(Consts.MetaFileExtension).Exists) .PMap(Queue, async f => new IndexedArchive(VFS.Index.ByRootPath[f]) { Name = (string)f.FileName, IniData = f.WithExtension(Consts.MetaFileExtension).LoadIniFile(), Meta = await f.WithExtension(Consts.MetaFileExtension).ReadAllTextAsync() })).ToList(); await IndexGameFileHashes(); IndexedArchives = IndexedArchives.DistinctBy(a => a.File.AbsoluteName).ToList(); await CleanInvalidArchivesAndFillState(); UpdateTracker.NextStep("Finding Install Files"); ModListOutputFolder.CreateDirectory(); var mo2Files = SourcePath.EnumerateFiles() .Where(p => p.IsFile) .Select(p => { if (!VFS.Index.ByRootPath.ContainsKey(p)) { Utils.Log($"WELL THERE'S YOUR PROBLEM: {p} {VFS.Index.ByRootPath.Count}"); } return(new RawSourceFile(VFS.Index.ByRootPath[p], p.RelativeTo(SourcePath))); }); // If Game Folder Files exists, ignore the game folder IndexedFiles = IndexedArchives.SelectMany(f => f.File.ThisAndAllChildren) .OrderBy(f => f.NestingFactor) .GroupBy(f => f.Hash) .ToDictionary(f => f.Key, f => f.AsEnumerable()); AllFiles.SetTo(mo2Files .DistinctBy(f => f.Path)); Info($"Found {AllFiles.Count} files to build into mod list"); if (cancel.IsCancellationRequested) { return(false); } UpdateTracker.NextStep("Verifying destinations"); var dups = AllFiles.GroupBy(f => f.Path) .Where(fs => fs.Count() > 1) .Select(fs => { Utils.Log( $"Duplicate files installed to {fs.Key} from : {String.Join(", ", fs.Select(f => f.AbsolutePath))}"); return(fs); }).ToList(); if (dups.Count > 0) { Error($"Found {dups.Count} duplicates, exiting"); } if (cancel.IsCancellationRequested) { return(false); } UpdateTracker.NextStep("Loading INIs"); ArchivesByFullPath = IndexedArchives.ToDictionary(a => a.File.AbsoluteName); if (cancel.IsCancellationRequested) { return(false); } var stack = MakeStack(); UpdateTracker.NextStep("Running Compilation Stack"); var results = await AllFiles.PMap(Queue, UpdateTracker, f => RunStack(stack, f)); // Add the extra files that were generated by the stack if (cancel.IsCancellationRequested) { return(false); } var noMatch = results.OfType <NoMatch>().ToArray(); PrintNoMatches(noMatch); if (CheckForNoMatchExit(noMatch)) { return(false); } foreach (var ignored in results.OfType <IgnoredDirectly>()) { Utils.Log($"Ignored {ignored.To} because {ignored.Reason}"); } InstallDirectives.SetTo(results.Where(i => !(i is IgnoredDirectly))); Info("Getting Nexus api_key, please click authorize if a browser window appears"); UpdateTracker.NextStep("Building Patches"); await BuildPatches(); UpdateTracker.NextStep("Gathering Archives"); await GatherArchives(); UpdateTracker.NextStep("Gathering Metadata"); await GatherMetaData(); ModList = new ModList { GameType = CompilingGame.Game, WabbajackVersion = Consts.CurrentMinimumWabbajackVersion, Archives = SelectedArchives.ToList(), ModManager = ModManager.MO2, Directives = InstallDirectives, Name = ModListName ?? "untitled", Author = ModListAuthor ?? "", Description = ModListDescription ?? "", Readme = ModlistReadme ?? "", Image = ModListImage != default ? ModListImage.FileName : default,
protected override async Task <bool> _Begin(CancellationToken cancel) { if (cancel.IsCancellationRequested) { return(false); } Info($"Starting Vortex compilation for {GameName} at {GamePath} with staging folder at {StagingFolder} and downloads folder at {DownloadsFolder}."); ConfigureProcessor(12, ConstructDynamicNumThreads(await RecommendQueueSize())); UpdateTracker.Reset(); if (cancel.IsCancellationRequested) { return(false); } UpdateTracker.NextStep("Parsing deployment file"); ParseDeploymentFile(); if (cancel.IsCancellationRequested) { return(false); } UpdateTracker.NextStep("Creating metas for archives"); await CreateMetaFiles(); Utils.Log($"VFS File Location: {VFSCacheName}"); if (cancel.IsCancellationRequested) { return(false); } await VFS.IntegrateFromFile(VFSCacheName); var roots = new List <AbsolutePath> { StagingFolder, GamePath, DownloadsFolder }; AddExternalFolder(ref roots); if (cancel.IsCancellationRequested) { return(false); } UpdateTracker.NextStep("Indexing folders"); await VFS.AddRoots(roots); await VFS.WriteToFile(VFSCacheName); if (cancel.IsCancellationRequested) { return(false); } UpdateTracker.NextStep("Cleaning output folder"); await ModListOutputFolder.DeleteDirectory(); ModListOutputFolder.CreateDirectory(); UpdateTracker.NextStep("Finding Install Files"); /* TODO * var vortexStagingFiles = Directory.EnumerateFiles(StagingFolder, "*", SearchOption.AllDirectories) * .Where(p => p.FileExists() && p != StagingMarkerName && !p.Contains(Consts.ManualGameFilesDir)) * .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p], p.RelativeTo(StagingFolder))); * * var vortexDownloads = Directory.EnumerateFiles(DownloadsFolder, "*", SearchOption.AllDirectories) * .Where(p => p.FileExists() && p != DownloadMarkerName) * .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p], p.RelativeTo(DownloadsFolder))); * * var gameFiles = Directory.EnumerateFiles(GamePath, "*", SearchOption.AllDirectories) * .Where(p => p.FileExists()) * .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p], Path.Combine(Consts.GameFolderFilesDir, p.RelativeTo(GamePath)))); * * Info("Indexing Archives"); * IndexedArchives = Directory.EnumerateFiles(DownloadsFolder) * .Where(f => File.Exists(f + Consts.MetaFileExtension)) * .Select(f => new IndexedArchive * { * File = VFS.Index.ByRootPath[f], * Name = Path.GetFileName(f), * IniData = (f + Consts.MetaFileExtension).LoadIniFile(), * Meta = File.ReadAllText(f + Consts.MetaFileExtension) * }) * .ToList(); * * Info("Indexing Files"); * IndexedFiles = IndexedArchives.SelectMany(f => f.File.ThisAndAllChildren) * .OrderBy(f => f.NestingFactor) * .GroupBy(f => f.Hash) * .ToDictionary(f => f.Key, f => f.AsEnumerable()); * * AllFiles = vortexStagingFiles.Concat(vortexDownloads) * .Concat(gameFiles) * .DistinctBy(f => f.Path) * .ToList(); * * Info($"Found {AllFiles.Count} files to build into mod list"); * * if (cancel.IsCancellationRequested) return false; * UpdateTracker.NextStep("Verifying destinations"); * var duplicates = AllFiles.GroupBy(f => f.Path) * .Where(fs => fs.Count() > 1) * .Select(fs => * { * Utils.Log($"Duplicate files installed to {fs.Key} from : {string.Join(", ", fs.Select(f => f.AbsolutePath))}"); * return fs; * }).ToList(); * * if (duplicates.Count > 0) * { * Error($"Found {duplicates.Count} duplicates, exiting"); * } * * for (var i = 0; i < AllFiles.Count; i++) * { * var f = AllFiles[i]; * if (!f.Path.StartsWith(Consts.GameFolderFilesDir) || !IndexedFiles.ContainsKey(f.Hash)) * continue; * * if (!IndexedFiles.TryGetValue(f.Hash, out var value)) * continue; * * var element = value.ElementAt(0); * * if (!f.Path.Contains(element.Name)) * continue; * * IndexedArchive targetArchive = null; * IndexedArchives.Where(a => a.File.ThisAndAllChildren.Contains(element)).Do(a => targetArchive = a); * * if (targetArchive == null) * continue; * * if(targetArchive.IniData?.General?.tag == null || targetArchive.IniData?.General?.tag != Consts.WABBAJACK_VORTEX_MANUAL) * continue; * #if DEBUG * Utils.Log($"Double hash for: {f.AbsolutePath}"); #endif * * var replace = f; * var name = replace.File.Name; * var archiveName = targetArchive.Name; * var elementPath = element.FullPath.Substring(element.FullPath.LastIndexOf('|')+1); * var gameToFile = name.Substring(GamePath.Length + 1).Replace(elementPath, ""); * if (gameToFile.EndsWith("\\")) * gameToFile = gameToFile.Substring(0, gameToFile.Length - 1); * //replace.Path = replace.Path.Replace(Consts.GameFolderFilesDir, Consts.ManualGameFilesDir); * replace.Path = Path.Combine(Consts.ManualGameFilesDir, archiveName, gameToFile, elementPath); * //replace.Path = Path.Combine(Consts.ManualGameFilesDir, element.FullPath.Substring(DownloadsFolder.Length + 1).Replace('|', '\\')); * AllFiles.RemoveAt(i); * AllFiles.Insert(i, replace); * //AllFiles.Replace(f, replace); * } * * var stack = MakeStack(); * * Info("Running Compilation Stack"); * var results = await AllFiles.PMap(Queue, f => RunStack(stack.Where(s => s != null), f)); * * var noMatch = results.OfType<NoMatch>().ToList(); * PrintNoMatches(noMatch); * if (CheckForNoMatchExit(noMatch)) return false; * * InstallDirectives = results.Where(i => !(i is IgnoredDirectly)).ToList(); * * Info("Getting Nexus api_key, please click authorize if a browser window appears"); * * if (cancel.IsCancellationRequested) return false; * UpdateTracker.NextStep("Gathering Archives"); * await GatherArchives(); * * ModList = new ModList * { * Name = ModListName ?? "", * Author = ModListAuthor ?? "", * Description = ModListDescription ?? "", * Readme = ModListReadme ?? "", * Image = ModListImage ?? "", * Website = ModListWebsite != null ? new Uri(ModListWebsite) : null, * Archives = SelectedArchives.ToList(), * ModManager = ModManager.Vortex, * Directives = InstallDirectives, * GameType = Game * }; * * UpdateTracker.NextStep("Running Validation"); * await ValidateModlist.RunValidation(ModList); * * UpdateTracker.NextStep("Generating Report"); * GenerateManifest(); * * UpdateTracker.NextStep("Exporting ModList"); * ExportModList(); * * ResetMembers(); * * UpdateTracker.NextStep("Done Building ModList"); */ return(true); }