public Manifest(ModList modlist) { Name = modlist.Name; Author = modlist.Author; Description = modlist.Description; GameType = modlist.GameType; GameName = GameType.ToString(); ModManager = modlist.ModManager; ModManagerName = ModManager.ToString(); DownloadSize = modlist.DownloadSize; InstallSize = modlist.InstallSize; // meta is being omitted due to it being useless and not very space friendly Archives = modlist.Archives.Select(a => new Archive { Hash = a.Hash, Name = a.Name, Size = a.Size, State = a.State }).ToList(); }
/// <summary> /// The user may already have some files in the OutputFolder. If so we can go through these and /// figure out which need to be updated, deleted, or left alone /// </summary> public async Task OptimizeModlist() { Utils.Log("Optimizing ModList directives"); // Clone the ModList so our changes don't modify the original data ModList = ModList.Clone(); var indexed = ModList.Directives.ToDictionary(d => d.To); var profileFolder = OutputFolder.Combine("profiles"); var savePath = (RelativePath)"saves"; UpdateTracker.NextStep("Looking for files to delete"); await OutputFolder.EnumerateFiles() .PMap(Queue, UpdateTracker, async f => { var relativeTo = f.RelativeTo(OutputFolder); if (indexed.ContainsKey(relativeTo) || f.InFolder(DownloadFolder)) { return; } if (f.InFolder(profileFolder) && f.Parent.FileName == savePath) { return; } Utils.Log($"Deleting {relativeTo} it's not part of this ModList"); await f.DeleteAsync(); }); Utils.Log("Cleaning empty folders"); var expectedFolders = indexed.Keys .Select(f => f.RelativeTo(OutputFolder)) // We ignore the last part of the path, so we need a dummy file name .Append(DownloadFolder.Combine("_")) .Where(f => f.InFolder(OutputFolder)) .SelectMany(path => { // Get all the folders and all the folder parents // so for foo\bar\baz\qux.txt this emits ["foo", "foo\\bar", "foo\\bar\\baz"] var split = ((string)path.RelativeTo(OutputFolder)).Split('\\'); return(Enumerable.Range(1, split.Length - 1).Select(t => string.Join("\\", split.Take(t)))); }) .Distinct() .Select(p => OutputFolder.Combine(p)) .ToHashSet(); try { var toDelete = OutputFolder.EnumerateDirectories(true) .Where(p => !expectedFolders.Contains(p)) .OrderByDescending(p => ((string)p).Length) .ToList(); foreach (var dir in toDelete) { await dir.DeleteDirectory(dontDeleteIfNotEmpty : true); } } catch (Exception) { // ignored because it's not worth throwing a fit over Utils.Log("Error when trying to clean empty folders. This doesn't really matter."); } var existingfiles = OutputFolder.EnumerateFiles().ToHashSet(); UpdateTracker.NextStep("Looking for unmodified files"); (await indexed.Values.PMap(Queue, UpdateTracker, async d => { // Bit backwards, but we want to return null for // all files we *want* installed. We return the files // to remove from the install list. var path = OutputFolder.Combine(d.To); if (!existingfiles.Contains(path)) { return(null); } return(await path.FileHashCachedAsync() == d.Hash ? d : null); })) .Do(d => { if (d != null) { indexed.Remove(d.To); } }); UpdateTracker.NextStep("Updating ModList"); Utils.Log($"Optimized {ModList.Directives.Count} directives to {indexed.Count} required"); var requiredArchives = indexed.Values.OfType <FromArchive>() .GroupBy(d => d.ArchiveHashPath.BaseHash) .Select(d => d.Key) .ToHashSet(); ModList.Archives = ModList.Archives.Where(a => requiredArchives.Contains(a.Hash)).ToList(); ModList.Directives = indexed.Values.ToList(); }
protected override async Task <bool> _Begin(CancellationToken cancel) { if (cancel.IsCancellationRequested) { return(false); } Info($"Starting Vortex compilation for {GameName} at {GamePath} with staging folder at {StagingFolder} and downloads folder at {DownloadsFolder}."); ConfigureProcessor(12, ConstructDynamicNumThreads(await RecommendQueueSize())); UpdateTracker.Reset(); if (cancel.IsCancellationRequested) { return(false); } UpdateTracker.NextStep("Parsing deployment file"); ParseDeploymentFile(); if (cancel.IsCancellationRequested) { return(false); } UpdateTracker.NextStep("Creating metas for archives"); await CreateMetaFiles(); Utils.Log($"VFS File Location: {VFSCacheName}"); if (cancel.IsCancellationRequested) { return(false); } await VFS.IntegrateFromFile(VFSCacheName); var roots = new List <string> { StagingFolder, GamePath, DownloadsFolder }; AddExternalFolder(ref roots); if (cancel.IsCancellationRequested) { return(false); } UpdateTracker.NextStep("Indexing folders"); await VFS.AddRoots(roots); await VFS.WriteToFile(VFSCacheName); if (cancel.IsCancellationRequested) { return(false); } UpdateTracker.NextStep("Cleaning output folder"); if (Directory.Exists(ModListOutputFolder)) { Utils.DeleteDirectory(ModListOutputFolder); } Directory.CreateDirectory(ModListOutputFolder); UpdateTracker.NextStep("Finding Install Files"); var vortexStagingFiles = Directory.EnumerateFiles(StagingFolder, "*", SearchOption.AllDirectories) .Where(p => p.FileExists() && p != StagingMarkerName && !p.Contains(Consts.ManualGameFilesDir)) .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p], p.RelativeTo(StagingFolder))); var vortexDownloads = Directory.EnumerateFiles(DownloadsFolder, "*", SearchOption.AllDirectories) .Where(p => p.FileExists() && p != DownloadMarkerName) .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p], p.RelativeTo(DownloadsFolder))); var gameFiles = Directory.EnumerateFiles(GamePath, "*", SearchOption.AllDirectories) .Where(p => p.FileExists()) .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p], Path.Combine(Consts.GameFolderFilesDir, p.RelativeTo(GamePath)))); Info("Indexing Archives"); IndexedArchives = Directory.EnumerateFiles(DownloadsFolder) .Where(f => File.Exists(f + Consts.MetaFileExtension)) .Select(f => new IndexedArchive { File = VFS.Index.ByRootPath[f], Name = Path.GetFileName(f), IniData = (f + Consts.MetaFileExtension).LoadIniFile(), Meta = File.ReadAllText(f + Consts.MetaFileExtension) }) .ToList(); Info("Indexing Files"); IndexedFiles = IndexedArchives.SelectMany(f => f.File.ThisAndAllChildren) .OrderBy(f => f.NestingFactor) .GroupBy(f => f.Hash) .ToDictionary(f => f.Key, f => f.AsEnumerable()); AllFiles = vortexStagingFiles.Concat(vortexDownloads) .Concat(gameFiles) .DistinctBy(f => f.Path) .ToList(); Info($"Found {AllFiles.Count} files to build into mod list"); if (cancel.IsCancellationRequested) { return(false); } UpdateTracker.NextStep("Verifying destinations"); var duplicates = AllFiles.GroupBy(f => f.Path) .Where(fs => fs.Count() > 1) .Select(fs => { Utils.Log($"Duplicate files installed to {fs.Key} from : {string.Join(", ", fs.Select(f => f.AbsolutePath))}"); return(fs); }).ToList(); if (duplicates.Count > 0) { Error($"Found {duplicates.Count} duplicates, exiting"); } for (var i = 0; i < AllFiles.Count; i++) { var f = AllFiles[i]; if (!f.Path.StartsWith(Consts.GameFolderFilesDir) || !IndexedFiles.ContainsKey(f.Hash)) { continue; } if (!IndexedFiles.TryGetValue(f.Hash, out var value)) { continue; } var element = value.ElementAt(0); if (!f.Path.Contains(element.Name)) { continue; } IndexedArchive targetArchive = null; IndexedArchives.Where(a => a.File.ThisAndAllChildren.Contains(element)).Do(a => targetArchive = a); if (targetArchive == null) { continue; } if (targetArchive.IniData?.General?.tag == null || targetArchive.IniData?.General?.tag != Consts.WABBAJACK_VORTEX_MANUAL) { continue; } #if DEBUG Utils.Log($"Double hash for: {f.AbsolutePath}"); #endif var replace = f; var name = replace.File.Name; var archiveName = targetArchive.Name; var elementPath = element.FullPath.Substring(element.FullPath.LastIndexOf('|') + 1); var gameToFile = name.Substring(GamePath.Length + 1).Replace(elementPath, ""); if (gameToFile.EndsWith("\\")) { gameToFile = gameToFile.Substring(0, gameToFile.Length - 1); } //replace.Path = replace.Path.Replace(Consts.GameFolderFilesDir, Consts.ManualGameFilesDir); replace.Path = Path.Combine(Consts.ManualGameFilesDir, archiveName, gameToFile, elementPath); //replace.Path = Path.Combine(Consts.ManualGameFilesDir, element.FullPath.Substring(DownloadsFolder.Length + 1).Replace('|', '\\')); AllFiles.RemoveAt(i); AllFiles.Insert(i, replace); //AllFiles.Replace(f, replace); } var stack = MakeStack(); Info("Running Compilation Stack"); var results = await AllFiles.PMap(Queue, f => RunStack(stack.Where(s => s != null), f)); var noMatch = results.OfType <NoMatch>().ToList(); PrintNoMatches(noMatch); if (CheckForNoMatchExit(noMatch)) { return(false); } InstallDirectives = results.Where(i => !(i is IgnoredDirectly)).ToList(); Info("Getting Nexus api_key, please click authorize if a browser window appears"); if (cancel.IsCancellationRequested) { return(false); } UpdateTracker.NextStep("Gathering Archives"); await GatherArchives(); ModList = new ModList { Name = ModListName ?? "", Author = ModListAuthor ?? "", Description = ModListDescription ?? "", Readme = ModListReadme ?? "", Image = ModListImage ?? "", Website = ModListWebsite ?? "", Archives = SelectedArchives.ToList(), ModManager = ModManager.Vortex, Directives = InstallDirectives, GameType = Game }; UpdateTracker.NextStep("Running Validation"); await ValidateModlist.RunValidation(Queue, ModList); UpdateTracker.NextStep("Generating Report"); GenerateManifest(); UpdateTracker.NextStep("Exporting ModList"); ExportModList(); ResetMembers(); UpdateTracker.NextStep("Done Building ModList"); return(true); }
/// <summary> /// The user may already have some files in the OutputFolder. If so we can go through these and /// figure out which need to be updated, deleted, or left alone /// </summary> public async Task OptimizeModlist() { Utils.Log("Optimizing ModList directives"); // Clone the ModList so our changes don't modify the original data ModList = ModList.Clone(); var indexed = ModList.Directives.ToDictionary(d => d.To); UpdateTracker.NextStep("Looking for files to delete"); await Directory.EnumerateFiles(OutputFolder, "*", DirectoryEnumerationOptions.Recursive) .PMap(Queue, UpdateTracker, f => { var relative_to = f.RelativeTo(OutputFolder); Utils.Status($"Checking if ModList file {relative_to}"); if (indexed.ContainsKey(relative_to) || f.IsInPath(DownloadFolder)) { return; } Utils.Log($"Deleting {relative_to} it's not part of this ModList"); File.Delete(f); }); UpdateTracker.NextStep("Looking for unmodified files"); (await indexed.Values.PMap(Queue, UpdateTracker, d => { // Bit backwards, but we want to return null for // all files we *want* installed. We return the files // to remove from the install list. Status($"Optimizing {d.To}"); var path = Path.Combine(OutputFolder, d.To); if (!File.Exists(path)) { return(null); } var fi = new FileInfo(path); if (fi.Length != d.Size) { return(null); } return(path.FileHash() == d.Hash ? d : null); })) .Where(d => d != null) .Do(d => indexed.Remove(d.To)); Utils.Log("Cleaning empty folders"); var expectedFolders = indexed.Keys // We ignore the last part of the path, so we need a dummy file name .Append(Path.Combine(DownloadFolder, "_")) .SelectMany(path => { // Get all the folders and all the folder parents // so for foo\bar\baz\qux.txt this emits ["foo", "foo\\bar", "foo\\bar\\baz"] var split = path.Split('\\'); return(Enumerable.Range(1, split.Length - 1).Select(t => string.Join("\\", split.Take(t)))); }).Distinct() .Select(p => Path.Combine(OutputFolder, p)) .ToHashSet(); try { Directory.EnumerateDirectories(OutputFolder, DirectoryEnumerationOptions.Recursive) .Where(p => !expectedFolders.Contains(p)) .OrderByDescending(p => p.Length) .Do(Utils.DeleteDirectory); } catch (Exception) { // ignored because it's not worth throwing a fit over Utils.Log("Error when trying to clean empty folders. This doesn't really matter."); } UpdateTracker.NextStep("Updating ModList"); Utils.Log($"Optimized {ModList.Directives.Count} directives to {indexed.Count} required"); var requiredArchives = indexed.Values.OfType <FromArchive>() .GroupBy(d => d.ArchiveHashPath[0]) .Select(d => d.Key) .ToHashSet(); ModList.Archives = ModList.Archives.Where(a => requiredArchives.Contains(a.Hash)).ToList(); ModList.Directives = indexed.Values.ToList(); }
public void Build(ACompiler c, ModList lst) { MO2Compiler compiler = null; if (lst.ModManager == ModManager.MO2) { compiler = (MO2Compiler)c; } Text($"### {lst.Name} by {lst.Author} - Installation Summary"); Text($"Build with Wabbajack Version {lst.WabbajackVersion}"); Text(lst.Description); Text("#### Website:"); NoWrapText($"[{lst.Website}]({lst.Website})"); Text($"Mod Manager: {lst.ModManager.ToString()}"); if (lst.ModManager == ModManager.MO2) { var readmeFile = Path.Combine(compiler?.MO2ProfileDir, "readme.md"); if (File.Exists(readmeFile)) { File.ReadAllLines(readmeFile) .Do(NoWrapText); } } var archiveCount = lst.Archives.Count + lst.Directives.Count(d => d is SteamMeta); var totalSize = lst.Archives.Sum(a => a.Size); totalSize += lst.Directives.Where(d => d is SteamMeta).Cast <SteamMeta>().Sum(s => s.Size); Text( $"#### Download Summary ({archiveCount} archives - {totalSize.ToFileSizeString()})"); foreach (var archive in SortArchives(lst.Archives)) { var hash = archive.Hash.FromBase64().ToHex(); NoWrapText(archive.State.GetReportEntry(archive)); NoWrapText($" * Size : {archive.Size.ToFileSizeString()}"); NoWrapText($" * SHA256 : [{hash}](https://www.virustotal.com/gui/file/{hash})"); } lst.Directives.Where(d => d is SteamMeta).Do(f => { if (!(f is SteamMeta s)) { return; } var link = $"https://steamcommunity.com/sharedfiles/filedetails/?id={s.ItemID}"; var size = ((long)s.Size).ToFileSizeString(); NoWrapText($"* Steam Workshop Item: [{s.ItemID}]({link}) | Size: {size}"); }); Text("\n\n"); var patched = lst.Directives.OfType <PatchedFromArchive>().OrderBy(p => p.To).ToList(); Text($"#### Summary of ({patched.Count}) patches"); foreach (var directive in patched) { NoWrapText( $"* Applying {SizeForID(directive.PatchID)} byte patch `{directive.FullPath}` to create `{directive.To}`"); } var files = lst.Directives.OrderBy(d => d.To).ToList(); Text($"\n\n### Install Plan of ({files.Count}) files"); Text("(ignoring files that are directly copied from archives or listed in the patches section above)"); foreach (var directive in files.OrderBy(f => f.GetType().Name).ThenByDescending(f => f.To)) { switch (directive) { case PropertyFile i: NoWrapText($"* `{i.SourceDataID}` as a `{Enum.GetName(typeof(PropertyType),i.Type)}`"); break; case FromArchive f: //NoWrapText($"* `{f.To}` from `{f.FullPath}`"); break; case CleanedESM i: NoWrapText($"* `{i.To}` by applying a patch to a game ESM ({i.SourceESMHash})"); break; case RemappedInlineFile i: NoWrapText($"* `{i.To}` by remapping the contents of an inline file"); break; case InlineFile i: NoWrapText($"* `{i.To}` from `{SizeForID(i.SourceDataID).ToFileSizeString()}` file included in modlist"); break; case CreateBSA i: NoWrapText( $"* `{i.To}` by creating a BSA of files found in `{Consts.BSACreationDir}\\{i.TempID}`"); break; } } var inlined = lst.Directives.OfType <InlineFile>() .Select(f => (f.To, "inlined", SizeForID(f.SourceDataID))) .Concat(lst.Directives .OfType <PatchedFromArchive>() .Select(f => (f.To, "patched", SizeForID(f.PatchID)))) .Distinct() .OrderByDescending(f => f.Item3); NoWrapText("\n\n### Summary of inlined files in this installer"); foreach (var inline in inlined) { NoWrapText($"* {inline.Item3.ToFileSizeString()} for {inline.Item2} file {inline.To}"); } }
public Installer(string archive, ModList mod_list, string output_folder) { ModListArchive = archive; Outputfolder = output_folder; ModList = mod_list; }