public Manifest(ModList modlist)
{
Name = modlist.Name;
Author = modlist.Author;
Description = modlist.Description;
GameType = modlist.GameType;
GameName = GameType.ToString();
ModManager = modlist.ModManager;
ModManagerName = ModManager.ToString();
DownloadSize = modlist.DownloadSize;
InstallSize = modlist.InstallSize;
// meta is being omitted due to it being useless and not very space friendly
Archives = modlist.Archives.Select(a => new Archive
{
Hash = a.Hash,
Name = a.Name,
Size = a.Size,
State = a.State
}).ToList();
}
/// <summary>
/// The user may already have some files in the OutputFolder. If so we can go through these and
/// figure out which need to be updated, deleted, or left alone
/// </summary>
public async Task OptimizeModlist()
{
Utils.Log("Optimizing ModList directives");
// Clone the ModList so our changes don't modify the original data
ModList = ModList.Clone();
var indexed = ModList.Directives.ToDictionary(d => d.To);
var profileFolder = OutputFolder.Combine("profiles");
var savePath = (RelativePath)"saves";
UpdateTracker.NextStep("Looking for files to delete");
await OutputFolder.EnumerateFiles()
.PMap(Queue, UpdateTracker, async f =>
{
var relativeTo = f.RelativeTo(OutputFolder);
if (indexed.ContainsKey(relativeTo) || f.InFolder(DownloadFolder))
{
return;
}
if (f.InFolder(profileFolder) && f.Parent.FileName == savePath)
{
return;
}
Utils.Log($"Deleting {relativeTo} it's not part of this ModList");
await f.DeleteAsync();
});
Utils.Log("Cleaning empty folders");
var expectedFolders = indexed.Keys
.Select(f => f.RelativeTo(OutputFolder))
// We ignore the last part of the path, so we need a dummy file name
.Append(DownloadFolder.Combine("_"))
.Where(f => f.InFolder(OutputFolder))
.SelectMany(path =>
{
// Get all the folders and all the folder parents
// so for foo\bar\baz\qux.txt this emits ["foo", "foo\\bar", "foo\\bar\\baz"]
var split = ((string)path.RelativeTo(OutputFolder)).Split('\\');
return(Enumerable.Range(1, split.Length - 1).Select(t => string.Join("\\", split.Take(t))));
})
.Distinct()
.Select(p => OutputFolder.Combine(p))
.ToHashSet();
try
{
var toDelete = OutputFolder.EnumerateDirectories(true)
.Where(p => !expectedFolders.Contains(p))
.OrderByDescending(p => ((string)p).Length)
.ToList();
foreach (var dir in toDelete)
{
await dir.DeleteDirectory(dontDeleteIfNotEmpty : true);
}
}
catch (Exception)
{
// ignored because it's not worth throwing a fit over
Utils.Log("Error when trying to clean empty folders. This doesn't really matter.");
}
var existingfiles = OutputFolder.EnumerateFiles().ToHashSet();
UpdateTracker.NextStep("Looking for unmodified files");
(await indexed.Values.PMap(Queue, UpdateTracker, async d =>
{
// Bit backwards, but we want to return null for
// all files we *want* installed. We return the files
// to remove from the install list.
var path = OutputFolder.Combine(d.To);
if (!existingfiles.Contains(path))
{
return(null);
}
return(await path.FileHashCachedAsync() == d.Hash ? d : null);
}))
.Do(d =>
{
if (d != null)
{
indexed.Remove(d.To);
}
});
UpdateTracker.NextStep("Updating ModList");
Utils.Log($"Optimized {ModList.Directives.Count} directives to {indexed.Count} required");
var requiredArchives = indexed.Values.OfType <FromArchive>()
.GroupBy(d => d.ArchiveHashPath.BaseHash)
.Select(d => d.Key)
.ToHashSet();
ModList.Archives = ModList.Archives.Where(a => requiredArchives.Contains(a.Hash)).ToList();
ModList.Directives = indexed.Values.ToList();
}
protected override async Task <bool> _Begin(CancellationToken cancel)
{
if (cancel.IsCancellationRequested)
{
return(false);
}
Info($"Starting Vortex compilation for {GameName} at {GamePath} with staging folder at {StagingFolder} and downloads folder at {DownloadsFolder}.");
ConfigureProcessor(12, ConstructDynamicNumThreads(await RecommendQueueSize()));
UpdateTracker.Reset();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Parsing deployment file");
ParseDeploymentFile();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Creating metas for archives");
await CreateMetaFiles();
Utils.Log($"VFS File Location: {VFSCacheName}");
if (cancel.IsCancellationRequested)
{
return(false);
}
await VFS.IntegrateFromFile(VFSCacheName);
var roots = new List <string> {
StagingFolder, GamePath, DownloadsFolder
};
AddExternalFolder(ref roots);
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Indexing folders");
await VFS.AddRoots(roots);
await VFS.WriteToFile(VFSCacheName);
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Cleaning output folder");
if (Directory.Exists(ModListOutputFolder))
{
Utils.DeleteDirectory(ModListOutputFolder);
}
Directory.CreateDirectory(ModListOutputFolder);
UpdateTracker.NextStep("Finding Install Files");
var vortexStagingFiles = Directory.EnumerateFiles(StagingFolder, "*", SearchOption.AllDirectories)
.Where(p => p.FileExists() && p != StagingMarkerName && !p.Contains(Consts.ManualGameFilesDir))
.Select(p => new RawSourceFile(VFS.Index.ByRootPath[p], p.RelativeTo(StagingFolder)));
var vortexDownloads = Directory.EnumerateFiles(DownloadsFolder, "*", SearchOption.AllDirectories)
.Where(p => p.FileExists() && p != DownloadMarkerName)
.Select(p => new RawSourceFile(VFS.Index.ByRootPath[p], p.RelativeTo(DownloadsFolder)));
var gameFiles = Directory.EnumerateFiles(GamePath, "*", SearchOption.AllDirectories)
.Where(p => p.FileExists())
.Select(p => new RawSourceFile(VFS.Index.ByRootPath[p], Path.Combine(Consts.GameFolderFilesDir, p.RelativeTo(GamePath))));
Info("Indexing Archives");
IndexedArchives = Directory.EnumerateFiles(DownloadsFolder)
.Where(f => File.Exists(f + Consts.MetaFileExtension))
.Select(f => new IndexedArchive
{
File = VFS.Index.ByRootPath[f],
Name = Path.GetFileName(f),
IniData = (f + Consts.MetaFileExtension).LoadIniFile(),
Meta = File.ReadAllText(f + Consts.MetaFileExtension)
})
.ToList();
Info("Indexing Files");
IndexedFiles = IndexedArchives.SelectMany(f => f.File.ThisAndAllChildren)
.OrderBy(f => f.NestingFactor)
.GroupBy(f => f.Hash)
.ToDictionary(f => f.Key, f => f.AsEnumerable());
AllFiles = vortexStagingFiles.Concat(vortexDownloads)
.Concat(gameFiles)
.DistinctBy(f => f.Path)
.ToList();
Info($"Found {AllFiles.Count} files to build into mod list");
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Verifying destinations");
var duplicates = AllFiles.GroupBy(f => f.Path)
.Where(fs => fs.Count() > 1)
.Select(fs =>
{
Utils.Log($"Duplicate files installed to {fs.Key} from : {string.Join(", ", fs.Select(f => f.AbsolutePath))}");
return(fs);
}).ToList();
if (duplicates.Count > 0)
{
Error($"Found {duplicates.Count} duplicates, exiting");
}
for (var i = 0; i < AllFiles.Count; i++)
{
var f = AllFiles[i];
if (!f.Path.StartsWith(Consts.GameFolderFilesDir) || !IndexedFiles.ContainsKey(f.Hash))
{
continue;
}
if (!IndexedFiles.TryGetValue(f.Hash, out var value))
{
continue;
}
var element = value.ElementAt(0);
if (!f.Path.Contains(element.Name))
{
continue;
}
IndexedArchive targetArchive = null;
IndexedArchives.Where(a => a.File.ThisAndAllChildren.Contains(element)).Do(a => targetArchive = a);
if (targetArchive == null)
{
continue;
}
if (targetArchive.IniData?.General?.tag == null || targetArchive.IniData?.General?.tag != Consts.WABBAJACK_VORTEX_MANUAL)
{
continue;
}
#if DEBUG
Utils.Log($"Double hash for: {f.AbsolutePath}");
#endif
var replace = f;
var name = replace.File.Name;
var archiveName = targetArchive.Name;
var elementPath = element.FullPath.Substring(element.FullPath.LastIndexOf('|') + 1);
var gameToFile = name.Substring(GamePath.Length + 1).Replace(elementPath, "");
if (gameToFile.EndsWith("\\"))
{
gameToFile = gameToFile.Substring(0, gameToFile.Length - 1);
}
//replace.Path = replace.Path.Replace(Consts.GameFolderFilesDir, Consts.ManualGameFilesDir);
replace.Path = Path.Combine(Consts.ManualGameFilesDir, archiveName, gameToFile, elementPath);
//replace.Path = Path.Combine(Consts.ManualGameFilesDir, element.FullPath.Substring(DownloadsFolder.Length + 1).Replace('|', '\\'));
AllFiles.RemoveAt(i);
AllFiles.Insert(i, replace);
//AllFiles.Replace(f, replace);
}
var stack = MakeStack();
Info("Running Compilation Stack");
var results = await AllFiles.PMap(Queue, f => RunStack(stack.Where(s => s != null), f));
var noMatch = results.OfType <NoMatch>().ToList();
PrintNoMatches(noMatch);
if (CheckForNoMatchExit(noMatch))
{
return(false);
}
InstallDirectives = results.Where(i => !(i is IgnoredDirectly)).ToList();
Info("Getting Nexus api_key, please click authorize if a browser window appears");
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Gathering Archives");
await GatherArchives();
ModList = new ModList
{
Name = ModListName ?? "",
Author = ModListAuthor ?? "",
Description = ModListDescription ?? "",
Readme = ModListReadme ?? "",
Image = ModListImage ?? "",
Website = ModListWebsite ?? "",
Archives = SelectedArchives.ToList(),
ModManager = ModManager.Vortex,
Directives = InstallDirectives,
GameType = Game
};
UpdateTracker.NextStep("Running Validation");
await ValidateModlist.RunValidation(Queue, ModList);
UpdateTracker.NextStep("Generating Report");
GenerateManifest();
UpdateTracker.NextStep("Exporting ModList");
ExportModList();
ResetMembers();
UpdateTracker.NextStep("Done Building ModList");
return(true);
}
/// <summary>
/// The user may already have some files in the OutputFolder. If so we can go through these and
/// figure out which need to be updated, deleted, or left alone
/// </summary>
public async Task OptimizeModlist()
{
Utils.Log("Optimizing ModList directives");
// Clone the ModList so our changes don't modify the original data
ModList = ModList.Clone();
var indexed = ModList.Directives.ToDictionary(d => d.To);
UpdateTracker.NextStep("Looking for files to delete");
await Directory.EnumerateFiles(OutputFolder, "*", DirectoryEnumerationOptions.Recursive)
.PMap(Queue, UpdateTracker, f =>
{
var relative_to = f.RelativeTo(OutputFolder);
Utils.Status($"Checking if ModList file {relative_to}");
if (indexed.ContainsKey(relative_to) || f.IsInPath(DownloadFolder))
{
return;
}
Utils.Log($"Deleting {relative_to} it's not part of this ModList");
File.Delete(f);
});
UpdateTracker.NextStep("Looking for unmodified files");
(await indexed.Values.PMap(Queue, UpdateTracker, d =>
{
// Bit backwards, but we want to return null for
// all files we *want* installed. We return the files
// to remove from the install list.
Status($"Optimizing {d.To}");
var path = Path.Combine(OutputFolder, d.To);
if (!File.Exists(path))
{
return(null);
}
var fi = new FileInfo(path);
if (fi.Length != d.Size)
{
return(null);
}
return(path.FileHash() == d.Hash ? d : null);
}))
.Where(d => d != null)
.Do(d => indexed.Remove(d.To));
Utils.Log("Cleaning empty folders");
var expectedFolders = indexed.Keys
// We ignore the last part of the path, so we need a dummy file name
.Append(Path.Combine(DownloadFolder, "_"))
.SelectMany(path =>
{
// Get all the folders and all the folder parents
// so for foo\bar\baz\qux.txt this emits ["foo", "foo\\bar", "foo\\bar\\baz"]
var split = path.Split('\\');
return(Enumerable.Range(1, split.Length - 1).Select(t => string.Join("\\", split.Take(t))));
}).Distinct()
.Select(p => Path.Combine(OutputFolder, p))
.ToHashSet();
try
{
Directory.EnumerateDirectories(OutputFolder, DirectoryEnumerationOptions.Recursive)
.Where(p => !expectedFolders.Contains(p))
.OrderByDescending(p => p.Length)
.Do(Utils.DeleteDirectory);
}
catch (Exception)
{
// ignored because it's not worth throwing a fit over
Utils.Log("Error when trying to clean empty folders. This doesn't really matter.");
}
UpdateTracker.NextStep("Updating ModList");
Utils.Log($"Optimized {ModList.Directives.Count} directives to {indexed.Count} required");
var requiredArchives = indexed.Values.OfType <FromArchive>()
.GroupBy(d => d.ArchiveHashPath[0])
.Select(d => d.Key)
.ToHashSet();
ModList.Archives = ModList.Archives.Where(a => requiredArchives.Contains(a.Hash)).ToList();
ModList.Directives = indexed.Values.ToList();
}
public void Build(ACompiler c, ModList lst)
{
MO2Compiler compiler = null;
if (lst.ModManager == ModManager.MO2)
{
compiler = (MO2Compiler)c;
}
Text($"### {lst.Name} by {lst.Author} - Installation Summary");
Text($"Build with Wabbajack Version {lst.WabbajackVersion}");
Text(lst.Description);
Text("#### Website:");
NoWrapText($"[{lst.Website}]({lst.Website})");
Text($"Mod Manager: {lst.ModManager.ToString()}");
if (lst.ModManager == ModManager.MO2)
{
var readmeFile = Path.Combine(compiler?.MO2ProfileDir, "readme.md");
if (File.Exists(readmeFile))
{
File.ReadAllLines(readmeFile)
.Do(NoWrapText);
}
}
var archiveCount = lst.Archives.Count + lst.Directives.Count(d => d is SteamMeta);
var totalSize = lst.Archives.Sum(a => a.Size);
totalSize += lst.Directives.Where(d => d is SteamMeta).Cast <SteamMeta>().Sum(s => s.Size);
Text(
$"#### Download Summary ({archiveCount} archives - {totalSize.ToFileSizeString()})");
foreach (var archive in SortArchives(lst.Archives))
{
var hash = archive.Hash.FromBase64().ToHex();
NoWrapText(archive.State.GetReportEntry(archive));
NoWrapText($" * Size : {archive.Size.ToFileSizeString()}");
NoWrapText($" * SHA256 : [{hash}](https://www.virustotal.com/gui/file/{hash})");
}
lst.Directives.Where(d => d is SteamMeta).Do(f =>
{
if (!(f is SteamMeta s))
{
return;
}
var link = $"https://steamcommunity.com/sharedfiles/filedetails/?id={s.ItemID}";
var size = ((long)s.Size).ToFileSizeString();
NoWrapText($"* Steam Workshop Item: [{s.ItemID}]({link}) | Size: {size}");
});
Text("\n\n");
var patched = lst.Directives.OfType <PatchedFromArchive>().OrderBy(p => p.To).ToList();
Text($"#### Summary of ({patched.Count}) patches");
foreach (var directive in patched)
{
NoWrapText(
$"* Applying {SizeForID(directive.PatchID)} byte patch `{directive.FullPath}` to create `{directive.To}`");
}
var files = lst.Directives.OrderBy(d => d.To).ToList();
Text($"\n\n### Install Plan of ({files.Count}) files");
Text("(ignoring files that are directly copied from archives or listed in the patches section above)");
foreach (var directive in files.OrderBy(f => f.GetType().Name).ThenByDescending(f => f.To))
{
switch (directive)
{
case PropertyFile i:
NoWrapText($"* `{i.SourceDataID}` as a `{Enum.GetName(typeof(PropertyType),i.Type)}`");
break;
case FromArchive f:
//NoWrapText($"* `{f.To}` from `{f.FullPath}`");
break;
case CleanedESM i:
NoWrapText($"* `{i.To}` by applying a patch to a game ESM ({i.SourceESMHash})");
break;
case RemappedInlineFile i:
NoWrapText($"* `{i.To}` by remapping the contents of an inline file");
break;
case InlineFile i:
NoWrapText($"* `{i.To}` from `{SizeForID(i.SourceDataID).ToFileSizeString()}` file included in modlist");
break;
case CreateBSA i:
NoWrapText(
$"* `{i.To}` by creating a BSA of files found in `{Consts.BSACreationDir}\\{i.TempID}`");
break;
}
}
var inlined = lst.Directives.OfType <InlineFile>()
.Select(f => (f.To, "inlined", SizeForID(f.SourceDataID)))
.Concat(lst.Directives
.OfType <PatchedFromArchive>()
.Select(f => (f.To, "patched", SizeForID(f.PatchID))))
.Distinct()
.OrderByDescending(f => f.Item3);
NoWrapText("\n\n### Summary of inlined files in this installer");
foreach (var inline in inlined)
{
NoWrapText($"* {inline.Item3.ToFileSizeString()} for {inline.Item2} file {inline.To}");
}
}
public Installer(string archive, ModList mod_list, string output_folder)
{
ModListArchive = archive;
Outputfolder = output_folder;
ModList = mod_list;
}
