/// <summary>
/// The user may already have some files in the OutputFolder. If so we can go through these and
/// figure out which need to be updated, deleted, or left alone
/// </summary>
public async Task OptimizeModlist()
{
Utils.Log("Optimizing Modlist directives");
var indexed = ModList.Directives.ToDictionary(d => d.To);
UpdateTracker.NextStep("Looking for files to delete");
await Directory.EnumerateFiles(OutputFolder, "*", DirectoryEnumerationOptions.Recursive)
.PMap(Queue, UpdateTracker, f =>
{
var relative_to = f.RelativeTo(OutputFolder);
Utils.Status($"Checking if modlist file {relative_to}");
if (indexed.ContainsKey(relative_to) || f.IsInPath(DownloadFolder))
{
return;
}
Utils.Log($"Deleting {relative_to} it's not part of this modlist");
File.Delete(f);
});
UpdateTracker.NextStep("Looking for unmodified files");
(await indexed.Values.PMap(Queue, UpdateTracker, d =>
{
// Bit backwards, but we want to return null for
// all files we *want* installed. We return the files
// to remove from the install list.
Status($"Optimizing {d.To}");
var path = Path.Combine(OutputFolder, d.To);
if (!File.Exists(path))
{
return(null);
}
var fi = new FileInfo(path);
if (fi.Length != d.Size)
{
return(null);
}
return(path.FileHash() == d.Hash ? d : null);
}))
.Where(d => d != null)
.Do(d => indexed.Remove(d.To));
UpdateTracker.NextStep("Updating Modlist");
Utils.Log($"Optimized {ModList.Directives.Count} directives to {indexed.Count} required");
var requiredArchives = indexed.Values.OfType <FromArchive>()
.GroupBy(d => d.ArchiveHashPath[0])
.Select(d => d.Key)
.ToHashSet();
ModList.Archives = ModList.Archives.Where(a => requiredArchives.Contains(a.Hash)).ToList();
ModList.Directives = indexed.Values.ToList();
}
protected override async Task <bool> _Begin(CancellationToken cancel)
{
await Metrics.Send("begin_compiling", ModListName ?? "unknown");
if (cancel.IsCancellationRequested)
{
return(false);
}
DesiredThreads.OnNext(DiskThreads);
FileExtractor2.FavorPerfOverRAM = FavorPerfOverRam;
UpdateTracker.Reset();
UpdateTracker.NextStep("Gathering information");
Utils.Log($"Compiling Game: {CompilingGame.Game}");
Utils.Log("Games from setting files:");
foreach (var game in Settings.IncludedGames)
{
Utils.Log($"- {game}");
}
Utils.Log($"VFS File Location: {VFSCacheName}");
Utils.Log($"MO2 Folder: {SourcePath}");
Utils.Log($"Downloads Folder: {DownloadsPath}");
Utils.Log($"Game Folder: {GamePath}");
var watcher = new DiskSpaceWatcher(cancel,
new[] { SourcePath, DownloadsPath, GamePath, AbsolutePath.EntryPoint }, (long)2 << 31,
drive =>
{
Utils.Log($"Aborting due to low space on {drive.Name}");
Abort();
});
var watcherTask = watcher.Start();
if (cancel.IsCancellationRequested)
{
return(false);
}
List <AbsolutePath> roots = new List <AbsolutePath> {
SourcePath, GamePath, DownloadsPath
};
roots.AddRange(Settings.IncludedGames.Select(g => g.MetaData().GameLocation()));
UpdateTracker.NextStep("Indexing folders");
if (cancel.IsCancellationRequested)
{
return(false);
}
await VFS.AddRoots(roots);
UpdateTracker.NextStep("Cleaning output folder");
await ModListOutputFolder.DeleteDirectory();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Inferring metas for game file downloads");
await InferMetas();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Reindexing downloads after meta inferring");
await VFS.AddRoot(DownloadsPath);
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Pre-validating Archives");
// Find all Downloads
IndexedArchives = (await DownloadsPath.EnumerateFiles()
.Where(f => f.WithExtension(Consts.MetaFileExtension).Exists)
.PMap(Queue,
async f => new IndexedArchive(VFS.Index.ByRootPath[f])
{
Name = (string)f.FileName,
IniData = f.WithExtension(Consts.MetaFileExtension).LoadIniFile(),
Meta = await f.WithExtension(Consts.MetaFileExtension).ReadAllTextAsync()
})).ToList();
await IndexGameFileHashes();
IndexedArchives = IndexedArchives.DistinctBy(a => a.File.AbsoluteName).ToList();
await CleanInvalidArchivesAndFillState();
UpdateTracker.NextStep("Finding Install Files");
ModListOutputFolder.CreateDirectory();
var mo2Files = SourcePath.EnumerateFiles()
.Where(p => p.IsFile)
.Select(p =>
{
if (!VFS.Index.ByRootPath.ContainsKey(p))
{
Utils.Log($"WELL THERE'S YOUR PROBLEM: {p} {VFS.Index.ByRootPath.Count}");
}
return(new RawSourceFile(VFS.Index.ByRootPath[p], p.RelativeTo(SourcePath)));
});
// If Game Folder Files exists, ignore the game folder
IndexedFiles = IndexedArchives.SelectMany(f => f.File.ThisAndAllChildren)
.OrderBy(f => f.NestingFactor)
.GroupBy(f => f.Hash)
.ToDictionary(f => f.Key, f => f.AsEnumerable());
AllFiles.SetTo(mo2Files
.DistinctBy(f => f.Path));
Info($"Found {AllFiles.Count} files to build into mod list");
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Verifying destinations");
var dups = AllFiles.GroupBy(f => f.Path)
.Where(fs => fs.Count() > 1)
.Select(fs =>
{
Utils.Log(
$"Duplicate files installed to {fs.Key} from : {String.Join(", ", fs.Select(f => f.AbsolutePath))}");
return(fs);
}).ToList();
if (dups.Count > 0)
{
Error($"Found {dups.Count} duplicates, exiting");
}
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Loading INIs");
ArchivesByFullPath = IndexedArchives.ToDictionary(a => a.File.AbsoluteName);
if (cancel.IsCancellationRequested)
{
return(false);
}
var stack = MakeStack();
UpdateTracker.NextStep("Running Compilation Stack");
var results = await AllFiles.PMap(Queue, UpdateTracker, f => RunStack(stack, f));
// Add the extra files that were generated by the stack
if (cancel.IsCancellationRequested)
{
return(false);
}
var noMatch = results.OfType <NoMatch>().ToArray();
PrintNoMatches(noMatch);
if (CheckForNoMatchExit(noMatch))
{
return(false);
}
foreach (var ignored in results.OfType <IgnoredDirectly>())
{
Utils.Log($"Ignored {ignored.To} because {ignored.Reason}");
}
InstallDirectives.SetTo(results.Where(i => !(i is IgnoredDirectly)));
Info("Getting Nexus api_key, please click authorize if a browser window appears");
UpdateTracker.NextStep("Building Patches");
await BuildPatches();
UpdateTracker.NextStep("Gathering Archives");
await GatherArchives();
UpdateTracker.NextStep("Gathering Metadata");
await GatherMetaData();
ModList = new ModList
{
GameType = CompilingGame.Game,
WabbajackVersion = Consts.CurrentMinimumWabbajackVersion,
Archives = SelectedArchives.ToList(),
ModManager = ModManager.MO2,
Directives = InstallDirectives,
Name = ModListName ?? "untitled",
Author = ModListAuthor ?? "",
Description = ModListDescription ?? "",
Readme = ModlistReadme ?? "",
Image = ModListImage != default ? ModListImage.FileName : default,
protected override async Task <bool> _Begin(CancellationToken cancel)
{
if (cancel.IsCancellationRequested)
{
return(false);
}
var metric = Metrics.Send(Metrics.BeginInstall, ModList.Name);
Utils.Log("Configuring Processor");
Queue.SetActiveThreadsObservable(ConstructDynamicNumThreads(await RecommendQueueSize()));
if (GameFolder == null)
{
GameFolder = Game.TryGetGameLocation();
}
if (GameFolder == null)
{
var otherGame = Game.CommonlyConfusedWith.Where(g => g.MetaData().IsInstalled).Select(g => g.MetaData()).FirstOrDefault();
if (otherGame != null)
{
await Utils.Log(new CriticalFailureIntervention(
$"In order to do a proper install Wabbajack needs to know where your {Game.HumanFriendlyGameName} folder resides. However this game doesn't seem to be installed, we did however find a installed " +
$"copy of {otherGame.HumanFriendlyGameName}, did you install the wrong game?",
$"Could not locate {Game.HumanFriendlyGameName}"))
.Task;
}
else
{
await Utils.Log(new CriticalFailureIntervention(
$"In order to do a proper install Wabbajack needs to know where your {Game.HumanFriendlyGameName} folder resides. However this game doesn't seem to be installed",
$"Could not locate {Game.HumanFriendlyGameName}"))
.Task;
}
Utils.Log("Exiting because we couldn't find the game folder.");
return(false);
}
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Validating Game ESMs");
ValidateGameESMs();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Validating Modlist");
await ValidateModlist.RunValidation(ModList);
OutputFolder.CreateDirectory();
DownloadFolder.CreateDirectory();
if (OutputFolder.Combine(Consts.MO2ModFolderName).IsDirectory&& WarnOnOverwrite)
{
if ((await Utils.Log(new ConfirmUpdateOfExistingInstall {
ModListName = ModList.Name, OutputFolder = OutputFolder
}).Task) == ConfirmUpdateOfExistingInstall.Choice.Abort)
{
Utils.Log("Exiting installation at the request of the user, existing mods folder found.");
return(false);
}
}
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Optimizing ModList");
await OptimizeModlist();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Hashing Archives");
await HashArchives();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Downloading Missing Archives");
await DownloadArchives();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Hashing Remaining Archives");
await HashArchives();
var missing = ModList.Archives.Where(a => !HashedArchives.ContainsKey(a.Hash)).ToList();
if (missing.Count > 0)
{
foreach (var a in missing)
{
Info($"Unable to download {a.Name}");
}
if (IgnoreMissingFiles)
{
Info("Missing some archives, but continuing anyways at the request of the user");
}
else
{
Error("Cannot continue, was unable to download one or more archives");
}
}
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Priming VFS");
await PrimeVFS();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Building Folder Structure");
BuildFolderStructure();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Installing Archives");
await InstallArchives();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Installing Included files");
await InstallIncludedFiles();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Installing Archive Metas");
await InstallIncludedDownloadMetas();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Building BSAs");
await BuildBSAs();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Generating Merges");
await zEditIntegration.GenerateMerges(this);
UpdateTracker.NextStep("Set MO2 into portable");
await ForcePortable();
UpdateTracker.NextStep("Create Empty Output Mods");
CreateOutputMods();
UpdateTracker.NextStep("Updating System-specific ini settings");
SetScreenSizeInPrefs();
UpdateTracker.NextStep("Installation complete! You may exit the program.");
var metric2 = Metrics.Send(Metrics.FinishInstall, ModList.Name);
return(true);
}
protected override async Task <bool> _Begin(CancellationToken cancel)
{
await Metrics.Send("begin_compiling", MO2Profile ?? "unknown");
if (cancel.IsCancellationRequested)
{
return(false);
}
DesiredThreads.OnNext(DiskThreads);
FileExtractor2.FavorPerfOverRAM = FavorPerfOverRam;
UpdateTracker.Reset();
UpdateTracker.NextStep("Gathering information");
Utils.Log("Loading compiler Settings");
Settings = await CompilerSettings.Load(MO2ProfileDir);
Settings.IncludedGames = Settings.IncludedGames.Add(CompilingGame.Game);
Info("Looking for other profiles");
var otherProfilesPath = MO2ProfileDir.Combine("otherprofiles.txt");
SelectedProfiles = new HashSet <string>();
if (otherProfilesPath.Exists)
{
SelectedProfiles = (await otherProfilesPath.ReadAllLinesAsync()).ToHashSet();
}
SelectedProfiles.Add(MO2Profile !);
Info("Using Profiles: " + string.Join(", ", SelectedProfiles.OrderBy(p => p)));
Utils.Log($"Compiling Game: {CompilingGame.Game}");
Utils.Log("Games from setting files:");
foreach (var game in Settings.IncludedGames)
{
Utils.Log($"- {game}");
}
Utils.Log($"VFS File Location: {VFSCacheName}");
Utils.Log($"MO2 Folder: {SourcePath}");
Utils.Log($"Downloads Folder: {DownloadsPath}");
Utils.Log($"Game Folder: {GamePath}");
var watcher = new DiskSpaceWatcher(cancel,
new[] { SourcePath, DownloadsPath, GamePath, AbsolutePath.EntryPoint }, (long)2 << 31,
drive =>
{
Utils.Log($"Aborting due to low space on {drive.Name}");
Abort();
});
var watcherTask = watcher.Start();
if (cancel.IsCancellationRequested)
{
return(false);
}
List <AbsolutePath> roots;
if (UseGamePaths)
{
roots = new List <AbsolutePath> {
SourcePath, GamePath, DownloadsPath
};
roots.AddRange(Settings.IncludedGames.Select(g => g.MetaData().GameLocation()));
}
else
{
roots = new List <AbsolutePath> {
SourcePath, DownloadsPath
};
}
// TODO: make this generic so we can add more paths
var lootPath = (AbsolutePath)Path.Combine(
Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),
"LOOT");
IEnumerable <RawSourceFile> lootFiles = new List <RawSourceFile>();
if (lootPath.Exists)
{
roots.Add(lootPath);
}
UpdateTracker.NextStep("Indexing folders");
if (cancel.IsCancellationRequested)
{
return(false);
}
await VFS.AddRoots(roots);
if (lootPath.Exists)
{
if (CompilingGame.MO2Name == null)
{
throw new ArgumentException("Compiling game had no MO2 name specified.");
}
var lootGameDirs = new[]
{
CompilingGame.MO2Name, // most of the games use the MO2 name
CompilingGame.MO2Name.Replace(" ", "") //eg: Fallout 4 -> Fallout4
};
var lootGameDir = lootGameDirs.Select(x => lootPath.Combine(x))
.FirstOrDefault(p => p.IsDirectory);
if (lootGameDir != default)
{
Utils.Log($"Found LOOT game folder at {lootGameDir}");
lootFiles = lootGameDir.EnumerateFiles(false)
.Where(p => p.FileName == (RelativePath)"userlist.yaml")
.Where(p => p.IsFile)
.Select(p => new RawSourceFile(VFS.Index.ByRootPath[p],
Consts.LOOTFolderFilesDir.Combine(p.RelativeTo(lootPath))));
if (!lootFiles.Any())
{
Utils.Log(
$"Found no LOOT user data for {CompilingGame.HumanFriendlyGameName} at {lootGameDir}!");
}
}
}
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Cleaning output folder");
await ModListOutputFolder.DeleteDirectory();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Inferring metas for game file downloads");
await InferMetas();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Reindexing downloads after meta inferring");
await VFS.AddRoot(DownloadsPath);
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Pre-validating Archives");
// Find all Downloads
IndexedArchives = (await DownloadsPath.EnumerateFiles()
.Where(f => f.WithExtension(Consts.MetaFileExtension).Exists)
.PMap(Queue, UpdateTracker,
async f => new IndexedArchive(VFS.Index.ByRootPath[f])
{
Name = (string)f.FileName,
IniData = f.WithExtension(Consts.MetaFileExtension).LoadIniFile(),
Meta = await f.WithExtension(Consts.MetaFileExtension).ReadAllTextAsync()
})).ToList();
await IndexGameFileHashes();
IndexedArchives = IndexedArchives.DistinctBy(a => a.File.AbsoluteName).ToList();
await CleanInvalidArchivesAndFillState();
UpdateTracker.NextStep("Finding Install Files");
ModListOutputFolder.CreateDirectory();
var mo2Files = SourcePath.EnumerateFiles()
.Where(p => p.IsFile)
.Select(p =>
{
if (!VFS.Index.ByRootPath.ContainsKey(p))
{
Utils.Log($"WELL THERE'S YOUR PROBLEM: {p} {VFS.Index.ByRootPath.Count}");
}
return(new RawSourceFile(VFS.Index.ByRootPath[p], p.RelativeTo(SourcePath)));
});
// If Game Folder Files exists, ignore the game folder
IndexedFiles = IndexedArchives.SelectMany(f => f.File.ThisAndAllChildren)
.OrderBy(f => f.NestingFactor)
.GroupBy(f => f.Hash)
.ToDictionary(f => f.Key, f => f.AsEnumerable());
AllFiles.SetTo(mo2Files
.Concat(lootFiles)
.DistinctBy(f => f.Path));
Info($"Found {AllFiles.Count} files to build into mod list");
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Verifying destinations");
var dups = AllFiles.GroupBy(f => f.Path)
.Where(fs => fs.Count() > 1)
.Select(fs =>
{
Utils.Log(
$"Duplicate files installed to {fs.Key} from : {String.Join(", ", fs.Select(f => f.AbsolutePath))}");
return(fs);
}).ToList();
if (dups.Count > 0)
{
Error($"Found {dups.Count} duplicates, exiting");
}
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Loading INIs");
ModInis.SetTo(SourcePath.Combine(Consts.MO2ModFolderName)
.EnumerateDirectories()
.Select(f =>
{
var modName = f.FileName;
var metaPath = f.Combine("meta.ini");
return(metaPath.Exists ? (mod_name: f, metaPath.LoadIniFile()) : default);
protected override async Task <bool> _Begin(CancellationToken cancel)
{
if (cancel.IsCancellationRequested)
{
return(false);
}
var metric = Metrics.Send(Metrics.BeginInstall, ModList.Name);
ConfigureProcessor(20, ConstructDynamicNumThreads(await RecommendQueueSize()));
var game = ModList.GameType.MetaData();
if (GameFolder == null)
{
GameFolder = game.GameLocation();
}
if (GameFolder == null)
{
await Utils.Log(new CriticalFailureIntervention(
$"In order to do a proper install Wabbajack needs to know where your {game.MO2Name} folder resides. We tried looking the " +
"game location up in the Windows Registry but were unable to find it, please make sure you launch the game once before running this installer. ",
"Could not find game location")).Task;
Utils.Log("Exiting because we couldn't find the game folder.");
return(false);
}
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Validating Game ESMs");
ValidateGameESMs();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Validating Modlist");
await ValidateModlist.RunValidation(Queue, ModList);
Directory.CreateDirectory(OutputFolder);
Directory.CreateDirectory(DownloadFolder);
if (Directory.Exists(Path.Combine(OutputFolder, "mods")) && WarnOnOverwrite)
{
if ((await Utils.Log(new ConfirmUpdateOfExistingInstall {
ModListName = ModList.Name, OutputFolder = OutputFolder
}).Task) == ConfirmUpdateOfExistingInstall.Choice.Abort)
{
Utils.Log("Exiting installation at the request of the user, existing mods folder found.");
return(false);
}
}
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Optimizing ModList");
await OptimizeModlist();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Hashing Archives");
await HashArchives();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Downloading Missing Archives");
await DownloadArchives();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Hashing Remaining Archives");
await HashArchives();
var missing = ModList.Archives.Where(a => !HashedArchives.ContainsKey(a.Hash)).ToList();
if (missing.Count > 0)
{
foreach (var a in missing)
{
Info($"Unable to download {a.Name}");
}
if (IgnoreMissingFiles)
{
Info("Missing some archives, but continuing anyways at the request of the user");
}
else
{
Error("Cannot continue, was unable to download one or more archives");
}
}
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Priming VFS");
await PrimeVFS();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Building Folder Structure");
BuildFolderStructure();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Installing Archives");
await InstallArchives();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Installing Included files");
await InstallIncludedFiles();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Installing Archive Metas");
await InstallIncludedDownloadMetas();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Building BSAs");
await BuildBSAs();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Generating Merges");
await zEditIntegration.GenerateMerges(this);
UpdateTracker.NextStep("Set MO2 into portable");
ForcePortable();
UpdateTracker.NextStep("Create Empty Output Mods");
CreateOutputMods();
UpdateTracker.NextStep("Updating System-specific ini settings");
SetScreenSizeInPrefs();
UpdateTracker.NextStep("Installation complete! You may exit the program.");
var metric2 = Metrics.Send(Metrics.FinishInstall, ModList.Name);
return(true);
}
/// <summary>
/// The user may already have some files in the OutputFolder. If so we can go through these and
/// figure out which need to be updated, deleted, or left alone
/// </summary>
public async Task OptimizeModlist()
{
Utils.Log("Optimizing ModList directives");
// Clone the ModList so our changes don't modify the original data
ModList = ModList.Clone();
var indexed = ModList.Directives.ToDictionary(d => d.To);
var profileFolder = OutputFolder.Combine("profiles");
var savePath = (RelativePath)"saves";
UpdateTracker.NextStep("Looking for files to delete");
await OutputFolder.EnumerateFiles()
.PMap(Queue, UpdateTracker, async f =>
{
var relativeTo = f.RelativeTo(OutputFolder);
if (indexed.ContainsKey(relativeTo) || f.InFolder(DownloadFolder))
{
return;
}
if (f.InFolder(profileFolder) && f.Parent.FileName == savePath)
{
return;
}
Utils.Log($"Deleting {relativeTo} it's not part of this ModList");
await f.DeleteAsync();
});
Utils.Log("Cleaning empty folders");
var expectedFolders = indexed.Keys
.Select(f => f.RelativeTo(OutputFolder))
// We ignore the last part of the path, so we need a dummy file name
.Append(DownloadFolder.Combine("_"))
.Where(f => f.InFolder(OutputFolder))
.SelectMany(path =>
{
// Get all the folders and all the folder parents
// so for foo\bar\baz\qux.txt this emits ["foo", "foo\\bar", "foo\\bar\\baz"]
var split = ((string)path.RelativeTo(OutputFolder)).Split('\\');
return(Enumerable.Range(1, split.Length - 1).Select(t => string.Join("\\", split.Take(t))));
})
.Distinct()
.Select(p => OutputFolder.Combine(p))
.ToHashSet();
try
{
var toDelete = OutputFolder.EnumerateDirectories(true)
.Where(p => !expectedFolders.Contains(p))
.OrderByDescending(p => ((string)p).Length)
.ToList();
foreach (var dir in toDelete)
{
await dir.DeleteDirectory(dontDeleteIfNotEmpty : true);
}
}
catch (Exception)
{
// ignored because it's not worth throwing a fit over
Utils.Log("Error when trying to clean empty folders. This doesn't really matter.");
}
var existingfiles = OutputFolder.EnumerateFiles().ToHashSet();
UpdateTracker.NextStep("Looking for unmodified files");
(await indexed.Values.PMap(Queue, UpdateTracker, async d =>
{
// Bit backwards, but we want to return null for
// all files we *want* installed. We return the files
// to remove from the install list.
var path = OutputFolder.Combine(d.To);
if (!existingfiles.Contains(path))
{
return(null);
}
return(await path.FileHashCachedAsync() == d.Hash ? d : null);
}))
.Do(d =>
{
if (d != null)
{
indexed.Remove(d.To);
}
});
UpdateTracker.NextStep("Updating ModList");
Utils.Log($"Optimized {ModList.Directives.Count} directives to {indexed.Count} required");
var requiredArchives = indexed.Values.OfType <FromArchive>()
.GroupBy(d => d.ArchiveHashPath.BaseHash)
.Select(d => d.Key)
.ToHashSet();
ModList.Archives = ModList.Archives.Where(a => requiredArchives.Contains(a.Hash)).ToList();
ModList.Directives = indexed.Values.ToList();
}
protected override async Task <bool> _Begin(CancellationToken cancel)
{
if (cancel.IsCancellationRequested)
{
return(false);
}
Info($"Starting Vortex compilation for {GameName} at {GamePath} with staging folder at {StagingFolder} and downloads folder at {DownloadsFolder}.");
ConfigureProcessor(12, ConstructDynamicNumThreads(await RecommendQueueSize()));
UpdateTracker.Reset();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Parsing deployment file");
ParseDeploymentFile();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Creating metas for archives");
await CreateMetaFiles();
Utils.Log($"VFS File Location: {VFSCacheName}");
if (cancel.IsCancellationRequested)
{
return(false);
}
await VFS.IntegrateFromFile(VFSCacheName);
var roots = new List <string> {
StagingFolder, GamePath, DownloadsFolder
};
AddExternalFolder(ref roots);
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Indexing folders");
await VFS.AddRoots(roots);
await VFS.WriteToFile(VFSCacheName);
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Cleaning output folder");
if (Directory.Exists(ModListOutputFolder))
{
Utils.DeleteDirectory(ModListOutputFolder);
}
Directory.CreateDirectory(ModListOutputFolder);
UpdateTracker.NextStep("Finding Install Files");
var vortexStagingFiles = Directory.EnumerateFiles(StagingFolder, "*", SearchOption.AllDirectories)
.Where(p => p.FileExists() && p != StagingMarkerName && !p.Contains(Consts.ManualGameFilesDir))
.Select(p => new RawSourceFile(VFS.Index.ByRootPath[p], p.RelativeTo(StagingFolder)));
var vortexDownloads = Directory.EnumerateFiles(DownloadsFolder, "*", SearchOption.AllDirectories)
.Where(p => p.FileExists() && p != DownloadMarkerName)
.Select(p => new RawSourceFile(VFS.Index.ByRootPath[p], p.RelativeTo(DownloadsFolder)));
var gameFiles = Directory.EnumerateFiles(GamePath, "*", SearchOption.AllDirectories)
.Where(p => p.FileExists())
.Select(p => new RawSourceFile(VFS.Index.ByRootPath[p], Path.Combine(Consts.GameFolderFilesDir, p.RelativeTo(GamePath))));
Info("Indexing Archives");
IndexedArchives = Directory.EnumerateFiles(DownloadsFolder)
.Where(f => File.Exists(f + Consts.MetaFileExtension))
.Select(f => new IndexedArchive
{
File = VFS.Index.ByRootPath[f],
Name = Path.GetFileName(f),
IniData = (f + Consts.MetaFileExtension).LoadIniFile(),
Meta = File.ReadAllText(f + Consts.MetaFileExtension)
})
.ToList();
Info("Indexing Files");
IndexedFiles = IndexedArchives.SelectMany(f => f.File.ThisAndAllChildren)
.OrderBy(f => f.NestingFactor)
.GroupBy(f => f.Hash)
.ToDictionary(f => f.Key, f => f.AsEnumerable());
AllFiles = vortexStagingFiles.Concat(vortexDownloads)
.Concat(gameFiles)
.DistinctBy(f => f.Path)
.ToList();
Info($"Found {AllFiles.Count} files to build into mod list");
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Verifying destinations");
var duplicates = AllFiles.GroupBy(f => f.Path)
.Where(fs => fs.Count() > 1)
.Select(fs =>
{
Utils.Log($"Duplicate files installed to {fs.Key} from : {string.Join(", ", fs.Select(f => f.AbsolutePath))}");
return(fs);
}).ToList();
if (duplicates.Count > 0)
{
Error($"Found {duplicates.Count} duplicates, exiting");
}
for (var i = 0; i < AllFiles.Count; i++)
{
var f = AllFiles[i];
if (!f.Path.StartsWith(Consts.GameFolderFilesDir) || !IndexedFiles.ContainsKey(f.Hash))
{
continue;
}
if (!IndexedFiles.TryGetValue(f.Hash, out var value))
{
continue;
}
var element = value.ElementAt(0);
if (!f.Path.Contains(element.Name))
{
continue;
}
IndexedArchive targetArchive = null;
IndexedArchives.Where(a => a.File.ThisAndAllChildren.Contains(element)).Do(a => targetArchive = a);
if (targetArchive == null)
{
continue;
}
if (targetArchive.IniData?.General?.tag == null || targetArchive.IniData?.General?.tag != Consts.WABBAJACK_VORTEX_MANUAL)
{
continue;
}
#if DEBUG
Utils.Log($"Double hash for: {f.AbsolutePath}");
#endif
var replace = f;
var name = replace.File.Name;
var archiveName = targetArchive.Name;
var elementPath = element.FullPath.Substring(element.FullPath.LastIndexOf('|') + 1);
var gameToFile = name.Substring(GamePath.Length + 1).Replace(elementPath, "");
if (gameToFile.EndsWith("\\"))
{
gameToFile = gameToFile.Substring(0, gameToFile.Length - 1);
}
//replace.Path = replace.Path.Replace(Consts.GameFolderFilesDir, Consts.ManualGameFilesDir);
replace.Path = Path.Combine(Consts.ManualGameFilesDir, archiveName, gameToFile, elementPath);
//replace.Path = Path.Combine(Consts.ManualGameFilesDir, element.FullPath.Substring(DownloadsFolder.Length + 1).Replace('|', '\\'));
AllFiles.RemoveAt(i);
AllFiles.Insert(i, replace);
//AllFiles.Replace(f, replace);
}
var stack = MakeStack();
Info("Running Compilation Stack");
var results = await AllFiles.PMap(Queue, f => RunStack(stack.Where(s => s != null), f));
var noMatch = results.OfType <NoMatch>().ToList();
PrintNoMatches(noMatch);
if (CheckForNoMatchExit(noMatch))
{
return(false);
}
InstallDirectives = results.Where(i => !(i is IgnoredDirectly)).ToList();
Info("Getting Nexus api_key, please click authorize if a browser window appears");
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Gathering Archives");
await GatherArchives();
ModList = new ModList
{
Name = ModListName ?? "",
Author = ModListAuthor ?? "",
Description = ModListDescription ?? "",
Readme = ModListReadme ?? "",
Image = ModListImage ?? "",
Website = ModListWebsite ?? "",
Archives = SelectedArchives.ToList(),
ModManager = ModManager.Vortex,
Directives = InstallDirectives,
GameType = Game
};
UpdateTracker.NextStep("Running Validation");
await ValidateModlist.RunValidation(Queue, ModList);
UpdateTracker.NextStep("Generating Report");
GenerateManifest();
UpdateTracker.NextStep("Exporting ModList");
ExportModList();
ResetMembers();
UpdateTracker.NextStep("Done Building ModList");
return(true);
}
/// <summary>
/// The user may already have some files in the OutputFolder. If so we can go through these and
/// figure out which need to be updated, deleted, or left alone
/// </summary>
public async Task OptimizeModlist()
{
Utils.Log("Optimizing ModList directives");
// Clone the ModList so our changes don't modify the original data
ModList = ModList.Clone();
var indexed = ModList.Directives.ToDictionary(d => d.To);
UpdateTracker.NextStep("Looking for files to delete");
await Directory.EnumerateFiles(OutputFolder, "*", DirectoryEnumerationOptions.Recursive)
.PMap(Queue, UpdateTracker, f =>
{
var relative_to = f.RelativeTo(OutputFolder);
Utils.Status($"Checking if ModList file {relative_to}");
if (indexed.ContainsKey(relative_to) || f.IsInPath(DownloadFolder))
{
return;
}
Utils.Log($"Deleting {relative_to} it's not part of this ModList");
File.Delete(f);
});
UpdateTracker.NextStep("Looking for unmodified files");
(await indexed.Values.PMap(Queue, UpdateTracker, d =>
{
// Bit backwards, but we want to return null for
// all files we *want* installed. We return the files
// to remove from the install list.
Status($"Optimizing {d.To}");
var path = Path.Combine(OutputFolder, d.To);
if (!File.Exists(path))
{
return(null);
}
var fi = new FileInfo(path);
if (fi.Length != d.Size)
{
return(null);
}
return(path.FileHash() == d.Hash ? d : null);
}))
.Where(d => d != null)
.Do(d => indexed.Remove(d.To));
Utils.Log("Cleaning empty folders");
var expectedFolders = indexed.Keys
// We ignore the last part of the path, so we need a dummy file name
.Append(Path.Combine(DownloadFolder, "_"))
.SelectMany(path =>
{
// Get all the folders and all the folder parents
// so for foo\bar\baz\qux.txt this emits ["foo", "foo\\bar", "foo\\bar\\baz"]
var split = path.Split('\\');
return(Enumerable.Range(1, split.Length - 1).Select(t => string.Join("\\", split.Take(t))));
}).Distinct()
.Select(p => Path.Combine(OutputFolder, p))
.ToHashSet();
try
{
Directory.EnumerateDirectories(OutputFolder, DirectoryEnumerationOptions.Recursive)
.Where(p => !expectedFolders.Contains(p))
.OrderByDescending(p => p.Length)
.Do(Utils.DeleteDirectory);
}
catch (Exception)
{
// ignored because it's not worth throwing a fit over
Utils.Log("Error when trying to clean empty folders. This doesn't really matter.");
}
UpdateTracker.NextStep("Updating ModList");
Utils.Log($"Optimized {ModList.Directives.Count} directives to {indexed.Count} required");
var requiredArchives = indexed.Values.OfType <FromArchive>()
.GroupBy(d => d.ArchiveHashPath[0])
.Select(d => d.Key)
.ToHashSet();
ModList.Archives = ModList.Archives.Where(a => requiredArchives.Contains(a.Hash)).ToList();
ModList.Directives = indexed.Values.ToList();
}
protected override async Task <bool> _Begin(CancellationToken cancel)
{
if (cancel.IsCancellationRequested)
{
return(false);
}
await Metrics.Send(Metrics.BeginInstall, ModList.Name);
Utils.Log("Configuring Processor");
FileExtractor2.FavorPerfOverRAM = FavorPerfOverRam;
if (GameFolder == null)
{
GameFolder = Game.TryGetGameLocation();
}
if (GameFolder == null)
{
var otherGame = Game.CommonlyConfusedWith.Where(g => g.MetaData().IsInstalled).Select(g => g.MetaData()).FirstOrDefault();
if (otherGame != null)
{
await Utils.Log(new CriticalFailureIntervention(
$"In order to do a proper install Wabbajack needs to know where your {Game.HumanFriendlyGameName} folder resides. However this game doesn't seem to be installed, we did however find a installed " +
$"copy of {otherGame.HumanFriendlyGameName}, did you install the wrong game?",
$"Could not locate {Game.HumanFriendlyGameName}"))
.Task;
}
else
{
await Utils.Log(new CriticalFailureIntervention(
$"In order to do a proper install Wabbajack needs to know where your {Game.HumanFriendlyGameName} folder resides. However this game doesn't seem to be installed",
$"Could not locate {Game.HumanFriendlyGameName}"))
.Task;
}
Utils.Log("Exiting because we couldn't find the game folder.");
return(false);
}
Utils.Log($"Install Folder: {OutputFolder}");
Utils.Log($"Downloads Folder: {DownloadFolder}");
Utils.Log($"Game Folder: {GameFolder.Value}");
Utils.Log($"Wabbajack Folder: {AbsolutePath.EntryPoint}");
var watcher = new DiskSpaceWatcher(cancel, new[] { OutputFolder, DownloadFolder, GameFolder.Value, AbsolutePath.EntryPoint }, (long)2 << 31,
drive =>
{
Utils.Log($"Aborting due to low space on {drive.Name}");
Abort();
});
var watcherTask = watcher.Start();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Validating Game ESMs");
await ValidateGameESMs();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Creating Output Folders");
OutputFolder.CreateDirectory();
DownloadFolder.CreateDirectory();
if (OutputFolder.Combine(Consts.MO2ModFolderName).IsDirectory&& WarnOnOverwrite)
{
if ((await Utils.Log(new ConfirmUpdateOfExistingInstall {
ModListName = ModList.Name, OutputFolder = OutputFolder
}).Task) == ConfirmUpdateOfExistingInstall.Choice.Abort)
{
Utils.Log("Exiting installation at the request of the user, existing mods folder found.");
return(false);
}
}
// Reduce to one thread if downloads on HDD, else use specified. Hashing on HDD has no benefit with more threads.
if (new PhysicalDisk(DownloadFolder.DriveInfo().Name).MediaType == PhysicalDisk.MediaTypes.HDD && ReduceHDDThreads)
{
DesiredThreads.OnNext(1);
}
else
{
DesiredThreads.OnNext(DiskThreads);
}
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Optimizing ModList");
await OptimizeModlist();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Hashing Archives");
await HashArchives();
// Set to download thread count.
DesiredThreads.OnNext(DownloadThreads);
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Downloading Missing Archives");
await DownloadArchives();
// Reduce to one thread if downloads on HDD, else use specified. Hashing on HDD has no benefit with more threads.
if (new PhysicalDisk(DownloadFolder.DriveInfo().Name).MediaType == PhysicalDisk.MediaTypes.HDD && ReduceHDDThreads)
{
DesiredThreads.OnNext(1);
}
else
{
DesiredThreads.OnNext(DiskThreads);
}
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Hashing Remaining Archives");
await HashArchives();
var missing = ModList.Archives.Where(a => !HashedArchives.ContainsKey(a.Hash)).ToList();
if (missing.Count > 0)
{
foreach (var a in missing)
{
Info($"Unable to download {a.Name} ({a.State.PrimaryKeyString})");
}
if (IgnoreMissingFiles)
{
Info("Missing some archives, but continuing anyways at the request of the user");
}
else
{
Error("Cannot continue, was unable to download one or more archives");
}
}
// Reduce to two threads if output on HDD, else use specified. Installing files seems to have a slight benefit with two threads.
if (new PhysicalDisk(OutputFolder.DriveInfo().Name).MediaType == PhysicalDisk.MediaTypes.HDD && ReduceHDDThreads)
{
DesiredThreads.OnNext(2);
}
else
{
DesiredThreads.OnNext(DiskThreads);
}
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Extracting Modlist contents");
await ExtractModlist();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Priming VFS");
await PrimeVFS();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Building Folder Structure");
BuildFolderStructure();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Installing Archives");
await InstallArchives();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Installing Included files");
await InstallIncludedFiles();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Installing Archive Metas");
await InstallIncludedDownloadMetas();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Building BSAs");
await BuildBSAs();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Generating Merges");
await zEditIntegration.GenerateMerges(this);
UpdateTracker.NextStep("Set MO2 into portable");
await ForcePortable();
UpdateTracker.NextStep("Create Empty Output Mods");
CreateOutputMods();
UpdateTracker.NextStep("Updating System-specific ini settings");
SetScreenSizeInPrefs();
UpdateTracker.NextStep("Compacting files");
await CompactFiles();
UpdateTracker.NextStep("Installation complete! You may exit the program.");
await ExtractedModlistFolder !.DisposeAsync();
await Metrics.Send(Metrics.FinishInstall, ModList.Name);
return(true);
}
protected override async Task <bool> _Begin(CancellationToken cancel)
{
if (cancel.IsCancellationRequested)
{
return(false);
}
var metric = Metrics.Send(Metrics.BeginInstall, ModList.Name);
var result = await Utils.Log(new YesNoIntervention(
"Vortex Support is still experimental and may produce unexpected results. " +
"If anything fails please go to the special Vortex support channels on the Wabbajack Discord and contact @erri120#2285 " +
"for support.", "Continue with experimental feature?")).Task;
if (result == ConfirmationIntervention.Choice.Abort)
{
Utils.Log("Exiting at request of user");
return(false);
}
if (cancel.IsCancellationRequested)
{
return(false);
}
ConfigureProcessor(10, ConstructDynamicNumThreads(await RecommendQueueSize()));
Directory.CreateDirectory(DownloadFolder);
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Hashing Archives");
await HashArchives();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Downloading Missing Archives");
await DownloadArchives();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Hashing Remaining Archives");
await HashArchives();
if (cancel.IsCancellationRequested)
{
return(false);
}
var missing = ModList.Archives.Where(a => !HashedArchives.ContainsKey(a.Hash)).ToList();
if (missing.Count > 0)
{
foreach (var a in missing)
{
Info($"Unable to download {a.Name}");
}
if (IgnoreMissingFiles)
{
Info("Missing some archives, but continuing anyways at the request of the user");
}
else
{
Error("Cannot continue, was unable to download one or more archives");
}
}
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Priming VFS");
await PrimeVFS();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Building Folder Structure");
BuildFolderStructure();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Installing Archives");
await InstallArchives();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Installing Included files");
await InstallIncludedFiles();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Installing Manual files");
await InstallManualGameFiles();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Installing SteamWorkshopItems");
await InstallSteamWorkshopItems();
//InstallIncludedDownloadMetas();
var metric2 = Metrics.Send(Metrics.FinishInstall, ModList.Name);
UpdateTracker.NextStep("Installation complete! You may exit the program.");
return(true);
}
protected override async Task <bool> _Begin(CancellationToken cancel)
{
if (cancel.IsCancellationRequested)
{
return(false);
}
var metric = Metrics.Send("begin_install", ModList.Name);
MessageBox.Show(
"Vortex Support is still experimental and may produce unexpected results. " +
"If anything fails go to the special vortex support channels on the discord. @erri120#2285 " +
"for support.", "Warning",
MessageBoxButton.OK);
if (cancel.IsCancellationRequested)
{
return(false);
}
ConfigureProcessor(10, await RecommendQueueSize());
Directory.CreateDirectory(DownloadFolder);
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Hashing Archives");
await HashArchives();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Downloading Missing Archives");
await DownloadArchives();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Hashing Remaining Archives");
await HashArchives();
if (cancel.IsCancellationRequested)
{
return(false);
}
var missing = ModList.Archives.Where(a => !HashedArchives.ContainsKey(a.Hash)).ToList();
if (missing.Count > 0)
{
foreach (var a in missing)
{
Info($"Unable to download {a.Name}");
}
if (IgnoreMissingFiles)
{
Info("Missing some archives, but continuing anyways at the request of the user");
}
else
{
Error("Cannot continue, was unable to download one or more archives");
}
}
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Priming VFS");
await PrimeVFS();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Building Folder Structure");
BuildFolderStructure();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Installing Archives");
await InstallArchives();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Installing Included files");
await InstallIncludedFiles();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Installing Manual files");
await InstallManualGameFiles();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Installing SteamWorkshopItems");
await InstallSteamWorkshopItems();
//InstallIncludedDownloadMetas();
var metric2 = Metrics.Send("finish_install", ModList.Name);
UpdateTracker.NextStep("Installation complete! You may exit the program.");
return(true);
}
