public async Task DownloadMissingArchives(List <Archive> missing, bool download = true)
{
if (download)
{
foreach (var a in missing.Where(a => a.State.GetType() == typeof(ManualDownloader.State)))
{
var outputPath = DownloadFolder.Combine(a.Name);
await a.State.Download(a, outputPath);
}
}
DesiredThreads.OnNext(DownloadThreads);
await missing.Where(a => a.State.GetType() != typeof(ManualDownloader.State))
.PMap(Queue, async archive =>
{
Info($"Downloading {archive.Name}");
var outputPath = DownloadFolder.Combine(archive.Name);
if (download)
{
if (outputPath.Exists)
{
var origName = Path.GetFileNameWithoutExtension(archive.Name);
var ext = Path.GetExtension(archive.Name);
var uniqueKey = archive.State.PrimaryKeyString.StringSha256Hex();
outputPath = DownloadFolder.Combine(origName + "_" + uniqueKey + "_" + ext);
await outputPath.DeleteAsync();
}
}
return(await DownloadArchive(archive, download, outputPath));
});
DesiredThreads.OnNext(DiskThreads);
}
protected override async Task <bool> _Begin(CancellationToken cancel)
{
await Metrics.Send("begin_compiling", ModListName ?? "unknown");
if (cancel.IsCancellationRequested)
{
return(false);
}
DesiredThreads.OnNext(DiskThreads);
FileExtractor2.FavorPerfOverRAM = FavorPerfOverRam;
UpdateTracker.Reset();
UpdateTracker.NextStep("Gathering information");
Utils.Log($"Compiling Game: {CompilingGame.Game}");
Utils.Log("Games from setting files:");
foreach (var game in Settings.IncludedGames)
{
Utils.Log($"- {game}");
}
Utils.Log($"VFS File Location: {VFSCacheName}");
Utils.Log($"MO2 Folder: {SourcePath}");
Utils.Log($"Downloads Folder: {DownloadsPath}");
Utils.Log($"Game Folder: {GamePath}");
var watcher = new DiskSpaceWatcher(cancel,
new[] { SourcePath, DownloadsPath, GamePath, AbsolutePath.EntryPoint }, (long)2 << 31,
drive =>
{
Utils.Log($"Aborting due to low space on {drive.Name}");
Abort();
});
var watcherTask = watcher.Start();
if (cancel.IsCancellationRequested)
{
return(false);
}
List <AbsolutePath> roots = new List <AbsolutePath> {
SourcePath, GamePath, DownloadsPath
};
roots.AddRange(Settings.IncludedGames.Select(g => g.MetaData().GameLocation()));
UpdateTracker.NextStep("Indexing folders");
if (cancel.IsCancellationRequested)
{
return(false);
}
await VFS.AddRoots(roots);
UpdateTracker.NextStep("Cleaning output folder");
await ModListOutputFolder.DeleteDirectory();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Inferring metas for game file downloads");
await InferMetas();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Reindexing downloads after meta inferring");
await VFS.AddRoot(DownloadsPath);
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Pre-validating Archives");
// Find all Downloads
IndexedArchives = (await DownloadsPath.EnumerateFiles()
.Where(f => f.WithExtension(Consts.MetaFileExtension).Exists)
.PMap(Queue,
async f => new IndexedArchive(VFS.Index.ByRootPath[f])
{
Name = (string)f.FileName,
IniData = f.WithExtension(Consts.MetaFileExtension).LoadIniFile(),
Meta = await f.WithExtension(Consts.MetaFileExtension).ReadAllTextAsync()
})).ToList();
await IndexGameFileHashes();
IndexedArchives = IndexedArchives.DistinctBy(a => a.File.AbsoluteName).ToList();
await CleanInvalidArchivesAndFillState();
UpdateTracker.NextStep("Finding Install Files");
ModListOutputFolder.CreateDirectory();
var mo2Files = SourcePath.EnumerateFiles()
.Where(p => p.IsFile)
.Select(p =>
{
if (!VFS.Index.ByRootPath.ContainsKey(p))
{
Utils.Log($"WELL THERE'S YOUR PROBLEM: {p} {VFS.Index.ByRootPath.Count}");
}
return(new RawSourceFile(VFS.Index.ByRootPath[p], p.RelativeTo(SourcePath)));
});
// If Game Folder Files exists, ignore the game folder
IndexedFiles = IndexedArchives.SelectMany(f => f.File.ThisAndAllChildren)
.OrderBy(f => f.NestingFactor)
.GroupBy(f => f.Hash)
.ToDictionary(f => f.Key, f => f.AsEnumerable());
AllFiles.SetTo(mo2Files
.DistinctBy(f => f.Path));
Info($"Found {AllFiles.Count} files to build into mod list");
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Verifying destinations");
var dups = AllFiles.GroupBy(f => f.Path)
.Where(fs => fs.Count() > 1)
.Select(fs =>
{
Utils.Log(
$"Duplicate files installed to {fs.Key} from : {String.Join(", ", fs.Select(f => f.AbsolutePath))}");
return(fs);
}).ToList();
if (dups.Count > 0)
{
Error($"Found {dups.Count} duplicates, exiting");
}
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Loading INIs");
ArchivesByFullPath = IndexedArchives.ToDictionary(a => a.File.AbsoluteName);
if (cancel.IsCancellationRequested)
{
return(false);
}
var stack = MakeStack();
UpdateTracker.NextStep("Running Compilation Stack");
var results = await AllFiles.PMap(Queue, UpdateTracker, f => RunStack(stack, f));
// Add the extra files that were generated by the stack
if (cancel.IsCancellationRequested)
{
return(false);
}
var noMatch = results.OfType <NoMatch>().ToArray();
PrintNoMatches(noMatch);
if (CheckForNoMatchExit(noMatch))
{
return(false);
}
foreach (var ignored in results.OfType <IgnoredDirectly>())
{
Utils.Log($"Ignored {ignored.To} because {ignored.Reason}");
}
InstallDirectives.SetTo(results.Where(i => !(i is IgnoredDirectly)));
Info("Getting Nexus api_key, please click authorize if a browser window appears");
UpdateTracker.NextStep("Building Patches");
await BuildPatches();
UpdateTracker.NextStep("Gathering Archives");
await GatherArchives();
UpdateTracker.NextStep("Gathering Metadata");
await GatherMetaData();
ModList = new ModList
{
GameType = CompilingGame.Game,
WabbajackVersion = Consts.CurrentMinimumWabbajackVersion,
Archives = SelectedArchives.ToList(),
ModManager = ModManager.MO2,
Directives = InstallDirectives,
Name = ModListName ?? "untitled",
Author = ModListAuthor ?? "",
Description = ModListDescription ?? "",
Readme = ModlistReadme ?? "",
Image = ModListImage != default ? ModListImage.FileName : default,
public TestInstaller(AbsolutePath archive, ModList modList, AbsolutePath outputFolder, AbsolutePath downloadFolder, SystemParameters parameters)
: base(archive, modList, outputFolder, downloadFolder, parameters, steps: 1, modList.GameType)
{
DesiredThreads.OnNext(1);
}
protected override async Task <bool> _Begin(CancellationToken cancel)
{
await Metrics.Send("begin_compiling", MO2Profile ?? "unknown");
if (cancel.IsCancellationRequested)
{
return(false);
}
DesiredThreads.OnNext(DiskThreads);
FileExtractor2.FavorPerfOverRAM = FavorPerfOverRam;
UpdateTracker.Reset();
UpdateTracker.NextStep("Gathering information");
Utils.Log("Loading compiler Settings");
Settings = await CompilerSettings.Load(MO2ProfileDir);
Settings.IncludedGames = Settings.IncludedGames.Add(CompilingGame.Game);
Info("Looking for other profiles");
var otherProfilesPath = MO2ProfileDir.Combine("otherprofiles.txt");
SelectedProfiles = new HashSet <string>();
if (otherProfilesPath.Exists)
{
SelectedProfiles = (await otherProfilesPath.ReadAllLinesAsync()).ToHashSet();
}
SelectedProfiles.Add(MO2Profile !);
Info("Using Profiles: " + string.Join(", ", SelectedProfiles.OrderBy(p => p)));
Utils.Log($"Compiling Game: {CompilingGame.Game}");
Utils.Log("Games from setting files:");
foreach (var game in Settings.IncludedGames)
{
Utils.Log($"- {game}");
}
Utils.Log($"VFS File Location: {VFSCacheName}");
Utils.Log($"MO2 Folder: {SourcePath}");
Utils.Log($"Downloads Folder: {DownloadsPath}");
Utils.Log($"Game Folder: {GamePath}");
var watcher = new DiskSpaceWatcher(cancel,
new[] { SourcePath, DownloadsPath, GamePath, AbsolutePath.EntryPoint }, (long)2 << 31,
drive =>
{
Utils.Log($"Aborting due to low space on {drive.Name}");
Abort();
});
var watcherTask = watcher.Start();
if (cancel.IsCancellationRequested)
{
return(false);
}
List <AbsolutePath> roots;
if (UseGamePaths)
{
roots = new List <AbsolutePath> {
SourcePath, GamePath, DownloadsPath
};
roots.AddRange(Settings.IncludedGames.Select(g => g.MetaData().GameLocation()));
}
else
{
roots = new List <AbsolutePath> {
SourcePath, DownloadsPath
};
}
// TODO: make this generic so we can add more paths
var lootPath = (AbsolutePath)Path.Combine(
Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),
"LOOT");
IEnumerable <RawSourceFile> lootFiles = new List <RawSourceFile>();
if (lootPath.Exists)
{
roots.Add(lootPath);
}
UpdateTracker.NextStep("Indexing folders");
if (cancel.IsCancellationRequested)
{
return(false);
}
await VFS.AddRoots(roots);
if (lootPath.Exists)
{
if (CompilingGame.MO2Name == null)
{
throw new ArgumentException("Compiling game had no MO2 name specified.");
}
var lootGameDirs = new[]
{
CompilingGame.MO2Name, // most of the games use the MO2 name
CompilingGame.MO2Name.Replace(" ", "") //eg: Fallout 4 -> Fallout4
};
var lootGameDir = lootGameDirs.Select(x => lootPath.Combine(x))
.FirstOrDefault(p => p.IsDirectory);
if (lootGameDir != default)
{
Utils.Log($"Found LOOT game folder at {lootGameDir}");
lootFiles = lootGameDir.EnumerateFiles(false)
.Where(p => p.FileName == (RelativePath)"userlist.yaml")
.Where(p => p.IsFile)
.Select(p => new RawSourceFile(VFS.Index.ByRootPath[p],
Consts.LOOTFolderFilesDir.Combine(p.RelativeTo(lootPath))));
if (!lootFiles.Any())
{
Utils.Log(
$"Found no LOOT user data for {CompilingGame.HumanFriendlyGameName} at {lootGameDir}!");
}
}
}
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Cleaning output folder");
await ModListOutputFolder.DeleteDirectory();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Inferring metas for game file downloads");
await InferMetas();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Reindexing downloads after meta inferring");
await VFS.AddRoot(DownloadsPath);
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Pre-validating Archives");
// Find all Downloads
IndexedArchives = (await DownloadsPath.EnumerateFiles()
.Where(f => f.WithExtension(Consts.MetaFileExtension).Exists)
.PMap(Queue, UpdateTracker,
async f => new IndexedArchive(VFS.Index.ByRootPath[f])
{
Name = (string)f.FileName,
IniData = f.WithExtension(Consts.MetaFileExtension).LoadIniFile(),
Meta = await f.WithExtension(Consts.MetaFileExtension).ReadAllTextAsync()
})).ToList();
await IndexGameFileHashes();
IndexedArchives = IndexedArchives.DistinctBy(a => a.File.AbsoluteName).ToList();
await CleanInvalidArchivesAndFillState();
UpdateTracker.NextStep("Finding Install Files");
ModListOutputFolder.CreateDirectory();
var mo2Files = SourcePath.EnumerateFiles()
.Where(p => p.IsFile)
.Select(p =>
{
if (!VFS.Index.ByRootPath.ContainsKey(p))
{
Utils.Log($"WELL THERE'S YOUR PROBLEM: {p} {VFS.Index.ByRootPath.Count}");
}
return(new RawSourceFile(VFS.Index.ByRootPath[p], p.RelativeTo(SourcePath)));
});
// If Game Folder Files exists, ignore the game folder
IndexedFiles = IndexedArchives.SelectMany(f => f.File.ThisAndAllChildren)
.OrderBy(f => f.NestingFactor)
.GroupBy(f => f.Hash)
.ToDictionary(f => f.Key, f => f.AsEnumerable());
AllFiles.SetTo(mo2Files
.Concat(lootFiles)
.DistinctBy(f => f.Path));
Info($"Found {AllFiles.Count} files to build into mod list");
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Verifying destinations");
var dups = AllFiles.GroupBy(f => f.Path)
.Where(fs => fs.Count() > 1)
.Select(fs =>
{
Utils.Log(
$"Duplicate files installed to {fs.Key} from : {String.Join(", ", fs.Select(f => f.AbsolutePath))}");
return(fs);
}).ToList();
if (dups.Count > 0)
{
Error($"Found {dups.Count} duplicates, exiting");
}
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Loading INIs");
ModInis.SetTo(SourcePath.Combine(Consts.MO2ModFolderName)
.EnumerateDirectories()
.Select(f =>
{
var modName = f.FileName;
var metaPath = f.Combine("meta.ini");
return(metaPath.Exists ? (mod_name: f, metaPath.LoadIniFile()) : default);
protected override async Task <bool> _Begin(CancellationToken cancel)
{
if (cancel.IsCancellationRequested)
{
return(false);
}
await Metrics.Send(Metrics.BeginInstall, ModList.Name);
Utils.Log("Configuring Processor");
FileExtractor2.FavorPerfOverRAM = FavorPerfOverRam;
if (GameFolder == null)
{
GameFolder = Game.TryGetGameLocation();
}
if (GameFolder == null)
{
var otherGame = Game.CommonlyConfusedWith.Where(g => g.MetaData().IsInstalled).Select(g => g.MetaData()).FirstOrDefault();
if (otherGame != null)
{
await Utils.Log(new CriticalFailureIntervention(
$"In order to do a proper install Wabbajack needs to know where your {Game.HumanFriendlyGameName} folder resides. However this game doesn't seem to be installed, we did however find a installed " +
$"copy of {otherGame.HumanFriendlyGameName}, did you install the wrong game?",
$"Could not locate {Game.HumanFriendlyGameName}"))
.Task;
}
else
{
await Utils.Log(new CriticalFailureIntervention(
$"In order to do a proper install Wabbajack needs to know where your {Game.HumanFriendlyGameName} folder resides. However this game doesn't seem to be installed",
$"Could not locate {Game.HumanFriendlyGameName}"))
.Task;
}
Utils.Log("Exiting because we couldn't find the game folder.");
return(false);
}
Utils.Log($"Install Folder: {OutputFolder}");
Utils.Log($"Downloads Folder: {DownloadFolder}");
Utils.Log($"Game Folder: {GameFolder.Value}");
Utils.Log($"Wabbajack Folder: {AbsolutePath.EntryPoint}");
var watcher = new DiskSpaceWatcher(cancel, new[] { OutputFolder, DownloadFolder, GameFolder.Value, AbsolutePath.EntryPoint }, (long)2 << 31,
drive =>
{
Utils.Log($"Aborting due to low space on {drive.Name}");
Abort();
});
var watcherTask = watcher.Start();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Validating Game ESMs");
await ValidateGameESMs();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Creating Output Folders");
OutputFolder.CreateDirectory();
DownloadFolder.CreateDirectory();
if (OutputFolder.Combine(Consts.MO2ModFolderName).IsDirectory&& WarnOnOverwrite)
{
if ((await Utils.Log(new ConfirmUpdateOfExistingInstall {
ModListName = ModList.Name, OutputFolder = OutputFolder
}).Task) == ConfirmUpdateOfExistingInstall.Choice.Abort)
{
Utils.Log("Exiting installation at the request of the user, existing mods folder found.");
return(false);
}
}
// Reduce to one thread if downloads on HDD, else use specified. Hashing on HDD has no benefit with more threads.
if (new PhysicalDisk(DownloadFolder.DriveInfo().Name).MediaType == PhysicalDisk.MediaTypes.HDD && ReduceHDDThreads)
{
DesiredThreads.OnNext(1);
}
else
{
DesiredThreads.OnNext(DiskThreads);
}
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Optimizing ModList");
await OptimizeModlist();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Hashing Archives");
await HashArchives();
// Set to download thread count.
DesiredThreads.OnNext(DownloadThreads);
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Downloading Missing Archives");
await DownloadArchives();
// Reduce to one thread if downloads on HDD, else use specified. Hashing on HDD has no benefit with more threads.
if (new PhysicalDisk(DownloadFolder.DriveInfo().Name).MediaType == PhysicalDisk.MediaTypes.HDD && ReduceHDDThreads)
{
DesiredThreads.OnNext(1);
}
else
{
DesiredThreads.OnNext(DiskThreads);
}
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Hashing Remaining Archives");
await HashArchives();
var missing = ModList.Archives.Where(a => !HashedArchives.ContainsKey(a.Hash)).ToList();
if (missing.Count > 0)
{
foreach (var a in missing)
{
Info($"Unable to download {a.Name} ({a.State.PrimaryKeyString})");
}
if (IgnoreMissingFiles)
{
Info("Missing some archives, but continuing anyways at the request of the user");
}
else
{
Error("Cannot continue, was unable to download one or more archives");
}
}
// Reduce to two threads if output on HDD, else use specified. Installing files seems to have a slight benefit with two threads.
if (new PhysicalDisk(OutputFolder.DriveInfo().Name).MediaType == PhysicalDisk.MediaTypes.HDD && ReduceHDDThreads)
{
DesiredThreads.OnNext(2);
}
else
{
DesiredThreads.OnNext(DiskThreads);
}
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Extracting Modlist contents");
await ExtractModlist();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Priming VFS");
await PrimeVFS();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Building Folder Structure");
BuildFolderStructure();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Installing Archives");
await InstallArchives();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Installing Included files");
await InstallIncludedFiles();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Installing Archive Metas");
await InstallIncludedDownloadMetas();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Building BSAs");
await BuildBSAs();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Generating Merges");
await zEditIntegration.GenerateMerges(this);
UpdateTracker.NextStep("Set MO2 into portable");
await ForcePortable();
UpdateTracker.NextStep("Create Empty Output Mods");
CreateOutputMods();
UpdateTracker.NextStep("Updating System-specific ini settings");
SetScreenSizeInPrefs();
UpdateTracker.NextStep("Compacting files");
await CompactFiles();
UpdateTracker.NextStep("Installation complete! You may exit the program.");
await ExtractedModlistFolder !.DisposeAsync();
await Metrics.Send(Metrics.FinishInstall, ModList.Name);
return(true);
}
