private void GatherArchives()
{
var archives = IndexedArchives.GroupBy(a => a.Hash).ToDictionary(k => k.Key, k => k.First());
var shas = InstallDirectives.OfType <FromArchive>()
.Select(a => a.ArchiveHash)
.Distinct();
SelectedArchives = shas.Select(sha => ResolveArchive(sha, archives)).ToList();
}
private void GatherArchives()
{
Info($"Building a list of archives based on the files required");
var shas = InstallDirectives.OfType <FromArchive>()
.Select(a => a.ArchiveHashPath[0])
.Distinct();
var archives = IndexedArchives.OrderByDescending(f => f.File.LastModified)
.GroupBy(f => f.File.Hash)
.ToDictionary(f => f.Key, f => f.First());
SelectedArchives = shas.PMap(sha => ResolveArchive(sha, archives));
}
/// <summary>
/// Fills in the Patch fields in files that require them
/// </summary>
private void BuildPatches()
{
var groups = InstallDirectives.OfType <PatchedFromArchive>()
.GroupBy(p => p.ArchiveHash)
.ToList();
Info("Patching building patches from {0} archives", groups.Count);
var absolute_paths = AllFiles.ToDictionary(e => e.Path, e => e.AbsolutePath);
groups.PMap(group => BuildArchivePatches(group.Key, group, absolute_paths));
if (InstallDirectives.OfType <PatchedFromArchive>().FirstOrDefault(f => f.Patch == null) != null)
{
Error("Missing patches after generation, this should not happen");
}
}
private byte[] LoadDataForTo(string to, Dictionary <string, string> absolute_paths)
{
if (absolute_paths.TryGetValue(to, out var absolute))
{
return(File.ReadAllBytes(absolute));
}
if (to.StartsWith(Consts.BSACreationDir))
{
var bsa_id = to.Split('\\')[1];
var bsa = InstallDirectives.OfType <CreateBSA>().First(b => b.TempID == bsa_id);
using (var a = new BSAReader(Path.Combine(MO2Folder, bsa.To)))
{
var file = a.Files.First(e => e.Path == Path.Combine(to.Split('\\').Skip(2).ToArray()));
return(file.GetData());
}
}
Error($"Couldn't load data for {to}");
return(null);
}
protected override async Task <bool> _Begin(CancellationToken cancel)
{
await Metrics.Send("begin_compiling", ModListName ?? "unknown");
if (cancel.IsCancellationRequested)
{
return(false);
}
DesiredThreads.OnNext(DiskThreads);
FileExtractor2.FavorPerfOverRAM = FavorPerfOverRam;
UpdateTracker.Reset();
UpdateTracker.NextStep("Gathering information");
Utils.Log($"Compiling Game: {CompilingGame.Game}");
Utils.Log("Games from setting files:");
foreach (var game in Settings.IncludedGames)
{
Utils.Log($"- {game}");
}
Utils.Log($"VFS File Location: {VFSCacheName}");
Utils.Log($"MO2 Folder: {SourcePath}");
Utils.Log($"Downloads Folder: {DownloadsPath}");
Utils.Log($"Game Folder: {GamePath}");
var watcher = new DiskSpaceWatcher(cancel,
new[] { SourcePath, DownloadsPath, GamePath, AbsolutePath.EntryPoint }, (long)2 << 31,
drive =>
{
Utils.Log($"Aborting due to low space on {drive.Name}");
Abort();
});
var watcherTask = watcher.Start();
if (cancel.IsCancellationRequested)
{
return(false);
}
List <AbsolutePath> roots = new List <AbsolutePath> {
SourcePath, GamePath, DownloadsPath
};
roots.AddRange(Settings.IncludedGames.Select(g => g.MetaData().GameLocation()));
UpdateTracker.NextStep("Indexing folders");
if (cancel.IsCancellationRequested)
{
return(false);
}
await VFS.AddRoots(roots);
UpdateTracker.NextStep("Cleaning output folder");
await ModListOutputFolder.DeleteDirectory();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Inferring metas for game file downloads");
await InferMetas();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Reindexing downloads after meta inferring");
await VFS.AddRoot(DownloadsPath);
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Pre-validating Archives");
// Find all Downloads
IndexedArchives = (await DownloadsPath.EnumerateFiles()
.Where(f => f.WithExtension(Consts.MetaFileExtension).Exists)
.PMap(Queue,
async f => new IndexedArchive(VFS.Index.ByRootPath[f])
{
Name = (string)f.FileName,
IniData = f.WithExtension(Consts.MetaFileExtension).LoadIniFile(),
Meta = await f.WithExtension(Consts.MetaFileExtension).ReadAllTextAsync()
})).ToList();
await IndexGameFileHashes();
IndexedArchives = IndexedArchives.DistinctBy(a => a.File.AbsoluteName).ToList();
await CleanInvalidArchivesAndFillState();
UpdateTracker.NextStep("Finding Install Files");
ModListOutputFolder.CreateDirectory();
var mo2Files = SourcePath.EnumerateFiles()
.Where(p => p.IsFile)
.Select(p =>
{
if (!VFS.Index.ByRootPath.ContainsKey(p))
{
Utils.Log($"WELL THERE'S YOUR PROBLEM: {p} {VFS.Index.ByRootPath.Count}");
}
return(new RawSourceFile(VFS.Index.ByRootPath[p], p.RelativeTo(SourcePath)));
});
// If Game Folder Files exists, ignore the game folder
IndexedFiles = IndexedArchives.SelectMany(f => f.File.ThisAndAllChildren)
.OrderBy(f => f.NestingFactor)
.GroupBy(f => f.Hash)
.ToDictionary(f => f.Key, f => f.AsEnumerable());
AllFiles.SetTo(mo2Files
.DistinctBy(f => f.Path));
Info($"Found {AllFiles.Count} files to build into mod list");
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Verifying destinations");
var dups = AllFiles.GroupBy(f => f.Path)
.Where(fs => fs.Count() > 1)
.Select(fs =>
{
Utils.Log(
$"Duplicate files installed to {fs.Key} from : {String.Join(", ", fs.Select(f => f.AbsolutePath))}");
return(fs);
}).ToList();
if (dups.Count > 0)
{
Error($"Found {dups.Count} duplicates, exiting");
}
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Loading INIs");
ArchivesByFullPath = IndexedArchives.ToDictionary(a => a.File.AbsoluteName);
if (cancel.IsCancellationRequested)
{
return(false);
}
var stack = MakeStack();
UpdateTracker.NextStep("Running Compilation Stack");
var results = await AllFiles.PMap(Queue, UpdateTracker, f => RunStack(stack, f));
// Add the extra files that were generated by the stack
if (cancel.IsCancellationRequested)
{
return(false);
}
var noMatch = results.OfType <NoMatch>().ToArray();
PrintNoMatches(noMatch);
if (CheckForNoMatchExit(noMatch))
{
return(false);
}
foreach (var ignored in results.OfType <IgnoredDirectly>())
{
Utils.Log($"Ignored {ignored.To} because {ignored.Reason}");
}
InstallDirectives.SetTo(results.Where(i => !(i is IgnoredDirectly)));
Info("Getting Nexus api_key, please click authorize if a browser window appears");
UpdateTracker.NextStep("Building Patches");
await BuildPatches();
UpdateTracker.NextStep("Gathering Archives");
await GatherArchives();
UpdateTracker.NextStep("Gathering Metadata");
await GatherMetaData();
ModList = new ModList
{
GameType = CompilingGame.Game,
WabbajackVersion = Consts.CurrentMinimumWabbajackVersion,
Archives = SelectedArchives.ToList(),
ModManager = ModManager.MO2,
Directives = InstallDirectives,
Name = ModListName ?? "untitled",
Author = ModListAuthor ?? "",
Description = ModListDescription ?? "",
Readme = ModlistReadme ?? "",
Image = ModListImage != default ? ModListImage.FileName : default,
