private static void ExtractAllWithBSA(string source, string dest)
{
try
{
using (var arch = BSADispatch.OpenRead(source))
{
arch.Files.PMap(f =>
{
var path = f.Path;
if (f.Path.StartsWith("\\"))
{
path = f.Path.Substring(1);
}
Utils.Status($"Extracting {path}");
var out_path = Path.Combine(dest, path);
var parent = Path.GetDirectoryName(out_path);
if (!Directory.Exists(parent))
{
Directory.CreateDirectory(parent);
}
using (var fs = File.OpenWrite(out_path))
{
f.CopyDataTo(fs);
}
});
}
}
catch (Exception ex)
{
Utils.Log($"While Extracting {source}");
throw ex;
}
}
private static async Task ExtractAllWithBSA(WorkQueue queue, string source, string dest)
{
try
{
using (var arch = BSADispatch.OpenRead(source))
{
await arch.Files
.PMap(queue, f =>
{
var path = f.Path;
if (f.Path.StartsWith("\\"))
{
path = f.Path.Substring(1);
}
Utils.Status($"Extracting {path}");
var outPath = Path.Combine(dest, path);
var parent = Path.GetDirectoryName(outPath);
if (!Directory.Exists(parent))
{
Directory.CreateDirectory(parent);
}
using (var fs = File.Open(outPath, System.IO.FileMode.Create))
{
f.CopyDataTo(fs);
}
});
}
}
catch (Exception ex)
{
Utils.ErrorThrow(ex, $"While Extracting {source}");
}
}
public static async Task <ExtractedFiles> ExtractAll(WorkQueue queue, AbsolutePath source, IEnumerable <RelativePath> OnlyFiles = null)
{
try
{
if (BSADispatch.MightBeBSA(source))
{
return(await ExtractAllWithBSA(queue, source));
}
else if (source.Extension == Consts.OMOD)
{
return(await ExtractAllWithOMOD(source));
}
else if (source.Extension == Consts.EXE)
{
return(await ExtractAllExe(source));
}
else
{
return(await ExtractAllWith7Zip(source, OnlyFiles));
}
}
catch (Exception ex)
{
Utils.ErrorThrow(ex, $"Error while extracting {source}");
throw new Exception();
}
}
private static async Task ExtractAllWithBSA(WorkQueue queue, AbsolutePath source, AbsolutePath dest)
{
try
{
using var arch = BSADispatch.OpenRead(source);
await arch.Files
.PMap(queue, f =>
{
Utils.Status($"Extracting {(string)f.Path}");
var outPath = f.Path.RelativeTo(dest);
var parent = outPath.Parent;
if (!parent.IsDirectory)
{
parent.CreateDirectory();
}
using var fs = outPath.Create();
f.CopyDataTo(fs);
});
}
catch (Exception ex)
{
Utils.ErrorThrow(ex, $"While Extracting {source}");
}
}
protected override async Task <ExitCode> Run()
{
var bsa = await BSADispatch.OpenRead(Input.RelativeTo(AbsolutePath.GetCurrentDirectory()));
bsa.Dump(Console.WriteLine);
return(ExitCode.Ok);
}
public override Directive Run(RawSourceFile source)
{
if (!Consts.SupportedBSAs.Contains(Path.GetExtension(source.Path).ToLower()))
{
return(null);
}
var defaultInclude = false;
if (source.Path.StartsWith("mods"))
{
if (_include_directly.Any(path => source.Path.StartsWith(path)))
{
defaultInclude = true;
}
}
var source_files = source.File.FileInArchive;
var stack = defaultInclude ? _microstackWithInclude : _microstack;
var id = Guid.NewGuid().ToString();
var matches = source_files.PMap(e => Compiler.RunStack(stack, new RawSourceFile(e)
{
Path = Path.Combine(Consts.BSACreationDir, id, e.Paths.Last())
}));
foreach (var match in matches)
{
if (match is IgnoredDirectly)
{
Utils.Error($"File required for BSA {source.Path} creation doesn't exist: {match.To}");
}
_compiler.ExtraFiles.Add(match);
}
CreateBSA directive;
using (var bsa = BSADispatch.OpenRead(source.AbsolutePath))
{
directive = new CreateBSA
{
To = source.Path,
TempID = id,
State = bsa.State,
FileStates = bsa.Files.Select(f => f.State).ToList()
};
}
return(directive);
}
private static async Task <ExtractedFiles> ExtractAllWithBSA(WorkQueue queue, AbsolutePath source)
{
try
{
await using var arch = BSADispatch.OpenRead(source);
var files = arch.Files.ToDictionary(f => f.Path, f => (IExtractedFile) new ExtractedBSAFile(f));
return(new ExtractedFiles(files, arch));
}
catch (Exception ex)
{
Utils.ErrorThrow(ex, $"While Extracting {source}");
throw new Exception();
}
}
public override async ValueTask <Directive?> Run(RawSourceFile source)
{
if (!Consts.SupportedBSAs.Contains(source.Path.Extension))
{
return(null);
}
var defaultInclude = false;
if (source.Path.RelativeTo(_mo2Compiler.MO2Folder).InFolder(_mo2Compiler.MO2Folder.Combine(Consts.MO2ModFolderName)))
{
if (_includeDirectly.Any(path => source.Path.StartsWith(path)))
{
defaultInclude = true;
}
}
var sourceFiles = source.File.Children;
var stack = defaultInclude ? _microstackWithInclude(source.File) : _microstack(source.File);
var id = Guid.NewGuid().ToString();
var matches = await sourceFiles.PMap(_mo2Compiler.Queue, e => _mo2Compiler.RunStack(stack, new RawSourceFile(e, Consts.BSACreationDir.Combine((RelativePath)id, e.Name.FileName))));
foreach (var match in matches)
{
if (match is IgnoredDirectly)
{
Utils.ErrorThrow(new UnconvertedError($"File required for BSA {source.Path} creation doesn't exist: {match.To}"));
}
_mo2Compiler.ExtraFiles.Add(match);
}
CreateBSA directive;
using (var bsa = BSADispatch.OpenRead(source.AbsolutePath))
{
directive = new CreateBSA(
state: bsa.State,
items: bsa.Files.Select(f => f.State).ToList())
{
To = source.Path,
TempID = (RelativePath)id,
};
}
return(directive);
}
protected override async Task <ExitCode> Run()
{
Console.WriteLine($"Extracting {InputFile} to {OutputFolder}");
var bsa = await BSADispatch.OpenRead((AbsolutePath)InputFile);
foreach (var file in bsa.Files)
{
Console.WriteLine($"Extracting {file.Path}");
var ofile = file.Path.RelativeTo((AbsolutePath)OutputFolder);
ofile.Parent.CreateDirectory();
await using var ostream = await ofile.Create();
await file.CopyDataTo(ostream);
}
return(ExitCode.Ok);
}
private static async Task <Dictionary <RelativePath, T> > GatheringExtractWithBSA <T>(IStreamFactory sFn, Definitions.FileType sig, Predicate <RelativePath> shouldExtract, Func <RelativePath, IStreamFactory, ValueTask <T> > mapfn)
{
var archive = await BSADispatch.OpenRead(sFn, sig);
var results = new Dictionary <RelativePath, T>();
foreach (var entry in archive.Files)
{
if (!shouldExtract(entry.Path))
{
continue;
}
var result = await mapfn(entry.Path, await entry.GetStreamFactory());
results.Add(entry.Path, result);
}
return(results);
}
public async Task CanRecompressAndResizeDDSImages()
{
var profile = utils.AddProfile();
var mod = await utils.AddMod();
var nativeFile = await utils.AddModFile(mod, @"native\whitestagbody.dds", 0);
var recompressedFile = await utils.AddModFile(mod, @"recompressed\whitestagbody.dds", 0);
var resizedFile = await utils.AddModFile(mod, @"resized\whitestagbody.dds", 0);
var gameBSA = Game.SkyrimSpecialEdition.MetaData().GameLocation().Combine(@"Data\Skyrim - Textures1.bsa");
var bsa = await BSADispatch.OpenRead(gameBSA);
var ddsExtension = new Extension(".dds");
var firstFile = bsa.Files.First(f => f.Path.Extension == ddsExtension);
await using (var nf = await nativeFile.OpenWrite())
{
await firstFile.CopyDataTo(nf);
}
{
var originalDDS = await ImageState.GetState(nativeFile);
await ImageState.ConvertImage(nativeFile, recompressedFile.Parent, originalDDS.Width, originalDDS.Height, DXGI_FORMAT.BC7_UNORM, recompressedFile.Extension);
await ImageState.ConvertImage(nativeFile, resizedFile.Parent, 128, 128, DXGI_FORMAT.BC7_UNORM, resizedFile.Extension);
}
await utils.Configure();
await CompileAndInstall(profile, true);
await utils.VerifyInstalledFile(mod, @"native\whitestagbody.dds");
Assert.True(0.99f <= (await ImageState.GetState(recompressedFile)).PerceptualHash.Similarity(await ImageState.GetPHash(utils.InstalledPath(mod, @"recompressed\whitestagbody.dds"))));
Assert.True(0.98f <= (await ImageState.GetState(resizedFile)).PerceptualHash.Similarity(await ImageState.GetPHash(utils.InstalledPath(mod, @"resized\whitestagbody.dds"))));
}
public override async ValueTask <Directive?> Run(RawSourceFile source)
{
if (!Consts.SupportedBSAs.Contains(source.Path.Extension))
{
return(null);
}
var defaultInclude = false;
if (source.Path.RelativeTo(_mo2Compiler.SourcePath).InFolder(_mo2Compiler.SourcePath.Combine(Consts.MO2ModFolderName)))
{
if (_includeDirectly.Any(path => source.Path.StartsWith(path)))
{
defaultInclude = true;
}
}
if (source.AbsolutePath.Size >= (long)2 << 31)
{
var bsaTest = await BSADispatch.OpenRead(source.AbsolutePath);
if (bsaTest.State is BSAStateObject)
{
Utils.Error(
$"BSA {source.AbsolutePath.FileName} is over 2GB in size, very few programs (Including Wabbajack) can create BSA files this large without causing CTD issues." +
$"Please re-compress this BSA into a more manageable size.");
}
}
var sourceFiles = source.File.Children;
var stack = defaultInclude ? _microstackWithInclude(source.File) : _microstack(source.File);
var id = Guid.NewGuid().ToString();
Func <Task>?_cleanup = null;
if (defaultInclude)
{
//_cleanup = await source.File.Context.Stage(source.File.Children);
}
var matches = await sourceFiles.PMap(_mo2Compiler.Queue, e => _mo2Compiler.RunStack(stack, new RawSourceFile(e, Consts.BSACreationDir.Combine((RelativePath)id, (RelativePath)e.Name))));
foreach (var match in matches)
{
if (match is IgnoredDirectly)
{
Utils.ErrorThrow(new UnconvertedError($"File required for BSA {source.Path} creation doesn't exist: {match.To}"));
}
_mo2Compiler.ExtraFiles.Add(match);
}
CreateBSA directive;
var bsa = await BSADispatch.OpenRead(source.AbsolutePath);
directive = new CreateBSA(
state: bsa.State,
items: bsa.Files.Select(f => f.State).ToList())
{
To = source.Path,
TempID = (RelativePath)id,
};
if (_cleanup != null)
{
await _cleanup();
}
return(directive);
}
[InlineData(Game.Fallout4, 43474)] // EM 2 Rifle
public async Task BSACompressionRecompression(Game game, int modid)
{
var filename = await DownloadMod(game, modid);
var folder = _bsaFolder.Combine(game.ToString(), modid.ToString());
await folder.DeleteDirectory();
folder.CreateDirectory();
await using var files = await FileExtractor.ExtractAll(Queue, filename);
await files.MoveAllTo(folder);
foreach (var bsa in folder.EnumerateFiles().Where(f => Consts.SupportedBSAs.Contains(f.Extension)))
{
TestContext.WriteLine($"From {bsa}");
TestContext.WriteLine("Cleaning Output Dir");
await _tempDir.DeleteDirectory();
_tempDir.CreateDirectory();
TestContext.WriteLine($"Reading {bsa}");
var tempFile = ((RelativePath)"tmp.bsa").RelativeToEntryPoint();
var size = bsa.Size;
await using var a = BSADispatch.OpenRead(bsa);
await a.Files.PMap(Queue, file =>
{
var absName = _tempDir.Combine(file.Path);
ViaJson(file.State);
absName.Parent.CreateDirectory();
using (var fs = absName.Create())
{
file.CopyDataTo(fs);
}
Assert.Equal(file.Size, absName.Size);
});
// Check Files should be case insensitive
Assert.Equal(a.Files.Count(), a.Files.Select(f => f.Path).ToHashSet().Count);
Assert.Equal(a.Files.Count(), a.Files.Select(f => f.Path.ToString().ToLowerInvariant()).ToHashSet().Count);
TestContext.WriteLine($"Building {bsa}");
await using (var w = ViaJson(a.State).MakeBuilder(size))
{
var streams = await a.Files.PMap(Queue, async file =>
{
var absPath = _tempDir.Combine(file.Path);
var str = absPath.OpenRead();
await w.AddFile(ViaJson(file.State), str);
return(str);
});
await w.Build(tempFile);
streams.Do(s => s.Dispose());
}
TestContext.WriteLine($"Verifying {bsa}");
await using var b = BSADispatch.OpenRead(tempFile);
TestContext.WriteLine($"Performing A/B tests on {bsa}");
Assert.Equal(a.State.ToJson(), b.State.ToJson());
// Check same number of files
Assert.Equal(a.Files.Count(), b.Files.Count());
await a.Files.Zip(b.Files, (ai, bi) => (ai, bi))
.PMap(Queue, pair =>
{
Assert.Equal(pair.ai.State.ToJson(), pair.bi.State.ToJson());
//Console.WriteLine($" - {pair.ai.Path}");
Assert.Equal(pair.ai.Path, pair.bi.Path);
//Equal(pair.ai.Compressed, pair.bi.Compressed);
Assert.Equal(pair.ai.Size, pair.bi.Size);
Assert.Equal(GetData(pair.ai), GetData(pair.bi));
});
}
}
//private const string Archive2Location = @"D:\Steam\steamapps\common\Fallout 4\Tools\Archive2\Archive2.exe";
private static void Main(string[] args)
{
foreach (var bsa in Directory.EnumerateFiles(TestDir, "*.ba2", SearchOption.AllDirectories)
//.Concat(Directory.EnumerateFiles(TestDir, "*.bsa", SearchOption.AllDirectories))
)
{
Console.WriteLine($"From {bsa}");
Console.WriteLine("Cleaning Output Dir");
if (Directory.Exists(TempDir))
{
Directory.Delete(TempDir, true);
}
if (Directory.Exists(ArchiveTempDir))
{
Directory.Delete(ArchiveTempDir, true);
}
Directory.CreateDirectory(TempDir);
Console.WriteLine($"Reading {bsa}");
using (var a = BSADispatch.OpenRead(bsa))
{
Parallel.ForEach(a.Files, file =>
{
var abs_name = Path.Combine(TempDir, file.Path);
ViaJson(file.State);
if (!Directory.Exists(Path.GetDirectoryName(abs_name)))
{
Directory.CreateDirectory(Path.GetDirectoryName(abs_name));
}
using (var fs = File.OpenWrite(abs_name))
{
file.CopyDataTo(fs);
}
Equal(file.Size, new FileInfo(abs_name).Length);
});
/*
* Console.WriteLine("Extracting via Archive.exe");
* if (bsa.ToLower().EndsWith(".ba2"))
* {
* var p = Process.Start(Archive2Location, $"\"{bsa}\" -e=\"{ArchiveTempDir}\"");
* p.WaitForExit();
*
* foreach (var file in a.Files)
* {
* var a_path = Path.Combine(TempDir, file.Path);
* var b_path = Path.Combine(ArchiveTempDir, file.Path);
* Equal(new FileInfo(a_path).Length, new FileInfo(b_path).Length);
* Equal(File.ReadAllBytes(a_path), File.ReadAllBytes(b_path));
* }
* }*/
Console.WriteLine($"Building {bsa}");
using (var w = ViaJson(a.State).MakeBuilder())
{
Parallel.ForEach(a.Files, file =>
{
var abs_path = Path.Combine(TempDir, file.Path);
using (var str = File.OpenRead(abs_path))
{
w.AddFile(ViaJson(file.State), str);
}
});
w.Build("c:\\tmp\\tmp.bsa");
}
Console.WriteLine($"Verifying {bsa}");
using (var b = BSADispatch.OpenRead("c:\\tmp\\tmp.bsa"))
{
Console.WriteLine($"Performing A/B tests on {bsa}");
Equal(JsonConvert.SerializeObject(a.State), JsonConvert.SerializeObject(b.State));
//Equal((uint) a.ArchiveFlags, (uint) b.ArchiveFlags);
//Equal((uint) a.FileFlags, (uint) b.FileFlags);
// Check same number of files
Equal(a.Files.Count(), b.Files.Count());
var idx = 0;
foreach (var pair in a.Files.Zip(b.Files, (ai, bi) => (ai, bi)))
{
idx++;
Equal(JsonConvert.SerializeObject(pair.ai.State),
JsonConvert.SerializeObject(pair.bi.State));
//Console.WriteLine($" - {pair.ai.Path}");
Equal(pair.ai.Path, pair.bi.Path);
//Equal(pair.ai.Compressed, pair.bi.Compressed);
Equal(pair.ai.Size, pair.bi.Size);
Equal(GetData(pair.ai), GetData(pair.bi));
}
}
}
}
}
