private static void ExtractAllWithBSA(string source, string dest)
{
try
{
using (var arch = new BSAReader(source))
{
arch.Files.PMap(f =>
{
var path = f.Path;
if (f.Path.StartsWith("\\"))
{
path = f.Path.Substring(1);
}
Utils.Status($"Extracting {path}");
var out_path = Path.Combine(dest, path);
var parent = Path.GetDirectoryName(out_path);
if (!Directory.Exists(parent))
{
Directory.CreateDirectory(parent);
}
using (var fs = File.OpenWrite(out_path))
{
f.CopyDataTo(fs);
}
});
}
}
catch (Exception ex)
{
Utils.Log($"While Extracting {source}");
throw ex;
}
}
public async Task BuildBSACacheAsync()
{
Utils.Log("Start building BSA cache");
var bsaList = await Task.Run(() => Directory.EnumerateFiles(_dataFolder, "*.bsa", SearchOption.TopDirectoryOnly).ToList());
Utils.Log($"Found {bsaList.Count} BSAs in {_dataFolder}");
bsaList.Do(async bsa =>
{
Utils.Log($"Start parsing BSA {bsa}");
await using var reader = new BSAReader(bsa);
var allowedFiles = reader.Files
.Where(x => AllowedExtensions.Contains(Path.GetExtension(x.Path)))
.Select(x => x.Path)
.Select(x => x.ToLower())
.Where(x => !_fileList.Contains(x))
.ToList();
Utils.Log($"BSA {bsa} has {allowedFiles.Count} allowed files");
_fileList.AddRange(allowedFiles);
Utils.Log($"Finished parsing BSA {bsa}");
});
Utils.Log($"Finished parsing of {bsaList.Count} BSAs");
}
/// <summary>
/// This function will search for a way to create a BSA in the installed mod list by assembling it from files
/// found in archives. To do this we hash all the files in side the BSA then try to find matches and patches for
/// all of the files.
/// </summary>
/// <returns></returns>
private Func <RawSourceFile, Directive> DeconstructBSAs()
{
var microstack = new List <Func <RawSourceFile, Directive> >()
{
DirectMatch(),
IncludePatches(),
DropAll()
};
return(source =>
{
if (!Consts.SupportedBSAs.Contains(Path.GetExtension(source.Path)))
{
return null;
}
var hashed = HashBSA(source.AbsolutePath);
var source_files = hashed.Select(e => new RawSourceFile()
{
Hash = e.Item2,
Path = e.Item1,
AbsolutePath = e.Item1
});
var matches = source_files.Select(e => RunStack(microstack, e));
var id = Guid.NewGuid().ToString();
foreach (var match in matches)
{
if (match is IgnoredDirectly)
{
Error($"File required for BSA creation doesn't exist: {match.To}");
}
match.To = Path.Combine(Consts.BSACreationDir, id, match.To);
ExtraFiles.Add(match);
}
;
CreateBSA directive;
using (var bsa = new BSAReader(source.AbsolutePath))
{
directive = new CreateBSA()
{
To = source.Path,
TempID = id,
Type = (uint)bsa.HeaderType,
FileFlags = (uint)bsa.FileFlags,
ArchiveFlags = (uint)bsa.ArchiveFlags,
};
};
return directive;
});
}
public static StringsFolderLookupOverlay?TypicalFactory(string referenceModPath, StringsReadParameters?instructions, ModKey modKey)
{
var ret = new StringsFolderLookupOverlay();
var stringsFolderPath = instructions?.StringsFolderOverride;
var dir = Path.GetDirectoryName(referenceModPath);
if (stringsFolderPath == null)
{
stringsFolderPath = Path.Combine(dir, "Strings");
}
if (stringsFolderPath.Value.Exists)
{
foreach (var file in stringsFolderPath.Value.Info.EnumerateFiles($"{modKey.Name}*{StringsUtility.StringsFileExtension}"))
{
if (!StringsUtility.TryRetrieveInfoFromString(file.Name, out var type, out var lang, out _))
{
continue;
}
var dict = ret.Get(type);
dict[lang] = new Lazy <IStringsLookup>(() => new StringsLookupOverlay(file.FullName, type), LazyThreadSafetyMode.ExecutionAndPublication);
}
}
foreach (var bsaFile in Directory.EnumerateFiles(dir, "*.bsa"))
{
var bsaReader = BSAReader.Load(new AbsolutePath(bsaFile, skipValidation: true));
foreach (var item in bsaReader.Files)
{
if (!StringsUtility.TryRetrieveInfoFromString(Path.GetFileName(item.Path.ToString()), out var type, out var lang, out var modName))
{
continue;
}
if (!MemoryExtensions.Equals(modKey.Name, modName, StringComparison.OrdinalIgnoreCase))
{
continue;
}
var dict = ret.Get(type);
if (dict.ContainsKey(lang))
{
continue;
}
dict[lang] = new Lazy <IStringsLookup>(() =>
{
byte[] bytes = new byte[item.Size];
using var stream = new MemoryStream(bytes);
item.CopyDataTo(stream).AsTask().Wait();
return(new StringsLookupOverlay(bytes, type));
}, LazyThreadSafetyMode.ExecutionAndPublication);
}
}
return(ret);
}
/// <summary>
/// Given a BSA on disk, index it and return a dictionary of SHA256 -> filename
/// </summary>
/// <param name="absolutePath"></param>
/// <returns></returns>
private List <(string, string)> HashBSA(string absolutePath)
{
Status($"Hashing BSA: {absolutePath}");
var results = new List <(string, string)>();
using (var a = new BSAReader(absolutePath))
{
foreach (var entry in a.Files)
{
Status($"Hashing BSA: {absolutePath} - {entry.Path}");
var data = entry.GetData();
results.Add((entry.Path, data.SHA256()));
}
}
return(results);
}
internal List <string> GetArchiveEntryNames(VirtualFile file)
{
if (!file.IsStaged)
{
throw new InvalidDataException("File is not staged");
}
if (file.Extension == ".bsa")
{
using (var ar = new BSAReader(file.StagedPath))
{
return(ar.Files.Select(f => f.Path).ToList());
}
}
if (file.Extension == ".zip")
{
using (var s = new ZipFile(File.OpenRead(file.StagedPath)))
{
s.IsStreamOwner = true;
s.UseZip64 = UseZip64.On;
if (s.OfType <ZipEntry>().FirstOrDefault(e => !e.CanDecompress) == null)
{
return(s.OfType <ZipEntry>()
.Where(f => f.IsFile)
.Select(f => f.Name.Replace('/', '\\'))
.ToList());
}
}
}
/*
* using (var e = new ArchiveFile(file.StagedPath))
* {
* return e.Entries
* .Where(f => !f.IsFolder)
* .Select(f => f.FileName).ToList();
* }*/
return(null);
}
private byte[] LoadDataForTo(string to, Dictionary <string, string> absolute_paths)
{
if (absolute_paths.TryGetValue(to, out var absolute))
{
return(File.ReadAllBytes(absolute));
}
if (to.StartsWith(Consts.BSACreationDir))
{
var bsa_id = to.Split('\\')[1];
var bsa = InstallDirectives.OfType <CreateBSA>().First(b => b.TempID == bsa_id);
using (var a = new BSAReader(Path.Combine(MO2Folder, bsa.To)))
{
var file = a.Files.First(e => e.Path == Path.Combine(to.Split('\\').Skip(2).ToArray()));
return(file.GetData());
}
}
Error($"Couldn't load data for {to}");
return(null);
}
public void TestBSACreation()
{
const string outputPath = "output.bsa";
const string expectedContent = "Hello World!";
const string dummyFile = "bsa-dummy-file.txt";
File.WriteAllText(dummyFile, expectedContent, Encoding.ASCII);
var fi = new FileInfo(dummyFile);
var expectedLength = fi.Length;
var creator = new BSACreator();
creator.AddFile(dummyFile, "text\\test.txt");
creator.WriteToFile(outputPath);
Assert.True(File.Exists(outputPath));
using var reader = new BSAReader(outputPath);
Assert.Single(reader.Folders);
Assert.Single(reader.Files);
var folder = reader.Folders.First();
var file = reader.Files.First();
Assert.Equal("text", folder.Name);
Assert.Equal("test.txt", file.Name);
Assert.Equal((uint)expectedLength, file.Size);
using var ms = new MemoryStream();
reader.CopyFileTo(file, ms);
var buffer = new byte[ms.Length];
ms.Read(buffer, 0, buffer.Length);
var actualContents = Encoding.ASCII.GetString(buffer);
Assert.Equal(expectedContent, actualContents);
}
public void TestBSAParsing()
{
const string file = "test-bsa.bsa";
var path = Path.Combine("files", file);
using var reader = new BSAReader(path);
Assert.Equal(115, reader.Files.Count);
Assert.Equal(10, reader.Folders.Count);
const string expectedContents = "@#$%";
var fileInfo = reader.Files.First(x => x.Name.Equals("s.txt"));
using var ms = new MemoryStream();
reader.CopyFileTo(fileInfo, ms);
var buffer = new byte[ms.Length];
ms.Read(buffer, 0, buffer.Length);
var actualContents = Encoding.UTF8.GetString(buffer);
Assert.Equal(expectedContents, actualContents);
}
static void Main(string[] args)
{
foreach (var bsa in Directory.EnumerateFiles(TestDir, "*.bsa", SearchOption.AllDirectories).Skip(0))
{
Console.WriteLine($"From {bsa}");
Console.WriteLine("Cleaning Output Dir");
if (Directory.Exists(TempDir))
{
Directory.Delete(TempDir, true);
}
Directory.CreateDirectory(TempDir);
Console.WriteLine($"Reading {bsa}");
using (var a = new BSAReader(bsa))
{
Parallel.ForEach(a.Files, file =>
{
var abs_name = Path.Combine(TempDir, file.Path);
if (!Directory.Exists(Path.GetDirectoryName(abs_name)))
{
Directory.CreateDirectory(Path.GetDirectoryName(abs_name));
}
using (var fs = File.OpenWrite(abs_name))
file.CopyDataTo(fs);
Equal((long)file.Size, new FileInfo(abs_name).Length);
});
Console.WriteLine($"Building {bsa}");
using (var w = new BSABuilder())
{
w.ArchiveFlags = a.ArchiveFlags;
w.FileFlags = a.FileFlags;
w.HeaderType = a.HeaderType;
Parallel.ForEach(a.Files, file =>
{
var abs_path = Path.Combine("c:\\tmp\\out", file.Path);
using (var str = File.OpenRead(abs_path))
{
var entry = w.AddFile(file.Path, str, file.FlipCompression);
}
});
w.Build("c:\\tmp\\tmp.bsa");
// Sanity Checks
Equal(a.Files.Count(), w.Files.Count());
Equal(a.Files.Select(f => f.Path).ToHashSet(), w.Files.Select(f => f.Path).ToHashSet());
/*foreach (var pair in Enumerable.Zip(a.Files, w.Files, (ai, bi) => (ai, bi)))
* {
* Console.WriteLine($"{pair.ai.Path}, {pair.ai.Hash}, {pair.bi.Path}, {pair.bi.Hash}");
* }*/
foreach (var pair in Enumerable.Zip(a.Files, w.Files, (ai, bi) => (ai, bi)))
{
Equal(pair.ai.Path, pair.bi.Path);
Equal(pair.ai.Hash, pair.bi.Hash);
}
}
Console.WriteLine($"Verifying {bsa}");
using (var b = new BSAReader("c:\\tmp\\tmp.bsa"))
{
Console.WriteLine($"Performing A/B tests on {bsa}");
Equal((uint)a.ArchiveFlags, (uint)b.ArchiveFlags);
Equal((uint)a.FileFlags, (uint)b.FileFlags);
// Check same number of files
Equal(a.Files.Count(), b.Files.Count());
int idx = 0;
foreach (var pair in Enumerable.Zip(a.Files, b.Files, (ai, bi) => (ai, bi)))
{
idx++;
//Console.WriteLine($" - {pair.ai.Path}");
Equal(pair.ai.Path, pair.bi.Path);
Equal(pair.ai.Compressed, pair.bi.Compressed);
Equal(pair.ai.Size, pair.bi.Size);
Equal(pair.ai.GetData(), pair.bi.GetData());
}
}
}
}
}
/// <summary>
/// This function will search for a way to create a BSA in the installed mod list by assembling it from files
/// found in archives. To do this we hash all the files in side the BSA then try to find matches and patches for
/// all of the files.
/// </summary>
/// <returns></returns>
private Func <RawSourceFile, Directive> DeconstructBSAs()
{
var include_directly = ModInis.Where(kv => {
var general = kv.Value.General;
if (general.notes != null && general.notes.Contains(Consts.WABBAJACK_INCLUDE))
{
return(true);
}
if (general.comments != null && general.comments.Contains(Consts.WABBAJACK_INCLUDE))
{
return(true);
}
return(false);
}).Select(kv => $"mods\\{kv.Key}\\");
var microstack = new List <Func <RawSourceFile, Directive> >()
{
DirectMatch(),
IncludePatches(),
DropAll()
};
var microstack_with_include = new List <Func <RawSourceFile, Directive> >()
{
DirectMatch(),
IncludePatches(),
IncludeALL()
};
return(source =>
{
if (!Consts.SupportedBSAs.Contains(Path.GetExtension(source.Path)))
{
return null;
}
bool default_include = false;
if (source.Path.StartsWith("mods"))
{
foreach (var modpath in include_directly)
{
if (source.Path.StartsWith(modpath))
{
default_include = true;
break;
}
}
}
var source_files = source.File.FileInArchive;
var stack = default_include ? microstack_with_include : microstack;
var id = Guid.NewGuid().ToString();
var matches = source_files.PMap(e => RunStack(stack, new RawSourceFile(e)
{
Path = Path.Combine(Consts.BSACreationDir, id, e.Paths.Last())
}));
foreach (var match in matches)
{
if (match is IgnoredDirectly)
{
Error($"File required for BSA creation doesn't exist: {match.To}");
}
ExtraFiles.Add(match);
}
;
CreateBSA directive;
using (var bsa = new BSAReader(source.AbsolutePath))
{
directive = new CreateBSA()
{
To = source.Path,
TempID = id,
Type = (uint)bsa.HeaderType,
FileFlags = (uint)bsa.FileFlags,
ArchiveFlags = (uint)bsa.ArchiveFlags,
};
};
return directive;
});
}
