private static void ExtractAllWithBSA(string source, string dest) { try { using (var arch = new BSAReader(source)) { arch.Files.PMap(f => { var path = f.Path; if (f.Path.StartsWith("\\")) { path = f.Path.Substring(1); } Utils.Status($"Extracting {path}"); var out_path = Path.Combine(dest, path); var parent = Path.GetDirectoryName(out_path); if (!Directory.Exists(parent)) { Directory.CreateDirectory(parent); } using (var fs = File.OpenWrite(out_path)) { f.CopyDataTo(fs); } }); } } catch (Exception ex) { Utils.Log($"While Extracting {source}"); throw ex; } }
public async Task BuildBSACacheAsync() { Utils.Log("Start building BSA cache"); var bsaList = await Task.Run(() => Directory.EnumerateFiles(_dataFolder, "*.bsa", SearchOption.TopDirectoryOnly).ToList()); Utils.Log($"Found {bsaList.Count} BSAs in {_dataFolder}"); bsaList.Do(async bsa => { Utils.Log($"Start parsing BSA {bsa}"); await using var reader = new BSAReader(bsa); var allowedFiles = reader.Files .Where(x => AllowedExtensions.Contains(Path.GetExtension(x.Path))) .Select(x => x.Path) .Select(x => x.ToLower()) .Where(x => !_fileList.Contains(x)) .ToList(); Utils.Log($"BSA {bsa} has {allowedFiles.Count} allowed files"); _fileList.AddRange(allowedFiles); Utils.Log($"Finished parsing BSA {bsa}"); }); Utils.Log($"Finished parsing of {bsaList.Count} BSAs"); }
/// <summary> /// This function will search for a way to create a BSA in the installed mod list by assembling it from files /// found in archives. To do this we hash all the files in side the BSA then try to find matches and patches for /// all of the files. /// </summary> /// <returns></returns> private Func <RawSourceFile, Directive> DeconstructBSAs() { var microstack = new List <Func <RawSourceFile, Directive> >() { DirectMatch(), IncludePatches(), DropAll() }; return(source => { if (!Consts.SupportedBSAs.Contains(Path.GetExtension(source.Path))) { return null; } var hashed = HashBSA(source.AbsolutePath); var source_files = hashed.Select(e => new RawSourceFile() { Hash = e.Item2, Path = e.Item1, AbsolutePath = e.Item1 }); var matches = source_files.Select(e => RunStack(microstack, e)); var id = Guid.NewGuid().ToString(); foreach (var match in matches) { if (match is IgnoredDirectly) { Error($"File required for BSA creation doesn't exist: {match.To}"); } match.To = Path.Combine(Consts.BSACreationDir, id, match.To); ExtraFiles.Add(match); } ; CreateBSA directive; using (var bsa = new BSAReader(source.AbsolutePath)) { directive = new CreateBSA() { To = source.Path, TempID = id, Type = (uint)bsa.HeaderType, FileFlags = (uint)bsa.FileFlags, ArchiveFlags = (uint)bsa.ArchiveFlags, }; }; return directive; }); }
public static StringsFolderLookupOverlay?TypicalFactory(string referenceModPath, StringsReadParameters?instructions, ModKey modKey) { var ret = new StringsFolderLookupOverlay(); var stringsFolderPath = instructions?.StringsFolderOverride; var dir = Path.GetDirectoryName(referenceModPath); if (stringsFolderPath == null) { stringsFolderPath = Path.Combine(dir, "Strings"); } if (stringsFolderPath.Value.Exists) { foreach (var file in stringsFolderPath.Value.Info.EnumerateFiles($"{modKey.Name}*{StringsUtility.StringsFileExtension}")) { if (!StringsUtility.TryRetrieveInfoFromString(file.Name, out var type, out var lang, out _)) { continue; } var dict = ret.Get(type); dict[lang] = new Lazy <IStringsLookup>(() => new StringsLookupOverlay(file.FullName, type), LazyThreadSafetyMode.ExecutionAndPublication); } } foreach (var bsaFile in Directory.EnumerateFiles(dir, "*.bsa")) { var bsaReader = BSAReader.Load(new AbsolutePath(bsaFile, skipValidation: true)); foreach (var item in bsaReader.Files) { if (!StringsUtility.TryRetrieveInfoFromString(Path.GetFileName(item.Path.ToString()), out var type, out var lang, out var modName)) { continue; } if (!MemoryExtensions.Equals(modKey.Name, modName, StringComparison.OrdinalIgnoreCase)) { continue; } var dict = ret.Get(type); if (dict.ContainsKey(lang)) { continue; } dict[lang] = new Lazy <IStringsLookup>(() => { byte[] bytes = new byte[item.Size]; using var stream = new MemoryStream(bytes); item.CopyDataTo(stream).AsTask().Wait(); return(new StringsLookupOverlay(bytes, type)); }, LazyThreadSafetyMode.ExecutionAndPublication); } } return(ret); }
/// <summary> /// Given a BSA on disk, index it and return a dictionary of SHA256 -> filename /// </summary> /// <param name="absolutePath"></param> /// <returns></returns> private List <(string, string)> HashBSA(string absolutePath) { Status($"Hashing BSA: {absolutePath}"); var results = new List <(string, string)>(); using (var a = new BSAReader(absolutePath)) { foreach (var entry in a.Files) { Status($"Hashing BSA: {absolutePath} - {entry.Path}"); var data = entry.GetData(); results.Add((entry.Path, data.SHA256())); } } return(results); }
internal List <string> GetArchiveEntryNames(VirtualFile file) { if (!file.IsStaged) { throw new InvalidDataException("File is not staged"); } if (file.Extension == ".bsa") { using (var ar = new BSAReader(file.StagedPath)) { return(ar.Files.Select(f => f.Path).ToList()); } } if (file.Extension == ".zip") { using (var s = new ZipFile(File.OpenRead(file.StagedPath))) { s.IsStreamOwner = true; s.UseZip64 = UseZip64.On; if (s.OfType <ZipEntry>().FirstOrDefault(e => !e.CanDecompress) == null) { return(s.OfType <ZipEntry>() .Where(f => f.IsFile) .Select(f => f.Name.Replace('/', '\\')) .ToList()); } } } /* * using (var e = new ArchiveFile(file.StagedPath)) * { * return e.Entries * .Where(f => !f.IsFolder) * .Select(f => f.FileName).ToList(); * }*/ return(null); }
private byte[] LoadDataForTo(string to, Dictionary <string, string> absolute_paths) { if (absolute_paths.TryGetValue(to, out var absolute)) { return(File.ReadAllBytes(absolute)); } if (to.StartsWith(Consts.BSACreationDir)) { var bsa_id = to.Split('\\')[1]; var bsa = InstallDirectives.OfType <CreateBSA>().First(b => b.TempID == bsa_id); using (var a = new BSAReader(Path.Combine(MO2Folder, bsa.To))) { var file = a.Files.First(e => e.Path == Path.Combine(to.Split('\\').Skip(2).ToArray())); return(file.GetData()); } } Error($"Couldn't load data for {to}"); return(null); }
public void TestBSACreation() { const string outputPath = "output.bsa"; const string expectedContent = "Hello World!"; const string dummyFile = "bsa-dummy-file.txt"; File.WriteAllText(dummyFile, expectedContent, Encoding.ASCII); var fi = new FileInfo(dummyFile); var expectedLength = fi.Length; var creator = new BSACreator(); creator.AddFile(dummyFile, "text\\test.txt"); creator.WriteToFile(outputPath); Assert.True(File.Exists(outputPath)); using var reader = new BSAReader(outputPath); Assert.Single(reader.Folders); Assert.Single(reader.Files); var folder = reader.Folders.First(); var file = reader.Files.First(); Assert.Equal("text", folder.Name); Assert.Equal("test.txt", file.Name); Assert.Equal((uint)expectedLength, file.Size); using var ms = new MemoryStream(); reader.CopyFileTo(file, ms); var buffer = new byte[ms.Length]; ms.Read(buffer, 0, buffer.Length); var actualContents = Encoding.ASCII.GetString(buffer); Assert.Equal(expectedContent, actualContents); }
public void TestBSAParsing() { const string file = "test-bsa.bsa"; var path = Path.Combine("files", file); using var reader = new BSAReader(path); Assert.Equal(115, reader.Files.Count); Assert.Equal(10, reader.Folders.Count); const string expectedContents = "@#$%"; var fileInfo = reader.Files.First(x => x.Name.Equals("s.txt")); using var ms = new MemoryStream(); reader.CopyFileTo(fileInfo, ms); var buffer = new byte[ms.Length]; ms.Read(buffer, 0, buffer.Length); var actualContents = Encoding.UTF8.GetString(buffer); Assert.Equal(expectedContents, actualContents); }
static void Main(string[] args) { foreach (var bsa in Directory.EnumerateFiles(TestDir, "*.bsa", SearchOption.AllDirectories).Skip(0)) { Console.WriteLine($"From {bsa}"); Console.WriteLine("Cleaning Output Dir"); if (Directory.Exists(TempDir)) { Directory.Delete(TempDir, true); } Directory.CreateDirectory(TempDir); Console.WriteLine($"Reading {bsa}"); using (var a = new BSAReader(bsa)) { Parallel.ForEach(a.Files, file => { var abs_name = Path.Combine(TempDir, file.Path); if (!Directory.Exists(Path.GetDirectoryName(abs_name))) { Directory.CreateDirectory(Path.GetDirectoryName(abs_name)); } using (var fs = File.OpenWrite(abs_name)) file.CopyDataTo(fs); Equal((long)file.Size, new FileInfo(abs_name).Length); }); Console.WriteLine($"Building {bsa}"); using (var w = new BSABuilder()) { w.ArchiveFlags = a.ArchiveFlags; w.FileFlags = a.FileFlags; w.HeaderType = a.HeaderType; Parallel.ForEach(a.Files, file => { var abs_path = Path.Combine("c:\\tmp\\out", file.Path); using (var str = File.OpenRead(abs_path)) { var entry = w.AddFile(file.Path, str, file.FlipCompression); } }); w.Build("c:\\tmp\\tmp.bsa"); // Sanity Checks Equal(a.Files.Count(), w.Files.Count()); Equal(a.Files.Select(f => f.Path).ToHashSet(), w.Files.Select(f => f.Path).ToHashSet()); /*foreach (var pair in Enumerable.Zip(a.Files, w.Files, (ai, bi) => (ai, bi))) * { * Console.WriteLine($"{pair.ai.Path}, {pair.ai.Hash}, {pair.bi.Path}, {pair.bi.Hash}"); * }*/ foreach (var pair in Enumerable.Zip(a.Files, w.Files, (ai, bi) => (ai, bi))) { Equal(pair.ai.Path, pair.bi.Path); Equal(pair.ai.Hash, pair.bi.Hash); } } Console.WriteLine($"Verifying {bsa}"); using (var b = new BSAReader("c:\\tmp\\tmp.bsa")) { Console.WriteLine($"Performing A/B tests on {bsa}"); Equal((uint)a.ArchiveFlags, (uint)b.ArchiveFlags); Equal((uint)a.FileFlags, (uint)b.FileFlags); // Check same number of files Equal(a.Files.Count(), b.Files.Count()); int idx = 0; foreach (var pair in Enumerable.Zip(a.Files, b.Files, (ai, bi) => (ai, bi))) { idx++; //Console.WriteLine($" - {pair.ai.Path}"); Equal(pair.ai.Path, pair.bi.Path); Equal(pair.ai.Compressed, pair.bi.Compressed); Equal(pair.ai.Size, pair.bi.Size); Equal(pair.ai.GetData(), pair.bi.GetData()); } } } } }
/// <summary> /// This function will search for a way to create a BSA in the installed mod list by assembling it from files /// found in archives. To do this we hash all the files in side the BSA then try to find matches and patches for /// all of the files. /// </summary> /// <returns></returns> private Func <RawSourceFile, Directive> DeconstructBSAs() { var include_directly = ModInis.Where(kv => { var general = kv.Value.General; if (general.notes != null && general.notes.Contains(Consts.WABBAJACK_INCLUDE)) { return(true); } if (general.comments != null && general.comments.Contains(Consts.WABBAJACK_INCLUDE)) { return(true); } return(false); }).Select(kv => $"mods\\{kv.Key}\\"); var microstack = new List <Func <RawSourceFile, Directive> >() { DirectMatch(), IncludePatches(), DropAll() }; var microstack_with_include = new List <Func <RawSourceFile, Directive> >() { DirectMatch(), IncludePatches(), IncludeALL() }; return(source => { if (!Consts.SupportedBSAs.Contains(Path.GetExtension(source.Path))) { return null; } bool default_include = false; if (source.Path.StartsWith("mods")) { foreach (var modpath in include_directly) { if (source.Path.StartsWith(modpath)) { default_include = true; break; } } } var source_files = source.File.FileInArchive; var stack = default_include ? microstack_with_include : microstack; var id = Guid.NewGuid().ToString(); var matches = source_files.PMap(e => RunStack(stack, new RawSourceFile(e) { Path = Path.Combine(Consts.BSACreationDir, id, e.Paths.Last()) })); foreach (var match in matches) { if (match is IgnoredDirectly) { Error($"File required for BSA creation doesn't exist: {match.To}"); } ExtraFiles.Add(match); } ; CreateBSA directive; using (var bsa = new BSAReader(source.AbsolutePath)) { directive = new CreateBSA() { To = source.Path, TempID = id, Type = (uint)bsa.HeaderType, FileFlags = (uint)bsa.FileFlags, ArchiveFlags = (uint)bsa.ArchiveFlags, }; }; return directive; }); }