internal static void SanityCheckEntry(Big.Entry entry, Big.Platform platform) { if (entry.CompressionScheme == Big.CompressionScheme.None) { if (platform != Big.Platform.X360 && entry.UncompressedSize != 0) { throw new FormatException("got entry with no compression with a non-zero uncompressed size"); } } else if (entry.CompressionScheme == Big.CompressionScheme.LZO1x || entry.CompressionScheme == Big.CompressionScheme.Zlib) { if (entry.CompressedSize == 0 && entry.UncompressedSize > 0) { throw new FormatException( "got entry with compression with a zero compressed size and a non-zero uncompressed size"); } } else { throw new FormatException("got entry with unsupported compression scheme"); } }
public static void Main(string[] args) { bool showHelp = false; bool extractUnknowns = true; bool extractFiles = true; bool extractSubFats = true; bool unpackSubFats = false; string filterPattern = null; bool overwriteFiles = false; bool verbose = false; var options = new OptionSet() { { "o|overwrite", "overwrite existing files", v => overwriteFiles = v != null }, { "nf|no-files", "don't extract files", v => extractFiles = v == null }, { "nu|no-unknowns", "don't extract unknown files", v => extractUnknowns = v == null }, { "ns|no-subfats", "don't extract subfats", v => extractSubFats = v == null }, { "us|unpack-subfats", "unpack files from subfats", v => unpackSubFats = v != null }, { "f|filter=", "only extract files using pattern", v => filterPattern = v }, { "v|verbose", "be verbose", v => verbose = v != null }, { "h|help", "show this message and exit", v => showHelp = v != null }, }; List <string> extras; try { extras = options.Parse(args); } catch (OptionException e) { Console.Write("{0}: ", GetExecutableName()); Console.WriteLine(e.Message); Console.WriteLine("Try `{0} --help' for more information.", GetExecutableName()); return; } if (extras.Count < 1 || extras.Count > 2 || showHelp == true) { Console.WriteLine("Usage: {0} [OPTIONS]+ input_fat [output_dir]", GetExecutableName()); Console.WriteLine(); Console.WriteLine("Unpack files from a Big File (FAT/DAT pair)."); Console.WriteLine(); Console.WriteLine("Options:"); options.WriteOptionDescriptions(Console.Out); return; } string fatPath = extras[0]; string outputPath = extras.Count > 1 ? extras[1] : Path.ChangeExtension(fatPath, null) + "_unpack"; string datPath; Regex filter = null; if (string.IsNullOrEmpty(filterPattern) == false) { filter = new Regex(filterPattern, RegexOptions.Compiled | RegexOptions.IgnoreCase); } if (Path.GetExtension(fatPath) == ".dat") { datPath = fatPath; fatPath = Path.ChangeExtension(fatPath, ".fat"); } else { datPath = Path.ChangeExtension(fatPath, ".dat"); } if (verbose == true) { Console.WriteLine("Loading project..."); } var manager = ProjectData.Manager.Load(); if (manager.ActiveProject == null) { Console.WriteLine("Warning: no active project loaded."); } if (verbose == true) { Console.WriteLine("Reading FAT..."); } BigFile fat; using (var input = File.OpenRead(fatPath)) { fat = new BigFile(); fat.Deserialize(input); } var hashes = manager.LoadListsFileNames(fat.Version); var subFatHashes = manager.LoadListsSubFatNames(fat.Version); using (var input = File.OpenRead(datPath)) { if (extractFiles == true) { Big.Entry[] entries; if (extractSubFats == true && unpackSubFats == true) { entries = fat.Entries.Concat(fat.SubFats.SelectMany(sf => sf.Entries)) .OrderBy(e => e.Offset) .ToArray(); } else { entries = fat.Entries.OrderBy(e => e.Offset).ToArray(); } if (entries.Length > 0) { if (verbose == true) { Console.WriteLine("Unpacking files..."); } long current = 0; long total = entries.Length; var padding = total.ToString(CultureInfo.InvariantCulture).Length; var duplicates = new Dictionary <ulong, int>(); foreach (var entry in entries) { current++; if (subFatHashes.Contains(entry.NameHash) == true) { continue; } string entryName; if (GetEntryName(input, fat, entry, hashes, extractUnknowns, out entryName) == false) { continue; } if (duplicates.ContainsKey(entry.NameHash) == true) { var number = duplicates[entry.NameHash]++; var e = Path.GetExtension(entryName); var nn = Path.ChangeExtension( Path.ChangeExtension(entryName, null) + "__DUPLICATE_" + number.ToString(CultureInfo.InvariantCulture), e); entryName = Path.Combine("__DUPLICATE", nn); } else { duplicates[entry.NameHash] = 0; } if (filter != null && filter.IsMatch(entryName) == false) { continue; } var entryPath = Path.Combine(outputPath, entryName); if (overwriteFiles == false && File.Exists(entryPath) == true) { continue; } if (verbose == true) { Console.WriteLine("[{0}/{1}] {2}", current.ToString(CultureInfo.InvariantCulture).PadLeft(padding), total, entryName); } input.Seek(entry.Offset, SeekOrigin.Begin); var entryParent = Path.GetDirectoryName(entryPath); if (string.IsNullOrEmpty(entryParent) == false) { Directory.CreateDirectory(entryParent); } using (var output = File.Create(entryPath)) { EntryDecompression.Decompress(entry, input, output); } } } } if (extractSubFats == true && unpackSubFats == false && fat.SubFats.Count > 0) { if (verbose == true) { Console.WriteLine("Unpacking subfats..."); } var subFatsFromFat = fat.SubFats.ToList(); long current = 0; long total = subFatsFromFat.Count; var padding = total.ToString(CultureInfo.InvariantCulture).Length; foreach (var headerEntry in fat.Entries.Where(e => subFatHashes.Contains(e.NameHash) == true)) { current++; var subFat = new SubFatFile(); using (var temp = new MemoryStream()) { EntryDecompression.Decompress(headerEntry, input, temp); temp.Position = 0; subFat.Deserialize(temp, fat); } var matchingSubFats = subFatsFromFat .Where(sf => subFat.Entries.SequenceEqual(sf.Entries)) .ToArray(); if (matchingSubFats.Length == 0) { continue; } if (matchingSubFats.Length > 1) { throw new InvalidOperationException(); } var entryName = subFatHashes[headerEntry.NameHash]; entryName = FilterEntryName(entryName); var entryHeaderPath = Path.Combine(outputPath, "__SUBFAT", entryName); if (overwriteFiles == false && File.Exists(entryHeaderPath) == true) { continue; } if (verbose == true) { Console.WriteLine("[{0}/{1}] {2}", current.ToString(CultureInfo.InvariantCulture).PadLeft(padding), total, entryName); } var entryParent = Path.GetDirectoryName(entryHeaderPath); if (string.IsNullOrEmpty(entryParent) == false) { Directory.CreateDirectory(entryParent); } var entryDataPath = Path.ChangeExtension(entryHeaderPath, ".dat"); var rebuiltFat = new BigFile { Version = fat.Version, Platform = fat.Platform, Unknown74 = fat.Unknown74 }; using (var output = File.Create(entryDataPath)) { var rebuiltEntries = new List <Big.Entry>(); foreach (var entry in subFat.Entries.OrderBy(e => e.Offset)) { var rebuiltEntry = new Big.Entry { NameHash = entry.NameHash, UncompressedSize = entry.UncompressedSize, CompressedSize = entry.CompressedSize, Offset = output.Position, CompressionScheme = entry.CompressionScheme }; input.Seek(entry.Offset, SeekOrigin.Begin); output.WriteFromStream(input, entry.CompressedSize); output.Seek(output.Position.Align(16), SeekOrigin.Begin); rebuiltEntries.Add(rebuiltEntry); } rebuiltFat.Entries.AddRange(rebuiltEntries.OrderBy(e => e.NameHash)); } using (var output = File.Create(entryHeaderPath)) { rebuiltFat.Serialize(output); } foreach (var matchingSubFat in matchingSubFats) { subFatsFromFat.Remove(matchingSubFat); } } if (subFatsFromFat.Count > 0) { Console.WriteLine("Warning: could not identify {0} subfats", subFatsFromFat.Count); } } } }
private static void Main(string[] args) { bool showHelp = false; bool verbose = false; bool compress = false; int packageVersion = 9; Big.Platform packagePlatform = Big.Platform.PC; var options = new OptionSet() { { "v|verbose", "be verbose", v => verbose = v != null }, { "c|compress", "compress data with LZO1x", v => compress = v != null }, { "pv|package-version", "package version (default 9)", v => packageVersion = ParsePackageVersion(v) }, { "pp|package-platform", "package platform (default PC)", v => packagePlatform = ParsePackagePlatform(v) }, { "h|help", "show this message and exit", v => showHelp = v != null }, }; List <string> extras; try { extras = options.Parse(args); } catch (OptionException e) { Console.Write("{0}: ", GetExecutableName()); Console.WriteLine(e.Message); Console.WriteLine("Try `{0} --help' for more information.", GetExecutableName()); return; } if (extras.Count < 1 || showHelp == true) { Console.WriteLine("Usage: {0} [OPTIONS]+ output_fat input_directory+", GetExecutableName()); Console.WriteLine(); Console.WriteLine("Pack files from input directories into a Big File (FAT/DAT pair)."); Console.WriteLine(); Console.WriteLine("Options:"); options.WriteOptionDescriptions(Console.Out); return; } var inputPaths = new List <string>(); string fatPath, datPath; if (extras.Count == 1) { inputPaths.Add(extras[0]); fatPath = Path.ChangeExtension(extras[0], ".fat"); datPath = Path.ChangeExtension(extras[0], ".dat"); } else { fatPath = extras[0]; if (Path.GetExtension(fatPath) != ".fat") { datPath = fatPath; fatPath = Path.ChangeExtension(datPath, ".fat"); } else { datPath = Path.ChangeExtension(fatPath, ".dat"); } inputPaths.AddRange(extras.Skip(1)); } var pendingEntries = new SortedDictionary <ulong, PendingEntry>(); if (verbose == true) { Console.WriteLine("Finding files..."); } foreach (var relativePath in inputPaths) { string inputPath = Path.GetFullPath(relativePath); if (inputPath.EndsWith(Path.DirectorySeparatorChar.ToString(CultureInfo.InvariantCulture)) == true) { inputPath = inputPath.Substring(0, inputPath.Length - 1); } foreach (string path in Directory.GetFiles(inputPath, "*", SearchOption.AllDirectories)) { PendingEntry pendingEntry; string fullPath = Path.GetFullPath(path); string partPath = fullPath.Substring(inputPath.Length + 1).ToLowerInvariant(); pendingEntry.FullPath = fullPath; pendingEntry.PartPath = partPath; var pieces = partPath.Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); int index = 0; if (index >= pieces.Length) { continue; } if (pieces[index].ToUpperInvariant() == "__SUBFAT") { Console.WriteLine("Sorry, packing of subfats is not currently implemented."); return; } if (index >= pieces.Length) { continue; } if (pieces[index].ToUpperInvariant() == "__UNKNOWN") { var partName = Path.GetFileNameWithoutExtension(partPath); if (string.IsNullOrEmpty(partName) == true) { continue; } if (partName.Length > 16) { partName = partName.Substring(0, 16); } pendingEntry.Name = null; pendingEntry.NameHash = ulong.Parse(partName, NumberStyles.AllowHexSpecifier); } else { pendingEntry.Name = string.Join("\\", pieces.Skip(index).ToArray()).ToLowerInvariant(); pendingEntry.NameHash = CRC64.Hash(pendingEntry.Name); } if (pendingEntries.ContainsKey(pendingEntry.NameHash) == true) { Console.WriteLine("Ignoring duplicate of {0:X}: {1}", pendingEntry.NameHash, partPath); if (verbose == true) { Console.WriteLine(" Previously added from: {0}", pendingEntries[pendingEntry.NameHash].PartPath); } continue; } pendingEntries[pendingEntry.NameHash] = pendingEntry; } } var fat = new BigFile { Version = packageVersion, Platform = packagePlatform }; // reasonable default? // need to figure out what this value actually does if (packagePlatform == Big.Platform.PC) { fat.Unknown74 = 3; } else if (packagePlatform == Big.Platform.PS3 || packagePlatform == Big.Platform.X360) { fat.Unknown74 = 4; } else { throw new InvalidOperationException(); } using (var output = File.Create(datPath)) { long current = 0; long total = pendingEntries.Count; var padding = total.ToString(CultureInfo.InvariantCulture).Length; foreach (var pendingEntry in pendingEntries.Select(kv => kv.Value)) { current++; if (verbose == true) { Console.WriteLine("[{0}/{1}] {2}", current.ToString(CultureInfo.InvariantCulture).PadLeft(padding), total, pendingEntry.PartPath); } var entry = new Big.Entry(); entry.NameHash = pendingEntry.NameHash; entry.Offset = output.Position; using (var input = File.OpenRead(pendingEntry.FullPath)) { EntryCompression.Compress(fat.Platform, ref entry, input, compress, output); output.Seek(output.Position.Align(16), SeekOrigin.Begin); } fat.Entries.Add(entry); } } using (var output = File.Create(fatPath)) { fat.Serialize(output); } }
private static bool GetEntryName(Stream input, BigFile fat, Big.Entry entry, ProjectData.HashList <ulong> hashes, bool extractUnknowns, out string entryName) { entryName = hashes[entry.NameHash]; if (entryName == null) { if (extractUnknowns == false) { return(false); } string type; string extension; { var guess = new byte[64]; int read = 0; if (entry.CompressionScheme == Big.CompressionScheme.None) { if (entry.CompressedSize > 0) { input.Seek(entry.Offset, SeekOrigin.Begin); read = input.Read(guess, 0, (int)Math.Min(entry.CompressedSize, guess.Length)); } } else { using (var temp = new MemoryStream()) { EntryDecompression.Decompress(entry, input, temp); temp.Position = 0; read = temp.Read(guess, 0, (int)Math.Min(temp.Length, guess.Length)); } } var tuple = FileExtensions.Detect(guess, Math.Min(guess.Length, read)); type = tuple != null ? tuple.Item1 : "unknown"; extension = tuple != null ? tuple.Item2 : null; } entryName = entry.NameHash.ToString(fat.Version >= 9 ? "X16" : "X8"); if (string.IsNullOrEmpty(extension) == false) { entryName = Path.ChangeExtension(entryName, "." + extension); } if (string.IsNullOrEmpty(type) == false) { entryName = Path.Combine(type, entryName); } entryName = Path.Combine("__UNKNOWN", entryName); } else { entryName = FilterEntryName(entryName); } return(true); }