internal static void SanityCheckEntry(Big.Entry entry, Big.Platform platform)
{
if (entry.CompressionScheme == Big.CompressionScheme.None)
{
if (platform != Big.Platform.X360 &&
entry.UncompressedSize != 0)
{
throw new FormatException("got entry with no compression with a non-zero uncompressed size");
}
}
else if (entry.CompressionScheme == Big.CompressionScheme.LZO1x ||
entry.CompressionScheme == Big.CompressionScheme.Zlib)
{
if (entry.CompressedSize == 0 &&
entry.UncompressedSize > 0)
{
throw new FormatException(
"got entry with compression with a zero compressed size and a non-zero uncompressed size");
}
}
else
{
throw new FormatException("got entry with unsupported compression scheme");
}
}
public static void Main(string[] args)
{
bool showHelp = false;
bool extractUnknowns = true;
bool extractFiles = true;
bool extractSubFats = true;
bool unpackSubFats = false;
string filterPattern = null;
bool overwriteFiles = false;
bool verbose = false;
var options = new OptionSet()
{
{ "o|overwrite", "overwrite existing files", v => overwriteFiles = v != null },
{ "nf|no-files", "don't extract files", v => extractFiles = v == null },
{ "nu|no-unknowns", "don't extract unknown files", v => extractUnknowns = v == null },
{ "ns|no-subfats", "don't extract subfats", v => extractSubFats = v == null },
{ "us|unpack-subfats", "unpack files from subfats", v => unpackSubFats = v != null },
{ "f|filter=", "only extract files using pattern", v => filterPattern = v },
{ "v|verbose", "be verbose", v => verbose = v != null },
{ "h|help", "show this message and exit", v => showHelp = v != null },
};
List <string> extras;
try
{
extras = options.Parse(args);
}
catch (OptionException e)
{
Console.Write("{0}: ", GetExecutableName());
Console.WriteLine(e.Message);
Console.WriteLine("Try `{0} --help' for more information.", GetExecutableName());
return;
}
if (extras.Count < 1 || extras.Count > 2 || showHelp == true)
{
Console.WriteLine("Usage: {0} [OPTIONS]+ input_fat [output_dir]", GetExecutableName());
Console.WriteLine();
Console.WriteLine("Unpack files from a Big File (FAT/DAT pair).");
Console.WriteLine();
Console.WriteLine("Options:");
options.WriteOptionDescriptions(Console.Out);
return;
}
string fatPath = extras[0];
string outputPath = extras.Count > 1 ? extras[1] : Path.ChangeExtension(fatPath, null) + "_unpack";
string datPath;
Regex filter = null;
if (string.IsNullOrEmpty(filterPattern) == false)
{
filter = new Regex(filterPattern, RegexOptions.Compiled | RegexOptions.IgnoreCase);
}
if (Path.GetExtension(fatPath) == ".dat")
{
datPath = fatPath;
fatPath = Path.ChangeExtension(fatPath, ".fat");
}
else
{
datPath = Path.ChangeExtension(fatPath, ".dat");
}
if (verbose == true)
{
Console.WriteLine("Loading project...");
}
var manager = ProjectData.Manager.Load();
if (manager.ActiveProject == null)
{
Console.WriteLine("Warning: no active project loaded.");
}
if (verbose == true)
{
Console.WriteLine("Reading FAT...");
}
BigFile fat;
using (var input = File.OpenRead(fatPath))
{
fat = new BigFile();
fat.Deserialize(input);
}
var hashes = manager.LoadListsFileNames(fat.Version);
var subFatHashes = manager.LoadListsSubFatNames(fat.Version);
using (var input = File.OpenRead(datPath))
{
if (extractFiles == true)
{
Big.Entry[] entries;
if (extractSubFats == true &&
unpackSubFats == true)
{
entries =
fat.Entries.Concat(fat.SubFats.SelectMany(sf => sf.Entries))
.OrderBy(e => e.Offset)
.ToArray();
}
else
{
entries = fat.Entries.OrderBy(e => e.Offset).ToArray();
}
if (entries.Length > 0)
{
if (verbose == true)
{
Console.WriteLine("Unpacking files...");
}
long current = 0;
long total = entries.Length;
var padding = total.ToString(CultureInfo.InvariantCulture).Length;
var duplicates = new Dictionary <ulong, int>();
foreach (var entry in entries)
{
current++;
if (subFatHashes.Contains(entry.NameHash) == true)
{
continue;
}
string entryName;
if (GetEntryName(input, fat, entry, hashes, extractUnknowns, out entryName) == false)
{
continue;
}
if (duplicates.ContainsKey(entry.NameHash) == true)
{
var number = duplicates[entry.NameHash]++;
var e = Path.GetExtension(entryName);
var nn =
Path.ChangeExtension(
Path.ChangeExtension(entryName, null) + "__DUPLICATE_" +
number.ToString(CultureInfo.InvariantCulture),
e);
entryName = Path.Combine("__DUPLICATE", nn);
}
else
{
duplicates[entry.NameHash] = 0;
}
if (filter != null &&
filter.IsMatch(entryName) == false)
{
continue;
}
var entryPath = Path.Combine(outputPath, entryName);
if (overwriteFiles == false &&
File.Exists(entryPath) == true)
{
continue;
}
if (verbose == true)
{
Console.WriteLine("[{0}/{1}] {2}",
current.ToString(CultureInfo.InvariantCulture).PadLeft(padding),
total,
entryName);
}
input.Seek(entry.Offset, SeekOrigin.Begin);
var entryParent = Path.GetDirectoryName(entryPath);
if (string.IsNullOrEmpty(entryParent) == false)
{
Directory.CreateDirectory(entryParent);
}
using (var output = File.Create(entryPath))
{
EntryDecompression.Decompress(entry, input, output);
}
}
}
}
if (extractSubFats == true &&
unpackSubFats == false &&
fat.SubFats.Count > 0)
{
if (verbose == true)
{
Console.WriteLine("Unpacking subfats...");
}
var subFatsFromFat = fat.SubFats.ToList();
long current = 0;
long total = subFatsFromFat.Count;
var padding = total.ToString(CultureInfo.InvariantCulture).Length;
foreach (var headerEntry in fat.Entries.Where(e => subFatHashes.Contains(e.NameHash) == true))
{
current++;
var subFat = new SubFatFile();
using (var temp = new MemoryStream())
{
EntryDecompression.Decompress(headerEntry, input, temp);
temp.Position = 0;
subFat.Deserialize(temp, fat);
}
var matchingSubFats = subFatsFromFat
.Where(sf => subFat.Entries.SequenceEqual(sf.Entries))
.ToArray();
if (matchingSubFats.Length == 0)
{
continue;
}
if (matchingSubFats.Length > 1)
{
throw new InvalidOperationException();
}
var entryName = subFatHashes[headerEntry.NameHash];
entryName = FilterEntryName(entryName);
var entryHeaderPath = Path.Combine(outputPath, "__SUBFAT", entryName);
if (overwriteFiles == false &&
File.Exists(entryHeaderPath) == true)
{
continue;
}
if (verbose == true)
{
Console.WriteLine("[{0}/{1}] {2}",
current.ToString(CultureInfo.InvariantCulture).PadLeft(padding),
total,
entryName);
}
var entryParent = Path.GetDirectoryName(entryHeaderPath);
if (string.IsNullOrEmpty(entryParent) == false)
{
Directory.CreateDirectory(entryParent);
}
var entryDataPath = Path.ChangeExtension(entryHeaderPath, ".dat");
var rebuiltFat = new BigFile
{
Version = fat.Version,
Platform = fat.Platform,
Unknown74 = fat.Unknown74
};
using (var output = File.Create(entryDataPath))
{
var rebuiltEntries = new List <Big.Entry>();
foreach (var entry in subFat.Entries.OrderBy(e => e.Offset))
{
var rebuiltEntry = new Big.Entry
{
NameHash = entry.NameHash,
UncompressedSize = entry.UncompressedSize,
CompressedSize = entry.CompressedSize,
Offset = output.Position,
CompressionScheme = entry.CompressionScheme
};
input.Seek(entry.Offset, SeekOrigin.Begin);
output.WriteFromStream(input, entry.CompressedSize);
output.Seek(output.Position.Align(16), SeekOrigin.Begin);
rebuiltEntries.Add(rebuiltEntry);
}
rebuiltFat.Entries.AddRange(rebuiltEntries.OrderBy(e => e.NameHash));
}
using (var output = File.Create(entryHeaderPath))
{
rebuiltFat.Serialize(output);
}
foreach (var matchingSubFat in matchingSubFats)
{
subFatsFromFat.Remove(matchingSubFat);
}
}
if (subFatsFromFat.Count > 0)
{
Console.WriteLine("Warning: could not identify {0} subfats", subFatsFromFat.Count);
}
}
}
}
private static void Main(string[] args)
{
bool showHelp = false;
bool verbose = false;
bool compress = false;
int packageVersion = 9;
Big.Platform packagePlatform = Big.Platform.PC;
var options = new OptionSet()
{
{ "v|verbose", "be verbose", v => verbose = v != null },
{ "c|compress", "compress data with LZO1x", v => compress = v != null },
{ "pv|package-version", "package version (default 9)", v => packageVersion = ParsePackageVersion(v) },
{ "pp|package-platform", "package platform (default PC)", v => packagePlatform = ParsePackagePlatform(v) },
{ "h|help", "show this message and exit", v => showHelp = v != null },
};
List <string> extras;
try
{
extras = options.Parse(args);
}
catch (OptionException e)
{
Console.Write("{0}: ", GetExecutableName());
Console.WriteLine(e.Message);
Console.WriteLine("Try `{0} --help' for more information.", GetExecutableName());
return;
}
if (extras.Count < 1 || showHelp == true)
{
Console.WriteLine("Usage: {0} [OPTIONS]+ output_fat input_directory+", GetExecutableName());
Console.WriteLine();
Console.WriteLine("Pack files from input directories into a Big File (FAT/DAT pair).");
Console.WriteLine();
Console.WriteLine("Options:");
options.WriteOptionDescriptions(Console.Out);
return;
}
var inputPaths = new List <string>();
string fatPath, datPath;
if (extras.Count == 1)
{
inputPaths.Add(extras[0]);
fatPath = Path.ChangeExtension(extras[0], ".fat");
datPath = Path.ChangeExtension(extras[0], ".dat");
}
else
{
fatPath = extras[0];
if (Path.GetExtension(fatPath) != ".fat")
{
datPath = fatPath;
fatPath = Path.ChangeExtension(datPath, ".fat");
}
else
{
datPath = Path.ChangeExtension(fatPath, ".dat");
}
inputPaths.AddRange(extras.Skip(1));
}
var pendingEntries = new SortedDictionary <ulong, PendingEntry>();
if (verbose == true)
{
Console.WriteLine("Finding files...");
}
foreach (var relativePath in inputPaths)
{
string inputPath = Path.GetFullPath(relativePath);
if (inputPath.EndsWith(Path.DirectorySeparatorChar.ToString(CultureInfo.InvariantCulture)) == true)
{
inputPath = inputPath.Substring(0, inputPath.Length - 1);
}
foreach (string path in Directory.GetFiles(inputPath, "*", SearchOption.AllDirectories))
{
PendingEntry pendingEntry;
string fullPath = Path.GetFullPath(path);
string partPath = fullPath.Substring(inputPath.Length + 1).ToLowerInvariant();
pendingEntry.FullPath = fullPath;
pendingEntry.PartPath = partPath;
var pieces = partPath.Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
int index = 0;
if (index >= pieces.Length)
{
continue;
}
if (pieces[index].ToUpperInvariant() == "__SUBFAT")
{
Console.WriteLine("Sorry, packing of subfats is not currently implemented.");
return;
}
if (index >= pieces.Length)
{
continue;
}
if (pieces[index].ToUpperInvariant() == "__UNKNOWN")
{
var partName = Path.GetFileNameWithoutExtension(partPath);
if (string.IsNullOrEmpty(partName) == true)
{
continue;
}
if (partName.Length > 16)
{
partName = partName.Substring(0, 16);
}
pendingEntry.Name = null;
pendingEntry.NameHash = ulong.Parse(partName, NumberStyles.AllowHexSpecifier);
}
else
{
pendingEntry.Name = string.Join("\\", pieces.Skip(index).ToArray()).ToLowerInvariant();
pendingEntry.NameHash = CRC64.Hash(pendingEntry.Name);
}
if (pendingEntries.ContainsKey(pendingEntry.NameHash) == true)
{
Console.WriteLine("Ignoring duplicate of {0:X}: {1}", pendingEntry.NameHash, partPath);
if (verbose == true)
{
Console.WriteLine(" Previously added from: {0}",
pendingEntries[pendingEntry.NameHash].PartPath);
}
continue;
}
pendingEntries[pendingEntry.NameHash] = pendingEntry;
}
}
var fat = new BigFile
{
Version = packageVersion,
Platform = packagePlatform
};
// reasonable default?
// need to figure out what this value actually does
if (packagePlatform == Big.Platform.PC)
{
fat.Unknown74 = 3;
}
else if (packagePlatform == Big.Platform.PS3 ||
packagePlatform == Big.Platform.X360)
{
fat.Unknown74 = 4;
}
else
{
throw new InvalidOperationException();
}
using (var output = File.Create(datPath))
{
long current = 0;
long total = pendingEntries.Count;
var padding = total.ToString(CultureInfo.InvariantCulture).Length;
foreach (var pendingEntry in pendingEntries.Select(kv => kv.Value))
{
current++;
if (verbose == true)
{
Console.WriteLine("[{0}/{1}] {2}",
current.ToString(CultureInfo.InvariantCulture).PadLeft(padding),
total,
pendingEntry.PartPath);
}
var entry = new Big.Entry();
entry.NameHash = pendingEntry.NameHash;
entry.Offset = output.Position;
using (var input = File.OpenRead(pendingEntry.FullPath))
{
EntryCompression.Compress(fat.Platform, ref entry, input, compress, output);
output.Seek(output.Position.Align(16), SeekOrigin.Begin);
}
fat.Entries.Add(entry);
}
}
using (var output = File.Create(fatPath))
{
fat.Serialize(output);
}
}
private static bool GetEntryName(Stream input,
BigFile fat,
Big.Entry entry,
ProjectData.HashList <ulong> hashes,
bool extractUnknowns,
out string entryName)
{
entryName = hashes[entry.NameHash];
if (entryName == null)
{
if (extractUnknowns == false)
{
return(false);
}
string type;
string extension;
{
var guess = new byte[64];
int read = 0;
if (entry.CompressionScheme == Big.CompressionScheme.None)
{
if (entry.CompressedSize > 0)
{
input.Seek(entry.Offset, SeekOrigin.Begin);
read = input.Read(guess, 0, (int)Math.Min(entry.CompressedSize, guess.Length));
}
}
else
{
using (var temp = new MemoryStream())
{
EntryDecompression.Decompress(entry, input, temp);
temp.Position = 0;
read = temp.Read(guess, 0, (int)Math.Min(temp.Length, guess.Length));
}
}
var tuple = FileExtensions.Detect(guess, Math.Min(guess.Length, read));
type = tuple != null ? tuple.Item1 : "unknown";
extension = tuple != null ? tuple.Item2 : null;
}
entryName = entry.NameHash.ToString(fat.Version >= 9 ? "X16" : "X8");
if (string.IsNullOrEmpty(extension) == false)
{
entryName = Path.ChangeExtension(entryName, "." + extension);
}
if (string.IsNullOrEmpty(type) == false)
{
entryName = Path.Combine(type, entryName);
}
entryName = Path.Combine("__UNKNOWN", entryName);
}
else
{
entryName = FilterEntryName(entryName);
}
return(true);
}
