private Package ReadPackageV10(FileStream mainStream, BinaryReader reader)
{
var package = new Package();
mainStream.Seek(4, SeekOrigin.Begin);
var header = BinUtils.ReadStruct <LSPKHeader10>(reader);
package.Metadata.Flags = (PackageFlags)header.Flags;
package.Metadata.Priority = header.Priority;
package.Version = PackageVersion.V10;
if (_metadataOnly)
{
return(package);
}
OpenStreams(mainStream, header.NumParts);
for (uint i = 0; i < header.NumFiles; i++)
{
var entry = BinUtils.ReadStruct <FileEntry13>(reader);
if (entry.ArchivePart == 0)
{
entry.OffsetInFile += header.DataOffset;
}
// Add missing compression level flags
entry.Flags = (entry.Flags & 0x0f) | 0x20;
package.Files.Add(PackagedFileInfo.CreateFromEntry(entry, _streams[entry.ArchivePart]));
}
return(package);
}
private Package ReadPackageV16(FileStream mainStream, BinaryReader reader)
{
var package = new Package();
var header = BinUtils.ReadStruct <LSPKHeader16>(reader);
if (header.Version != (ulong)PackageVersion.V16)
{
string msg = $"Unsupported package version {header.Version}; this layout is only supported for V16";
throw new InvalidDataException(msg);
}
package.Metadata.Flags = (PackageFlags)header.Flags;
package.Metadata.Priority = header.Priority;
package.Version = PackageVersion.V16;
if (_metadataOnly)
{
return(package);
}
OpenStreams(mainStream, header.NumParts);
mainStream.Seek((long)header.FileListOffset, SeekOrigin.Begin);
ReadFileListV15(reader, package);
return(package);
}
private Package ReadPackageV7(FileStream mainStream, BinaryReader reader)
{
var package = new Package();
mainStream.Seek(0, SeekOrigin.Begin);
var header = BinUtils.ReadStruct <LSPKHeader7>(reader);
package.Metadata.Flags = 0;
package.Metadata.Priority = 0;
package.Version = PackageVersion.V7;
if (_metadataOnly)
{
return(package);
}
OpenStreams(mainStream, (int)header.NumParts);
for (uint i = 0; i < header.NumFiles; i++)
{
var entry = BinUtils.ReadStruct <FileEntry7>(reader);
if (entry.ArchivePart == 0)
{
entry.OffsetInFile += header.DataOffset;
}
package.Files.Add(PackagedFileInfo.CreateFromEntry(entry, _streams[entry.ArchivePart]));
}
return(package);
}
/// <summary>
/// Reads the V3 attribute headers for the LSOF resource
/// </summary>
/// <param name="s">Stream to read the attribute headers from</param>
private void ReadAttributesV3(Stream s)
{
using (var reader = new BinaryReader(s))
{
while (s.Position < s.Length)
{
var attribute = BinUtils.ReadStruct <AttributeEntryV3>(reader);
var resolved = new AttributeInfo();
resolved.NameIndex = attribute.NameIndex;
resolved.NameOffset = attribute.NameOffset;
resolved.TypeId = attribute.TypeId;
resolved.Length = attribute.Length;
resolved.DataOffset = attribute.Offset;
resolved.NextAttributeIndex = attribute.NextAttributeIndex;
Attributes.Add(resolved);
}
#if DEBUG_LSF_SERIALIZATION
Console.WriteLine(" ----- DUMP OF V3 ATTRIBUTE TABLE -----");
for (int i = 0; i < Attributes.Count; i++)
{
var resolved = Attributes[i];
var debug = String.Format(
"{0}: {1} (offset {2:X}, typeId {3}, nextAttribute {4})",
i, Names[resolved.NameIndex][resolved.NameOffset], resolved.DataOffset,
resolved.TypeId, resolved.NextAttributeIndex
);
Console.WriteLine(debug);
}
#endif
}
}
public Resource Read()
{
using (this.reader = new BinaryReader(stream))
{
// Check for BG3 header
var header = BinUtils.ReadStruct <LSBHeader>(reader);
if (header.Signature != BitConverter.ToUInt32(LSBHeader.SignatureBG3, 0) && header.Signature != LSBHeader.SignatureFW3)
{
throw new InvalidFormatException(String.Format("Illegal signature in LSB header ({1})", header.Signature));
}
if (stream.Length != header.TotalSize)
{
throw new InvalidFormatException(String.Format("Invalid LSB file size; expected {0}, got {1}", header.TotalSize, stream.Length));
}
// The game only uses little-endian files on all platforms currently and big-endian support isn't worth the hassle
if (header.BigEndian != 0)
{
throw new InvalidFormatException("Big-endian LSB files are not supported");
}
IsBG3 = (header.Signature == BitConverter.ToUInt32(LSBHeader.SignatureBG3, 0));
ReadStaticStrings();
Resource rsrc = new Resource();
rsrc.Metadata = header.Metadata;
ReadRegions(rsrc);
return(rsrc);
}
}
private Package ReadPackageV15(FileStream mainStream, BinaryReader reader)
{
var package = new Package();
var header = BinUtils.ReadStruct <LSPKHeader15>(reader);
if (header.Version != (ulong)PackageVersion.V15)
{
string msg = $"Unsupported package version {header.Version}; this layout is only supported for {PackageVersion.V15}";
throw new InvalidDataException(msg);
}
package.Metadata.Flags = (PackageFlags)header.Flags;
package.Metadata.Priority = header.Priority;
package.Version = PackageVersion.V15;
if (_metadataOnly)
{
return(package);
}
OpenStreams(mainStream, 1);
mainStream.Seek((long)header.FileListOffset, SeekOrigin.Begin);
int numFiles = reader.ReadInt32();
int compressedSize = reader.ReadInt32();
byte[] compressedFileList = reader.ReadBytes(compressedSize);
int fileBufferSize = Marshal.SizeOf(typeof(FileEntry15)) * numFiles;
var uncompressedList = new byte[fileBufferSize];
int uncompressedSize = LZ4Codec.Decode(compressedFileList, 0, compressedFileList.Length, uncompressedList, 0, fileBufferSize, true);
if (uncompressedSize != fileBufferSize)
{
string msg = $"LZ4 compressor disagrees about the size of file headers; expected {fileBufferSize}, got {uncompressedSize}";
throw new InvalidDataException(msg);
}
var ms = new MemoryStream(uncompressedList);
var msr = new BinaryReader(ms);
var entries = new FileEntry15[numFiles];
BinUtils.ReadStructs(msr, entries);
foreach (var entry in entries)
{
package.Files.Add(PackagedFileInfo.CreateFromEntry(entry, _streams[0]));
}
return(package);
}
/// <summary>
/// Reads the structure headers for the LSOF resource
/// </summary>
/// <param name="s">Stream to read the node headers from</param>
/// <param name="longNodes">Use the long (V3) on-disk node format</param>
private void ReadNodes(Stream s, bool longNodes)
{
#if DEBUG_LSF_SERIALIZATION
Console.WriteLine(" ----- DUMP OF NODE TABLE -----");
#endif
Nodes = new List <NodeInfo>();
using (var reader = new BinaryReader(s))
{
Int32 index = 0;
while (s.Position < s.Length)
{
var resolved = new NodeInfo();
#if DEBUG_LSF_SERIALIZATION
var pos = s.Position;
#endif
if (longNodes)
{
var item = BinUtils.ReadStruct <NodeEntryV3>(reader);
resolved.ParentIndex = item.ParentIndex;
resolved.NameIndex = item.NameIndex;
resolved.NameOffset = item.NameOffset;
resolved.FirstAttributeIndex = item.FirstAttributeIndex;
}
else
{
var item = BinUtils.ReadStruct <NodeEntryV2>(reader);
resolved.ParentIndex = item.ParentIndex;
resolved.NameIndex = item.NameIndex;
resolved.NameOffset = item.NameOffset;
resolved.FirstAttributeIndex = item.FirstAttributeIndex;
}
#if DEBUG_LSF_SERIALIZATION
Console.WriteLine(String.Format(
"{0}: {1} @ {2:X} (parent {3}, firstAttribute {4})",
index, Names[resolved.NameIndex][resolved.NameOffset], pos, resolved.ParentIndex,
resolved.FirstAttributeIndex
));
#endif
Nodes.Add(resolved);
index++;
}
}
}
private Package ReadPackageV13(FileStream mainStream, BinaryReader reader)
{
var package = new Package();
var header = BinUtils.ReadStruct <LSPKHeader13>(reader);
if (header.Version != Package.CurrentVersion)
{
var msg = String.Format("Unsupported package version {0}; this extractor only supports {1}", header.Version, Package.CurrentVersion);
throw new InvalidDataException(msg);
}
OpenStreams(mainStream, header.NumParts);
mainStream.Seek(header.FileListOffset, SeekOrigin.Begin);
int numFiles = reader.ReadInt32();
int fileBufferSize = Marshal.SizeOf(typeof(FileEntry13)) * numFiles;
byte[] compressedFileList = reader.ReadBytes((int)header.FileListSize - 4);
var uncompressedList = new byte[fileBufferSize];
var uncompressedSize = LZ4Codec.Decode(compressedFileList, 0, compressedFileList.Length, uncompressedList, 0, fileBufferSize, true);
if (uncompressedSize != fileBufferSize)
{
var msg = String.Format("LZ4 compressor disagrees about the size of file headers; expected {0}, got {1}", fileBufferSize, uncompressedSize);
throw new InvalidDataException(msg);
}
var ms = new MemoryStream(uncompressedList);
var msr = new BinaryReader(ms);
for (int i = 0; i < numFiles; i++)
{
var entry = BinUtils.ReadStruct <FileEntry13>(msr);
package.Files.Add(PackagedFileInfo.CreateFromEntry(entry, streams[entry.ArchivePart]));
}
return(package);
}
public Resource Read()
{
using (var reader = new BinaryReader(Stream))
{
var hdr = BinUtils.ReadStruct <Header>(reader);
if (hdr.Magic != BitConverter.ToUInt32(Header.Signature, 0))
{
var msg = String.Format(
"Invalid LSF signature; expected {0,8:X}, got {1,8:X}",
BitConverter.ToUInt32(Header.Signature, 0), hdr.Magic
);
throw new InvalidDataException(msg);
}
if (hdr.Version < (ulong)FileVersion.VerInitial || hdr.Version > (ulong)FileVersion.CurrentVersion)
{
var msg = String.Format("LSF version {0} is not supported", hdr.Version);
throw new InvalidDataException(msg);
}
bool isCompressed = BinUtils.CompressionFlagsToMethod(hdr.CompressionFlags) != CompressionMethod.None;
if (hdr.StringsSizeOnDisk > 0 || hdr.StringsUncompressedSize > 0)
{
uint onDiskSize = isCompressed ? hdr.StringsSizeOnDisk : hdr.StringsUncompressedSize;
byte[] compressed = reader.ReadBytes((int)onDiskSize);
byte[] uncompressed;
if (isCompressed)
{
uncompressed = BinUtils.Decompress(compressed, (int)hdr.StringsUncompressedSize, hdr.CompressionFlags);
}
else
{
uncompressed = compressed;
}
#if DUMP_LSF_SERIALIZATION
using (var nodesFile = new FileStream("names.bin", FileMode.Create, FileAccess.Write))
{
nodesFile.Write(uncompressed, 0, uncompressed.Length);
}
#endif
using (var namesStream = new MemoryStream(uncompressed))
{
ReadNames(namesStream);
}
}
if (hdr.NodesSizeOnDisk > 0 || hdr.NodesUncompressedSize > 0)
{
uint onDiskSize = isCompressed ? hdr.NodesSizeOnDisk : hdr.NodesUncompressedSize;
var uncompressed = Decompress(reader, onDiskSize, hdr.NodesUncompressedSize, hdr);
#if DUMP_LSF_SERIALIZATION
using (var nodesFile = new FileStream("nodes.bin", FileMode.Create, FileAccess.Write))
{
nodesFile.Write(uncompressed, 0, uncompressed.Length);
}
#endif
using (var nodesStream = new MemoryStream(uncompressed))
{
var longNodes = hdr.Version >= (ulong)FileVersion.VerExtendedNodes &&
hdr.Extended == 1;
ReadNodes(nodesStream, longNodes);
}
}
if (hdr.AttributesSizeOnDisk > 0 || hdr.AttributesUncompressedSize > 0)
{
uint onDiskSize = isCompressed ? hdr.AttributesSizeOnDisk : hdr.AttributesUncompressedSize;
var uncompressed = Decompress(reader, onDiskSize, hdr.AttributesUncompressedSize, hdr);
#if DUMP_LSF_SERIALIZATION
using (var attributesFile = new FileStream("attributes.bin", FileMode.Create, FileAccess.Write))
{
attributesFile.Write(uncompressed, 0, uncompressed.Length);
}
#endif
using (var attributesStream = new MemoryStream(uncompressed))
{
var longAttributes = hdr.Version >= (ulong)FileVersion.VerExtendedNodes &&
hdr.Extended == 1;
if (longAttributes)
{
ReadAttributesV3(attributesStream);
}
else
{
ReadAttributesV2(attributesStream);
}
}
}
if (hdr.ValuesSizeOnDisk > 0 || hdr.ValuesUncompressedSize > 0)
{
uint onDiskSize = isCompressed ? hdr.ValuesSizeOnDisk : hdr.ValuesUncompressedSize;
var uncompressed = Decompress(reader, onDiskSize, hdr.ValuesUncompressedSize, hdr);
var valueStream = new MemoryStream(uncompressed);
this.Values = valueStream;
#if DUMP_LSF_SERIALIZATION
using (var valuesFile = new FileStream("values.bin", FileMode.Create, FileAccess.Write))
{
valuesFile.Write(uncompressed, 0, uncompressed.Length);
}
#endif
}
else
{
this.Values = new MemoryStream();
}
Resource resource = new Resource();
ReadRegions(resource);
resource.Metadata.majorVersion = (hdr.EngineVersion & 0xff000000) >> 24;
resource.Metadata.minorVersion = (hdr.EngineVersion & 0xff0000) >> 16;
resource.Metadata.revision = (hdr.EngineVersion & 0xff00) >> 8;
resource.Metadata.buildNumber = (hdr.EngineVersion & 0xff);
return(resource);
}
}
/// <summary>
/// Reads the V2 attribute headers for the LSOF resource
/// </summary>
/// <param name="s">Stream to read the attribute headers from</param>
private void ReadAttributesV2(Stream s)
{
Attributes = new List <AttributeInfo>();
using (var reader = new BinaryReader(s))
{
#if DEBUG_LSF_SERIALIZATION
var rawAttributes = new List <AttributeEntryV2>();
#endif
var prevAttributeRefs = new List <Int32>();
UInt32 dataOffset = 0;
Int32 index = 0;
while (s.Position < s.Length)
{
var attribute = BinUtils.ReadStruct <AttributeEntryV2>(reader);
var resolved = new AttributeInfo();
resolved.NameIndex = attribute.NameIndex;
resolved.NameOffset = attribute.NameOffset;
resolved.TypeId = attribute.TypeId;
resolved.Length = attribute.Length;
resolved.DataOffset = dataOffset;
resolved.NextAttributeIndex = -1;
var nodeIndex = attribute.NodeIndex + 1;
if (prevAttributeRefs.Count > nodeIndex)
{
if (prevAttributeRefs[nodeIndex] != -1)
{
Attributes[prevAttributeRefs[nodeIndex]].NextAttributeIndex = index;
}
prevAttributeRefs[nodeIndex] = index;
}
else
{
while (prevAttributeRefs.Count < nodeIndex)
{
prevAttributeRefs.Add(-1);
}
prevAttributeRefs.Add(index);
}
#if DEBUG_LSF_SERIALIZATION
rawAttributes.Add(attribute);
#endif
dataOffset += resolved.Length;
Attributes.Add(resolved);
index++;
}
#if DEBUG_LSF_SERIALIZATION
Console.WriteLine(" ----- DUMP OF ATTRIBUTE REFERENCES -----");
for (int i = 0; i < prevAttributeRefs.Count; i++)
{
Console.WriteLine(String.Format("Node {0}: last attribute {1}", i, prevAttributeRefs[i]));
}
Console.WriteLine(" ----- DUMP OF V2 ATTRIBUTE TABLE -----");
for (int i = 0; i < Attributes.Count; i++)
{
var resolved = Attributes[i];
var attribute = rawAttributes[i];
var debug = String.Format(
"{0}: {1} (offset {2:X}, typeId {3}, nextAttribute {4}, node {5})",
i, Names[resolved.NameIndex][resolved.NameOffset], resolved.DataOffset,
resolved.TypeId, resolved.NextAttributeIndex, attribute.NodeIndex
);
Console.WriteLine(debug);
}
#endif
}
}
private Package ReadPackageV13(FileStream mainStream, BinaryReader reader)
{
var package = new Package();
var header = BinUtils.ReadStruct <LSPKHeader13>(reader);
if (header.Version != (ulong)PackageVersion.V13)
{
string msg = $"Unsupported package version {header.Version}; this package layout is only supported for {PackageVersion.V13}";
throw new InvalidDataException(msg);
}
package.Metadata.Flags = (PackageFlags)header.Flags;
package.Metadata.Priority = header.Priority;
package.Version = PackageVersion.V13;
if (_metadataOnly)
{
return(package);
}
OpenStreams(mainStream, header.NumParts);
mainStream.Seek(header.FileListOffset, SeekOrigin.Begin);
int numFiles = reader.ReadInt32();
int fileBufferSize = Marshal.SizeOf(typeof(FileEntry13)) * numFiles;
byte[] compressedFileList = reader.ReadBytes((int)header.FileListSize - 4);
var uncompressedList = new byte[fileBufferSize];
int uncompressedSize = LZ4Codec.Decode(compressedFileList, 0, compressedFileList.Length, uncompressedList, 0, fileBufferSize, true);
if (uncompressedSize != fileBufferSize)
{
string msg = $"LZ4 compressor disagrees about the size of file headers; expected {fileBufferSize}, got {uncompressedSize}";
throw new InvalidDataException(msg);
}
var ms = new MemoryStream(uncompressedList);
var msr = new BinaryReader(ms);
var entries = new FileEntry13[numFiles];
BinUtils.ReadStructs(msr, entries);
if ((package.Metadata.Flags & PackageFlags.Solid) == PackageFlags.Solid && numFiles > 0)
{
// Calculate compressed frame offset and bounds
uint totalUncompressedSize = 0;
uint totalSizeOnDisk = 0;
uint firstOffset = 0xffffffff;
uint lastOffset = 0;
foreach (var entry in entries)
{
totalUncompressedSize += entry.UncompressedSize;
totalSizeOnDisk += entry.SizeOnDisk;
if (entry.OffsetInFile < firstOffset)
{
firstOffset = entry.OffsetInFile;
}
if (entry.OffsetInFile + entry.SizeOnDisk > lastOffset)
{
lastOffset = entry.OffsetInFile + entry.SizeOnDisk;
}
}
if (firstOffset != 7 || lastOffset - firstOffset != totalSizeOnDisk)
{
string msg = $"Incorrectly compressed solid archive; offsets {firstOffset}/{lastOffset}, bytes {totalSizeOnDisk}";
throw new InvalidDataException(msg);
}
// Decompress all files as a single frame (solid)
byte[] frame = new byte[lastOffset];
mainStream.Seek(0, SeekOrigin.Begin);
mainStream.Read(frame, 0, (int)lastOffset);
byte[] decompressed = Native.LZ4FrameCompressor.Decompress(frame);
var decompressedStream = new MemoryStream(decompressed);
// Update offsets to point to the decompressed chunk
uint offset = 7;
uint compressedOffset = 0;
foreach (var entry in entries)
{
if (entry.OffsetInFile != offset)
{
throw new InvalidDataException("File list in solid archive not contiguous");
}
var file = PackagedFileInfo.CreateSolidFromEntry(entry, _streams[entry.ArchivePart], compressedOffset, decompressedStream);
package.Files.Add(file);
offset += entry.SizeOnDisk;
compressedOffset += entry.UncompressedSize;
}
}
else
{
foreach (var entry in entries)
{
package.Files.Add(PackagedFileInfo.CreateFromEntry(entry, _streams[entry.ArchivePart]));
}
}
return(package);
}