internal static PackagedFileInfo CreateFromEntry(FileEntry13 entry, Stream dataStream)
{
var info = new PackagedFileInfo();
info.PackageStream = dataStream;
var nameLen = 0;
for (nameLen = 0; nameLen < entry.Name.Length && entry.Name[nameLen] != 0; nameLen++)
{
}
info.Name = Encoding.UTF8.GetString(entry.Name, 0, nameLen);
var compressionMethod = entry.Flags & 0x0F;
if (compressionMethod > 2 || (entry.Flags & ~0x7F) != 0)
{
var msg = String.Format("File '{0}' has unsupported flags: {1}", info.Name, entry.Flags);
throw new InvalidDataException(msg);
}
info.OffsetInFile = entry.OffsetInFile;
info.SizeOnDisk = entry.SizeOnDisk;
info.UncompressedSize = entry.UncompressedSize;
info.ArchivePart = entry.ArchivePart;
info.Flags = entry.Flags;
info.Crc = entry.Crc;
return(info);
}
internal static PackagedFileInfo CreateFromEntry(FileEntry7 entry, Stream dataStream)
{
var info = new PackagedFileInfo();
info.PackageStream = dataStream;
var nameLen = 0;
for (nameLen = 0; nameLen < entry.Name.Length && entry.Name[nameLen] != 0; nameLen++)
{
}
info.Name = Encoding.UTF8.GetString(entry.Name, 0, nameLen);
info.OffsetInFile = entry.OffsetInFile;
info.SizeOnDisk = entry.SizeOnDisk;
info.UncompressedSize = entry.UncompressedSize;
info.ArchivePart = entry.ArchivePart;
info.Crc = 0;
if (entry.UncompressedSize > 0)
{
info.Flags = BinUtils.MakeCompressionFlags(CompressionMethod.Zlib, CompressionLevel.DefaultCompression);
}
else
{
info.Flags = 0;
}
return(info);
}
internal static PackagedFileInfo CreateFromEntry(FileEntry13 entry, Stream dataStream)
{
var info = new PackagedFileInfo
{
PackageStream = dataStream,
OffsetInFile = entry.OffsetInFile,
SizeOnDisk = entry.SizeOnDisk,
UncompressedSize = entry.UncompressedSize,
ArchivePart = entry.ArchivePart,
Flags = entry.Flags,
Crc = entry.Crc,
Solid = false
};
int nameLen;
for (nameLen = 0; nameLen < entry.Name.Length && entry.Name[nameLen] != 0; nameLen++)
{
}
info.Name = Encoding.UTF8.GetString(entry.Name, 0, nameLen);
uint compressionMethod = entry.Flags & 0x0F;
if (compressionMethod > 2 || (entry.Flags & ~0x7F) != 0)
{
string msg = $"File '{info.Name}' has unsupported flags: {entry.Flags}";
throw new InvalidDataException(msg);
}
return(info);
}
private Package ReadPackageV7(FileStream mainStream, BinaryReader reader)
{
var package = new Package();
mainStream.Seek(0, SeekOrigin.Begin);
var header = BinUtils.ReadStruct <LSPKHeader7>(reader);
package.Metadata.Flags = 0;
package.Metadata.Priority = 0;
package.Version = PackageVersion.V7;
if (_metadataOnly)
{
return(package);
}
OpenStreams(mainStream, (int)header.NumParts);
for (uint i = 0; i < header.NumFiles; i++)
{
var entry = BinUtils.ReadStruct <FileEntry7>(reader);
if (entry.ArchivePart == 0)
{
entry.OffsetInFile += header.DataOffset;
}
package.Files.Add(PackagedFileInfo.CreateFromEntry(entry, _streams[entry.ArchivePart]));
}
return(package);
}
private Package ReadPackageV10(FileStream mainStream, BinaryReader reader)
{
var package = new Package();
mainStream.Seek(4, SeekOrigin.Begin);
var header = BinUtils.ReadStruct <LSPKHeader10>(reader);
package.Metadata.Flags = (PackageFlags)header.Flags;
package.Metadata.Priority = header.Priority;
package.Version = PackageVersion.V10;
if (_metadataOnly)
{
return(package);
}
OpenStreams(mainStream, header.NumParts);
for (uint i = 0; i < header.NumFiles; i++)
{
var entry = BinUtils.ReadStruct <FileEntry13>(reader);
if (entry.ArchivePart == 0)
{
entry.OffsetInFile += header.DataOffset;
}
// Add missing compression level flags
entry.Flags = (entry.Flags & 0x0f) | 0x20;
package.Files.Add(PackagedFileInfo.CreateFromEntry(entry, _streams[entry.ArchivePart]));
}
return(package);
}
private void ReadFileListV15(BinaryReader reader, Package package)
{
int numFiles = reader.ReadInt32();
int compressedSize = reader.ReadInt32();
byte[] compressedFileList = reader.ReadBytes(compressedSize);
int fileBufferSize = Marshal.SizeOf(typeof(FileEntry15)) * numFiles;
var uncompressedList = new byte[fileBufferSize];
int uncompressedSize = LZ4Codec.Decode(compressedFileList, 0, compressedFileList.Length, uncompressedList, 0, fileBufferSize, true);
if (uncompressedSize != fileBufferSize)
{
string msg = $"LZ4 compressor disagrees about the size of file headers; expected {fileBufferSize}, got {uncompressedSize}";
throw new InvalidDataException(msg);
}
var ms = new MemoryStream(uncompressedList);
var msr = new BinaryReader(ms);
var entries = new FileEntry15[numFiles];
BinUtils.ReadStructs(msr, entries);
foreach (var entry in entries)
{
package.Files.Add(PackagedFileInfo.CreateFromEntry(entry, _streams[entry.ArchivePart]));
}
}
public UncompressedPackagedFileStream(Stream packageStream, PackagedFileInfo fileInfo)
{
PackageStream = packageStream;
FileInfo = fileInfo;
PackageStream.Seek((long)fileInfo.OffsetInFile, SeekOrigin.Begin);
if ((CompressionMethod)(FileInfo.Flags & 0x0F) != CompressionMethod.None)
{
throw new ArgumentException("We only support uncompressed files!");
}
}
private Package ReadPackageV15(FileStream mainStream, BinaryReader reader)
{
var package = new Package();
var header = BinUtils.ReadStruct <LSPKHeader15>(reader);
if (header.Version != (ulong)PackageVersion.V15)
{
string msg = $"Unsupported package version {header.Version}; this layout is only supported for {PackageVersion.V15}";
throw new InvalidDataException(msg);
}
package.Metadata.Flags = (PackageFlags)header.Flags;
package.Metadata.Priority = header.Priority;
package.Version = PackageVersion.V15;
if (_metadataOnly)
{
return(package);
}
OpenStreams(mainStream, 1);
mainStream.Seek((long)header.FileListOffset, SeekOrigin.Begin);
int numFiles = reader.ReadInt32();
int compressedSize = reader.ReadInt32();
byte[] compressedFileList = reader.ReadBytes(compressedSize);
int fileBufferSize = Marshal.SizeOf(typeof(FileEntry15)) * numFiles;
var uncompressedList = new byte[fileBufferSize];
int uncompressedSize = LZ4Codec.Decode(compressedFileList, 0, compressedFileList.Length, uncompressedList, 0, fileBufferSize, true);
if (uncompressedSize != fileBufferSize)
{
string msg = $"LZ4 compressor disagrees about the size of file headers; expected {fileBufferSize}, got {uncompressedSize}";
throw new InvalidDataException(msg);
}
var ms = new MemoryStream(uncompressedList);
var msr = new BinaryReader(ms);
var entries = new FileEntry15[numFiles];
BinUtils.ReadStructs(msr, entries);
foreach (var entry in entries)
{
package.Files.Add(PackagedFileInfo.CreateFromEntry(entry, _streams[0]));
}
return(package);
}
public PackagedFileInfo WriteFile(FileInfo info)
{
// Assume that all files are written uncompressed (worst-case) when calculating package sizes
var size = info.Size();
if (streams.Last().Position + size > MaxPackageSize)
{
// Start a new package file if the current one is full.
var partPath = Package.MakePartFilename(path, streams.Count);
var nextPart = new FileStream(partPath, FileMode.Create, FileAccess.Write);
streams.Add(nextPart);
}
var stream = streams.Last();
var packaged = new PackagedFileInfo();
packaged.PackageStream = stream;
packaged.Name = info.Name;
packaged.UncompressedSize = size;
packaged.ArchivePart = (UInt32)(streams.Count - 1);
packaged.OffsetInFile = (UInt32)stream.Position;
packaged.Flags = BinUtils.MakeCompressionFlags(Compression, CompressionLevel);
var reader = info.MakeReader();
var uncompressed = reader.ReadBytes((int)reader.BaseStream.Length);
var compressed = BinUtils.Compress(uncompressed, Compression, CompressionLevel);
stream.Write(compressed, 0, compressed.Length);
reader.Dispose();
packaged.SizeOnDisk = (UInt32)(stream.Position - packaged.OffsetInFile);
packaged.Crc = Crc32.Compute(compressed);
var padLength = PaddingLength();
if (stream.Position % padLength > 0)
{
// Pad the file to a multiple of 64 bytes
byte[] pad = new byte[padLength - (stream.Position % padLength)];
for (int i = 0; i < pad.Length; i++)
{
pad[i] = 0xAD;
}
stream.Write(pad, 0, pad.Length);
}
return(packaged);
}
public PackagedFileInfo WriteFile(FileInfo info)
{
// Assume that all files are written uncompressed (worst-case) when calculating package sizes
var size = info.Size();
if (streams.Last().Position + size > MaxPackageSize)
{
// Start a new package file if the current one is full.
var partPath = Package.MakePartFilename(path, streams.Count);
var nextPart = new FileStream(partPath, FileMode.Create, FileAccess.Write);
streams.Add(nextPart);
}
var stream = streams.Last();
var packaged = new PackagedFileInfo();
packaged.PackageStream = stream;
packaged.Name = info.Name;
packaged.UncompressedSize = size;
packaged.ArchivePart = (UInt32)(streams.Count - 1);
packaged.OffsetInFile = (UInt32)stream.Position;
packaged.Flags = BinUtils.MakeCompressionFlags(Compression, CompressionLevel);
var reader = info.MakeReader();
var uncompressed = reader.ReadBytes((int)reader.BaseStream.Length);
var compressed = BinUtils.Compress(uncompressed, Compression, CompressionLevel);
stream.Write(compressed, 0, compressed.Length);
reader.Dispose();
packaged.SizeOnDisk = (UInt32)(stream.Position - packaged.OffsetInFile);
packaged.Crc = Crc32.Compute(compressed);
var padLength = PaddingLength();
if (stream.Position % padLength > 0)
{
// Pad the file to a multiple of 64 bytes
byte[] pad = new byte[padLength - (stream.Position % padLength)];
for (int i = 0; i < pad.Length; i++)
{
pad[i] = 0xAD;
}
stream.Write(pad, 0, pad.Length);
}
return packaged;
}
private Package ReadPackageV13(FileStream mainStream, BinaryReader reader)
{
var package = new Package();
var header = BinUtils.ReadStruct <LSPKHeader13>(reader);
if (header.Version != Package.CurrentVersion)
{
var msg = String.Format("Unsupported package version {0}; this extractor only supports {1}", header.Version, Package.CurrentVersion);
throw new InvalidDataException(msg);
}
OpenStreams(mainStream, header.NumParts);
mainStream.Seek(header.FileListOffset, SeekOrigin.Begin);
int numFiles = reader.ReadInt32();
int fileBufferSize = Marshal.SizeOf(typeof(FileEntry13)) * numFiles;
byte[] compressedFileList = reader.ReadBytes((int)header.FileListSize - 4);
var uncompressedList = new byte[fileBufferSize];
var uncompressedSize = LZ4Codec.Decode(compressedFileList, 0, compressedFileList.Length, uncompressedList, 0, fileBufferSize, true);
if (uncompressedSize != fileBufferSize)
{
var msg = String.Format("LZ4 compressor disagrees about the size of file headers; expected {0}, got {1}", fileBufferSize, uncompressedSize);
throw new InvalidDataException(msg);
}
var ms = new MemoryStream(uncompressedList);
var msr = new BinaryReader(ms);
for (int i = 0; i < numFiles; i++)
{
var entry = BinUtils.ReadStruct <FileEntry13>(msr);
package.Files.Add(PackagedFileInfo.CreateFromEntry(entry, streams[entry.ArchivePart]));
}
return(package);
}
public PackagedFileInfo WriteFile(AbstractFileInfo info)
{
// Assume that all files are written uncompressed (worst-case) when calculating package sizes
uint size = info.Size();
if (_streams.Last().Position + size > MaxPackageSize)
{
// Start a new package file if the current one is full.
string partPath = Package.MakePartFilename(_path, _streams.Count);
var nextPart = File.Open(partPath, FileMode.Create, FileAccess.Write);
_streams.Add(nextPart);
}
Stream stream = _streams.Last();
var packaged = new PackagedFileInfo
{
PackageStream = stream,
Name = info.Name,
UncompressedSize = size,
ArchivePart = (UInt32)(_streams.Count - 1),
OffsetInFile = (UInt32)stream.Position,
Flags = BinUtils.MakeCompressionFlags(Compression, CompressionLevel)
};
Stream packagedStream = info.MakeStream();
byte[] compressed;
try
{
using (var reader = new BinaryReader(packagedStream, Encoding.UTF8, true))
{
byte[] uncompressed = reader.ReadBytes((int)reader.BaseStream.Length);
compressed = BinUtils.Compress(uncompressed, Compression, CompressionLevel);
stream.Write(compressed, 0, compressed.Length);
}
}
finally
{
info.ReleaseStream();
}
packaged.SizeOnDisk = (UInt32)(stream.Position - packaged.OffsetInFile);
packaged.Crc = Crc32.Compute(compressed, 0);
int padLength = PaddingLength();
if (stream.Position % padLength <= 0)
{
return(packaged);
}
if ((_package.Metadata.Flags & PackageFlags.Solid) == 0)
{
// Pad the file to a multiple of 64 bytes
var pad = new byte[padLength - stream.Position % padLength];
for (var i = 0; i < pad.Length; i++)
{
pad[i] = 0xAD;
}
stream.Write(pad, 0, pad.Length);
}
return(packaged);
}
private Package ReadPackageV13(FileStream mainStream, BinaryReader reader)
{
var package = new Package();
var header = BinUtils.ReadStruct <LSPKHeader13>(reader);
if (header.Version != (ulong)PackageVersion.V13)
{
string msg = $"Unsupported package version {header.Version}; this package layout is only supported for {PackageVersion.V13}";
throw new InvalidDataException(msg);
}
package.Metadata.Flags = (PackageFlags)header.Flags;
package.Metadata.Priority = header.Priority;
package.Version = PackageVersion.V13;
if (_metadataOnly)
{
return(package);
}
OpenStreams(mainStream, header.NumParts);
mainStream.Seek(header.FileListOffset, SeekOrigin.Begin);
int numFiles = reader.ReadInt32();
int fileBufferSize = Marshal.SizeOf(typeof(FileEntry13)) * numFiles;
byte[] compressedFileList = reader.ReadBytes((int)header.FileListSize - 4);
var uncompressedList = new byte[fileBufferSize];
int uncompressedSize = LZ4Codec.Decode(compressedFileList, 0, compressedFileList.Length, uncompressedList, 0, fileBufferSize, true);
if (uncompressedSize != fileBufferSize)
{
string msg = $"LZ4 compressor disagrees about the size of file headers; expected {fileBufferSize}, got {uncompressedSize}";
throw new InvalidDataException(msg);
}
var ms = new MemoryStream(uncompressedList);
var msr = new BinaryReader(ms);
var entries = new FileEntry13[numFiles];
BinUtils.ReadStructs(msr, entries);
if ((package.Metadata.Flags & PackageFlags.Solid) == PackageFlags.Solid && numFiles > 0)
{
// Calculate compressed frame offset and bounds
uint totalUncompressedSize = 0;
uint totalSizeOnDisk = 0;
uint firstOffset = 0xffffffff;
uint lastOffset = 0;
foreach (var entry in entries)
{
totalUncompressedSize += entry.UncompressedSize;
totalSizeOnDisk += entry.SizeOnDisk;
if (entry.OffsetInFile < firstOffset)
{
firstOffset = entry.OffsetInFile;
}
if (entry.OffsetInFile + entry.SizeOnDisk > lastOffset)
{
lastOffset = entry.OffsetInFile + entry.SizeOnDisk;
}
}
if (firstOffset != 7 || lastOffset - firstOffset != totalSizeOnDisk)
{
string msg = $"Incorrectly compressed solid archive; offsets {firstOffset}/{lastOffset}, bytes {totalSizeOnDisk}";
throw new InvalidDataException(msg);
}
// Decompress all files as a single frame (solid)
byte[] frame = new byte[lastOffset];
mainStream.Seek(0, SeekOrigin.Begin);
mainStream.Read(frame, 0, (int)lastOffset);
byte[] decompressed = Native.LZ4FrameCompressor.Decompress(frame);
var decompressedStream = new MemoryStream(decompressed);
// Update offsets to point to the decompressed chunk
uint offset = 7;
uint compressedOffset = 0;
foreach (var entry in entries)
{
if (entry.OffsetInFile != offset)
{
throw new InvalidDataException("File list in solid archive not contiguous");
}
var file = PackagedFileInfo.CreateSolidFromEntry(entry, _streams[entry.ArchivePart], compressedOffset, decompressedStream);
package.Files.Add(file);
offset += entry.SizeOnDisk;
compressedOffset += entry.UncompressedSize;
}
}
else
{
foreach (var entry in entries)
{
package.Files.Add(PackagedFileInfo.CreateFromEntry(entry, _streams[entry.ArchivePart]));
}
}
return(package);
}
internal static PackagedFileInfo CreateFromEntry(FileEntry7 entry, Stream dataStream)
{
var info = new PackagedFileInfo();
info.PackageStream = dataStream;
var nameLen = 0;
for (nameLen = 0; nameLen < entry.Name.Length && entry.Name[nameLen] != 0; nameLen++) { }
info.Name = Encoding.UTF8.GetString(entry.Name, 0, nameLen);
info.OffsetInFile = entry.OffsetInFile;
info.SizeOnDisk = entry.SizeOnDisk;
info.UncompressedSize = entry.UncompressedSize;
info.ArchivePart = entry.ArchivePart;
info.Crc = 0;
if (entry.UncompressedSize > 0)
{
info.Flags = BinUtils.MakeCompressionFlags(CompressionMethod.Zlib, CompressionLevel.DefaultCompression);
}
else
{
info.Flags = 0;
}
return info;
}
internal static PackagedFileInfo CreateFromEntry(FileEntry13 entry, Stream dataStream)
{
var info = new PackagedFileInfo();
info.PackageStream = dataStream;
var nameLen = 0;
for (nameLen = 0; nameLen < entry.Name.Length && entry.Name[nameLen] != 0; nameLen++) { }
info.Name = Encoding.UTF8.GetString(entry.Name, 0, nameLen);
var compressionMethod = entry.Flags & 0x0F;
if (compressionMethod > 2 || (entry.Flags & ~0x7F) != 0)
{
var msg = String.Format("File '{0}' has unsupported flags: {1}", info.Name, entry.Flags);
throw new InvalidDataException(msg);
}
info.OffsetInFile = entry.OffsetInFile;
info.SizeOnDisk = entry.SizeOnDisk;
info.UncompressedSize = entry.UncompressedSize;
info.ArchivePart = entry.ArchivePart;
info.Flags = entry.Flags;
info.Crc = entry.Crc;
return info;
}
