internal static PackagedFileInfo CreateFromEntry(FileEntry13 entry, Stream dataStream)
{
var info = new PackagedFileInfo
{
PackageStream = dataStream,
OffsetInFile = entry.OffsetInFile,
SizeOnDisk = entry.SizeOnDisk,
UncompressedSize = entry.UncompressedSize,
ArchivePart = entry.ArchivePart,
Flags = entry.Flags,
Crc = entry.Crc,
Solid = false
};
int nameLen;
for (nameLen = 0; nameLen < entry.Name.Length && entry.Name[nameLen] != 0; nameLen++)
{
}
info.Name = Encoding.UTF8.GetString(entry.Name, 0, nameLen);
uint compressionMethod = entry.Flags & 0x0F;
if (compressionMethod > 2 || (entry.Flags & ~0x7F) != 0)
{
string msg = $"File '{info.Name}' has unsupported flags: {entry.Flags}";
throw new InvalidDataException(msg);
}
return(info);
}
internal static PackagedFileInfo CreateFromEntry(FileEntry13 entry, Stream dataStream)
{
var info = new PackagedFileInfo();
info.PackageStream = dataStream;
var nameLen = 0;
for (nameLen = 0; nameLen < entry.Name.Length && entry.Name[nameLen] != 0; nameLen++)
{
}
info.Name = Encoding.UTF8.GetString(entry.Name, 0, nameLen);
var compressionMethod = entry.Flags & 0x0F;
if (compressionMethod > 2 || (entry.Flags & ~0x7F) != 0)
{
var msg = String.Format("File '{0}' has unsupported flags: {1}", info.Name, entry.Flags);
throw new InvalidDataException(msg);
}
info.OffsetInFile = entry.OffsetInFile;
info.SizeOnDisk = entry.SizeOnDisk;
info.UncompressedSize = entry.UncompressedSize;
info.ArchivePart = entry.ArchivePart;
info.Flags = entry.Flags;
info.Crc = entry.Crc;
return(info);
}
internal static PackagedFileInfo CreateSolidFromEntry(FileEntry13 entry, Stream dataStream, uint solidOffset, Stream solidStream)
{
var info = CreateFromEntry(entry, dataStream);
info.Solid = true;
info.SolidOffset = solidOffset;
info.SolidStream = solidStream;
return(info);
}
public void WriteV10(FileStream mainStream)
{
using (var writer = new BinaryWriter(mainStream, new UTF8Encoding(), true))
{
var header = new LSPKHeader10
{
Version = (uint)Version,
NumFiles = (UInt32)_package.Files.Count,
FileListSize = (UInt32)(Marshal.SizeOf(typeof(FileEntry13)) * _package.Files.Count)
};
header.DataOffset = (UInt32)Marshal.SizeOf(typeof(LSPKHeader10)) + 4 + header.FileListSize;
int paddingLength = PaddingLength();
if (header.DataOffset % paddingLength > 0)
{
header.DataOffset += (UInt32)(paddingLength - header.DataOffset % paddingLength);
}
// Write a placeholder instead of the actual headers; we'll write them after we
// compressed and flushed all files to disk
var placeholder = new byte[header.DataOffset];
writer.Write(placeholder);
long totalSize = _package.Files.Sum(p => (long)p.Size());
long currentSize = 0;
var writtenFiles = new List <PackagedFileInfo>();
foreach (AbstractFileInfo file in _package.Files)
{
WriteProgress(file, currentSize, totalSize);
writtenFiles.Add(WriteFile(file));
currentSize += file.Size();
}
mainStream.Seek(0, SeekOrigin.Begin);
writer.Write(Package.Signature);
header.NumParts = (UInt16)_streams.Count;
header.Priority = _package.Metadata.Priority;
header.Flags = (byte)_package.Metadata.Flags;
BinUtils.WriteStruct(writer, ref header);
foreach (PackagedFileInfo file in writtenFiles)
{
FileEntry13 entry = file.MakeEntryV13();
if (entry.ArchivePart == 0)
{
entry.OffsetInFile -= header.DataOffset;
}
// v10 packages don't support compression level in the flags field
entry.Flags &= 0x0f;
BinUtils.WriteStruct(writer, ref entry);
}
}
}
public void WriteV13(FileStream mainStream)
{
long totalSize = _package.Files.Sum(p => (long)p.Size());
long currentSize = 0;
var writtenFiles = new List <PackagedFileInfo>();
foreach (AbstractFileInfo file in _package.Files)
{
WriteProgress(file, currentSize, totalSize);
writtenFiles.Add(WriteFile(file));
currentSize += file.Size();
}
using (var writer = new BinaryWriter(mainStream, new UTF8Encoding(), true))
{
var header = new LSPKHeader13
{
Version = (uint)Version,
FileListOffset = (UInt32)mainStream.Position
};
writer.Write((UInt32)writtenFiles.Count);
var fileList = new MemoryStream();
var fileListWriter = new BinaryWriter(fileList);
foreach (PackagedFileInfo file in writtenFiles)
{
FileEntry13 entry = file.MakeEntryV13();
BinUtils.WriteStruct(fileListWriter, ref entry);
}
byte[] fileListBuf = fileList.ToArray();
fileListWriter.Dispose();
byte[] compressedFileList = LZ4Codec.EncodeHC(fileListBuf, 0, fileListBuf.Length);
writer.Write(compressedFileList);
header.FileListSize = (UInt32)mainStream.Position - header.FileListOffset;
header.NumParts = (UInt16)_streams.Count;
header.Priority = _package.Metadata.Priority;
header.Flags = (byte)_package.Metadata.Flags;
header.Md5 = ComputeArchiveHash();
BinUtils.WriteStruct(writer, ref header);
writer.Write((UInt32)(8 + Marshal.SizeOf(typeof(LSPKHeader13))));
writer.Write(Package.Signature);
}
}
private Package ReadPackageV13(FileStream mainStream, BinaryReader reader)
{
var package = new Package();
var header = BinUtils.ReadStruct <LSPKHeader13>(reader);
if (header.Version != (ulong)Package.CurrentVersion)
{
string msg = $"Unsupported package version {header.Version}; this extractor only supports {Package.CurrentVersion}";
throw new InvalidDataException(msg);
}
package.Metadata.Flags = (PackageFlags)header.Flags;
package.Metadata.Priority = header.Priority;
if (_metadataOnly)
{
return(package);
}
OpenStreams(mainStream, header.NumParts);
mainStream.Seek(header.FileListOffset, SeekOrigin.Begin);
int numFiles = reader.ReadInt32();
int fileBufferSize = Marshal.SizeOf(typeof(FileEntry13)) * numFiles;
byte[] compressedFileList = reader.ReadBytes((int)header.FileListSize - 4);
var uncompressedList = new byte[fileBufferSize];
int uncompressedSize = LZ4Codec.Decode(compressedFileList, 0, compressedFileList.Length, uncompressedList, 0, fileBufferSize, true);
if (uncompressedSize != fileBufferSize)
{
string msg = $"LZ4 compressor disagrees about the size of file headers; expected {fileBufferSize}, got {uncompressedSize}";
throw new InvalidDataException(msg);
}
var ms = new MemoryStream(uncompressedList);
var msr = new BinaryReader(ms);
var entries = new FileEntry13[numFiles];
BinUtils.ReadStructs(msr, entries);
foreach (var entry in entries)
{
package.Files.Add(PackagedFileInfo.CreateFromEntry(entry, _streams[entry.ArchivePart]));
}
return(package);
}
internal FileEntry13 MakeEntryV13()
{
var entry = new FileEntry13();
entry.Name = new byte[256];
var encodedName = Encoding.UTF8.GetBytes(Name.Replace('\\', '/'));
Array.Copy(encodedName, entry.Name, encodedName.Length);
entry.OffsetInFile = OffsetInFile;
entry.SizeOnDisk = SizeOnDisk;
entry.UncompressedSize = ((Flags & 0x0F) == 0) ? 0 : UncompressedSize;
entry.ArchivePart = ArchivePart;
entry.Flags = Flags;
entry.Crc = Crc;
return(entry);
}
private Package ReadPackageV13(FileStream mainStream, BinaryReader reader)
{
var package = new Package();
var header = BinUtils.ReadStruct <LSPKHeader13>(reader);
if (header.Version != (ulong)PackageVersion.V13)
{
string msg = $"Unsupported package version {header.Version}; this package layout is only supported for {PackageVersion.V13}";
throw new InvalidDataException(msg);
}
package.Metadata.Flags = (PackageFlags)header.Flags;
package.Metadata.Priority = header.Priority;
package.Version = PackageVersion.V13;
if (_metadataOnly)
{
return(package);
}
OpenStreams(mainStream, header.NumParts);
mainStream.Seek(header.FileListOffset, SeekOrigin.Begin);
int numFiles = reader.ReadInt32();
int fileBufferSize = Marshal.SizeOf(typeof(FileEntry13)) * numFiles;
byte[] compressedFileList = reader.ReadBytes((int)header.FileListSize - 4);
var uncompressedList = new byte[fileBufferSize];
int uncompressedSize = LZ4Codec.Decode(compressedFileList, 0, compressedFileList.Length, uncompressedList, 0, fileBufferSize, true);
if (uncompressedSize != fileBufferSize)
{
string msg = $"LZ4 compressor disagrees about the size of file headers; expected {fileBufferSize}, got {uncompressedSize}";
throw new InvalidDataException(msg);
}
var ms = new MemoryStream(uncompressedList);
var msr = new BinaryReader(ms);
var entries = new FileEntry13[numFiles];
BinUtils.ReadStructs(msr, entries);
if ((package.Metadata.Flags & PackageFlags.Solid) == PackageFlags.Solid && numFiles > 0)
{
// Calculate compressed frame offset and bounds
uint totalUncompressedSize = 0;
uint totalSizeOnDisk = 0;
uint firstOffset = 0xffffffff;
uint lastOffset = 0;
foreach (var entry in entries)
{
totalUncompressedSize += entry.UncompressedSize;
totalSizeOnDisk += entry.SizeOnDisk;
if (entry.OffsetInFile < firstOffset)
{
firstOffset = entry.OffsetInFile;
}
if (entry.OffsetInFile + entry.SizeOnDisk > lastOffset)
{
lastOffset = entry.OffsetInFile + entry.SizeOnDisk;
}
}
if (firstOffset != 7 || lastOffset - firstOffset != totalSizeOnDisk)
{
string msg = $"Incorrectly compressed solid archive; offsets {firstOffset}/{lastOffset}, bytes {totalSizeOnDisk}";
throw new InvalidDataException(msg);
}
// Decompress all files as a single frame (solid)
byte[] frame = new byte[lastOffset];
mainStream.Seek(0, SeekOrigin.Begin);
mainStream.Read(frame, 0, (int)lastOffset);
byte[] decompressed = Native.LZ4FrameCompressor.Decompress(frame);
var decompressedStream = new MemoryStream(decompressed);
// Update offsets to point to the decompressed chunk
uint offset = 7;
uint compressedOffset = 0;
foreach (var entry in entries)
{
if (entry.OffsetInFile != offset)
{
throw new InvalidDataException("File list in solid archive not contiguous");
}
var file = PackagedFileInfo.CreateSolidFromEntry(entry, _streams[entry.ArchivePart], compressedOffset, decompressedStream);
package.Files.Add(file);
offset += entry.SizeOnDisk;
compressedOffset += entry.UncompressedSize;
}
}
else
{
foreach (var entry in entries)
{
package.Files.Add(PackagedFileInfo.CreateFromEntry(entry, _streams[entry.ArchivePart]));
}
}
return(package);
}
internal FileEntry13 MakeEntryV13()
{
var entry = new FileEntry13();
entry.Name = new byte[256];
var encodedName = Encoding.UTF8.GetBytes(Name.Replace('\\', '/'));
Array.Copy(encodedName, entry.Name, encodedName.Length);
entry.OffsetInFile = OffsetInFile;
entry.SizeOnDisk = SizeOnDisk;
entry.UncompressedSize = ((Flags & 0x0F) == 0) ? 0 : UncompressedSize;
entry.ArchivePart = ArchivePart;
entry.Flags = Flags;
entry.Crc = Crc;
return entry;
}
internal static PackagedFileInfo CreateFromEntry(FileEntry13 entry, Stream dataStream)
{
var info = new PackagedFileInfo();
info.PackageStream = dataStream;
var nameLen = 0;
for (nameLen = 0; nameLen < entry.Name.Length && entry.Name[nameLen] != 0; nameLen++) { }
info.Name = Encoding.UTF8.GetString(entry.Name, 0, nameLen);
var compressionMethod = entry.Flags & 0x0F;
if (compressionMethod > 2 || (entry.Flags & ~0x7F) != 0)
{
var msg = String.Format("File '{0}' has unsupported flags: {1}", info.Name, entry.Flags);
throw new InvalidDataException(msg);
}
info.OffsetInFile = entry.OffsetInFile;
info.SizeOnDisk = entry.SizeOnDisk;
info.UncompressedSize = entry.UncompressedSize;
info.ArchivePart = entry.ArchivePart;
info.Flags = entry.Flags;
info.Crc = entry.Crc;
return info;
}