internal static PackagedFileInfo CreateFromEntry(FileEntry13 entry, Stream dataStream) { var info = new PackagedFileInfo(); info.PackageStream = dataStream; var nameLen = 0; for (nameLen = 0; nameLen < entry.Name.Length && entry.Name[nameLen] != 0; nameLen++) { } info.Name = Encoding.UTF8.GetString(entry.Name, 0, nameLen); var compressionMethod = entry.Flags & 0x0F; if (compressionMethod > 2 || (entry.Flags & ~0x7F) != 0) { var msg = String.Format("File '{0}' has unsupported flags: {1}", info.Name, entry.Flags); throw new InvalidDataException(msg); } info.OffsetInFile = entry.OffsetInFile; info.SizeOnDisk = entry.SizeOnDisk; info.UncompressedSize = entry.UncompressedSize; info.ArchivePart = entry.ArchivePart; info.Flags = entry.Flags; info.Crc = entry.Crc; return(info); }
internal static PackagedFileInfo CreateFromEntry(FileEntry7 entry, Stream dataStream) { var info = new PackagedFileInfo(); info.PackageStream = dataStream; var nameLen = 0; for (nameLen = 0; nameLen < entry.Name.Length && entry.Name[nameLen] != 0; nameLen++) { } info.Name = Encoding.UTF8.GetString(entry.Name, 0, nameLen); info.OffsetInFile = entry.OffsetInFile; info.SizeOnDisk = entry.SizeOnDisk; info.UncompressedSize = entry.UncompressedSize; info.ArchivePart = entry.ArchivePart; info.Crc = 0; if (entry.UncompressedSize > 0) { info.Flags = BinUtils.MakeCompressionFlags(CompressionMethod.Zlib, CompressionLevel.DefaultCompression); } else { info.Flags = 0; } return(info); }
internal static PackagedFileInfo CreateFromEntry(FileEntry13 entry, Stream dataStream) { var info = new PackagedFileInfo { PackageStream = dataStream, OffsetInFile = entry.OffsetInFile, SizeOnDisk = entry.SizeOnDisk, UncompressedSize = entry.UncompressedSize, ArchivePart = entry.ArchivePart, Flags = entry.Flags, Crc = entry.Crc, Solid = false }; int nameLen; for (nameLen = 0; nameLen < entry.Name.Length && entry.Name[nameLen] != 0; nameLen++) { } info.Name = Encoding.UTF8.GetString(entry.Name, 0, nameLen); uint compressionMethod = entry.Flags & 0x0F; if (compressionMethod > 2 || (entry.Flags & ~0x7F) != 0) { string msg = $"File '{info.Name}' has unsupported flags: {entry.Flags}"; throw new InvalidDataException(msg); } return(info); }
private Package ReadPackageV7(FileStream mainStream, BinaryReader reader) { var package = new Package(); mainStream.Seek(0, SeekOrigin.Begin); var header = BinUtils.ReadStruct <LSPKHeader7>(reader); package.Metadata.Flags = 0; package.Metadata.Priority = 0; package.Version = PackageVersion.V7; if (_metadataOnly) { return(package); } OpenStreams(mainStream, (int)header.NumParts); for (uint i = 0; i < header.NumFiles; i++) { var entry = BinUtils.ReadStruct <FileEntry7>(reader); if (entry.ArchivePart == 0) { entry.OffsetInFile += header.DataOffset; } package.Files.Add(PackagedFileInfo.CreateFromEntry(entry, _streams[entry.ArchivePart])); } return(package); }
private Package ReadPackageV10(FileStream mainStream, BinaryReader reader) { var package = new Package(); mainStream.Seek(4, SeekOrigin.Begin); var header = BinUtils.ReadStruct <LSPKHeader10>(reader); package.Metadata.Flags = (PackageFlags)header.Flags; package.Metadata.Priority = header.Priority; package.Version = PackageVersion.V10; if (_metadataOnly) { return(package); } OpenStreams(mainStream, header.NumParts); for (uint i = 0; i < header.NumFiles; i++) { var entry = BinUtils.ReadStruct <FileEntry13>(reader); if (entry.ArchivePart == 0) { entry.OffsetInFile += header.DataOffset; } // Add missing compression level flags entry.Flags = (entry.Flags & 0x0f) | 0x20; package.Files.Add(PackagedFileInfo.CreateFromEntry(entry, _streams[entry.ArchivePart])); } return(package); }
private void ReadFileListV15(BinaryReader reader, Package package) { int numFiles = reader.ReadInt32(); int compressedSize = reader.ReadInt32(); byte[] compressedFileList = reader.ReadBytes(compressedSize); int fileBufferSize = Marshal.SizeOf(typeof(FileEntry15)) * numFiles; var uncompressedList = new byte[fileBufferSize]; int uncompressedSize = LZ4Codec.Decode(compressedFileList, 0, compressedFileList.Length, uncompressedList, 0, fileBufferSize, true); if (uncompressedSize != fileBufferSize) { string msg = $"LZ4 compressor disagrees about the size of file headers; expected {fileBufferSize}, got {uncompressedSize}"; throw new InvalidDataException(msg); } var ms = new MemoryStream(uncompressedList); var msr = new BinaryReader(ms); var entries = new FileEntry15[numFiles]; BinUtils.ReadStructs(msr, entries); foreach (var entry in entries) { package.Files.Add(PackagedFileInfo.CreateFromEntry(entry, _streams[entry.ArchivePart])); } }
public UncompressedPackagedFileStream(Stream packageStream, PackagedFileInfo fileInfo) { PackageStream = packageStream; FileInfo = fileInfo; PackageStream.Seek((long)fileInfo.OffsetInFile, SeekOrigin.Begin); if ((CompressionMethod)(FileInfo.Flags & 0x0F) != CompressionMethod.None) { throw new ArgumentException("We only support uncompressed files!"); } }
private Package ReadPackageV15(FileStream mainStream, BinaryReader reader) { var package = new Package(); var header = BinUtils.ReadStruct <LSPKHeader15>(reader); if (header.Version != (ulong)PackageVersion.V15) { string msg = $"Unsupported package version {header.Version}; this layout is only supported for {PackageVersion.V15}"; throw new InvalidDataException(msg); } package.Metadata.Flags = (PackageFlags)header.Flags; package.Metadata.Priority = header.Priority; package.Version = PackageVersion.V15; if (_metadataOnly) { return(package); } OpenStreams(mainStream, 1); mainStream.Seek((long)header.FileListOffset, SeekOrigin.Begin); int numFiles = reader.ReadInt32(); int compressedSize = reader.ReadInt32(); byte[] compressedFileList = reader.ReadBytes(compressedSize); int fileBufferSize = Marshal.SizeOf(typeof(FileEntry15)) * numFiles; var uncompressedList = new byte[fileBufferSize]; int uncompressedSize = LZ4Codec.Decode(compressedFileList, 0, compressedFileList.Length, uncompressedList, 0, fileBufferSize, true); if (uncompressedSize != fileBufferSize) { string msg = $"LZ4 compressor disagrees about the size of file headers; expected {fileBufferSize}, got {uncompressedSize}"; throw new InvalidDataException(msg); } var ms = new MemoryStream(uncompressedList); var msr = new BinaryReader(ms); var entries = new FileEntry15[numFiles]; BinUtils.ReadStructs(msr, entries); foreach (var entry in entries) { package.Files.Add(PackagedFileInfo.CreateFromEntry(entry, _streams[0])); } return(package); }
public PackagedFileInfo WriteFile(FileInfo info) { // Assume that all files are written uncompressed (worst-case) when calculating package sizes var size = info.Size(); if (streams.Last().Position + size > MaxPackageSize) { // Start a new package file if the current one is full. var partPath = Package.MakePartFilename(path, streams.Count); var nextPart = new FileStream(partPath, FileMode.Create, FileAccess.Write); streams.Add(nextPart); } var stream = streams.Last(); var packaged = new PackagedFileInfo(); packaged.PackageStream = stream; packaged.Name = info.Name; packaged.UncompressedSize = size; packaged.ArchivePart = (UInt32)(streams.Count - 1); packaged.OffsetInFile = (UInt32)stream.Position; packaged.Flags = BinUtils.MakeCompressionFlags(Compression, CompressionLevel); var reader = info.MakeReader(); var uncompressed = reader.ReadBytes((int)reader.BaseStream.Length); var compressed = BinUtils.Compress(uncompressed, Compression, CompressionLevel); stream.Write(compressed, 0, compressed.Length); reader.Dispose(); packaged.SizeOnDisk = (UInt32)(stream.Position - packaged.OffsetInFile); packaged.Crc = Crc32.Compute(compressed); var padLength = PaddingLength(); if (stream.Position % padLength > 0) { // Pad the file to a multiple of 64 bytes byte[] pad = new byte[padLength - (stream.Position % padLength)]; for (int i = 0; i < pad.Length; i++) { pad[i] = 0xAD; } stream.Write(pad, 0, pad.Length); } return(packaged); }
public PackagedFileInfo WriteFile(FileInfo info) { // Assume that all files are written uncompressed (worst-case) when calculating package sizes var size = info.Size(); if (streams.Last().Position + size > MaxPackageSize) { // Start a new package file if the current one is full. var partPath = Package.MakePartFilename(path, streams.Count); var nextPart = new FileStream(partPath, FileMode.Create, FileAccess.Write); streams.Add(nextPart); } var stream = streams.Last(); var packaged = new PackagedFileInfo(); packaged.PackageStream = stream; packaged.Name = info.Name; packaged.UncompressedSize = size; packaged.ArchivePart = (UInt32)(streams.Count - 1); packaged.OffsetInFile = (UInt32)stream.Position; packaged.Flags = BinUtils.MakeCompressionFlags(Compression, CompressionLevel); var reader = info.MakeReader(); var uncompressed = reader.ReadBytes((int)reader.BaseStream.Length); var compressed = BinUtils.Compress(uncompressed, Compression, CompressionLevel); stream.Write(compressed, 0, compressed.Length); reader.Dispose(); packaged.SizeOnDisk = (UInt32)(stream.Position - packaged.OffsetInFile); packaged.Crc = Crc32.Compute(compressed); var padLength = PaddingLength(); if (stream.Position % padLength > 0) { // Pad the file to a multiple of 64 bytes byte[] pad = new byte[padLength - (stream.Position % padLength)]; for (int i = 0; i < pad.Length; i++) { pad[i] = 0xAD; } stream.Write(pad, 0, pad.Length); } return packaged; }
private Package ReadPackageV13(FileStream mainStream, BinaryReader reader) { var package = new Package(); var header = BinUtils.ReadStruct <LSPKHeader13>(reader); if (header.Version != Package.CurrentVersion) { var msg = String.Format("Unsupported package version {0}; this extractor only supports {1}", header.Version, Package.CurrentVersion); throw new InvalidDataException(msg); } OpenStreams(mainStream, header.NumParts); mainStream.Seek(header.FileListOffset, SeekOrigin.Begin); int numFiles = reader.ReadInt32(); int fileBufferSize = Marshal.SizeOf(typeof(FileEntry13)) * numFiles; byte[] compressedFileList = reader.ReadBytes((int)header.FileListSize - 4); var uncompressedList = new byte[fileBufferSize]; var uncompressedSize = LZ4Codec.Decode(compressedFileList, 0, compressedFileList.Length, uncompressedList, 0, fileBufferSize, true); if (uncompressedSize != fileBufferSize) { var msg = String.Format("LZ4 compressor disagrees about the size of file headers; expected {0}, got {1}", fileBufferSize, uncompressedSize); throw new InvalidDataException(msg); } var ms = new MemoryStream(uncompressedList); var msr = new BinaryReader(ms); for (int i = 0; i < numFiles; i++) { var entry = BinUtils.ReadStruct <FileEntry13>(msr); package.Files.Add(PackagedFileInfo.CreateFromEntry(entry, streams[entry.ArchivePart])); } return(package); }
public PackagedFileInfo WriteFile(AbstractFileInfo info) { // Assume that all files are written uncompressed (worst-case) when calculating package sizes uint size = info.Size(); if (_streams.Last().Position + size > MaxPackageSize) { // Start a new package file if the current one is full. string partPath = Package.MakePartFilename(_path, _streams.Count); var nextPart = File.Open(partPath, FileMode.Create, FileAccess.Write); _streams.Add(nextPart); } Stream stream = _streams.Last(); var packaged = new PackagedFileInfo { PackageStream = stream, Name = info.Name, UncompressedSize = size, ArchivePart = (UInt32)(_streams.Count - 1), OffsetInFile = (UInt32)stream.Position, Flags = BinUtils.MakeCompressionFlags(Compression, CompressionLevel) }; Stream packagedStream = info.MakeStream(); byte[] compressed; try { using (var reader = new BinaryReader(packagedStream, Encoding.UTF8, true)) { byte[] uncompressed = reader.ReadBytes((int)reader.BaseStream.Length); compressed = BinUtils.Compress(uncompressed, Compression, CompressionLevel); stream.Write(compressed, 0, compressed.Length); } } finally { info.ReleaseStream(); } packaged.SizeOnDisk = (UInt32)(stream.Position - packaged.OffsetInFile); packaged.Crc = Crc32.Compute(compressed, 0); int padLength = PaddingLength(); if (stream.Position % padLength <= 0) { return(packaged); } if ((_package.Metadata.Flags & PackageFlags.Solid) == 0) { // Pad the file to a multiple of 64 bytes var pad = new byte[padLength - stream.Position % padLength]; for (var i = 0; i < pad.Length; i++) { pad[i] = 0xAD; } stream.Write(pad, 0, pad.Length); } return(packaged); }
private Package ReadPackageV13(FileStream mainStream, BinaryReader reader) { var package = new Package(); var header = BinUtils.ReadStruct <LSPKHeader13>(reader); if (header.Version != (ulong)PackageVersion.V13) { string msg = $"Unsupported package version {header.Version}; this package layout is only supported for {PackageVersion.V13}"; throw new InvalidDataException(msg); } package.Metadata.Flags = (PackageFlags)header.Flags; package.Metadata.Priority = header.Priority; package.Version = PackageVersion.V13; if (_metadataOnly) { return(package); } OpenStreams(mainStream, header.NumParts); mainStream.Seek(header.FileListOffset, SeekOrigin.Begin); int numFiles = reader.ReadInt32(); int fileBufferSize = Marshal.SizeOf(typeof(FileEntry13)) * numFiles; byte[] compressedFileList = reader.ReadBytes((int)header.FileListSize - 4); var uncompressedList = new byte[fileBufferSize]; int uncompressedSize = LZ4Codec.Decode(compressedFileList, 0, compressedFileList.Length, uncompressedList, 0, fileBufferSize, true); if (uncompressedSize != fileBufferSize) { string msg = $"LZ4 compressor disagrees about the size of file headers; expected {fileBufferSize}, got {uncompressedSize}"; throw new InvalidDataException(msg); } var ms = new MemoryStream(uncompressedList); var msr = new BinaryReader(ms); var entries = new FileEntry13[numFiles]; BinUtils.ReadStructs(msr, entries); if ((package.Metadata.Flags & PackageFlags.Solid) == PackageFlags.Solid && numFiles > 0) { // Calculate compressed frame offset and bounds uint totalUncompressedSize = 0; uint totalSizeOnDisk = 0; uint firstOffset = 0xffffffff; uint lastOffset = 0; foreach (var entry in entries) { totalUncompressedSize += entry.UncompressedSize; totalSizeOnDisk += entry.SizeOnDisk; if (entry.OffsetInFile < firstOffset) { firstOffset = entry.OffsetInFile; } if (entry.OffsetInFile + entry.SizeOnDisk > lastOffset) { lastOffset = entry.OffsetInFile + entry.SizeOnDisk; } } if (firstOffset != 7 || lastOffset - firstOffset != totalSizeOnDisk) { string msg = $"Incorrectly compressed solid archive; offsets {firstOffset}/{lastOffset}, bytes {totalSizeOnDisk}"; throw new InvalidDataException(msg); } // Decompress all files as a single frame (solid) byte[] frame = new byte[lastOffset]; mainStream.Seek(0, SeekOrigin.Begin); mainStream.Read(frame, 0, (int)lastOffset); byte[] decompressed = Native.LZ4FrameCompressor.Decompress(frame); var decompressedStream = new MemoryStream(decompressed); // Update offsets to point to the decompressed chunk uint offset = 7; uint compressedOffset = 0; foreach (var entry in entries) { if (entry.OffsetInFile != offset) { throw new InvalidDataException("File list in solid archive not contiguous"); } var file = PackagedFileInfo.CreateSolidFromEntry(entry, _streams[entry.ArchivePart], compressedOffset, decompressedStream); package.Files.Add(file); offset += entry.SizeOnDisk; compressedOffset += entry.UncompressedSize; } } else { foreach (var entry in entries) { package.Files.Add(PackagedFileInfo.CreateFromEntry(entry, _streams[entry.ArchivePart])); } } return(package); }
internal static PackagedFileInfo CreateFromEntry(FileEntry7 entry, Stream dataStream) { var info = new PackagedFileInfo(); info.PackageStream = dataStream; var nameLen = 0; for (nameLen = 0; nameLen < entry.Name.Length && entry.Name[nameLen] != 0; nameLen++) { } info.Name = Encoding.UTF8.GetString(entry.Name, 0, nameLen); info.OffsetInFile = entry.OffsetInFile; info.SizeOnDisk = entry.SizeOnDisk; info.UncompressedSize = entry.UncompressedSize; info.ArchivePart = entry.ArchivePart; info.Crc = 0; if (entry.UncompressedSize > 0) { info.Flags = BinUtils.MakeCompressionFlags(CompressionMethod.Zlib, CompressionLevel.DefaultCompression); } else { info.Flags = 0; } return info; }
internal static PackagedFileInfo CreateFromEntry(FileEntry13 entry, Stream dataStream) { var info = new PackagedFileInfo(); info.PackageStream = dataStream; var nameLen = 0; for (nameLen = 0; nameLen < entry.Name.Length && entry.Name[nameLen] != 0; nameLen++) { } info.Name = Encoding.UTF8.GetString(entry.Name, 0, nameLen); var compressionMethod = entry.Flags & 0x0F; if (compressionMethod > 2 || (entry.Flags & ~0x7F) != 0) { var msg = String.Format("File '{0}' has unsupported flags: {1}", info.Name, entry.Flags); throw new InvalidDataException(msg); } info.OffsetInFile = entry.OffsetInFile; info.SizeOnDisk = entry.SizeOnDisk; info.UncompressedSize = entry.UncompressedSize; info.ArchivePart = entry.ArchivePart; info.Flags = entry.Flags; info.Crc = entry.Crc; return info; }