private static byte[] Decode(byte[] input, int blockSize, int extraBlocks) { using (var outputStream = new MemoryStream()) using (var inputStream = new MemoryStream(input)) { using (var inputReader = new BinaryReader(inputStream, Encoding.UTF8, false)) using (var outputWriter = new BinaryWriter(outputStream, Encoding.UTF8, true)) using (var decoder = new LZ4ChainDecoder(blockSize, extraBlocks)) { var maximumInputBlock = LZ4Codec.MaximumOutputSize(blockSize); var inputBuffer = new byte[maximumInputBlock]; var outputBuffer = new byte[blockSize]; while (true) { var length = inputReader.ReadInt32(); if (length < 0) { break; } Assert.True(length <= inputBuffer.Length); inputReader.Read(inputBuffer, 0, length); decoder.DecodeAndDrain( inputBuffer, 0, length, outputBuffer, 0, outputBuffer.Length, out var decoded); outputWriter.Write(outputBuffer, 0, decoded); } } return(outputStream.ToArray()); } }
private void ReadVersion3(BinaryReader reader, BinaryWriter outWrite, BinaryReader outRead) { Format = new Guid(reader.ReadBytes(16)); var compressionMethod = reader.ReadInt32(); var something = reader.ReadInt32(); var countOfBinaryBytes = reader.ReadInt32(); // how many bytes (binary blobs) var countOfIntegers = reader.ReadInt32(); // how many 4 byte values (ints) var countOfEightByteValues = reader.ReadInt32(); // how many 8 byte values (doubles) // 8 bytes that help valve preallocate, useless for us reader.BaseStream.Position += 8; var uncompressedSize = reader.ReadInt32(); var compressedSize = reader.ReadInt32(); var blockCount = reader.ReadInt32(); var blockTotalSize = reader.ReadInt32(); if (compressionMethod == 0) { if (something != 0) { throw new UnexpectedMagicException("Unhandled", something, nameof(something)); } var output = new Span <byte>(new byte[compressedSize]); reader.Read(output); outWrite.Write(output); } else if (compressionMethod == 1) { if (something != 0x40000000) { throw new UnexpectedMagicException("Unhandled", something, nameof(something)); } var input = reader.ReadBytes(compressedSize); var output = new Span <byte>(new byte[uncompressedSize]); LZ4Codec.Decode(input, output); outWrite.Write(output); outWrite.BaseStream.Position = 0; } else { throw new UnexpectedMagicException("Unknown compression method", compressionMethod, nameof(compressionMethod)); } currentBinaryBytesOffset = 0; outRead.BaseStream.Position = countOfBinaryBytes; if (outRead.BaseStream.Position % 4 != 0) { // Align to % 4 after binary blobs outRead.BaseStream.Position += 4 - (outRead.BaseStream.Position % 4); } var countOfStrings = outRead.ReadInt32(); var kvDataOffset = outRead.BaseStream.Position; // Subtract one integer since we already read it (countOfStrings) outRead.BaseStream.Position += (countOfIntegers - 1) * 4; if (outRead.BaseStream.Position % 8 != 0) { // Align to % 8 for the start of doubles outRead.BaseStream.Position += 8 - (outRead.BaseStream.Position % 8); } currentEightBytesOffset = outRead.BaseStream.Position; outRead.BaseStream.Position += countOfEightByteValues * 8; stringArray = new string[countOfStrings]; for (var i = 0; i < countOfStrings; i++) { stringArray[i] = outRead.ReadNullTermString(System.Text.Encoding.UTF8); } // 0xFFEEDD00 trailer + size of lz4 compressed block sizes (short) + size of lz4 decompressed block sizes (int) var typesArrayEnd = 4 + (blockCount * 2) + (blockCount * 4); var typesLength = outRead.BaseStream.Length - outRead.BaseStream.Position - typesArrayEnd; typesArray = new byte[typesLength]; for (var i = 0; i < typesLength; i++) { typesArray[i] = outRead.ReadByte(); } if (blockCount == 0) { if (outRead.ReadUInt32() != 0xFFEEDD00) { throw new InvalidDataException("Invalid trailer"); } // Move back to the start of the KV data for reading. outRead.BaseStream.Position = kvDataOffset; Data = ParseBinaryKV3(outRead, null, true); return; } uncompressedBlockLengthArray = new int[blockCount]; for (var i = 0; i < blockCount; i++) { uncompressedBlockLengthArray[i] = outRead.ReadInt32(); } if (outRead.ReadUInt32() != 0xFFEEDD00) { throw new InvalidDataException("Invalid trailer"); } try { using var uncompressedBlocks = new MemoryStream(blockTotalSize); using var uncompressedBlockDataWriter = new BinaryWriter(uncompressedBlocks); uncompressedBlockDataReader = new BinaryReader(uncompressedBlocks); // TODO: Do we need to pass blockTotalSize here? var lz4decoder = new LZ4ChainDecoder(blockTotalSize, 0); for (var i = 0; i < blockCount; i++) { var compressedBlockLength = outRead.ReadUInt16(); var input = new Span <byte>(new byte[compressedBlockLength]); var output = new Span <byte>(new byte[uncompressedBlockLengthArray[i]]); RawBinaryReader.Read(input); lz4decoder.DecodeAndDrain(input, output, out _); uncompressedBlockDataWriter.Write(output); } uncompressedBlocks.Position = 0; // Move back to the start of the KV data for reading. outRead.BaseStream.Position = kvDataOffset; Data = ParseBinaryKV3(outRead, null, true); } finally { uncompressedBlockDataReader.Dispose(); } }
private void ReadVersion3(BinaryReader reader, BinaryWriter outWrite, BinaryReader outRead) { Format = new Guid(reader.ReadBytes(16)); var compressionMethod = reader.ReadUInt32(); var compressionDictionaryId = reader.ReadUInt16(); var compressionFrameSize = reader.ReadUInt16(); var countOfBinaryBytes = reader.ReadUInt32(); // how many bytes (binary blobs) var countOfIntegers = reader.ReadUInt32(); // how many 4 byte values (ints) var countOfEightByteValues = reader.ReadUInt32(); // how many 8 byte values (doubles) // 8 bytes that help valve preallocate, useless for us var stringAndTypesBufferSize = reader.ReadUInt32(); var b = reader.ReadUInt16(); var c = reader.ReadUInt16(); var uncompressedSize = reader.ReadUInt32(); var compressedSize = reader.ReadInt32(); // uint32 var blockCount = reader.ReadUInt32(); var blockTotalSize = reader.ReadInt32(); // uint32 if (compressionMethod == 0) { if (compressionDictionaryId != 0) { throw new UnexpectedMagicException("Unhandled", compressionDictionaryId, nameof(compressionDictionaryId)); } if (compressionFrameSize != 0) { throw new UnexpectedMagicException("Unhandled", compressionFrameSize, nameof(compressionFrameSize)); } var output = new Span <byte>(new byte[compressedSize]); reader.Read(output); outWrite.Write(output); } else if (compressionMethod == 1) { if (compressionDictionaryId != 0) { throw new UnexpectedMagicException("Unhandled", compressionDictionaryId, nameof(compressionDictionaryId)); } if (compressionFrameSize != 16384) { throw new UnexpectedMagicException("Unhandled", compressionFrameSize, nameof(compressionFrameSize)); } var input = reader.ReadBytes(compressedSize); var output = new Span <byte>(new byte[uncompressedSize]); LZ4Codec.Decode(input, output); outWrite.Write(output); outWrite.BaseStream.Position = 0; } else if (compressionMethod == 2) { if (compressionDictionaryId != 0) { throw new UnexpectedMagicException("Unhandled", compressionDictionaryId, nameof(compressionDictionaryId)); } if (compressionFrameSize != 0) { throw new UnexpectedMagicException("Unhandled", compressionFrameSize, nameof(compressionFrameSize)); } var input = reader.ReadBytes(compressedSize); var zstd = new ZstdSharp.Decompressor(); var output = zstd.Unwrap(input); outWrite.Write(output); outWrite.BaseStream.Position = 0; } else { throw new UnexpectedMagicException("Unknown compression method", compressionMethod, nameof(compressionMethod)); } currentBinaryBytesOffset = 0; outRead.BaseStream.Position = countOfBinaryBytes; if (outRead.BaseStream.Position % 4 != 0) { // Align to % 4 after binary blobs outRead.BaseStream.Position += 4 - (outRead.BaseStream.Position % 4); } var countOfStrings = outRead.ReadInt32(); var kvDataOffset = outRead.BaseStream.Position; // Subtract one integer since we already read it (countOfStrings) outRead.BaseStream.Position += (countOfIntegers - 1) * 4; if (outRead.BaseStream.Position % 8 != 0) { // Align to % 8 for the start of doubles outRead.BaseStream.Position += 8 - (outRead.BaseStream.Position % 8); } currentEightBytesOffset = outRead.BaseStream.Position; outRead.BaseStream.Position += countOfEightByteValues * 8; var stringArrayStartPosition = outRead.BaseStream.Position; stringArray = new string[countOfStrings]; for (var i = 0; i < countOfStrings; i++) { stringArray[i] = outRead.ReadNullTermString(System.Text.Encoding.UTF8); } var typesLength = stringAndTypesBufferSize - (outRead.BaseStream.Position - stringArrayStartPosition); typesArray = new byte[typesLength]; for (var i = 0; i < typesLength; i++) { typesArray[i] = outRead.ReadByte(); } if (blockCount == 0) { var noBlocksTrailer = outRead.ReadUInt32(); if (noBlocksTrailer != 0xFFEEDD00) { throw new UnexpectedMagicException("Invalid trailer", noBlocksTrailer, nameof(noBlocksTrailer)); } // Move back to the start of the KV data for reading. outRead.BaseStream.Position = kvDataOffset; Data = ParseBinaryKV3(outRead, null, true); return; } uncompressedBlockLengthArray = new int[blockCount]; for (var i = 0; i < blockCount; i++) { uncompressedBlockLengthArray[i] = outRead.ReadInt32(); } var trailer = outRead.ReadUInt32(); if (trailer != 0xFFEEDD00) { throw new UnexpectedMagicException("Invalid trailer", trailer, nameof(trailer)); } try { using var uncompressedBlocks = new MemoryStream(blockTotalSize); uncompressedBlockDataReader = new BinaryReader(uncompressedBlocks); if (compressionMethod == 0) { for (var i = 0; i < blockCount; i++) { RawBinaryReader.BaseStream.CopyTo(uncompressedBlocks, uncompressedBlockLengthArray[i]); } } else if (compressionMethod == 1) { // TODO: Do we need to pass blockTotalSize here? using var lz4decoder = new LZ4ChainDecoder(blockTotalSize, 0); for (var i = 0; i < blockCount; i++) { var compressedBlockLength = outRead.ReadUInt16(); var input = new Span <byte>(new byte[compressedBlockLength]); var output = new Span <byte>(new byte[uncompressedBlockLengthArray[i]]); RawBinaryReader.Read(input); lz4decoder.DecodeAndDrain(input, output, out _); uncompressedBlocks.Write(output); } } else if (compressionMethod == 2) { // This is supposed to be a streaming decompress using ZSTD_decompressStream, // but as it turns out, zstd unwrap above already decompressed all of the blocks for us, // so all we need to do is just copy the buffer. // It's possible that Valve's code needs extra decompress because they set ZSTD_d_stableOutBuffer parameter. outRead.BaseStream.CopyTo(uncompressedBlocks); } else { throw new UnexpectedMagicException("Unimplemented compression method in block decoder", compressionMethod, nameof(compressionMethod)); } uncompressedBlocks.Position = 0; // Move back to the start of the KV data for reading. outRead.BaseStream.Position = kvDataOffset; Data = ParseBinaryKV3(outRead, null, true); } finally { uncompressedBlockDataReader.Dispose(); } }