private static byte[] Decode(byte[] input, int blockSize, int extraBlocks)
{
using (var outputStream = new MemoryStream())
using (var inputStream = new MemoryStream(input))
{
using (var inputReader = new BinaryReader(inputStream, Encoding.UTF8, false))
using (var outputWriter = new BinaryWriter(outputStream, Encoding.UTF8, true))
using (var decoder = new LZ4ChainDecoder(blockSize, extraBlocks))
{
var maximumInputBlock = LZ4Codec.MaximumOutputSize(blockSize);
var inputBuffer = new byte[maximumInputBlock];
var outputBuffer = new byte[blockSize];
while (true)
{
var length = inputReader.ReadInt32();
if (length < 0)
{
break;
}
Assert.True(length <= inputBuffer.Length);
inputReader.Read(inputBuffer, 0, length);
decoder.DecodeAndDrain(
inputBuffer,
0,
length,
outputBuffer,
0,
outputBuffer.Length,
out var decoded);
outputWriter.Write(outputBuffer, 0, decoded);
}
}
return(outputStream.ToArray());
}
}
private void ReadVersion3(BinaryReader reader, BinaryWriter outWrite, BinaryReader outRead)
{
Format = new Guid(reader.ReadBytes(16));
var compressionMethod = reader.ReadInt32();
var something = reader.ReadInt32();
var countOfBinaryBytes = reader.ReadInt32(); // how many bytes (binary blobs)
var countOfIntegers = reader.ReadInt32(); // how many 4 byte values (ints)
var countOfEightByteValues = reader.ReadInt32(); // how many 8 byte values (doubles)
// 8 bytes that help valve preallocate, useless for us
reader.BaseStream.Position += 8;
var uncompressedSize = reader.ReadInt32();
var compressedSize = reader.ReadInt32();
var blockCount = reader.ReadInt32();
var blockTotalSize = reader.ReadInt32();
if (compressionMethod == 0)
{
if (something != 0)
{
throw new UnexpectedMagicException("Unhandled", something, nameof(something));
}
var output = new Span <byte>(new byte[compressedSize]);
reader.Read(output);
outWrite.Write(output);
}
else if (compressionMethod == 1)
{
if (something != 0x40000000)
{
throw new UnexpectedMagicException("Unhandled", something, nameof(something));
}
var input = reader.ReadBytes(compressedSize);
var output = new Span <byte>(new byte[uncompressedSize]);
LZ4Codec.Decode(input, output);
outWrite.Write(output);
outWrite.BaseStream.Position = 0;
}
else
{
throw new UnexpectedMagicException("Unknown compression method", compressionMethod, nameof(compressionMethod));
}
currentBinaryBytesOffset = 0;
outRead.BaseStream.Position = countOfBinaryBytes;
if (outRead.BaseStream.Position % 4 != 0)
{
// Align to % 4 after binary blobs
outRead.BaseStream.Position += 4 - (outRead.BaseStream.Position % 4);
}
var countOfStrings = outRead.ReadInt32();
var kvDataOffset = outRead.BaseStream.Position;
// Subtract one integer since we already read it (countOfStrings)
outRead.BaseStream.Position += (countOfIntegers - 1) * 4;
if (outRead.BaseStream.Position % 8 != 0)
{
// Align to % 8 for the start of doubles
outRead.BaseStream.Position += 8 - (outRead.BaseStream.Position % 8);
}
currentEightBytesOffset = outRead.BaseStream.Position;
outRead.BaseStream.Position += countOfEightByteValues * 8;
stringArray = new string[countOfStrings];
for (var i = 0; i < countOfStrings; i++)
{
stringArray[i] = outRead.ReadNullTermString(System.Text.Encoding.UTF8);
}
// 0xFFEEDD00 trailer + size of lz4 compressed block sizes (short) + size of lz4 decompressed block sizes (int)
var typesArrayEnd = 4 + (blockCount * 2) + (blockCount * 4);
var typesLength = outRead.BaseStream.Length - outRead.BaseStream.Position - typesArrayEnd;
typesArray = new byte[typesLength];
for (var i = 0; i < typesLength; i++)
{
typesArray[i] = outRead.ReadByte();
}
if (blockCount == 0)
{
if (outRead.ReadUInt32() != 0xFFEEDD00)
{
throw new InvalidDataException("Invalid trailer");
}
// Move back to the start of the KV data for reading.
outRead.BaseStream.Position = kvDataOffset;
Data = ParseBinaryKV3(outRead, null, true);
return;
}
uncompressedBlockLengthArray = new int[blockCount];
for (var i = 0; i < blockCount; i++)
{
uncompressedBlockLengthArray[i] = outRead.ReadInt32();
}
if (outRead.ReadUInt32() != 0xFFEEDD00)
{
throw new InvalidDataException("Invalid trailer");
}
try
{
using var uncompressedBlocks = new MemoryStream(blockTotalSize);
using var uncompressedBlockDataWriter = new BinaryWriter(uncompressedBlocks);
uncompressedBlockDataReader = new BinaryReader(uncompressedBlocks);
// TODO: Do we need to pass blockTotalSize here?
var lz4decoder = new LZ4ChainDecoder(blockTotalSize, 0);
for (var i = 0; i < blockCount; i++)
{
var compressedBlockLength = outRead.ReadUInt16();
var input = new Span <byte>(new byte[compressedBlockLength]);
var output = new Span <byte>(new byte[uncompressedBlockLengthArray[i]]);
RawBinaryReader.Read(input);
lz4decoder.DecodeAndDrain(input, output, out _);
uncompressedBlockDataWriter.Write(output);
}
uncompressedBlocks.Position = 0;
// Move back to the start of the KV data for reading.
outRead.BaseStream.Position = kvDataOffset;
Data = ParseBinaryKV3(outRead, null, true);
}
finally
{
uncompressedBlockDataReader.Dispose();
}
}
private void ReadVersion3(BinaryReader reader, BinaryWriter outWrite, BinaryReader outRead)
{
Format = new Guid(reader.ReadBytes(16));
var compressionMethod = reader.ReadUInt32();
var compressionDictionaryId = reader.ReadUInt16();
var compressionFrameSize = reader.ReadUInt16();
var countOfBinaryBytes = reader.ReadUInt32(); // how many bytes (binary blobs)
var countOfIntegers = reader.ReadUInt32(); // how many 4 byte values (ints)
var countOfEightByteValues = reader.ReadUInt32(); // how many 8 byte values (doubles)
// 8 bytes that help valve preallocate, useless for us
var stringAndTypesBufferSize = reader.ReadUInt32();
var b = reader.ReadUInt16();
var c = reader.ReadUInt16();
var uncompressedSize = reader.ReadUInt32();
var compressedSize = reader.ReadInt32(); // uint32
var blockCount = reader.ReadUInt32();
var blockTotalSize = reader.ReadInt32(); // uint32
if (compressionMethod == 0)
{
if (compressionDictionaryId != 0)
{
throw new UnexpectedMagicException("Unhandled", compressionDictionaryId, nameof(compressionDictionaryId));
}
if (compressionFrameSize != 0)
{
throw new UnexpectedMagicException("Unhandled", compressionFrameSize, nameof(compressionFrameSize));
}
var output = new Span <byte>(new byte[compressedSize]);
reader.Read(output);
outWrite.Write(output);
}
else if (compressionMethod == 1)
{
if (compressionDictionaryId != 0)
{
throw new UnexpectedMagicException("Unhandled", compressionDictionaryId, nameof(compressionDictionaryId));
}
if (compressionFrameSize != 16384)
{
throw new UnexpectedMagicException("Unhandled", compressionFrameSize, nameof(compressionFrameSize));
}
var input = reader.ReadBytes(compressedSize);
var output = new Span <byte>(new byte[uncompressedSize]);
LZ4Codec.Decode(input, output);
outWrite.Write(output);
outWrite.BaseStream.Position = 0;
}
else if (compressionMethod == 2)
{
if (compressionDictionaryId != 0)
{
throw new UnexpectedMagicException("Unhandled", compressionDictionaryId, nameof(compressionDictionaryId));
}
if (compressionFrameSize != 0)
{
throw new UnexpectedMagicException("Unhandled", compressionFrameSize, nameof(compressionFrameSize));
}
var input = reader.ReadBytes(compressedSize);
var zstd = new ZstdSharp.Decompressor();
var output = zstd.Unwrap(input);
outWrite.Write(output);
outWrite.BaseStream.Position = 0;
}
else
{
throw new UnexpectedMagicException("Unknown compression method", compressionMethod, nameof(compressionMethod));
}
currentBinaryBytesOffset = 0;
outRead.BaseStream.Position = countOfBinaryBytes;
if (outRead.BaseStream.Position % 4 != 0)
{
// Align to % 4 after binary blobs
outRead.BaseStream.Position += 4 - (outRead.BaseStream.Position % 4);
}
var countOfStrings = outRead.ReadInt32();
var kvDataOffset = outRead.BaseStream.Position;
// Subtract one integer since we already read it (countOfStrings)
outRead.BaseStream.Position += (countOfIntegers - 1) * 4;
if (outRead.BaseStream.Position % 8 != 0)
{
// Align to % 8 for the start of doubles
outRead.BaseStream.Position += 8 - (outRead.BaseStream.Position % 8);
}
currentEightBytesOffset = outRead.BaseStream.Position;
outRead.BaseStream.Position += countOfEightByteValues * 8;
var stringArrayStartPosition = outRead.BaseStream.Position;
stringArray = new string[countOfStrings];
for (var i = 0; i < countOfStrings; i++)
{
stringArray[i] = outRead.ReadNullTermString(System.Text.Encoding.UTF8);
}
var typesLength = stringAndTypesBufferSize - (outRead.BaseStream.Position - stringArrayStartPosition);
typesArray = new byte[typesLength];
for (var i = 0; i < typesLength; i++)
{
typesArray[i] = outRead.ReadByte();
}
if (blockCount == 0)
{
var noBlocksTrailer = outRead.ReadUInt32();
if (noBlocksTrailer != 0xFFEEDD00)
{
throw new UnexpectedMagicException("Invalid trailer", noBlocksTrailer, nameof(noBlocksTrailer));
}
// Move back to the start of the KV data for reading.
outRead.BaseStream.Position = kvDataOffset;
Data = ParseBinaryKV3(outRead, null, true);
return;
}
uncompressedBlockLengthArray = new int[blockCount];
for (var i = 0; i < blockCount; i++)
{
uncompressedBlockLengthArray[i] = outRead.ReadInt32();
}
var trailer = outRead.ReadUInt32();
if (trailer != 0xFFEEDD00)
{
throw new UnexpectedMagicException("Invalid trailer", trailer, nameof(trailer));
}
try
{
using var uncompressedBlocks = new MemoryStream(blockTotalSize);
uncompressedBlockDataReader = new BinaryReader(uncompressedBlocks);
if (compressionMethod == 0)
{
for (var i = 0; i < blockCount; i++)
{
RawBinaryReader.BaseStream.CopyTo(uncompressedBlocks, uncompressedBlockLengthArray[i]);
}
}
else if (compressionMethod == 1)
{
// TODO: Do we need to pass blockTotalSize here?
using var lz4decoder = new LZ4ChainDecoder(blockTotalSize, 0);
for (var i = 0; i < blockCount; i++)
{
var compressedBlockLength = outRead.ReadUInt16();
var input = new Span <byte>(new byte[compressedBlockLength]);
var output = new Span <byte>(new byte[uncompressedBlockLengthArray[i]]);
RawBinaryReader.Read(input);
lz4decoder.DecodeAndDrain(input, output, out _);
uncompressedBlocks.Write(output);
}
}
else if (compressionMethod == 2)
{
// This is supposed to be a streaming decompress using ZSTD_decompressStream,
// but as it turns out, zstd unwrap above already decompressed all of the blocks for us,
// so all we need to do is just copy the buffer.
// It's possible that Valve's code needs extra decompress because they set ZSTD_d_stableOutBuffer parameter.
outRead.BaseStream.CopyTo(uncompressedBlocks);
}
else
{
throw new UnexpectedMagicException("Unimplemented compression method in block decoder", compressionMethod, nameof(compressionMethod));
}
uncompressedBlocks.Position = 0;
// Move back to the start of the KV data for reading.
outRead.BaseStream.Position = kvDataOffset;
Data = ParseBinaryKV3(outRead, null, true);
}
finally
{
uncompressedBlockDataReader.Dispose();
}
}
