private Record ReadFragments(Stream stream)
{
// Blocks may be padded if size left is less than the header
int sizeLeft = (int)(BlockSize - stream.Position % BlockSize);
if (sizeLeft < 7)
{
stream.Seek(sizeLeft, SeekOrigin.Current);
}
// Header is checksum (4 bytes), length (2 bytes), type (1 byte).
byte[] header = new byte[4 + 2 + 1];
if (stream.Read(header, 0, header.Length) != header.Length)
{
return(new Record(LogRecordType.BadRecord));
}
uint expectedCrc = BitConverter.ToUInt32(header, 0);
ushort length = BitConverter.ToUInt16(header, 4);
byte type = header[6];
if (length > stream.Length - stream.Position)
{
throw new Exception("Not enough data in stream to read");
}
byte[] data = new byte[length];
int read = stream.Read(data, 0, data.Length);
uint actualCrc = Crc32C.Compute(type);
actualCrc = Crc32C.Mask(Crc32C.Append(actualCrc, data));
Record rec = new Record()
{
Checksum = expectedCrc,
Length = length,
LogRecordType = (LogRecordType)type,
Data = data
};
if (rec.LogRecordType != LogRecordType.Zero && expectedCrc != actualCrc)
{
throw new InvalidDataException($"Corrupted data. Failed checksum test. Excpeted {expectedCrc}, but calculated actual {actualCrc}");
}
return(rec);
}
private static BlockHandle WriteBlock(Stream stream, BlockCreator blockCreator, bool forceNoCompression = false)
{
byte[] dataBlock = blockCreator.Finish();
if (dataBlock.Length == 0)
{
return(null);
}
// Compress here
//byte compressionType = 0; // none
//byte compressionType = 2; // zlib
//memStream.WriteByte(0x87);
//memStream.WriteByte(0x9C);
byte compressionType = 0; // none
if (!forceNoCompression !)
{
compressionType = 4; // zlib raw
using var memStream = new MemoryStream();
using var compStream = new DeflateStream(memStream, CompressionLevel.Fastest, true);
compStream.Write(dataBlock);
compStream.Flush();
compStream.Close();
dataBlock = memStream.ToArray();
}
uint checkCrc = Crc32C.Compute(dataBlock);
checkCrc = Crc32C.Mask(Crc32C.Append(checkCrc, compressionType));
long offset = stream.Position;
stream.Write(dataBlock);
stream.WriteByte(compressionType);
stream.Write(BitConverter.GetBytes(checkCrc));
stream.Flush();
return(new BlockHandle((ulong)offset, (ulong)dataBlock.Length));
}
internal static Crc32C Compute(byte[] buffer, int offset, int length) => Crc32C.Compute(buffer, offset, length);
private byte[] ReadBlock(Stream stream, ulong length, bool verifyChecksum)
{
// File format contains a sequence of blocks where each block has:
// block_data: uint8[n]
// type: uint8
// crc: uint32
verifyChecksum = verifyChecksum || Database.ParanoidMode;
var data = new byte[length];
stream.Seek((long)0, SeekOrigin.Begin);
stream.Read(data, 0, (int)length);
byte compressionType = (byte)stream.ReadByte();
byte[] checksum = new byte[4];
stream.Read(checksum, 0, checksum.Length);
uint crc = BitConverter.ToUInt32(checksum);
if (verifyChecksum)
{
uint checkCrc = Crc32C.Compute(data);
//checkCrc = Crc32C.Mask(Crc32C.Append(checkCrc, new[] { compressionType }));
checkCrc = Crc32C.Mask(Crc32C.Append(checkCrc, compressionType));
if (crc != checkCrc)
{
throw new InvalidDataException($"Corrupted data. Failed checksum test. expected={crc}, actual={checkCrc}");
}
}
if (compressionType == 0)
{
// uncompressed
}
else if (compressionType == 1)
{
// Snapp, i can't read that
throw new NotSupportedException("Can't read snappy compressed data");
}
else if (compressionType >= 2)
{
using (var dataStream = new MemoryStream(data))
{
if (compressionType == 2)
{
if (dataStream.ReadByte() != 0x78)
{
throw new InvalidDataException("Incorrect ZLib header. Expected 0x78 0xnn");
}
dataStream.ReadByte();
}
using (var defStream2 = new DeflateStream(dataStream, CompressionMode.Decompress))
{
// Get actual package out of bytes
using (var destination = new MemoryStream())
{
defStream2.CopyTo(destination);
data = destination.ToArray();
}
}
}
}
return(data);
}