private static uint ComputeCrc(byte[] data)
{
var crc = new Crc32C();
crc.Update(data, 0, data.Length);
return(crc.GetIntValue());
}
public byte[] ToByteArray()
{
using (MemoryStream ms = new MemoryStream())
using (BinaryWriter writer = new BinaryWriter(ms))
{
writer.Write((byte)(((byte)packetType) | (usingChecksum ? 0x80 : 0)));
writer.Write(retryCount);
writer.Write(from);
writer.Write(to);
writer.Write(channel);
writer.Write(fragment);
writer.Write(seq);
writer.Write(timestamp);
writer.Write(payload);
writer.Flush();
if (usingChecksum)
{
byte[] data = ms.ToArray();
Crc32C crc = new Crc32C();
crc.Update(data, 0, data.Length);
uint checksum = crc.GetIntValue();
writer.Write(checksum);
writer.Flush();
}
return(ms.ToArray());
}
}
public unsafe void Crc32CHashIncrementalBenchmark()
{
var rng = new Random(42);
var len = 8;
var offset = 0;
var mem = BufferPool.Retain(len + offset);
var ptr = (byte *)mem.Pointer + offset;
rng.NextBytes(mem.GetSpan().Slice(offset));
var count = 100_000_000;
var sum = 0UL;
var mod8 = (ulong)ptr % 8;
uint digest = 0;
using (Benchmark.Run("CRC32 Intrinsic", count))
{
for (int i = 0; i < count; i++)
{
digest = Crc32C.CalculateCrc32C(ptr, len, digest);
}
}
Benchmark.Dump();
Console.WriteLine(digest);
}
public override int GetHashCode()
{
int hash = 1;
if (Dtype != 0)
{
hash ^= Dtype.GetHashCode();
}
if (shape_ != null)
{
hash ^= Shape.GetHashCode();
}
if (ShardId != 0)
{
hash ^= ShardId.GetHashCode();
}
if (Offset != 0L)
{
hash ^= Offset.GetHashCode();
}
if (Size != 0L)
{
hash ^= Size.GetHashCode();
}
if (Crc32C != 0)
{
hash ^= Crc32C.GetHashCode();
}
hash ^= slices_.GetHashCode();
return(hash);
}
public void TestComposes()
{
var crc = new Crc32C();
crc.Update(Encoding.ASCII.GetBytes("hello "), 0, 6);
crc.Update(Encoding.ASCII.GetBytes("world"), 0, 5);
Assert.Equal(crc.GetIntValue(), ComputeCrc(Encoding.ASCII.GetBytes("hello world")));
}
public void TestMask()
{
var crc = new Crc32C();
crc.Update(Encoding.ASCII.GetBytes("foo"), 0, 3);
Assert.Equal(crc.GetMaskedValue(), Mask(crc.GetIntValue()));
Assert.False(crc.GetIntValue() == crc.GetMaskedValue(), "crc should not match masked crc");
Assert.False(crc.GetIntValue() == Mask(crc.GetMaskedValue()), "crc should not match double masked crc");
Assert.Equal(crc.GetIntValue(), Unmask(crc.GetMaskedValue()));
Assert.Equal(crc.GetIntValue(), Unmask(Unmask(Mask(crc.GetMaskedValue()))));
}
public void Calculate_GivenSequenceWithSingleSegment_ShouldReturnExpectedChecksum()
{
//Arrange
var segment = new byte[] { 0x10, 0x01, 0x18, 0xc9, 0xf8, 0x86, 0x94, 0xeb, 0x2c };
var sequence = new SequenceBuilder <byte>().Append(segment).Build();
//Act
var actual = Crc32C.Calculate(sequence);
//Assert
const uint expected = 2355953212;
Assert.Equal(expected, actual);
}
private Record ReadFragments(Stream stream)
{
// Blocks may be padded if size left is less than the header
int sizeLeft = (int)(BlockSize - stream.Position % BlockSize);
if (sizeLeft < 7)
{
stream.Seek(sizeLeft, SeekOrigin.Current);
}
// Header is checksum (4 bytes), length (2 bytes), type (1 byte).
byte[] header = new byte[4 + 2 + 1];
if (stream.Read(header, 0, header.Length) != header.Length)
{
return(new Record(LogRecordType.BadRecord));
}
uint expectedCrc = BitConverter.ToUInt32(header, 0);
ushort length = BitConverter.ToUInt16(header, 4);
byte type = header[6];
if (length > stream.Length - stream.Position)
{
throw new Exception("Not enough data in stream to read");
}
byte[] data = new byte[length];
int read = stream.Read(data, 0, data.Length);
uint actualCrc = Crc32C.Compute(type);
actualCrc = Crc32C.Mask(Crc32C.Append(actualCrc, data));
Record rec = new Record()
{
Checksum = expectedCrc,
Length = length,
LogRecordType = (LogRecordType)type,
Data = data
};
if (rec.LogRecordType != LogRecordType.Zero && expectedCrc != actualCrc)
{
throw new InvalidDataException($"Corrupted data. Failed checksum test. Excpeted {expectedCrc}, but calculated actual {actualCrc}");
}
return(rec);
}
public void Calculate_GivenSequenceWithMultipleSegments_ShouldReturnExpectedChecksum()
{
//Arrange
var s1 = new byte[] { 0x0a, 0x0f, 0x73, 0x74, 0x61, 0x6e, 0x64, 0x61, 0x6c, 0x6f, 0x6e, 0x65, 0x2d, 0x33, 0x30, 0x2d, 0x35, 0x10, 0x00, 0x18, 0xc7, 0xee, 0xa3, 0x93, 0xeb, 0x2c, 0x58, 0x01 };
var s2 = new byte[] { 0x10, 0x01, 0x18, 0xc9, 0xf8, 0x86, 0x94, 0xeb, 0x2c };
var sequence = new SequenceBuilder <byte>().Append(s1).Append(s2).Build();
//Act
var actual = Crc32C.Calculate(sequence);
//Assert
const uint expected = 1079987866;
Assert.Equal(expected, actual);
}
private static BlockHandle WriteBlock(Stream stream, BlockCreator blockCreator, bool forceNoCompression = false)
{
byte[] dataBlock = blockCreator.Finish();
if (dataBlock.Length == 0)
{
return(null);
}
// Compress here
//byte compressionType = 0; // none
//byte compressionType = 2; // zlib
//memStream.WriteByte(0x87);
//memStream.WriteByte(0x9C);
byte compressionType = 0; // none
if (!forceNoCompression !)
{
compressionType = 4; // zlib raw
using var memStream = new MemoryStream();
using var compStream = new DeflateStream(memStream, CompressionLevel.Fastest, true);
compStream.Write(dataBlock);
compStream.Flush();
compStream.Close();
dataBlock = memStream.ToArray();
}
uint checkCrc = Crc32C.Compute(dataBlock);
checkCrc = Crc32C.Mask(Crc32C.Append(checkCrc, compressionType));
long offset = stream.Position;
stream.Write(dataBlock);
stream.WriteByte(compressionType);
stream.Write(BitConverter.GetBytes(checkCrc));
stream.Flush();
return(new BlockHandle((ulong)offset, (ulong)dataBlock.Length));
}
public unsafe void Crc32CManagedHashIncrementalBenchmark2()
{
var rng = new Random(42);
var len = 15;
var mem = BufferPool.Retain(len);
var ptr = (byte *)mem.Pointer;
rng.NextBytes(mem.GetSpan());
var count = 1_000_000;
var sum = 0UL;
uint digest = 0;
using (Benchmark.Run("CRC32 Managed", count))
{
for (int i = 0; i < count; i++)
{
digest = Crc32C.CalculateCrc32CManaged(ptr, len, digest);
}
}
Benchmark.Dump();
Console.WriteLine(digest);
}
public static NetworkPacket FromByteArray(byte[] data)
{
NetworkPacket packet = new NetworkPacket();
using (MemoryStream ms = new MemoryStream(data, false))
using (BinaryReader reader = new BinaryReader(ms))
{
byte packetFlags = reader.ReadByte();
packet.usingChecksum = (packetFlags & 0x80) != 0;
packet.packetType = (PacketType)(byte)(packetFlags & ~0x80);
packet.retryCount = reader.ReadByte();
packet.from = reader.ReadByte();
packet.to = reader.ReadByte();
packet.channel = reader.ReadByte();
packet.fragment = reader.ReadUInt16();
packet.seq = reader.ReadUInt16();
packet.timestamp = reader.ReadUInt32();
if (packet.usingChecksum)
{
packet.payload = reader.ReadBytes(data.Length - 13 - 4);
uint checksum = reader.ReadUInt32();
Crc32C crc = new Crc32C();
crc.Update(data, 0, data.Length - 4);
uint validChecksum = crc.GetIntValue();
if (checksum != validChecksum)
{
throw new IOException("Invalid checksum! Got 0x" + checksum.ToString("X8") + ", should be 0x" + validChecksum.ToString("X8"));
}
}
else
{
packet.payload = reader.ReadBytes(data.Length - 13);
}
return(packet);
}
}
public unsafe void CRCBenchmark()
{
var lens = new int[] { 1, 2, 3, 4, 7, 8, 9, 15, 16, 31, 32, 63, 64, 255, 256, 511, 512, 1023, 1024 };
foreach (var len in lens)
{
var arr = new byte[len];
var memory = (Memory <byte>)arr;
var handle = memory.Pin();
var ptr = (byte *)handle.Pointer;
var rng = new Random(42);
rng.NextBytes(arr);
var count = 1_00_000_000;
var cnt = count / len;
var sum = 0UL;
{
uint digest = 0;
using (Benchmark.Run("xxHash", count, true))
{
for (int i = 0; i < cnt; i++)
{
digest = XxHash.CalculateHash(ptr, len, digest);
}
}
}
uint crc0, crc1, crc2;
{
uint digest = 0;
using (Benchmark.Run("CRC32 Intrinsic", count, true))
{
for (int i = 0; i < cnt; i++)
{
digest = Crc32C.CalculateCrc32C(ptr, len, digest);
}
}
crc0 = digest;
}
{
uint digest = 0;
using (Benchmark.Run("CRC32 Managed", count, true))
{
for (int i = 0; i < cnt; i++)
{
digest = Crc32C.CalculateCrc32CManaged(ptr, len, digest);
}
}
crc1 = digest;
}
{
var crc32C = new Ralph.Crc32C.Crc32C();
uint digest = 0;
using (Benchmark.Run("CRC32 Ralph", count, true))
{
for (int i = 0; i < cnt; i++)
{
crc32C.Update(arr, 0, arr.Length);
digest = crc32C.GetIntValue();
}
}
crc2 = digest;
}
Assert.AreEqual(crc0, crc1);
Assert.AreEqual(crc0, crc2);
Benchmark.Dump("LEN: " + len);
}
}
internal static Crc32C Compute(byte[] buffer, int offset, int length) => Crc32C.Compute(buffer, offset, length);
internal static bool Verify(byte[] buffer, int offset, int length) => Crc32C.Verify(buffer, offset, length);
public unsafe void CRCCopyBenchmark()
{
var lens = new int[] { 1, 2, 3, 4, 7, 8, 9, 15, 16, 31, 32, 63, 64, 255, 256, 511, 512, 1023, 1024 };
foreach (var len in lens)
{
var arr = new byte[len];
var memory = (Memory <byte>)arr;
var handle = memory.Pin();
var ptr = (byte *)handle.Pointer;
var arrDest = new byte[len];
var memoryDest = (Memory <byte>)arrDest;
var handleDest = memoryDest.Pin();
var copyTarget = (byte *)handleDest.Pointer;
var rng = new Random(42);
rng.NextBytes(arr);
var count = 1_000_000_000;
var cnt = count / len;
var sum = 0UL;
uint crc0, crc1, crc2, crc3;
{
uint digest = 0;
using (Benchmark.Run("CRC32+Copy Intrinsic", count, true))
{
for (int i = 0; i < cnt; i++)
{
digest = Crc32C.CopyWithCrc32C(ptr, len, copyTarget, digest);
}
}
crc0 = digest;
}
//{
// uint digest = 0;
// using (Benchmark.Run("CRC32 Only Intrinsic", count, true))
// {
// for (int i = 0; i < cnt; i++)
// {
// digest = Crc32C.CalculateCrc32C(ptr, len, digest);
// }
// }
// crc1 = digest;
//}
{
uint digest = 0;
using (Benchmark.Run("CRC32+Copy Manual", count, true))
{
for (int i = 0; i < cnt; i++)
{
digest = Crc32C.CopyWithCrc32CManual(ptr, len, copyTarget, digest);
}
}
crc3 = digest;
}
//{
// using (Benchmark.Run("Copy only", count, true))
// {
// for (int i = 0; i < cnt; i++)
// {
// System.Runtime.CompilerServices.Unsafe.CopyBlockUnaligned(copyTarget, ptr, (uint)len);
// }
// }
//}
//{
// var crc32C = new Ralph.Crc32C.Crc32C();
// uint digest = 0;
// using (Benchmark.Run("CRC32 Checksum Ralph", count, true))
// {
// for (int i = 0; i < cnt; i++)
// {
// crc32C.Update(arr, 0, arr.Length);
// digest = crc32C.GetIntValue();
// }
// }
// crc2 = digest;
//}
//Assert.AreEqual(crc0, crc1);
//Assert.AreEqual(crc0, crc2);
Assert.AreEqual(crc0, crc3);
Benchmark.Dump("LEN: " + len);
}
}
private static bool HasValidCheckSum(ReadOnlySequence <byte> input) => input.ReadUInt32(Constants.MagicNumber.Length, true) == Crc32C.Calculate(input.Slice(Constants.MetadataSizeOffset));
private byte[] ReadBlock(Stream stream, ulong length, bool verifyChecksum)
{
// File format contains a sequence of blocks where each block has:
// block_data: uint8[n]
// type: uint8
// crc: uint32
verifyChecksum = verifyChecksum || Database.ParanoidMode;
var data = new byte[length];
stream.Seek((long)0, SeekOrigin.Begin);
stream.Read(data, 0, (int)length);
byte compressionType = (byte)stream.ReadByte();
byte[] checksum = new byte[4];
stream.Read(checksum, 0, checksum.Length);
uint crc = BitConverter.ToUInt32(checksum);
if (verifyChecksum)
{
uint checkCrc = Crc32C.Compute(data);
//checkCrc = Crc32C.Mask(Crc32C.Append(checkCrc, new[] { compressionType }));
checkCrc = Crc32C.Mask(Crc32C.Append(checkCrc, compressionType));
if (crc != checkCrc)
{
throw new InvalidDataException($"Corrupted data. Failed checksum test. expected={crc}, actual={checkCrc}");
}
}
if (compressionType == 0)
{
// uncompressed
}
else if (compressionType == 1)
{
// Snapp, i can't read that
throw new NotSupportedException("Can't read snappy compressed data");
}
else if (compressionType >= 2)
{
using (var dataStream = new MemoryStream(data))
{
if (compressionType == 2)
{
if (dataStream.ReadByte() != 0x78)
{
throw new InvalidDataException("Incorrect ZLib header. Expected 0x78 0xnn");
}
dataStream.ReadByte();
}
using (var defStream2 = new DeflateStream(dataStream, CompressionMode.Decompress))
{
// Get actual package out of bytes
using (var destination = new MemoryStream())
{
defStream2.CopyTo(destination);
data = destination.ToArray();
}
}
}
}
return(data);
}
