public unsafe void CouldSerializeRegularSortedMapWithZstd()
{
var rng = new Random();
var dest = (Memory <byte>) new byte[1000000];
var buffer = dest;
var handle = buffer.Pin();
var ptr = (IntPtr)handle.Pointer;
var sm = new SortedMap <DateTime, decimal>();
for (var i = 0; i < 1000; i++)
{
sm.Add(DateTime.Today.AddSeconds(i), (decimal)Math.Round(i + rng.NextDouble(), 2));
}
var sizeOf = BinarySerializer.SizeOf(sm, out var tmp);
var written = BinarySerializer.Write(sm, dest.Span, tmp);
Assert.AreEqual(sizeOf, written);
Console.WriteLine($"Useful: {sm.Count * 24}");
Console.WriteLine($"Total: {written}");
// NB interesting that with converting double to decimal savings go from 65% to 85%,
// even calculated from (8+8) base size not decimal's 16 size
Console.WriteLine($"Savings: {1.0 - ((written * 1.0) / (sm.Count * 24.0))}");
SortedMap <DateTime, decimal> sm2 = null;
var len2 = BinarySerializer.Read(buffer.Span, out sm2);
Assert.AreEqual(written, len2);
Assert.IsTrue(sm2.Keys.SequenceEqual(sm.Keys));
Assert.IsTrue(sm2.Values.SequenceEqual(sm.Values));
}
public unsafe void CouldSerializeSortedMap2()
{
var rng = new Random();
var dest = (Memory <byte>) new byte[1000000];
var buffer = dest;
var handle = buffer.Pin();
var ptr = (IntPtr)handle.Pointer;
var sm = new SortedMap <int, int>();
for (var i = 0; i < 10000; i++)
{
sm.Add(i, i);
}
var len = BinarySerializer.SizeOf(sm, out var temp);
var len2 = BinarySerializer.Write(sm, buffer.Span, temp);
Assert.AreEqual(len, len2);
Console.WriteLine($"Useful: {sm.Count * 8}");
Console.WriteLine($"Total: {len}");
// NB interesting that with converting double to decimal savings go from 65% to 85%,
// even calculated from (8+8) base size not decimal's 16 size
Console.WriteLine($"Savings: {1.0 - ((len * 1.0) / (sm.Count * 8.0))}");
SortedMap <int, int> sm2 = null;
var len3 = BinarySerializer.Read(buffer.Span, out sm2);
Assert.AreEqual(len, len3);
Assert.IsTrue(sm2.Keys.SequenceEqual(sm.Keys));
Assert.IsTrue(sm2.Values.SequenceEqual(sm.Values));
}
public void CouldSerializeOrderBag()
{
QuoteDecimal[] bids = new QuoteDecimal[10000];
for (int i = 0; i < bids.Length; i++)
{
bids[i] = new QuoteDecimal(new SmallDecimal(15000000.45, 8), new SmallDecimal((double)i, 8));
}
QuoteDecimal[] asks = new QuoteDecimal[2];
asks[0] = new QuoteDecimal(new SmallDecimal(123.89, 2), new SmallDecimal(0.00159748M, 8));
asks[1] = new QuoteDecimal(new SmallDecimal(123.99M, 2), new SmallDecimal(0.00128948M, 8));
var ob = new OrderBagDecimal(bids, asks);
var size = BinarySerializer.SizeOf(ob, out var segment, SerializationFormat.Json);
var rm = BufferPool.Retain(size);
var written = BinarySerializer.Write(ob, rm.Span, segment, SerializationFormat.Json);
Assert.AreEqual(size, written);
var str = JsonSerializer.ToJsonString(ob);
var ob2 = JsonSerializer.Deserialize <OrderBagDecimal>(str);
Console.WriteLine(str);
// var ob2 = JsonSerializer.Deserialize<OrderBagX>(str);
rm.Dispose();
}
public unsafe int SizeOf(TElement[] value, int valueOffset, int valueCount, out MemoryStream temporaryStream,
CompressionMethod compression = CompressionMethod.DefaultOrNone)
{
if (ItemSize > 0)
{
var maxSize = 8 + (16 + BloscMethods.ProcessorCount * 4) + ItemSize * valueCount;
var buffer = BufferPool <byte> .Rent(maxSize);
fixed(byte *ptr = &buffer[0])
{
var directBuffer = new DirectBuffer(maxSize, ptr);
var totalSize = Write(value, valueOffset, valueCount, ref directBuffer, 0, null, compression);
temporaryStream = new RentedMemoryStream(buffer, 0, totalSize);
return(totalSize);
}
}
else
{
// compress bytes array
MemoryStream tempStream;
var segment = new ArraySegment <TElement>(value, valueOffset, valueCount);
var bytesSize = BinarySerializer.SizeOf(segment, out tempStream, compression);
var buffer = BufferPool <byte> .Rent(bytesSize);
var writtenBytes = BinarySerializer.Write(segment, buffer, 0, tempStream);
tempStream?.Dispose();
Debug.Assert(bytesSize == writtenBytes);
var size = CompressedArrayBinaryConverter <byte> .Instance.SizeOf(buffer, 0, writtenBytes, out temporaryStream, compression);
BufferPool <byte> .Return(buffer);
return(size);
}
}
public void CouldSerializeDateTimeArrayAsJson()
{
// not compressed even when requested
var formats = new[]
{
SerializationFormat.Json,
SerializationFormat.JsonGZip,
SerializationFormat.JsonLz4,
SerializationFormat.JsonZstd
};
var lens = new[] { 1, 2, 10, 20, 50, 100, 200, 300, 400, 500, 600, 700, 1000, 10000 };
var rng = new Random(42);
foreach (var len in lens)
{
var dta = new DateTime[len];
for (int l = 0; l < len; l++)
{
dta[l] = DateTime.Today.ToUniversalTime().AddMinutes(l).AddMilliseconds(rng.Next(0, 1000));
}
var uncompressed = 0.0;
Console.WriteLine("---------------");
foreach (var serializationFormat in formats)
{
var sizeOf = BinarySerializer.SizeOf(dta, out var segment, serializationFormat);
var bytes = BufferPool <byte> .Rent(4 + sizeOf);
var written = BinarySerializer.Write(dta, bytes, segment, serializationFormat);
if (serializationFormat == SerializationFormat.Json)
{
uncompressed = written;
}
Console.WriteLine(
$"len: {len}, format: {serializationFormat}, written bytes: {written}, ratio: {Math.Round(uncompressed / written, 2)}");
if (sizeOf != written)
{
Assert.Fail($"sizeOf {sizeOf} != written {written}");
}
DateTime[] dta2 = null;
var read = BinarySerializer.Read(bytes, out dta2);
Assert.AreEqual(written, read);
Assert.IsTrue(dta.SequenceEqual(dta2));
}
}
}
public void KnownScalarsVariantHeader()
{
TestType <sbyte>(TypeEnum.Int8);
TestType <short>(TypeEnum.Int16);
TestType <int>(TypeEnum.Int32);
TestType <long>(TypeEnum.Int64);
TestType <byte>(TypeEnum.UInt8);
TestType <ushort>(TypeEnum.UInt16);
TestType <uint>(TypeEnum.UInt32);
TestType <ulong>(TypeEnum.UInt64);
TestType <float>(TypeEnum.Float32);
TestType <double>(TypeEnum.Float64);
TestType <decimal>(TypeEnum.Decimal);
TestType <SmallDecimal>(TypeEnum.SmallDecimal);
TestType <bool>(TypeEnum.Bool);
TestType <char>(TypeEnum.Utf16Char);
TestType <UUID>(TypeEnum.UUID);
TestType <DateTime>(TypeEnum.DateTime);
TestType <Timestamp>(TypeEnum.Timestamp);
TestType <Symbol>(TypeEnum.Symbol);
TestType <Symbol32>(TypeEnum.Symbol32);
TestType <Symbol64>(TypeEnum.Symbol64);
TestType <Symbol128>(TypeEnum.Symbol128);
TestType <Symbol256>(TypeEnum.Symbol256);
void TestType <T>(TypeEnum expectedTe)
{
var h = TypeEnumHelper <T> .DataTypeHeader;
Assert.AreEqual(expectedTe, h.TEOFS.TypeEnum);
Assert.IsTrue(h.TEOFS1 == default && h.TEOFS2 == default);
Assert.IsTrue(h.IsScalar);
var teofs = new TypeEnumOrFixedSize(expectedTe);
Assert.AreEqual(BinarySerializer.SizeOf <T>(default(T), out _, SerializationFormat.Binary), DataTypeHeader.Size + teofs.Size);
Assert.AreEqual(Unsafe.SizeOf <T>(), h.TEOFS.Size);
Assert.AreEqual(Unsafe.SizeOf <T>(), TypeEnumHelper <T> .FixedSize);
if (typeof(T) != typeof(DateTime))
{
Assert.AreEqual(TypeHelper <T> .PinnedSize, TypeEnumHelper <T> .FixedSize);
}
}
}
public void CouldSerializeSortedMapWithStrings()
{
var rng = new Random();
var dest = (Memory <byte>) new byte[10000000];
var buffer = dest;
var handle = buffer.Pin();
var ptr = (IntPtr)handle.Pointer;
var valueLens = 0;
var sm = new MutableSeries <int, string>();
for (var i = 0; i < 100000; i++)
{
var str = i.ToString();
valueLens += str.Length;
sm.Add(i, str);
}
var len = BinarySerializer.SizeOf(sm, out var temp);
var len2 = BinarySerializer.Write(sm, buffer.Span, temp);
Assert.AreEqual(len, len2);
var usefulLen = ((int)sm.RowCount * 4) + valueLens;
Console.WriteLine($"Useful: {usefulLen}");
Console.WriteLine($"Total: {len}");
// NB interesting that with converting double to decimal savings go from 65% to 85%,
// even calculated from (8+8) base size not decimal's 16 size
Console.WriteLine($"Savings: {1.0 - ((len * 1.0) / (usefulLen))}");
Series <int, string> sm2 = null;
var len3 = BinarySerializer.Read(buffer.Span, out sm2);
Assert.AreEqual(len, len3);
Assert.IsTrue(sm2.Keys.SequenceEqual(sm.Keys));
Assert.IsTrue(sm2.Values.SequenceEqual(sm.Values));
}
public unsafe int SizeOf(TElement[] value, int valueOffset, int valueCount, out MemoryStream temporaryStream,
CompressionMethod compression = CompressionMethod.DefaultOrNone)
{
if (ItemSize > 0)
{
var maxSize = 8 + (16 + BloscMethods.ProcessorCount * 4) + ItemSize * valueCount;
var ownedBuffer = BufferPool <byte> .RentOwnedBuffer(maxSize);
ownedBuffer.AddReference();
var buffer = ownedBuffer.Buffer;
var totalSize = Write(value, valueOffset, valueCount, ref buffer, 0, null, compression);
temporaryStream = new RentedBufferStream(ownedBuffer, totalSize);
return(totalSize);
}
else if (Buffers.BufferPool.IsPreservedBuffer <TElement>())
{
throw new NotImplementedException();
}
else
{
// compress bytes array
MemoryStream tempStream;
var segment = new ArraySegment <TElement>(value, valueOffset, valueCount);
var bytesSize = BinarySerializer.SizeOf(segment, out tempStream, compression);
var buffer = BufferPool <byte> .Rent(bytesSize);
var writtenBytes = BinarySerializer.Write(segment, buffer, 0, tempStream);
tempStream?.Dispose();
Debug.Assert(bytesSize == writtenBytes);
var size = CompressedArrayBinaryConverter <byte> .Instance.SizeOf(buffer, 0, writtenBytes, out temporaryStream, compression);
BufferPool <byte> .Return(buffer);
return(size);
}
}
public unsafe int Write(TElement[] value, int valueOffset, int valueCount, ref Buffer <byte> destination,
uint destinationOffset = 0u, MemoryStream temporaryStream = null,
CompressionMethod compression = CompressionMethod.DefaultOrNone)
{
// NB Blosc calls below are visually large - many LOCs with comments, but this is only a single method call
if (value == null)
{
throw new ArgumentNullException(nameof(value));
}
var handle = destination.Pin();
try
{
var ptr = (IntPtr)handle.PinnedPointer + (int)destinationOffset;
if (temporaryStream != null)
{
var len = temporaryStream.Length;
if (destination.Length < destinationOffset + len)
{
return((int)BinaryConverterErrorCode.NotEnoughCapacity);
}
temporaryStream.WriteToPtr(ptr);
temporaryStream.Dispose();
return(checked ((int)len));
}
bool isDiffable = false;
var compressionMethod = compression == CompressionMethod.DefaultOrNone
? BloscSettings.defaultCompressionMethod
: (compression == CompressionMethod.LZ4 ? "lz4" : "zstd");
var position = 8;
if (valueCount > 0)
{
int compressedSize;
if (ItemSize > 0)
{
if (typeof(TElement) == typeof(DateTime))
{
var buffer = BufferPool <byte> .Rent(valueCount * 8);
var dtArray = (DateTime[])(object)value;
fixed(byte *srcPtr = &buffer[0])
{
for (var i = 0; i < valueCount; i++)
{
*(DateTime *)(srcPtr + i * 8) = dtArray[i + valueOffset];
}
compressedSize = BloscMethods.blosc_compress_ctx(
new IntPtr(9), // max compression 9
new IntPtr(1), // do byte shuffle 1
new UIntPtr((uint)ItemSize), // type size
new UIntPtr((uint)(valueCount * ItemSize)), // number of input bytes
(IntPtr)srcPtr,
ptr + position, // destination
new UIntPtr((uint)(destination.Length - position)), // destination length
compressionMethod,
new UIntPtr((uint)0), // default block size
BloscMethods.ProcessorCount //
);
}
BufferPool <byte> .Return(buffer);
}
else if (value[0] is IDiffable <TElement> diffableFirst)
{
isDiffable = true;
// TODO (!) this is probably inefficient... some generic method caching or dynamic dispatch?
// however there is only a single pattern match with IDiffable boxing
var first = value[0];
var buffer = BufferPool <byte> .Rent(valueCount *ItemSize);
fixed(byte *srcPtr = &buffer[0])
{
Unsafe.Write(srcPtr, first);
for (var i = 1; i < valueCount; i++)
{
var current = value[i];
var diff = diffableFirst.GetDelta(current);
Unsafe.Write(srcPtr + i * ItemSize, diff);
}
compressedSize = BloscMethods.blosc_compress_ctx(
new IntPtr(9), // max compression 9
new IntPtr(1), // do byte shuffle 1
new UIntPtr((uint)ItemSize), // type size
new UIntPtr((uint)(valueCount * ItemSize)), // number of input bytes
(IntPtr)srcPtr,
ptr + position, // destination
new UIntPtr((uint)(destination.Length - position)), // destination length
compressionMethod,
new UIntPtr((uint)0), // default block size
BloscMethods.ProcessorCount //
);
}
BufferPool <byte> .Return(buffer);
}
else
{
var pinnedArray = GCHandle.Alloc(value, GCHandleType.Pinned);
var srcPtr = Marshal.UnsafeAddrOfPinnedArrayElement(value, valueOffset);
compressedSize = BloscMethods.blosc_compress_ctx(
new IntPtr(9), // max compression 9
new IntPtr(1), // do byte shuffle 1
new UIntPtr((uint)ItemSize), // type size
new UIntPtr((uint)(valueCount * ItemSize)), // number of input bytes
srcPtr,
ptr + position, // destination
new UIntPtr((uint)(destination.Length - position)), // destination length
compressionMethod,
new UIntPtr((uint)0), // default block size
BloscMethods.ProcessorCount //
);
pinnedArray.Free();
}
}
else if (Buffers.BufferPool.IsPreservedBuffer <TElement>())
{
throw new NotImplementedException();
}
else
{
MemoryStream tempStream;
var bytesSize =
BinarySerializer.SizeOf(new ArraySegment <TElement>(value, valueOffset, valueCount),
out tempStream, compression);
var buffer = BufferPool <byte> .Rent(bytesSize);
var writtenBytes =
BinarySerializer.Write(new ArraySegment <TElement>(value, valueOffset, valueCount), buffer, 0,
tempStream);
tempStream?.Dispose();
Debug.Assert(bytesSize == writtenBytes);
compressedSize = CompressedArrayBinaryConverter <byte> .Instance.Write(buffer, 0, writtenBytes,
ref destination, destinationOffset, null, compression);
BufferPool <byte> .Return(buffer);
}
if (compressedSize > 0)
{
position += compressedSize;
}
else
{
return((int)BinaryConverterErrorCode.NotEnoughCapacity);
}
}
// length
Marshal.WriteInt32(ptr, position);
// version & flags
Marshal.WriteByte(ptr + 4, (byte)((Version << 4) | (isDiffable ? 0b0000_0011 : 0b0000_0001)));
return(position);
}
finally
{
handle.Free();
}
}
public unsafe int Write(TElement[] value, int valueOffset, int valueCount, ref Memory <byte> destination,
uint destinationOffset = 0u, MemoryStream temporaryStream = null,
CompressionMethod compression = CompressionMethod.DefaultOrNone)
{
// NB Blosc calls below are visually large - many LOCs with comments, but this is only a single method call
if (value == null)
{
throw new ArgumentNullException(nameof(value));
}
var handle = destination.Retain(true);
try
{
var ptr = (IntPtr)handle.PinnedPointer + (int)destinationOffset;
if (temporaryStream != null)
{
var len = temporaryStream.Length;
if (destination.Length < destinationOffset + len)
{
return((int)BinaryConverterErrorCode.NotEnoughCapacity);
}
temporaryStream.WriteToPtr(ptr);
temporaryStream.Dispose();
return(checked ((int)len));
}
var compressionMethod = compression == CompressionMethod.DefaultOrNone
? BloscSettings.defaultCompressionMethod
: (compression == CompressionMethod.LZ4 ? "lz4" : "zstd");
var isDelta = IsIDelta;
var position = 8;
if (valueCount > 0)
{
int compressedSize;
if (ItemSize > 0)
{
if (typeof(TElement) == typeof(DateTime))
{
isDelta = true;
Trace.Assert(ItemSize == 8);
var buffer = BufferPool <byte> .Rent(valueCount * 8);
var dtArray = (DateTime[])(object)value;
var first = dtArray[valueOffset];
// NB For DateTime we calculate delta not from the first but
// from the previous value. This is a special case for the
// fact that DT[] is usually increasing by a similar (regular) step
// and the deltas are always positive, small and close to each other.
// In contrast, Price/Decimal could fluctuate in a small range
// and delta from previous could often change its sign, which
// leads to a very different bits and significantly reduces
// the Blosc shuffling benefits. For stationary time series
// deltas from the first value are also stationary and their sign
// changes less frequently that the sign of deltas from previous.
var previousLong = (long *)&first;
fixed(byte *srcPtr = &buffer[0])
{
Unsafe.WriteUnaligned(srcPtr, *previousLong);
for (var i = 1; i < valueCount; i++)
{
var current = dtArray[i + valueOffset];
var currentLong = (long *)(¤t);
var diff = currentLong - previousLong;
Unsafe.WriteUnaligned(srcPtr + i * ItemSize, diff);
previousLong = currentLong;
}
compressedSize = BloscMethods.blosc_compress_ctx(
new IntPtr(9), // max compression 9
new IntPtr(1), // do byte shuffle 1
new UIntPtr((uint)ItemSize), // type size
new UIntPtr((uint)(valueCount * ItemSize)), // number of input bytes
(IntPtr)srcPtr,
ptr + position, // destination
new UIntPtr((uint)(destination.Length - position)), // destination length
compressionMethod,
new UIntPtr((uint)0), // default block size
BloscMethods.ProcessorCount //
);
}
BufferPool <byte> .Return(buffer);
}
else if (IsIDelta)
{
var first = value[valueOffset];
var buffer = BufferPool <byte> .Rent(valueCount *ItemSize);
fixed(byte *srcPtr = &buffer[0])
{
Unsafe.WriteUnaligned(srcPtr, first);
for (var i = 1; i < valueCount; i++)
{
var diff = Unsafe.GetDeltaConstrained(ref first, ref value[valueOffset + i]);
Unsafe.WriteUnaligned(srcPtr + i * ItemSize, diff);
}
compressedSize = BloscMethods.blosc_compress_ctx(
new IntPtr(9), // max compression 9
new IntPtr(1), // do byte shuffle 1
new UIntPtr((uint)ItemSize), // type size
new UIntPtr((uint)(valueCount * ItemSize)), // number of input bytes
(IntPtr)srcPtr,
ptr + position, // destination
new UIntPtr((uint)(destination.Length - position)), // destination length
compressionMethod,
new UIntPtr((uint)0), // default block size
BloscMethods.ProcessorCount //
);
}
BufferPool <byte> .Return(buffer);
}
else
{
var pinnedArray = GCHandle.Alloc(value, GCHandleType.Pinned);
var srcPtr = Marshal.UnsafeAddrOfPinnedArrayElement(value, valueOffset);
compressedSize = BloscMethods.blosc_compress_ctx(
new IntPtr(9), // max compression 9
new IntPtr(1), // do byte shuffle 1
new UIntPtr((uint)ItemSize), // type size
new UIntPtr((uint)(valueCount * ItemSize)), // number of input bytes
srcPtr,
ptr + position, // destination
new UIntPtr((uint)(destination.Length - position)), // destination length
compressionMethod,
new UIntPtr((uint)0), // default block size
BloscMethods.ProcessorCount //
);
pinnedArray.Free();
}
}
else if (Buffers.BufferPool.IsPreservedBuffer <TElement>())
{
throw new NotImplementedException();
}
else
{
MemoryStream tempStream;
var bytesSize =
BinarySerializer.SizeOf(new ArraySegment <TElement>(value, valueOffset, valueCount),
out tempStream, compression);
var buffer = BufferPool <byte> .Rent(bytesSize);
var writtenBytes =
BinarySerializer.Write(new ArraySegment <TElement>(value, valueOffset, valueCount), buffer, 0,
tempStream);
tempStream?.Dispose();
Debug.Assert(bytesSize == writtenBytes);
compressedSize = CompressedArrayBinaryConverter <byte> .Instance.Write(buffer, 0, writtenBytes,
ref destination, destinationOffset, null, compression);
BufferPool <byte> .Return(buffer);
}
if (compressedSize > 0)
{
position += compressedSize;
}
else
{
return((int)BinaryConverterErrorCode.NotEnoughCapacity);
}
}
// length
Marshal.WriteInt32(ptr, position);
// version & flags
Marshal.WriteByte(ptr + 4, (byte)((Version << 4) | (isDelta ? 0b0000_0011 : 0b0000_0001)));
return(position);
}
finally
{
handle.Dispose();
}
}
