public unsafe void CouldSerializeRegularSortedMapWithZstd()
{
var rng = new Random();
var dest = (Memory <byte>) new byte[1000000];
var buffer = dest;
var handle = buffer.Pin();
var ptr = (IntPtr)handle.Pointer;
var sm = new SortedMap <DateTime, decimal>();
for (var i = 0; i < 1000; i++)
{
sm.Add(DateTime.Today.AddSeconds(i), (decimal)Math.Round(i + rng.NextDouble(), 2));
}
var sizeOf = BinarySerializer.SizeOf(sm, out var tmp);
var written = BinarySerializer.Write(sm, dest.Span, tmp);
Assert.AreEqual(sizeOf, written);
Console.WriteLine($"Useful: {sm.Count * 24}");
Console.WriteLine($"Total: {written}");
// NB interesting that with converting double to decimal savings go from 65% to 85%,
// even calculated from (8+8) base size not decimal's 16 size
Console.WriteLine($"Savings: {1.0 - ((written * 1.0) / (sm.Count * 24.0))}");
SortedMap <DateTime, decimal> sm2 = null;
var len2 = BinarySerializer.Read(buffer.Span, out sm2);
Assert.AreEqual(written, len2);
Assert.IsTrue(sm2.Keys.SequenceEqual(sm.Keys));
Assert.IsTrue(sm2.Values.SequenceEqual(sm.Values));
}
public unsafe void CouldSerializeSortedMap2()
{
var rng = new Random();
var dest = (Memory <byte>) new byte[1000000];
var buffer = dest;
var handle = buffer.Pin();
var ptr = (IntPtr)handle.Pointer;
var sm = new SortedMap <int, int>();
for (var i = 0; i < 10000; i++)
{
sm.Add(i, i);
}
var len = BinarySerializer.SizeOf(sm, out var temp);
var len2 = BinarySerializer.Write(sm, buffer.Span, temp);
Assert.AreEqual(len, len2);
Console.WriteLine($"Useful: {sm.Count * 8}");
Console.WriteLine($"Total: {len}");
// NB interesting that with converting double to decimal savings go from 65% to 85%,
// even calculated from (8+8) base size not decimal's 16 size
Console.WriteLine($"Savings: {1.0 - ((len * 1.0) / (sm.Count * 8.0))}");
SortedMap <int, int> sm2 = null;
var len3 = BinarySerializer.Read(buffer.Span, out sm2);
Assert.AreEqual(len, len3);
Assert.IsTrue(sm2.Keys.SequenceEqual(sm.Keys));
Assert.IsTrue(sm2.Values.SequenceEqual(sm.Values));
}
public void CouldSerializeBlittableStructArray()
{
var bytes = new byte[1000];
var value = new BlittableStruct[2];
value[0] = new BlittableStruct
{
Value1 = 123,
Value2 = 1230
};
value[1] = new BlittableStruct
{
Value1 = 456,
Value2 = 4560
};
var len0 = BinarySerializer.SizeOf(in value, out var pl, SerializationFormat.Binary);
var len = BinarySerializer.Write(value, bytes, in pl, format: SerializationFormat.Binary);
Assert.AreEqual(8 + 4 + 12 * 2, len);
BlittableStruct[] arr2 = null;
var len2 = BinarySerializer.Read(bytes, out arr2);
Assert.AreEqual(len, len2);
Assert.IsTrue(value.SequenceEqual(arr2));
}
public void CouldSerializeBlittable <T>(T value) where T : IEquatable <T>
{
var bytes = BufferPool <byte> .Rent(1000);
var mem = (Memory <byte>)bytes;
var h = mem.Pin();
var db = new DirectBuffer(bytes.Length, (byte *)h.Pointer);
foreach (var format in Formats)
{
var val = value;
var len0 = BinarySerializer.SizeOf(in val, out var pl, format);
var len = BinarySerializer.Write(val, db, in pl, format);
// for (int x = 0; x < 100; x++)
{
var len2 = BinarySerializer.Read(db, out T val2);
if (len != len2)
{
Assert.Fail($"len {len} != len2 {len2}");
}
if (!val.Equals(val2))
{
Assert.Fail();
}
}
}
foreach (var format in Formats)
{
var val = value;
var len0 = BinarySerializer.SizeOf(in val, out var segment, format);
var len1 = BinarySerializer.Write(in val, db, segment, format);
if (len0 != len1)
{
Assert.Fail($"len0 {len0} != len1 {len1}");
}
//Assert.IsTrue(len1 > 8);
var len2 = BinarySerializer.Read(db, out T val2);
if (len1 != len2)
{
Assert.Fail($"len1 {len1} != len2 {len2}");
}
if (!val.Equals(val2))
{
Assert.Fail();
}
}
h.Dispose();
BufferPool <byte> .Return(bytes);
}
public unsafe void CouldSerializeTickDecimal()
{
var count = 1_00;
var rounds = 1;
var rm = BufferPool.Retain(512);
var db = new DirectBuffer(rm.Length, (byte *)rm.Pointer);
var formats = new[] { SerializationFormat.Binary, SerializationFormat.Json };
for (int r = 0; r < rounds; r++)
{
foreach (var serializationFormat in formats)
{
using (Benchmark.Run(serializationFormat.ToString(), count, true))
{
for (int i = 0; i < count; i++)
{
var tick = new TickDecimal((Timestamp)(long)(i + 1), new SmallDecimal((double)i, 8), new SmallDecimal((double)i, 8), i % 3 - 1, i);
var sizeOf = BinarySerializer.SizeOf(in tick, out var segment, serializationFormat);
if (serializationFormat == SerializationFormat.Binary && sizeOf != 32 + 4)
{
Assert.Fail("size != 32");
}
var written = BinarySerializer.Write(tick, db, segment, serializationFormat);
if (sizeOf != written)
{
Assert.Fail("size != written");
}
var readSource = db.Slice(0, written);
var consumed = BinarySerializer.Read <TickDecimal>(readSource, out var tick1);
if (consumed != written)
{
Assert.Fail("readSize != written");
}
if (!tick1.Equals(tick))
{
Assert.Fail("!tick1.Equals(tick)");
}
}
}
}
}
Benchmark.Dump();
var tick2 = new TickDecimal(TimeService.Default.CurrentTime, new SmallDecimal((double)123.45, 8), new SmallDecimal((double)56.789, 8), 123, 7894561253);
var str = JsonSerializer.ToJsonString(tick2);
Console.WriteLine(str);
rm.Dispose();
}
public void CouldSerializeDateTimeArrayAsJson()
{
// not compressed even when requested
var formats = new[]
{
SerializationFormat.Json,
SerializationFormat.JsonGZip,
SerializationFormat.JsonLz4,
SerializationFormat.JsonZstd
};
var lens = new[] { 1, 2, 10, 20, 50, 100, 200, 300, 400, 500, 600, 700, 1000, 10000 };
var rng = new Random(42);
foreach (var len in lens)
{
var dta = new DateTime[len];
for (int l = 0; l < len; l++)
{
dta[l] = DateTime.Today.ToUniversalTime().AddMinutes(l).AddMilliseconds(rng.Next(0, 1000));
}
var uncompressed = 0.0;
Console.WriteLine("---------------");
foreach (var serializationFormat in formats)
{
var sizeOf = BinarySerializer.SizeOf(dta, out var segment, serializationFormat);
var bytes = BufferPool <byte> .Rent(4 + sizeOf);
var written = BinarySerializer.Write(dta, bytes, segment, serializationFormat);
if (serializationFormat == SerializationFormat.Json)
{
uncompressed = written;
}
Console.WriteLine(
$"len: {len}, format: {serializationFormat}, written bytes: {written}, ratio: {Math.Round(uncompressed / written, 2)}");
if (sizeOf != written)
{
Assert.Fail($"sizeOf {sizeOf} != written {written}");
}
DateTime[] dta2 = null;
var read = BinarySerializer.Read(bytes, out dta2);
Assert.AreEqual(written, read);
Assert.IsTrue(dta.SequenceEqual(dta2));
}
}
}
public void CouldSerializeStringArray()
{
var bytes = new byte[1000];
var value = new string[2];
value[0] = "123";
value[1] = "456";
var len0 = BinarySerializer.SizeOf(in value, out var pl);
var len = BinarySerializer.Write(value, bytes, in pl);
string[] arr2 = null;
var len2 = BinarySerializer.Read(bytes, out arr2);
Assert.AreEqual(len, len2);
Assert.IsTrue(value.SequenceEqual(arr2));
}
public void CouldSerializeDateTimeArray()
{
var bytes = new byte[1000];
var dta = new DateTime[2];
dta[0] = DateTime.Today;
dta[1] = DateTime.Today.AddDays(1);
var len0 = BinarySerializer.SizeOf(in dta, out var pl, SerializationFormat.Binary);
var len = BinarySerializer.Write(dta, bytes, in pl, format: SerializationFormat.Binary);
Assert.AreEqual(8 + 4 + 8 * 2, len);
DateTime[] dta2 = null;
var len2 = BinarySerializer.Read(bytes, out dta2);
Assert.AreEqual(len, len2);
Assert.IsTrue(dta.SequenceEqual(dta2));
}
public void CouldSerializeDecimalArray()
{
Assert.AreEqual(16, TypeHelper <decimal> .FixedSize);
var bytes = new byte[1000];
var value = new decimal[2];
value[0] = 123;
value[1] = 456;
var len0 = BinarySerializer.SizeOf(in value, out var pl, SerializationFormat.Binary);
var len = BinarySerializer.Write(value, bytes, in pl, format: SerializationFormat.Binary);
Assert.AreEqual(8 + 4 + 16 * 2, len);
decimal[] decimals2 = null;
var len2 = BinarySerializer.Read(bytes, out decimals2);
Assert.IsTrue(value.SequenceEqual(decimals2));
Assert.AreEqual(len, len2);
}
public void CouldSerializeIntArray()
{
var bytes = new byte[1000];
var value = new int[2];
value[0] = 123;
value[1] = 456;
var len0 = BinarySerializer.SizeOf(in value, out var pl, SerializationFormat.Binary);
var len = BinarySerializer.Write(in value, bytes, in pl, format: SerializationFormat.Binary);
Assert.AreEqual(len0, len);
Assert.AreEqual(8 + 4 + 4 * 2, len);
int[] ints2 = null;
var len2 = BinarySerializer.Read(bytes, out ints2);
Assert.AreEqual(len, len2);
Assert.IsTrue(value.SequenceEqual(ints2));
}
public void CouldSerializeSortedMapWithStrings()
{
var rng = new Random();
var dest = (Memory <byte>) new byte[10000000];
var buffer = dest;
var handle = buffer.Pin();
var ptr = (IntPtr)handle.Pointer;
var valueLens = 0;
var sm = new MutableSeries <int, string>();
for (var i = 0; i < 100000; i++)
{
var str = i.ToString();
valueLens += str.Length;
sm.Add(i, str);
}
var len = BinarySerializer.SizeOf(sm, out var temp);
var len2 = BinarySerializer.Write(sm, buffer.Span, temp);
Assert.AreEqual(len, len2);
var usefulLen = ((int)sm.RowCount * 4) + valueLens;
Console.WriteLine($"Useful: {usefulLen}");
Console.WriteLine($"Total: {len}");
// NB interesting that with converting double to decimal savings go from 65% to 85%,
// even calculated from (8+8) base size not decimal's 16 size
Console.WriteLine($"Savings: {1.0 - ((len * 1.0) / (usefulLen))}");
Series <int, string> sm2 = null;
var len3 = BinarySerializer.Read(buffer.Span, out sm2);
Assert.AreEqual(len, len3);
Assert.IsTrue(sm2.Keys.SequenceEqual(sm.Keys));
Assert.IsTrue(sm2.Values.SequenceEqual(sm.Values));
}
public void CouldSerializePocoArray()
{
var bytes = new byte[1000];
var value = new SimplePoco[2];
value[0] = new SimplePoco
{
Value1 = 123,
Value2 = "1230"
};
value[1] = new SimplePoco
{
Value1 = 456,
Value2 = "4560"
};
var len0 = BinarySerializer.SizeOf(in value, out var pl);
var len = BinarySerializer.Write(value, bytes, in pl);
SimplePoco[] arr2 = null;
var len2 = BinarySerializer.Read(bytes, out arr2);
Assert.IsTrue(value.SequenceEqual(arr2), "Items are not equal");
Assert.AreEqual(len, len2);
}
public unsafe int Read(IntPtr ptr, out TElement[] value, out int length, bool exactSize = false)
{
var totalSize = Marshal.ReadInt32(ptr);
var versionFlag = Marshal.ReadByte(ptr + 4);
var version = (byte)(versionFlag >> 4);
var isDiffable = (versionFlag & 0b0000_0010) != 0;
var isCompressed = (versionFlag & 0b0000_0001) != 0;
if (!isCompressed)
{
throw new InvalidOperationException("Wrong compressed flag. CompressedArrayBinaryConverter.Read works only with compressed methods.");
}
if (version != Version)
{
throw new NotSupportedException($"CompressedBinaryConverter work only with version {Version}");
}
if (ItemSize <= 0)
{
// first decompress bytes
var size = CompressedArrayBinaryConverter <byte> .Instance.Read(ptr, out byte[] decompressedBytes, out length);
// NB the length is encoded in the header and is returned as a part of ArraySegment
Debug.Assert(length == BitConverter.ToInt32(decompressedBytes, 0));
// then deserialize
// NB the size of the array will be exact, BinarySerializer.Read does not support non-exact buffers
BinarySerializer.Read(decompressedBytes, out value);
BufferPool <byte> .Return(decompressedBytes);
return(size);
}
else if (Buffers.BufferPool.IsPreservedBuffer <TElement>())
{
throw new NotImplementedException();
}
else
{
if (totalSize <= 8 + 16)
{
value = EmptyArray <TElement> .Instance;
length = 0;
return(totalSize);
}
var source = ptr + 8;
// avoid additional P/Invoke call, read header directly
// https://github.com/Blosc/c-blosc/blob/master/README_HEADER.rst
var nbytes = *(int *)(source + 4);
#if DEBUG
var blocksize = *(int *)(source + 8);
var cbytes = *(int *)(source + 12);
var nbytes2 = new UIntPtr();
var cbytes2 = new UIntPtr();
var blocksize2 = new UIntPtr();
BloscMethods.blosc_cbuffer_sizes(source, ref nbytes2, ref cbytes2, ref blocksize2);
Debug.Assert(nbytes == nbytes2.ToUInt32());
Debug.Assert(cbytes == cbytes2.ToUInt32());
Debug.Assert(blocksize == blocksize2.ToUInt32());
#endif
var arraySize = nbytes / ItemSize;
// when caller provides an empty AS, it could require exact size, e.g. DateTimeArrayBinaryConverter
var array = BufferPool <TElement> .Rent(arraySize, exactSize);
length = arraySize;
value = array;
if (arraySize > 0)
{
if (typeof(TElement) == typeof(DateTime))
{
var buffer = BufferPool <byte> .Rent(arraySize * 8);
var dtArray = new DateTime[arraySize];
fixed(byte *tgtPtr = &buffer[0])
{
var destination = (IntPtr)tgtPtr;
var decompSize = BloscMethods.blosc_decompress_ctx(
source, destination, new UIntPtr((uint)nbytes), BloscMethods.ProcessorCount);
if (decompSize <= 0)
{
throw new ArgumentException("Invalid compressed input");
}
Debug.Assert(decompSize == nbytes);
for (var i = 0; i < arraySize; i++)
{
dtArray[i] = *(DateTime *)(destination + i * 8);
}
}
value = (TElement[])(object)(dtArray);
BufferPool <byte> .Return(buffer);
}
else if (isDiffable && typeof(IDiffable <TElement>).GetTypeInfo().IsAssignableFrom(typeof(TElement).GetTypeInfo()))
{
var buffer = BufferPool <byte> .Rent(arraySize *ItemSize);
// TODO Pool these as well. Pool implementation could decide to use just new
var targetArray = BufferPool <TElement> .Rent(arraySize);
fixed(byte *tgtPtr = &buffer[0])
{
var destination = tgtPtr;
var decompSize = BloscMethods.blosc_decompress_ctx(
source, (IntPtr)destination, new UIntPtr((uint)nbytes), BloscMethods.ProcessorCount);
if (decompSize <= 0)
{
throw new ArgumentException("Invalid compressed input");
}
Debug.Assert(decompSize == nbytes);
var first = Unsafe.Read <TElement>(destination);
var diffableFirst = (IDiffable <TElement>)first;
targetArray[0] = first;
for (var i = 1; i < arraySize; i++)
{
var currentDelta = Unsafe.Read <TElement>(destination + i * ItemSize);
var current = diffableFirst.AddDelta(currentDelta);
targetArray[i] = current;
}
}
value = targetArray; // (<TElement>)(object)(new ArraySegment<TElement>(targetArray, 0, arraySize));
BufferPool <byte> .Return(buffer);
}
else
{
var pinnedArray = GCHandle.Alloc(value, GCHandleType.Pinned);
var destination = pinnedArray.AddrOfPinnedObject();
int decompSize = 0;
// TODO remove this try/catch and debugger stuff, it was used tp catch an eror that disappeared after adding
// try/catch. Probably some reordering, maybe add a memory barrier before the call
try
{
decompSize = BloscMethods.blosc_decompress_ctx(
source, destination, new UIntPtr((uint)nbytes), BloscMethods.ProcessorCount);
if (decompSize <= 0)
{
throw new ArgumentException("Invalid compressed input");
}
Debug.Assert(decompSize == nbytes);
}
catch (Exception ex)
{
Debugger.Launch();
UIntPtr nb = UIntPtr.Zero;
UIntPtr cb = UIntPtr.Zero;
UIntPtr bl = UIntPtr.Zero;
BloscMethods.blosc_cbuffer_sizes(source, ref nb, ref cb, ref bl);
//}
Trace.WriteLine($"Blosc error: nbytes: {nbytes}, nbytes2: {nb}, cbytes: {cb} arr size: {value.Length}, \n\r exeption: {ex.Message + Environment.NewLine + ex.ToString()}");
throw;
}
finally
{
pinnedArray.Free();
}
}
}
else
{
// BufferPool returns an empty array
}
return(totalSize);
}
}
