public unsafe void CouldSerializeRegularSortedMapWithZstd() { var rng = new Random(); var dest = (Memory <byte>) new byte[1000000]; var buffer = dest; var handle = buffer.Pin(); var ptr = (IntPtr)handle.Pointer; var sm = new SortedMap <DateTime, decimal>(); for (var i = 0; i < 1000; i++) { sm.Add(DateTime.Today.AddSeconds(i), (decimal)Math.Round(i + rng.NextDouble(), 2)); } var sizeOf = BinarySerializer.SizeOf(sm, out var tmp); var written = BinarySerializer.Write(sm, dest.Span, tmp); Assert.AreEqual(sizeOf, written); Console.WriteLine($"Useful: {sm.Count * 24}"); Console.WriteLine($"Total: {written}"); // NB interesting that with converting double to decimal savings go from 65% to 85%, // even calculated from (8+8) base size not decimal's 16 size Console.WriteLine($"Savings: {1.0 - ((written * 1.0) / (sm.Count * 24.0))}"); SortedMap <DateTime, decimal> sm2 = null; var len2 = BinarySerializer.Read(buffer.Span, out sm2); Assert.AreEqual(written, len2); Assert.IsTrue(sm2.Keys.SequenceEqual(sm.Keys)); Assert.IsTrue(sm2.Values.SequenceEqual(sm.Values)); }
public void CouldSerializeOrderBag() { QuoteDecimal[] bids = new QuoteDecimal[10000]; for (int i = 0; i < bids.Length; i++) { bids[i] = new QuoteDecimal(new SmallDecimal(15000000.45, 8), new SmallDecimal((double)i, 8)); } QuoteDecimal[] asks = new QuoteDecimal[2]; asks[0] = new QuoteDecimal(new SmallDecimal(123.89, 2), new SmallDecimal(0.00159748M, 8)); asks[1] = new QuoteDecimal(new SmallDecimal(123.99M, 2), new SmallDecimal(0.00128948M, 8)); var ob = new OrderBagDecimal(bids, asks); var size = BinarySerializer.SizeOf(ob, out var segment, SerializationFormat.Json); var rm = BufferPool.Retain(size); var written = BinarySerializer.Write(ob, rm.Span, segment, SerializationFormat.Json); Assert.AreEqual(size, written); var str = JsonSerializer.ToJsonString(ob); var ob2 = JsonSerializer.Deserialize <OrderBagDecimal>(str); Console.WriteLine(str); // var ob2 = JsonSerializer.Deserialize<OrderBagX>(str); rm.Dispose(); }
public unsafe void CouldSerializeSortedMap2() { var rng = new Random(); var dest = (Memory <byte>) new byte[1000000]; var buffer = dest; var handle = buffer.Pin(); var ptr = (IntPtr)handle.Pointer; var sm = new SortedMap <int, int>(); for (var i = 0; i < 10000; i++) { sm.Add(i, i); } var len = BinarySerializer.SizeOf(sm, out var temp); var len2 = BinarySerializer.Write(sm, buffer.Span, temp); Assert.AreEqual(len, len2); Console.WriteLine($"Useful: {sm.Count * 8}"); Console.WriteLine($"Total: {len}"); // NB interesting that with converting double to decimal savings go from 65% to 85%, // even calculated from (8+8) base size not decimal's 16 size Console.WriteLine($"Savings: {1.0 - ((len * 1.0) / (sm.Count * 8.0))}"); SortedMap <int, int> sm2 = null; var len3 = BinarySerializer.Read(buffer.Span, out sm2); Assert.AreEqual(len, len3); Assert.IsTrue(sm2.Keys.SequenceEqual(sm.Keys)); Assert.IsTrue(sm2.Values.SequenceEqual(sm.Values)); }
public void CouldUseCompression() { // this also tests write without SizeOf, but only happy path // TODO unhappy path with corrupted data and small buffers var formats = new[] { SerializationFormat.Binary, SerializationFormat.BinaryZstd, SerializationFormat.BinaryLz4, SerializationFormat.BinaryGZip, SerializationFormat.Json, SerializationFormat.JsonGZip, SerializationFormat.JsonLz4, SerializationFormat.JsonZstd }; foreach (var serializationFormat in formats) { Console.WriteLine(serializationFormat); var count = 10000; var rng = new Random(42); Memory <byte> bytes = new byte[count * 16 + 20]; var source = new decimal[count]; source[0] = 123.4567M; for (var i = 1; i < count; i++) { source[i] = source[i - 1] + Math.Round((decimal)rng.NextDouble() * 10, 4); } var len = BinarySerializer.Write(source, bytes.Span, default,
public void CouldSerializeBlittableStructArray() { var bytes = new byte[1000]; var value = new BlittableStruct[2]; value[0] = new BlittableStruct { Value1 = 123, Value2 = 1230 }; value[1] = new BlittableStruct { Value1 = 456, Value2 = 4560 }; var len0 = BinarySerializer.SizeOf(in value, out var pl, SerializationFormat.Binary); var len = BinarySerializer.Write(value, bytes, in pl, format: SerializationFormat.Binary); Assert.AreEqual(8 + 4 + 12 * 2, len); BlittableStruct[] arr2 = null; var len2 = BinarySerializer.Read(bytes, out arr2); Assert.AreEqual(len, len2); Assert.IsTrue(value.SequenceEqual(arr2)); }
public void CouldSerializeBlittable <T>(T value) where T : IEquatable <T> { var bytes = BufferPool <byte> .Rent(1000); var mem = (Memory <byte>)bytes; var h = mem.Pin(); var db = new DirectBuffer(bytes.Length, (byte *)h.Pointer); foreach (var format in Formats) { var val = value; var len0 = BinarySerializer.SizeOf(in val, out var pl, format); var len = BinarySerializer.Write(val, db, in pl, format); // for (int x = 0; x < 100; x++) { var len2 = BinarySerializer.Read(db, out T val2); if (len != len2) { Assert.Fail($"len {len} != len2 {len2}"); } if (!val.Equals(val2)) { Assert.Fail(); } } } foreach (var format in Formats) { var val = value; var len0 = BinarySerializer.SizeOf(in val, out var segment, format); var len1 = BinarySerializer.Write(in val, db, segment, format); if (len0 != len1) { Assert.Fail($"len0 {len0} != len1 {len1}"); } //Assert.IsTrue(len1 > 8); var len2 = BinarySerializer.Read(db, out T val2); if (len1 != len2) { Assert.Fail($"len1 {len1} != len2 {len2}"); } if (!val.Equals(val2)) { Assert.Fail(); } } h.Dispose(); BufferPool <byte> .Return(bytes); }
public unsafe void CouldSerializeTickDecimal() { var count = 1_00; var rounds = 1; var rm = BufferPool.Retain(512); var db = new DirectBuffer(rm.Length, (byte *)rm.Pointer); var formats = new[] { SerializationFormat.Binary, SerializationFormat.Json }; for (int r = 0; r < rounds; r++) { foreach (var serializationFormat in formats) { using (Benchmark.Run(serializationFormat.ToString(), count, true)) { for (int i = 0; i < count; i++) { var tick = new TickDecimal((Timestamp)(long)(i + 1), new SmallDecimal((double)i, 8), new SmallDecimal((double)i, 8), i % 3 - 1, i); var sizeOf = BinarySerializer.SizeOf(in tick, out var segment, serializationFormat); if (serializationFormat == SerializationFormat.Binary && sizeOf != 32 + 4) { Assert.Fail("size != 32"); } var written = BinarySerializer.Write(tick, db, segment, serializationFormat); if (sizeOf != written) { Assert.Fail("size != written"); } var readSource = db.Slice(0, written); var consumed = BinarySerializer.Read <TickDecimal>(readSource, out var tick1); if (consumed != written) { Assert.Fail("readSize != written"); } if (!tick1.Equals(tick)) { Assert.Fail("!tick1.Equals(tick)"); } } } } } Benchmark.Dump(); var tick2 = new TickDecimal(TimeService.Default.CurrentTime, new SmallDecimal((double)123.45, 8), new SmallDecimal((double)56.789, 8), 123, 7894561253); var str = JsonSerializer.ToJsonString(tick2); Console.WriteLine(str); rm.Dispose(); }
public void CouldSerializeDateTimeArrayAsJson() { // not compressed even when requested var formats = new[] { SerializationFormat.Json, SerializationFormat.JsonGZip, SerializationFormat.JsonLz4, SerializationFormat.JsonZstd }; var lens = new[] { 1, 2, 10, 20, 50, 100, 200, 300, 400, 500, 600, 700, 1000, 10000 }; var rng = new Random(42); foreach (var len in lens) { var dta = new DateTime[len]; for (int l = 0; l < len; l++) { dta[l] = DateTime.Today.ToUniversalTime().AddMinutes(l).AddMilliseconds(rng.Next(0, 1000)); } var uncompressed = 0.0; Console.WriteLine("---------------"); foreach (var serializationFormat in formats) { var sizeOf = BinarySerializer.SizeOf(dta, out var segment, serializationFormat); var bytes = BufferPool <byte> .Rent(4 + sizeOf); var written = BinarySerializer.Write(dta, bytes, segment, serializationFormat); if (serializationFormat == SerializationFormat.Json) { uncompressed = written; } Console.WriteLine( $"len: {len}, format: {serializationFormat}, written bytes: {written}, ratio: {Math.Round(uncompressed / written, 2)}"); if (sizeOf != written) { Assert.Fail($"sizeOf {sizeOf} != written {written}"); } DateTime[] dta2 = null; var read = BinarySerializer.Read(bytes, out dta2); Assert.AreEqual(written, read); Assert.IsTrue(dta.SequenceEqual(dta2)); } } }
public void KnownScalarsVariantHeader() { TestType <sbyte>(TypeEnum.Int8); TestType <short>(TypeEnum.Int16); TestType <int>(TypeEnum.Int32); TestType <long>(TypeEnum.Int64); TestType <byte>(TypeEnum.UInt8); TestType <ushort>(TypeEnum.UInt16); TestType <uint>(TypeEnum.UInt32); TestType <ulong>(TypeEnum.UInt64); TestType <float>(TypeEnum.Float32); TestType <double>(TypeEnum.Float64); TestType <decimal>(TypeEnum.Decimal); TestType <SmallDecimal>(TypeEnum.SmallDecimal); TestType <bool>(TypeEnum.Bool); TestType <char>(TypeEnum.Utf16Char); TestType <UUID>(TypeEnum.UUID); TestType <DateTime>(TypeEnum.DateTime); TestType <Timestamp>(TypeEnum.Timestamp); TestType <Symbol>(TypeEnum.Symbol); TestType <Symbol32>(TypeEnum.Symbol32); TestType <Symbol64>(TypeEnum.Symbol64); TestType <Symbol128>(TypeEnum.Symbol128); TestType <Symbol256>(TypeEnum.Symbol256); void TestType <T>(TypeEnum expectedTe) { var h = TypeEnumHelper <T> .DataTypeHeader; Assert.AreEqual(expectedTe, h.TEOFS.TypeEnum); Assert.IsTrue(h.TEOFS1 == default && h.TEOFS2 == default); Assert.IsTrue(h.IsScalar); var teofs = new TypeEnumOrFixedSize(expectedTe); Assert.AreEqual(BinarySerializer.SizeOf <T>(default(T), out _, SerializationFormat.Binary), DataTypeHeader.Size + teofs.Size); Assert.AreEqual(Unsafe.SizeOf <T>(), h.TEOFS.Size); Assert.AreEqual(Unsafe.SizeOf <T>(), TypeEnumHelper <T> .FixedSize); if (typeof(T) != typeof(DateTime)) { Assert.AreEqual(TypeHelper <T> .PinnedSize, TypeEnumHelper <T> .FixedSize); } } }
public void CouldSerializeStringArray() { var bytes = new byte[1000]; var value = new string[2]; value[0] = "123"; value[1] = "456"; var len0 = BinarySerializer.SizeOf(in value, out var pl); var len = BinarySerializer.Write(value, bytes, in pl); string[] arr2 = null; var len2 = BinarySerializer.Read(bytes, out arr2); Assert.AreEqual(len, len2); Assert.IsTrue(value.SequenceEqual(arr2)); }
public void CouldSerializeDateTimeArray() { var bytes = new byte[1000]; var dta = new DateTime[2]; dta[0] = DateTime.Today; dta[1] = DateTime.Today.AddDays(1); var len0 = BinarySerializer.SizeOf(in dta, out var pl, SerializationFormat.Binary); var len = BinarySerializer.Write(dta, bytes, in pl, format: SerializationFormat.Binary); Assert.AreEqual(8 + 4 + 8 * 2, len); DateTime[] dta2 = null; var len2 = BinarySerializer.Read(bytes, out dta2); Assert.AreEqual(len, len2); Assert.IsTrue(dta.SequenceEqual(dta2)); }
public void CouldSerializeDecimalArray() { Assert.AreEqual(16, TypeHelper <decimal> .FixedSize); var bytes = new byte[1000]; var value = new decimal[2]; value[0] = 123; value[1] = 456; var len0 = BinarySerializer.SizeOf(in value, out var pl, SerializationFormat.Binary); var len = BinarySerializer.Write(value, bytes, in pl, format: SerializationFormat.Binary); Assert.AreEqual(8 + 4 + 16 * 2, len); decimal[] decimals2 = null; var len2 = BinarySerializer.Read(bytes, out decimals2); Assert.IsTrue(value.SequenceEqual(decimals2)); Assert.AreEqual(len, len2); }
public void CouldSerializeIntArray() { var bytes = new byte[1000]; var value = new int[2]; value[0] = 123; value[1] = 456; var len0 = BinarySerializer.SizeOf(in value, out var pl, SerializationFormat.Binary); var len = BinarySerializer.Write(in value, bytes, in pl, format: SerializationFormat.Binary); Assert.AreEqual(len0, len); Assert.AreEqual(8 + 4 + 4 * 2, len); int[] ints2 = null; var len2 = BinarySerializer.Read(bytes, out ints2); Assert.AreEqual(len, len2); Assert.IsTrue(value.SequenceEqual(ints2)); }
public void CouldSerializeSortedMapWithStrings() { var rng = new Random(); var dest = (Memory <byte>) new byte[10000000]; var buffer = dest; var handle = buffer.Pin(); var ptr = (IntPtr)handle.Pointer; var valueLens = 0; var sm = new MutableSeries <int, string>(); for (var i = 0; i < 100000; i++) { var str = i.ToString(); valueLens += str.Length; sm.Add(i, str); } var len = BinarySerializer.SizeOf(sm, out var temp); var len2 = BinarySerializer.Write(sm, buffer.Span, temp); Assert.AreEqual(len, len2); var usefulLen = ((int)sm.RowCount * 4) + valueLens; Console.WriteLine($"Useful: {usefulLen}"); Console.WriteLine($"Total: {len}"); // NB interesting that with converting double to decimal savings go from 65% to 85%, // even calculated from (8+8) base size not decimal's 16 size Console.WriteLine($"Savings: {1.0 - ((len * 1.0) / (usefulLen))}"); Series <int, string> sm2 = null; var len3 = BinarySerializer.Read(buffer.Span, out sm2); Assert.AreEqual(len, len3); Assert.IsTrue(sm2.Keys.SequenceEqual(sm.Keys)); Assert.IsTrue(sm2.Values.SequenceEqual(sm.Values)); }
public void CouldSerializePocoArray() { var bytes = new byte[1000]; var value = new SimplePoco[2]; value[0] = new SimplePoco { Value1 = 123, Value2 = "1230" }; value[1] = new SimplePoco { Value1 = 456, Value2 = "4560" }; var len0 = BinarySerializer.SizeOf(in value, out var pl); var len = BinarySerializer.Write(value, bytes, in pl); SimplePoco[] arr2 = null; var len2 = BinarySerializer.Read(bytes, out arr2); Assert.IsTrue(value.SequenceEqual(arr2), "Items are not equal"); Assert.AreEqual(len, len2); }
public unsafe int Write(TElement[] value, int valueOffset, int valueCount, ref Buffer <byte> destination, uint destinationOffset = 0u, MemoryStream temporaryStream = null, CompressionMethod compression = CompressionMethod.DefaultOrNone) { // NB Blosc calls below are visually large - many LOCs with comments, but this is only a single method call if (value == null) { throw new ArgumentNullException(nameof(value)); } var handle = destination.Pin(); try { var ptr = (IntPtr)handle.PinnedPointer + (int)destinationOffset; if (temporaryStream != null) { var len = temporaryStream.Length; if (destination.Length < destinationOffset + len) { return((int)BinaryConverterErrorCode.NotEnoughCapacity); } temporaryStream.WriteToPtr(ptr); temporaryStream.Dispose(); return(checked ((int)len)); } bool isDiffable = false; var compressionMethod = compression == CompressionMethod.DefaultOrNone ? BloscSettings.defaultCompressionMethod : (compression == CompressionMethod.LZ4 ? "lz4" : "zstd"); var position = 8; if (valueCount > 0) { int compressedSize; if (ItemSize > 0) { if (typeof(TElement) == typeof(DateTime)) { var buffer = BufferPool <byte> .Rent(valueCount * 8); var dtArray = (DateTime[])(object)value; fixed(byte *srcPtr = &buffer[0]) { for (var i = 0; i < valueCount; i++) { *(DateTime *)(srcPtr + i * 8) = dtArray[i + valueOffset]; } compressedSize = BloscMethods.blosc_compress_ctx( new IntPtr(9), // max compression 9 new IntPtr(1), // do byte shuffle 1 new UIntPtr((uint)ItemSize), // type size new UIntPtr((uint)(valueCount * ItemSize)), // number of input bytes (IntPtr)srcPtr, ptr + position, // destination new UIntPtr((uint)(destination.Length - position)), // destination length compressionMethod, new UIntPtr((uint)0), // default block size BloscMethods.ProcessorCount // ); } BufferPool <byte> .Return(buffer); } else if (value[0] is IDiffable <TElement> diffableFirst) { isDiffable = true; // TODO (!) this is probably inefficient... some generic method caching or dynamic dispatch? // however there is only a single pattern match with IDiffable boxing var first = value[0]; var buffer = BufferPool <byte> .Rent(valueCount *ItemSize); fixed(byte *srcPtr = &buffer[0]) { Unsafe.Write(srcPtr, first); for (var i = 1; i < valueCount; i++) { var current = value[i]; var diff = diffableFirst.GetDelta(current); Unsafe.Write(srcPtr + i * ItemSize, diff); } compressedSize = BloscMethods.blosc_compress_ctx( new IntPtr(9), // max compression 9 new IntPtr(1), // do byte shuffle 1 new UIntPtr((uint)ItemSize), // type size new UIntPtr((uint)(valueCount * ItemSize)), // number of input bytes (IntPtr)srcPtr, ptr + position, // destination new UIntPtr((uint)(destination.Length - position)), // destination length compressionMethod, new UIntPtr((uint)0), // default block size BloscMethods.ProcessorCount // ); } BufferPool <byte> .Return(buffer); } else { var pinnedArray = GCHandle.Alloc(value, GCHandleType.Pinned); var srcPtr = Marshal.UnsafeAddrOfPinnedArrayElement(value, valueOffset); compressedSize = BloscMethods.blosc_compress_ctx( new IntPtr(9), // max compression 9 new IntPtr(1), // do byte shuffle 1 new UIntPtr((uint)ItemSize), // type size new UIntPtr((uint)(valueCount * ItemSize)), // number of input bytes srcPtr, ptr + position, // destination new UIntPtr((uint)(destination.Length - position)), // destination length compressionMethod, new UIntPtr((uint)0), // default block size BloscMethods.ProcessorCount // ); pinnedArray.Free(); } } else if (Buffers.BufferPool.IsPreservedBuffer <TElement>()) { throw new NotImplementedException(); } else { MemoryStream tempStream; var bytesSize = BinarySerializer.SizeOf(new ArraySegment <TElement>(value, valueOffset, valueCount), out tempStream, compression); var buffer = BufferPool <byte> .Rent(bytesSize); var writtenBytes = BinarySerializer.Write(new ArraySegment <TElement>(value, valueOffset, valueCount), buffer, 0, tempStream); tempStream?.Dispose(); Debug.Assert(bytesSize == writtenBytes); compressedSize = CompressedArrayBinaryConverter <byte> .Instance.Write(buffer, 0, writtenBytes, ref destination, destinationOffset, null, compression); BufferPool <byte> .Return(buffer); } if (compressedSize > 0) { position += compressedSize; } else { return((int)BinaryConverterErrorCode.NotEnoughCapacity); } } // length Marshal.WriteInt32(ptr, position); // version & flags Marshal.WriteByte(ptr + 4, (byte)((Version << 4) | (isDiffable ? 0b0000_0011 : 0b0000_0001))); return(position); } finally { handle.Free(); } }
public unsafe int Read(IntPtr ptr, out TElement[] value, out int length, bool exactSize = false) { var totalSize = Marshal.ReadInt32(ptr); var versionFlag = Marshal.ReadByte(ptr + 4); var version = (byte)(versionFlag >> 4); var isDiffable = (versionFlag & 0b0000_0010) != 0; var isCompressed = (versionFlag & 0b0000_0001) != 0; if (!isCompressed) { throw new InvalidOperationException("Wrong compressed flag. CompressedArrayBinaryConverter.Read works only with compressed methods."); } if (version != Version) { throw new NotSupportedException($"CompressedBinaryConverter work only with version {Version}"); } if (ItemSize <= 0) { // first decompress bytes var size = CompressedArrayBinaryConverter <byte> .Instance.Read(ptr, out byte[] decompressedBytes, out length); // NB the length is encoded in the header and is returned as a part of ArraySegment Debug.Assert(length == BitConverter.ToInt32(decompressedBytes, 0)); // then deserialize // NB the size of the array will be exact, BinarySerializer.Read does not support non-exact buffers BinarySerializer.Read(decompressedBytes, out value); BufferPool <byte> .Return(decompressedBytes); return(size); } else if (Buffers.BufferPool.IsPreservedBuffer <TElement>()) { throw new NotImplementedException(); } else { if (totalSize <= 8 + 16) { value = EmptyArray <TElement> .Instance; length = 0; return(totalSize); } var source = ptr + 8; // avoid additional P/Invoke call, read header directly // https://github.com/Blosc/c-blosc/blob/master/README_HEADER.rst var nbytes = *(int *)(source + 4); #if DEBUG var blocksize = *(int *)(source + 8); var cbytes = *(int *)(source + 12); var nbytes2 = new UIntPtr(); var cbytes2 = new UIntPtr(); var blocksize2 = new UIntPtr(); BloscMethods.blosc_cbuffer_sizes(source, ref nbytes2, ref cbytes2, ref blocksize2); Debug.Assert(nbytes == nbytes2.ToUInt32()); Debug.Assert(cbytes == cbytes2.ToUInt32()); Debug.Assert(blocksize == blocksize2.ToUInt32()); #endif var arraySize = nbytes / ItemSize; // when caller provides an empty AS, it could require exact size, e.g. DateTimeArrayBinaryConverter var array = BufferPool <TElement> .Rent(arraySize, exactSize); length = arraySize; value = array; if (arraySize > 0) { if (typeof(TElement) == typeof(DateTime)) { var buffer = BufferPool <byte> .Rent(arraySize * 8); var dtArray = new DateTime[arraySize]; fixed(byte *tgtPtr = &buffer[0]) { var destination = (IntPtr)tgtPtr; var decompSize = BloscMethods.blosc_decompress_ctx( source, destination, new UIntPtr((uint)nbytes), BloscMethods.ProcessorCount); if (decompSize <= 0) { throw new ArgumentException("Invalid compressed input"); } Debug.Assert(decompSize == nbytes); for (var i = 0; i < arraySize; i++) { dtArray[i] = *(DateTime *)(destination + i * 8); } } value = (TElement[])(object)(dtArray); BufferPool <byte> .Return(buffer); } else if (isDiffable && typeof(IDiffable <TElement>).GetTypeInfo().IsAssignableFrom(typeof(TElement).GetTypeInfo())) { var buffer = BufferPool <byte> .Rent(arraySize *ItemSize); // TODO Pool these as well. Pool implementation could decide to use just new var targetArray = BufferPool <TElement> .Rent(arraySize); fixed(byte *tgtPtr = &buffer[0]) { var destination = tgtPtr; var decompSize = BloscMethods.blosc_decompress_ctx( source, (IntPtr)destination, new UIntPtr((uint)nbytes), BloscMethods.ProcessorCount); if (decompSize <= 0) { throw new ArgumentException("Invalid compressed input"); } Debug.Assert(decompSize == nbytes); var first = Unsafe.Read <TElement>(destination); var diffableFirst = (IDiffable <TElement>)first; targetArray[0] = first; for (var i = 1; i < arraySize; i++) { var currentDelta = Unsafe.Read <TElement>(destination + i * ItemSize); var current = diffableFirst.AddDelta(currentDelta); targetArray[i] = current; } } value = targetArray; // (<TElement>)(object)(new ArraySegment<TElement>(targetArray, 0, arraySize)); BufferPool <byte> .Return(buffer); } else { var pinnedArray = GCHandle.Alloc(value, GCHandleType.Pinned); var destination = pinnedArray.AddrOfPinnedObject(); int decompSize = 0; // TODO remove this try/catch and debugger stuff, it was used tp catch an eror that disappeared after adding // try/catch. Probably some reordering, maybe add a memory barrier before the call try { decompSize = BloscMethods.blosc_decompress_ctx( source, destination, new UIntPtr((uint)nbytes), BloscMethods.ProcessorCount); if (decompSize <= 0) { throw new ArgumentException("Invalid compressed input"); } Debug.Assert(decompSize == nbytes); } catch (Exception ex) { Debugger.Launch(); UIntPtr nb = UIntPtr.Zero; UIntPtr cb = UIntPtr.Zero; UIntPtr bl = UIntPtr.Zero; BloscMethods.blosc_cbuffer_sizes(source, ref nb, ref cb, ref bl); //} Trace.WriteLine($"Blosc error: nbytes: {nbytes}, nbytes2: {nb}, cbytes: {cb} arr size: {value.Length}, \n\r exeption: {ex.Message + Environment.NewLine + ex.ToString()}"); throw; } finally { pinnedArray.Free(); } } } else { // BufferPool returns an empty array } return(totalSize); } }
public unsafe int Write(TElement[] value, int valueOffset, int valueCount, ref Memory <byte> destination, uint destinationOffset = 0u, MemoryStream temporaryStream = null, CompressionMethod compression = CompressionMethod.DefaultOrNone) { // NB Blosc calls below are visually large - many LOCs with comments, but this is only a single method call if (value == null) { throw new ArgumentNullException(nameof(value)); } var handle = destination.Retain(true); try { var ptr = (IntPtr)handle.PinnedPointer + (int)destinationOffset; if (temporaryStream != null) { var len = temporaryStream.Length; if (destination.Length < destinationOffset + len) { return((int)BinaryConverterErrorCode.NotEnoughCapacity); } temporaryStream.WriteToPtr(ptr); temporaryStream.Dispose(); return(checked ((int)len)); } var compressionMethod = compression == CompressionMethod.DefaultOrNone ? BloscSettings.defaultCompressionMethod : (compression == CompressionMethod.LZ4 ? "lz4" : "zstd"); var isDelta = IsIDelta; var position = 8; if (valueCount > 0) { int compressedSize; if (ItemSize > 0) { if (typeof(TElement) == typeof(DateTime)) { isDelta = true; Trace.Assert(ItemSize == 8); var buffer = BufferPool <byte> .Rent(valueCount * 8); var dtArray = (DateTime[])(object)value; var first = dtArray[valueOffset]; // NB For DateTime we calculate delta not from the first but // from the previous value. This is a special case for the // fact that DT[] is usually increasing by a similar (regular) step // and the deltas are always positive, small and close to each other. // In contrast, Price/Decimal could fluctuate in a small range // and delta from previous could often change its sign, which // leads to a very different bits and significantly reduces // the Blosc shuffling benefits. For stationary time series // deltas from the first value are also stationary and their sign // changes less frequently that the sign of deltas from previous. var previousLong = (long *)&first; fixed(byte *srcPtr = &buffer[0]) { Unsafe.WriteUnaligned(srcPtr, *previousLong); for (var i = 1; i < valueCount; i++) { var current = dtArray[i + valueOffset]; var currentLong = (long *)(¤t); var diff = currentLong - previousLong; Unsafe.WriteUnaligned(srcPtr + i * ItemSize, diff); previousLong = currentLong; } compressedSize = BloscMethods.blosc_compress_ctx( new IntPtr(9), // max compression 9 new IntPtr(1), // do byte shuffle 1 new UIntPtr((uint)ItemSize), // type size new UIntPtr((uint)(valueCount * ItemSize)), // number of input bytes (IntPtr)srcPtr, ptr + position, // destination new UIntPtr((uint)(destination.Length - position)), // destination length compressionMethod, new UIntPtr((uint)0), // default block size BloscMethods.ProcessorCount // ); } BufferPool <byte> .Return(buffer); } else if (IsIDelta) { var first = value[valueOffset]; var buffer = BufferPool <byte> .Rent(valueCount *ItemSize); fixed(byte *srcPtr = &buffer[0]) { Unsafe.WriteUnaligned(srcPtr, first); for (var i = 1; i < valueCount; i++) { var diff = Unsafe.GetDeltaConstrained(ref first, ref value[valueOffset + i]); Unsafe.WriteUnaligned(srcPtr + i * ItemSize, diff); } compressedSize = BloscMethods.blosc_compress_ctx( new IntPtr(9), // max compression 9 new IntPtr(1), // do byte shuffle 1 new UIntPtr((uint)ItemSize), // type size new UIntPtr((uint)(valueCount * ItemSize)), // number of input bytes (IntPtr)srcPtr, ptr + position, // destination new UIntPtr((uint)(destination.Length - position)), // destination length compressionMethod, new UIntPtr((uint)0), // default block size BloscMethods.ProcessorCount // ); } BufferPool <byte> .Return(buffer); } else { var pinnedArray = GCHandle.Alloc(value, GCHandleType.Pinned); var srcPtr = Marshal.UnsafeAddrOfPinnedArrayElement(value, valueOffset); compressedSize = BloscMethods.blosc_compress_ctx( new IntPtr(9), // max compression 9 new IntPtr(1), // do byte shuffle 1 new UIntPtr((uint)ItemSize), // type size new UIntPtr((uint)(valueCount * ItemSize)), // number of input bytes srcPtr, ptr + position, // destination new UIntPtr((uint)(destination.Length - position)), // destination length compressionMethod, new UIntPtr((uint)0), // default block size BloscMethods.ProcessorCount // ); pinnedArray.Free(); } } else if (Buffers.BufferPool.IsPreservedBuffer <TElement>()) { throw new NotImplementedException(); } else { MemoryStream tempStream; var bytesSize = BinarySerializer.SizeOf(new ArraySegment <TElement>(value, valueOffset, valueCount), out tempStream, compression); var buffer = BufferPool <byte> .Rent(bytesSize); var writtenBytes = BinarySerializer.Write(new ArraySegment <TElement>(value, valueOffset, valueCount), buffer, 0, tempStream); tempStream?.Dispose(); Debug.Assert(bytesSize == writtenBytes); compressedSize = CompressedArrayBinaryConverter <byte> .Instance.Write(buffer, 0, writtenBytes, ref destination, destinationOffset, null, compression); BufferPool <byte> .Return(buffer); } if (compressedSize > 0) { position += compressedSize; } else { return((int)BinaryConverterErrorCode.NotEnoughCapacity); } } // length Marshal.WriteInt32(ptr, position); // version & flags Marshal.WriteByte(ptr + 4, (byte)((Version << 4) | (isDelta ? 0b0000_0011 : 0b0000_0001))); return(position); } finally { handle.Dispose(); } }