public void CouldUseIDeltaMethods() { var first = new IntDelta { Value = 123 }; var second = new IntDelta { Value = 456 }; var delta = new IntDelta { Value = 456 - 123 }; Assert.AreEqual(delta, Unsafe.GetDeltaConstrained(ref first, ref second)); Assert.AreEqual(second, Unsafe.AddDeltaConstrained(ref first, ref delta)); }
public unsafe int Read(IntPtr ptr, out TElement[] value, out int length, bool exactSize = false) { var totalSize = Marshal.ReadInt32(ptr); var versionFlag = Marshal.ReadByte(ptr + 4); var version = (byte)(versionFlag >> 4); var isDelta = (versionFlag & 0b0000_0010) != 0; var isCompressed = (versionFlag & 0b0000_0001) != 0; if (!isCompressed) { throw new InvalidOperationException("Wrong compressed flag. CompressedArrayBinaryConverter.Read works only with compressed data."); } if (version != Version) { throw new NotSupportedException($"CompressedBinaryConverter work only with version {Version}"); } if (ItemSize <= 0) { // first decompress bytes var size = CompressedArrayBinaryConverter <byte> .Instance.Read(ptr, out byte[] decompressedBytes, out length); // NB the length is encoded in the header and is returned as a part of ArraySegment Debug.Assert(length == BitConverter.ToInt32(decompressedBytes, 0)); // then deserialize // NB the size of the array will be exact, BinarySerializer.Read does not support non-exact buffers BinarySerializer.Read(decompressedBytes, out value); BufferPool <byte> .Return(decompressedBytes); return(size); } else if (Buffers.BufferPool.IsPreservedBuffer <TElement>()) { throw new NotImplementedException(); } else { if (totalSize <= 8 + 16) { value = EmptyArray <TElement> .Instance; length = 0; return(totalSize); } var source = ptr + 8; // avoid additional P/Invoke call, read header directly // https://github.com/Blosc/c-blosc/blob/master/README_HEADER.rst var nbytes = *(int *)(source + 4); #if DEBUG var blocksize = *(int *)(source + 8); var cbytes = *(int *)(source + 12); var nbytes2 = new UIntPtr(); var cbytes2 = new UIntPtr(); var blocksize2 = new UIntPtr(); BloscMethods.blosc_cbuffer_sizes(source, ref nbytes2, ref cbytes2, ref blocksize2); Debug.Assert(nbytes == nbytes2.ToUInt32()); Debug.Assert(cbytes == cbytes2.ToUInt32()); Debug.Assert(blocksize == blocksize2.ToUInt32()); #endif var arraySize = nbytes / ItemSize; // when caller provides an empty AS, it could require exact size, e.g. DateTimeArrayBinaryConverter var array = BufferPool <TElement> .Rent(arraySize, exactSize); length = arraySize; value = array; if (arraySize > 0) { if (typeof(TElement) == typeof(DateTime)) { var buffer = BufferPool <byte> .Rent(arraySize * 8); var dtArray = new DateTime[arraySize]; fixed(byte *tgtPtr = &buffer[0]) { var destination = (IntPtr)tgtPtr; var decompSize = BloscMethods.blosc_decompress_ctx( source, destination, new UIntPtr((uint)nbytes), BloscMethods.ProcessorCount); if (decompSize <= 0) { throw new ArgumentException("Invalid compressed input"); } Debug.Assert(decompSize == nbytes); // NB a lot of data was stored without diff for DateTime, // should just check the flag if (isDelta) { var previousLong = *(long *)destination; var first = *(DateTime *)&previousLong; dtArray[0] = first; for (var i = 1; i < arraySize; i++) { var deltaLong = *(long *)(destination + i * 8); var currentLong = previousLong + deltaLong; dtArray[i] = *(DateTime *)¤tLong; previousLong = currentLong; } } else { for (var i = 0; i < arraySize; i++) { dtArray[i] = *(DateTime *)(destination + i * 8); } } } value = (TElement[])(object)(dtArray); BufferPool <byte> .Return(buffer); } else if (isDelta) { if (!IsIDelta) { ThrowHelper.ThrowInvalidOperationException("Delta flag is set for a type that does not implement IDelta interface."); } var buffer = BufferPool <byte> .Rent(arraySize *ItemSize); var targetArray = BufferPool <TElement> .Rent(arraySize); fixed(byte *tgtPtr = &buffer[0]) { var destination = tgtPtr; var decompSize = BloscMethods.blosc_decompress_ctx( source, (IntPtr)destination, new UIntPtr((uint)nbytes), BloscMethods.ProcessorCount); if (decompSize <= 0) { throw new ArgumentException("Invalid compressed input"); } Debug.Assert(decompSize == nbytes); var first = Unsafe.ReadUnaligned <TElement>(destination); targetArray[0] = first; for (var i = 1; i < arraySize; i++) { var currentDelta = Unsafe.Read <TElement>(destination + i * ItemSize); var current = Unsafe.AddDeltaConstrained(ref first, ref currentDelta); targetArray[i] = current; } } value = targetArray; BufferPool <byte> .Return(buffer); } else { var pinnedArray = GCHandle.Alloc(value, GCHandleType.Pinned); var destination = pinnedArray.AddrOfPinnedObject(); // TODO remove this try/catch and debugger stuff, it was used to catch an eror that disappeared after adding // try/catch. Probably some reordering, maybe add a memory barrier before the call try { var decompSize = BloscMethods.blosc_decompress_ctx( source, destination, new UIntPtr((uint)nbytes), BloscMethods.ProcessorCount); if (decompSize <= 0) { throw new ArgumentException("Invalid compressed input"); } Debug.Assert(decompSize == nbytes); } catch (Exception ex) { Debugger.Launch(); UIntPtr nb = UIntPtr.Zero; UIntPtr cb = UIntPtr.Zero; UIntPtr bl = UIntPtr.Zero; BloscMethods.blosc_cbuffer_sizes(source, ref nb, ref cb, ref bl); //} Trace.WriteLine($"Blosc error: nbytes: {nbytes}, nbytes2: {nb}, cbytes: {cb} arr size: {value.Length}, \n\r exeption: {ex.Message + Environment.NewLine + ex}"); throw; } finally { pinnedArray.Free(); } } } else { // BufferPool returns an empty array } return(totalSize); } }