public void CouldUseIDeltaMethods()
{
var first = new IntDelta {
Value = 123
};
var second = new IntDelta {
Value = 456
};
var delta = new IntDelta {
Value = 456 - 123
};
Assert.AreEqual(delta, Unsafe.GetDeltaConstrained(ref first, ref second));
Assert.AreEqual(second, Unsafe.AddDeltaConstrained(ref first, ref delta));
}
public unsafe int Read(IntPtr ptr, out TElement[] value, out int length, bool exactSize = false)
{
var totalSize = Marshal.ReadInt32(ptr);
var versionFlag = Marshal.ReadByte(ptr + 4);
var version = (byte)(versionFlag >> 4);
var isDelta = (versionFlag & 0b0000_0010) != 0;
var isCompressed = (versionFlag & 0b0000_0001) != 0;
if (!isCompressed)
{
throw new InvalidOperationException("Wrong compressed flag. CompressedArrayBinaryConverter.Read works only with compressed data.");
}
if (version != Version)
{
throw new NotSupportedException($"CompressedBinaryConverter work only with version {Version}");
}
if (ItemSize <= 0)
{
// first decompress bytes
var size = CompressedArrayBinaryConverter <byte> .Instance.Read(ptr, out byte[] decompressedBytes, out length);
// NB the length is encoded in the header and is returned as a part of ArraySegment
Debug.Assert(length == BitConverter.ToInt32(decompressedBytes, 0));
// then deserialize
// NB the size of the array will be exact, BinarySerializer.Read does not support non-exact buffers
BinarySerializer.Read(decompressedBytes, out value);
BufferPool <byte> .Return(decompressedBytes);
return(size);
}
else if (Buffers.BufferPool.IsPreservedBuffer <TElement>())
{
throw new NotImplementedException();
}
else
{
if (totalSize <= 8 + 16)
{
value = EmptyArray <TElement> .Instance;
length = 0;
return(totalSize);
}
var source = ptr + 8;
// avoid additional P/Invoke call, read header directly
// https://github.com/Blosc/c-blosc/blob/master/README_HEADER.rst
var nbytes = *(int *)(source + 4);
#if DEBUG
var blocksize = *(int *)(source + 8);
var cbytes = *(int *)(source + 12);
var nbytes2 = new UIntPtr();
var cbytes2 = new UIntPtr();
var blocksize2 = new UIntPtr();
BloscMethods.blosc_cbuffer_sizes(source, ref nbytes2, ref cbytes2, ref blocksize2);
Debug.Assert(nbytes == nbytes2.ToUInt32());
Debug.Assert(cbytes == cbytes2.ToUInt32());
Debug.Assert(blocksize == blocksize2.ToUInt32());
#endif
var arraySize = nbytes / ItemSize;
// when caller provides an empty AS, it could require exact size, e.g. DateTimeArrayBinaryConverter
var array = BufferPool <TElement> .Rent(arraySize, exactSize);
length = arraySize;
value = array;
if (arraySize > 0)
{
if (typeof(TElement) == typeof(DateTime))
{
var buffer = BufferPool <byte> .Rent(arraySize * 8);
var dtArray = new DateTime[arraySize];
fixed(byte *tgtPtr = &buffer[0])
{
var destination = (IntPtr)tgtPtr;
var decompSize = BloscMethods.blosc_decompress_ctx(
source, destination, new UIntPtr((uint)nbytes), BloscMethods.ProcessorCount);
if (decompSize <= 0)
{
throw new ArgumentException("Invalid compressed input");
}
Debug.Assert(decompSize == nbytes);
// NB a lot of data was stored without diff for DateTime,
// should just check the flag
if (isDelta)
{
var previousLong = *(long *)destination;
var first = *(DateTime *)&previousLong;
dtArray[0] = first;
for (var i = 1; i < arraySize; i++)
{
var deltaLong = *(long *)(destination + i * 8);
var currentLong = previousLong + deltaLong;
dtArray[i] = *(DateTime *)¤tLong;
previousLong = currentLong;
}
}
else
{
for (var i = 0; i < arraySize; i++)
{
dtArray[i] = *(DateTime *)(destination + i * 8);
}
}
}
value = (TElement[])(object)(dtArray);
BufferPool <byte> .Return(buffer);
}
else if (isDelta)
{
if (!IsIDelta)
{
ThrowHelper.ThrowInvalidOperationException("Delta flag is set for a type that does not implement IDelta interface.");
}
var buffer = BufferPool <byte> .Rent(arraySize *ItemSize);
var targetArray = BufferPool <TElement> .Rent(arraySize);
fixed(byte *tgtPtr = &buffer[0])
{
var destination = tgtPtr;
var decompSize = BloscMethods.blosc_decompress_ctx(
source, (IntPtr)destination, new UIntPtr((uint)nbytes), BloscMethods.ProcessorCount);
if (decompSize <= 0)
{
throw new ArgumentException("Invalid compressed input");
}
Debug.Assert(decompSize == nbytes);
var first = Unsafe.ReadUnaligned <TElement>(destination);
targetArray[0] = first;
for (var i = 1; i < arraySize; i++)
{
var currentDelta = Unsafe.Read <TElement>(destination + i * ItemSize);
var current = Unsafe.AddDeltaConstrained(ref first, ref currentDelta);
targetArray[i] = current;
}
}
value = targetArray;
BufferPool <byte> .Return(buffer);
}
else
{
var pinnedArray = GCHandle.Alloc(value, GCHandleType.Pinned);
var destination = pinnedArray.AddrOfPinnedObject();
// TODO remove this try/catch and debugger stuff, it was used to catch an eror that disappeared after adding
// try/catch. Probably some reordering, maybe add a memory barrier before the call
try
{
var decompSize = BloscMethods.blosc_decompress_ctx(
source, destination, new UIntPtr((uint)nbytes), BloscMethods.ProcessorCount);
if (decompSize <= 0)
{
throw new ArgumentException("Invalid compressed input");
}
Debug.Assert(decompSize == nbytes);
}
catch (Exception ex)
{
Debugger.Launch();
UIntPtr nb = UIntPtr.Zero;
UIntPtr cb = UIntPtr.Zero;
UIntPtr bl = UIntPtr.Zero;
BloscMethods.blosc_cbuffer_sizes(source, ref nb, ref cb, ref bl);
//}
Trace.WriteLine($"Blosc error: nbytes: {nbytes}, nbytes2: {nb}, cbytes: {cb} arr size: {value.Length}, \n\r exeption: {ex.Message + Environment.NewLine + ex}");
throw;
}
finally
{
pinnedArray.Free();
}
}
}
else
{
// BufferPool returns an empty array
}
return(totalSize);
}
}
