public void MemoryAccessVecViaDbVecStorageUnsafe(RetainableMemory <int> rm)
{
var rounds = TestUtils.GetBenchCount(100_000, 100);
long sum = 0;
using (Benchmark.Run("DbVecStorageUnsafe (+)", rm.Length * rounds))
{
DataBlockLike db = new DataBlockLike {
Rm = rm, Vec = rm.GetVec().AsVec(), RetainedVec = RetainedVec.Create(rm, 0, rm.Length, true)
};
for (int r = 0; r < rounds; r++)
{
for (long i = 0; i < rm.Length; i++)
{
sum += db.RetainedVec.Vec.UnsafeGetUnaligned <int>((IntPtr)i);
}
}
}
if (sum < 1000)
{
throw new InvalidOperationException();
}
}
public void SliceDisposeBenchmark()
{
// 6.3 MOPS
var count = (int)TestUtils.GetBenchCount(1_000_000, 100);
var rounds = 10;
var bufferSize = 1000;
var rm = BuffersTestHelper.CreateFilledRM(bufferSize);
var vs = RetainedVec.Create(rm, 0, rm.Length);
Assert.AreEqual(rm.Length, vs.Vec.Length);
for (int r = 0; r < rounds; r++)
{
using (Benchmark.Run("Slice/Dispose", count))
{
for (int i = 0; i < count; i++)
{
var vs1 = vs.Slice(0, vs.Vec.Length, externallyOwned: true);
vs1.Dispose();
}
}
}
Benchmark.Dump();
vs.Dispose();
}
public void MemoryFieldBench()
{
var count = (int)4 * 1024; // TestUtils.GetBenchCount(1_000_000);
var rm = PrivateMemory <int> .Create(count);
var arr = new int[count];
DataBlockLike db = new DataBlockLike {
Rm = rm, Vec = rm.GetVec().AsVec(), RetainedVec = RetainedVec.Create(rm, 0, rm.Length, true), Length = rm.Length
};
for (int i = 0; i < count; i++)
{
db.RetainedVec.UnsafeWriteUnaligned((IntPtr)i, i);
arr[i] = i;
}
for (int r = 0; r < 10; r++)
{
MemoryFieldBench_Field(rm);
MemoryFieldBench_CreateMem(rm);
}
Benchmark.Dump();
rm.Dispose();
}
public RetainedVec(RetainedVec storage)
{
if (VecTypeHelper <T> .RuntimeTypeId != storage.Vec.RuntimeTypeId)
{
VecThrowHelper.ThrowVecTypeMismatchException();
}
Storage = storage;
}
internal Series(TKey[] keys, TValue[] values)
{
if (keys == null)
{
throw new ArgumentNullException(nameof(keys));
}
if (values == null)
{
throw new ArgumentNullException(nameof(values));
}
if (keys.Length != values.Length)
{
throw new ArgumentException("Different keys and values length");
}
var ks = KeySorting.Strong;
if (keys.Length > 1)
{
var cmp = KeyComparer <TKey> .Default;
for (int i = 1; i < keys.Length; i++)
{
var c = cmp.Compare(keys[i], keys[i - 1]);
if (c == 0)
{
ks = KeySorting.Weak;
}
else if (c < 0)
{
ks = KeySorting.NotSorted;
break;
}
}
}
Flags = new Flags((byte)((byte)Mutability.ReadOnly | (byte)ks));
if (keys.Length == 0 && values.Length == 0)
{
Debug.Assert(Data == DataBlock.Empty);
return;
}
var keyMemory = ArrayMemory <TKey> .Create(keys);
var keyVs = RetainedVec.Create(keyMemory, 0, keyMemory.Length);
var valMemory = ArrayMemory <TValue> .Create(values);
var valVs = RetainedVec.Create(valMemory, 0, valMemory.Length);
var block = DataBlock.SeriesCreate(rowIndex: keyVs, values: valVs, rowLength: keys.Length);
Data = block;
}
private static BaseContainer <int> CreateIntBaseContainer(int capacity, int length)
{
var bc = new BaseContainer <int>();
var rm = ArrayMemory <int> .Create(Enumerable.Range(0, capacity).ToArray());
var vs = RetainedVec.Create(rm, 0, rm.Length);
var block = DataBlock.Create(rowIndex: vs, rowLength: length);
bc.Data = block;
return(bc);
}
public void CouldTryGetBlockSingleChunkBench()
{
var count = (int)TestUtils.GetBenchCount(50_000_000, 50_000);
var rounds = TestUtils.GetBenchCount(20, 2);
// for this test capacity is irrelevant - interpolation search hits exact position on first try
var capacity = count / 100;
var bc = new BaseContainer <long>();
var rm = BuffersTestHelper.CreateFilledRM(capacity);
var vs = RetainedVec.Create(rm, 0, rm.Length);
var block = DataBlock.Create(rowIndex: vs, rowLength: capacity);
bc.Data = block;
for (int r = 0; r < rounds; r++)
{
using (Benchmark.Run("TryGetBlock", count))
{
var m = count / capacity;
for (int _ = 0; _ < m; _++)
{
for (long i = 0; i < capacity; i++)
{
var ival = vs.UnsafeReadUnaligned <long>((IntPtr)i);
var ival2 = vs.UnsafeReadUnaligned <long>((IntPtr)capacity - 1);
var ival3 = Unsafe.Add <long>(ref vs.UnsafeGetRef <long>(), capacity - 1);
var ival4 = Unsafe.Add <long>(ref (bc.Data as DataBlock).RowKeys.UnsafeGetRef <long>(), capacity - 1);
var found = bc.TryGetBlock(i, out var c, out var ci);
if (!found ||
!ReferenceEquals(block, c) ||
i != ci
)
{
Assert.Fail();
}
}
}
}
}
Benchmark.Dump();
bc.Dispose();
}
public void CouldTryFindBlockAtSingleChunkBench()
{
var count = (int)TestUtils.GetBenchCount(50_000_000, 50_000);
var rounds = TestUtils.GetBenchCount(20, 2);
// for this test capacity is irrelevant - interpolation search hits exact position on first try
var capacity = count / 100;
var bc = new BaseContainer <long>();
var rm = BuffersTestHelper.CreateFilledRM(capacity);
var vs = RetainedVec.Create(rm, 0, rm.Length);
var block = DataBlock.Create(rowIndex: vs, rowLength: capacity);
bc.Data = block;
for (int r = 0; r < rounds; r++)
{
using (Benchmark.Run("TryFindChunkAt", count))
{
var m = count / capacity;
for (int _ = 0; _ < m; _++)
{
for (long i = 1; i < capacity; i++)
{
var searchIndexRef = i;
var found = bc.TryFindBlockAt(ref searchIndexRef, Lookup.LE, out var c, out var ci);
if (!found ||
!ReferenceEquals(block, c) ||
i != ci ||
i != searchIndexRef
)
{
Assert.Fail();
}
}
}
}
}
Benchmark.Dump();
bc.Dispose();
}
public void CouldTryGetBlockAtSingleChunk()
{
var capacity = 100;
var bc = new BaseContainer <int>();
var rm = BuffersTestHelper.CreateFilledRM(capacity);
var vs = RetainedVec.Create(rm, 0, rm.Length);
var block = DataBlock.Create(rowIndex: vs, rowLength: vs.Vec.Length / 2);
bc.Data = block;
var searchIndex = 40;
var found = bc.TryGetBlockAt(searchIndex, out var dataBlock, out var ci);
Assert.IsTrue(found);
Assert.AreSame(block, dataBlock);
Assert.AreEqual(searchIndex, ci);
bc.Dispose();
}
public void WrappedLookup()
{
var count = 10_000;
var arr = Enumerable.Range(0, count).Select(x => (long)x).ToArray();
var r = ArrayMemory <long> .Create(arr,
0, arr.Length, externallyOwned : true);
var keys = RetainedVec.Create(r, 0, r.Length);
var values = keys.Slice(0, count, true);
var block = DataBlock.SeriesCreate(keys, values, count);
for (int i = 0; i < count; i++)
{
var ii = (long)i;
Assert.AreEqual(i, block.LookupKey(ref ii, Lookup.EQ));
}
block.Dispose();
}
public void Equality()
{
RetainedVec vs1 = default;
RetainedVec vs2 = default;
Assert.AreEqual(vs1, vs2);
Assert.AreEqual(vs1.Vec.Length, 0);
var count = 1000;
var rm = BuffersTestHelper.CreateFilledRM(count);
var vs = RetainedVec.Create(rm, 0, rm.Length);
Assert.AreNotEqual(vs1, vs);
var vsCopy = vs.Slice(0, vs.Vec.Length, true);
var vsSlice = vs.Slice(0, vs.Vec.Length - 1, true);
Assert.AreEqual(vs, vsCopy);
Assert.AreNotEqual(vs, vsSlice);
vs.Dispose();
}
public void VectorStorageReadBench()
{
var count = (int)TestUtils.GetBenchCount(1_000_000, 100);
var rounds = 10;
var mult = 500;
var rm = BuffersTestHelper.CreateFilledRM(count);
var vs = RetainedVec.Create(rm, 0, rm.Length);
Assert.AreEqual(rm.Length, vs.Vec.Length);
int sum = 0;
for (int r = 0; r < rounds; r++)
{
using (Benchmark.Run("VS Read", vs.Vec.Length * mult))
{
for (int _ = 0; _ < mult; _++)
{
for (int i = 0; i < vs.Vec.Length; i++)
{
var vi = vs.Vec.DangerousGetUnaligned <int>(i);
//if (vi != i)
//{
// Assert.Fail("vi != i");
//}
unchecked
{
sum += vi;
}
}
}
}
}
Benchmark.Dump();
Console.WriteLine(sum);
}
public unsafe void SetUp()
{
var rng = new Random();
_count = Count;
_idx = rng.Next(_count / 4, _count / 2);
_idxLong = _idx;
_rm = PrivateMemory <int> .Create(_count);
_arr = new int[_count];
_db = new DataBlockLike
{
Rm = _rm, Vec = _rm.GetVec().AsVec(), RetainedVec = RetainedVec.Create(_rm, 0, _rm.Length, true), Length = _rm.Length, arr = _arr, Ptr = (int *)_rm.Pointer
};
_vec = _rm.GetVec().AsVec();
for (int i = 0; i < _count; i++)
{
_db.RetainedVec.UnsafeWriteUnaligned((IntPtr)i, i + 1);
_arr[i] = i + 1;
}
}
public void MemoryAccessBench()
{
var count = (int)4 * 1024; // TestUtils.GetBenchCount(1_000_000);
var rm = PrivateMemory <int> .Create(count);
var arr = new int[count];
DataBlockLike db = new DataBlockLike {
Rm = rm, Vec = rm.GetVec().AsVec(), RetainedVec = RetainedVec.Create(rm, 0, rm.Length, true), Length = rm.Length
};
for (int i = 0; i < count; i++)
{
db.RetainedVec.UnsafeWriteUnaligned((IntPtr)i, i);
arr[i] = i;
}
for (int r = 0; r < 10; r++)
{
Sum(rm);
MemoryAccessViaPointer(rm);
MemoryAccessViaArray(rm, arr);
// MemoryAccessVecViaMemSpan(rm);
// MemoryAccessVecViaLocalDangerous(rm);
// MemoryAccessVecViaLocalUnsafe(rm);
MemoryAccessVecViaDbVecUnsafe(rm);
MemoryAccessVecViaDbVecStorageRead(rm);
// MemoryAccessVecViaDbVecUnsafeX(rm);
MemoryAccessVecViaDbVecPointer(rm);
// MemoryAccessVecViaDbVecDangerous(rm);
MemoryAccessVecViaDbVecStorageUnsafe(rm);
MemoryAccessVecViaDbVecStorageDangerous(rm);
Thread.Sleep(100);
}
Benchmark.Dump();
rm.Dispose();
}
public void CouldTryFindBlockAtSingleChunk()
{
var capacity = 100;
var bc = new BaseContainer <long>();
var rm = BuffersTestHelper.CreateFilledRM(capacity);
var vs = RetainedVec.Create(rm, 0, rm.Length);
var block = DataBlock.Create(rowIndex: vs, rowLength: vs.Vec.Length / 2);
bc.Data = block;
var searchIndex = 40L;
var searchIndexRef = searchIndex;
var found = bc.TryFindBlockAt(ref searchIndexRef, Lookup.LT, out var c, out var ci);
Assert.IsTrue(found);
Assert.AreSame(block, c);
Assert.AreEqual(searchIndex - 1, ci);
Assert.AreEqual(searchIndex - 1, searchIndexRef);
bc.Dispose();
}
public void RingVecIndexing()
{
var count = (int)TestUtils.GetBenchCount(128 * 1024, 128);
var rm = PrivateMemory <int> .Create(count);
var rv = RetainedVec.Create(rm, 0, rm.Length, true);
for (int i = 0; i < count; i++)
{
rv.UnsafeWriteUnaligned((IntPtr)i, i);
}
for (int r = 0; r < 10; r++)
{
RingVecUtilIndexToOffset(rm);
DirectPointer(rm);
Modulo(rm);
Binary(rm);
}
Benchmark.Dump();
rm.Dispose();
}
public void CouldCreateVsAndReadElements()
{
var count = 1000;
var rm = BuffersTestHelper.CreateFilledRM(count);
var vs = RetainedVec.Create(rm, 0, rm.Length);
Assert.AreEqual(rm.Length, vs.Vec.Length);
long sum = 0L;
for (int i = 0; i < rm.Length; i++)
{
var vi = vs.Vec.DangerousGetUnaligned <long>(i);
if (vi != i)
{
Assert.Fail("vi != i");
}
sum += vs.Vec.DangerousGetUnaligned <int>(i);
}
Console.WriteLine(sum);
vs.Dispose();
}
public void CouldSerializeVectorStorage()
{
var rng = new Random(42);
var count = 100_000;
var arr = new SmallDecimal[count];
var r = PrivateMemory <SmallDecimal> .Create(count);
var vec = r.GetVec();
vec[0] = new SmallDecimal(1000 * 1.0, 4);
for (int i = 1; i < count; i++)
{
vec[i] = vec[i - 1] + new SmallDecimal((double)vec[i - 1] * (0.02 + -0.04 * rng.NextDouble()), 4);
}
var vs = RetainedVec.Create(r, 0, r.Length);
var vsT = new RetainedVec <SmallDecimal>(vs);
var payload = count * Unsafe.SizeOf <double>() + 4;
foreach (SerializationFormat format in ((SerializationFormat[])Enum.GetValues(typeof(SerializationFormat))).OrderBy(e => e.ToString()))
{
var len = BinarySerializer.SizeOf(in vsT, out var rm, format);
var destination = BufferPool.Retain(len);
var destinationDb = new DirectBuffer(destination);
var len1 = BinarySerializer.Write(in vsT, destinationDb, rm, format);
Assert.AreEqual(destination.Length, destinationDb.Length);
Assert.AreEqual(len, len1);
var flags = destinationDb.Read <VersionAndFlags>(0);
Assert.AreEqual(format, flags.SerializationFormat);
var header = destinationDb.Read <DataTypeHeader>(0);
Assert.AreEqual(TypeEnum.Array, header.TEOFS.TypeEnum);
Assert.AreEqual(TypeEnum.SmallDecimal, header.TEOFS1.TypeEnum);
Assert.AreEqual(Unsafe.SizeOf <SmallDecimal>(), header.TEOFS1.Size);
var len2 = BinarySerializer.Read(destinationDb, out RetainedVec <SmallDecimal> value);
Assert.AreEqual(destination.Length, destinationDb.Length);
Assert.AreEqual(len, len2);
Assert.AreEqual(vs.Vec.Length, value.Storage.Vec.Length);
for (int i = 0; i < count; i++)
{
SmallDecimal left;
SmallDecimal right;
if ((left = vs.Vec.DangerousGetUnaligned <SmallDecimal>(i)) != (right = value.Storage.Vec.DangerousGetUnaligned <SmallDecimal>(i)))
{
Console.WriteLine("Not equals");
}
}
Assert.IsTrue(vs.Vec.Slice(0, vs.Vec.Length).AsSpan <SmallDecimal>().SequenceEqual(value.Storage.Vec.Slice(0, value.Storage.Vec.Length).AsSpan <SmallDecimal>()));
Console.WriteLine($"{format} len: {len:N0} x{Math.Round((double) payload / len, 2)}");
destination.Dispose();
value.Storage.Dispose();
}
vs.Dispose();
}
internal static DataBlock Create(RetainedVec rowIndex = default, RetainedVec values = default, RetainedVec[]?columns = null, int rowLength = -1)
{
var block = ObjectPool.Rent();
block.EnsureDisposed();
var rowCapacity = -1;
if (rowIndex != default)
{
block._rowKeys = rowIndex;
rowCapacity = rowIndex.Vec.Length;
}
if (values != default)
{
block._values = values;
if (rowCapacity >= 0 && values.Vec.Length < rowCapacity)
{
ThrowHelper.ThrowArgumentException("");
}
else
{
rowCapacity = values.Vec.Length;
}
rowCapacity = Math.Min(rowCapacity, values.Vec.Length);
}
if (columns != null)
{
if (columns.Length == 0)
{
ThrowHelper.ThrowArgumentException("Empty columns array. Pass null instead.");
}
block._columns = columns;
foreach (var column in columns)
{
rowCapacity = Math.Min(rowCapacity, column.Vec.Length);
}
}
if (rowLength == -1)
{
block._rowCount = rowCapacity;
}
else
{
if ((uint)rowLength > rowCapacity)
{
ThrowHelper.ThrowArgumentOutOfRangeException("rowLength");
}
else
{
block._rowCount = rowLength;
}
}
block._head = 0;
block._refCount = 0;
ThrowHelper.DebugAssert(!block.IsDisposed, "!block.IsDisposed");
return(block);
}
