public void CouldCreateVsAndReadElements() { var count = 1000; var arr = Enumerable.Range(0, count).ToArray(); var r = ArrayMemory <int> .Create(arr); var vs = VecStorage.Create(r, 0, r.Length); Assert.AreEqual(arr.Length, vs.Vec.Length); long sum = 0L; for (int i = 0; i < arr.Length; i++) { var vi = vs.Vec.DangerousGetUnaligned <int>(i); if (vi != i) { Assert.Fail("vi != i"); } sum += vs.Vec.DangerousGetUnaligned <int>(i); } Console.WriteLine(sum); vs.Dispose(); }
public void SliceDisposeBenchmark() { // 6.3 MOPS var count = 1_000_000; var rounds = 10; var arrSize = 1000; var arr = Enumerable.Range(0, arrSize).ToArray(); var mem = ArrayMemory <int> .Create(arr); var vs = VecStorage.Create(mem, 0, mem.Length); Assert.AreEqual(arr.Length, vs.Vec.Length); for (int r = 0; r < rounds; r++) { using (Benchmark.Run("Slice/Dispose", count)) { for (int i = 0; i < count; i++) { var vs1 = vs.Slice(0, vs.Vec.Length, externallyOwned: true); vs1.Dispose(); } } } Benchmark.Dump(); vs.Dispose(); }
internal Series(TKey[] keys, TValue[] values) { if (keys == null) { throw new ArgumentNullException(nameof(keys)); } if (values == null) { throw new ArgumentNullException(nameof(values)); } if (keys.Length != values.Length) { throw new ArgumentException("Different keys and values length"); } var ks = KeySorting.Strong; if (keys.Length > 1) { var cmp = KeyComparer <TKey> .Default; for (int i = 1; i < keys.Length; i++) { var c = cmp.Compare(keys[i], keys[i - 1]); if (c == 0) { ks = KeySorting.Weak; } else if (c < 0) { ks = KeySorting.NotSorted; break; } } } Flags = new Flags((byte)((byte)Mutability.ReadOnly | (byte)ks)); if (keys.Length == 0 && values.Length == 0) { Debug.Assert(Data == DataBlock.Empty); return; } var keyMemory = ArrayMemory <TKey> .Create(keys); var keyVs = VecStorage.Create(keyMemory, 0, keyMemory.Length); var valMemory = ArrayMemory <TValue> .Create(values); var valVs = VecStorage.Create(valMemory, 0, valMemory.Length); var block = DataBlock.SeriesCreate(rowIndex: keyVs, values: valVs, rowLength: keys.Length); Data = block; }
private static BaseContainer <int> CreateIntBaseContainer(int capacity, int length) { var bc = new BaseContainer <int>(); var rm = ArrayMemory <int> .Create(Enumerable.Range(0, capacity).ToArray()); var vs = VecStorage.Create(rm, 0, rm.Length); var block = DataBlock.Create(rowIndex: vs, rowLength: length); bc.Data = block; return(bc); }
public void CouldTryFindBlockAtSingleChunkBench() { var count = 50_000_000; var rounds = 20; // for this test capacity is irrelevant - interpolation search hits exact position on first try var capacity = count / 100; var bc = new BaseContainer <int>(); var rm = ArrayMemory <int> .Create(Enumerable.Range(0, capacity).ToArray()); var vs = VecStorage.Create(rm, 0, rm.Length); var block = DataBlock.Create(rowIndex: vs, rowLength: vs.Vec.Length); bc.Data = block; for (int r = 0; r < rounds; r++) { using (Benchmark.Run("TryFindChunkAt", count)) { var m = count / capacity; for (int _ = 0; _ < m; _++) { for (int i = 1; i < capacity; i++) { var searchIndexRef = i; var found = bc.TryFindBlockAt(ref searchIndexRef, Lookup.LE, out var c, out var ci); if (!found || !ReferenceEquals(block, c) || i != ci || i != searchIndexRef ) { Assert.Fail(); } } } } } Benchmark.Dump(); bc.Dispose(); }
public void MemoryAccessVecViaDbVecStorageDangerous(RetainableMemory <int> rm) { long sum = 0; using (Benchmark.Run("DbVecStorageDangerous (_)", rm.Length)) { DataBlockLike db = new DataBlockLike { Rm = rm, Vec = rm.Vec, VecStorage = VecStorage.Create(rm, 0, rm.Length, true) }; for (int i = 0; i < rm.Length; i++) { sum += db.VecStorage.Vec.DangerousGetUnaligned <int>(i); } } if (sum < 1000) { throw new InvalidOperationException(); } }
public void VectorStorageReadBench() { var count = 1_000_000; var rounds = 10; var mult = 500; var arr = Enumerable.Range(0, count).ToArray(); var mem = ArrayMemory <int> .Create(arr); var vs = VecStorage.Create(mem, 0, mem.Length); Assert.AreEqual(arr.Length, vs.Vec.Length); int sum = 0; for (int r = 0; r < rounds; r++) { using (Benchmark.Run("VS Read", vs.Vec.Length * mult)) { for (int _ = 0; _ < mult; _++) { for (int i = 0; i < vs.Vec.Length; i++) { var vi = vs.Vec.DangerousGetUnaligned <int>(i); //if (vi != i) //{ // Assert.Fail("vi != i"); //} unchecked { sum += vi; } } } } } Benchmark.Dump(); Console.WriteLine(sum); }
public void WrappedLookup() { var count = 10_000; var arr = Enumerable.Range(0, count).Select(x => (long)x).ToArray(); var r = ArrayMemory <long> .Create(arr, 0, arr.Length, externallyOwned : true); var keys = VecStorage.Create(r, 0, r.Length); var values = keys.Slice(0, count, true); var block = DataBlock.SeriesCreate(keys, values, count); for (int i = 0; i < count; i++) { var ii = (long)i; Assert.AreEqual(i, block.LookupKey(ref ii, Lookup.EQ)); } block.Dispose(); }
public void CouldTryGetBlockAtSingleChunk() { var capacity = 100; var bc = new BaseContainer <int>(); var rm = ArrayMemory <int> .Create(Enumerable.Range(0, capacity).ToArray()); var vs = VecStorage.Create(rm, 0, rm.Length); var block = DataBlock.Create(rowIndex: vs, rowLength: vs.Vec.Length / 2); bc.Data = block; var searchIndex = 40; var found = bc.TryGetBlockAt(searchIndex, out var dataBlock, out var ci); Assert.IsTrue(found); Assert.AreSame(block, dataBlock); Assert.AreEqual(searchIndex, ci); bc.Dispose(); }
public void CouldTryGetBlockSingleChunk() { var capacity = 100; var bc = new BaseContainer <long>(); var rm = ArrayMemory <long> .Create(Enumerable.Range(0, capacity).Select(x => (long)x).ToArray()); var vs = VecStorage.Create(rm, 0, rm.Length); var block = DataBlock.Create(rowIndex: vs, rowLength: vs.Vec.Length / 2); bc.Data = block; var searchIndex = 40L; var searchIndexRef = searchIndex; var found = bc.TryGetBlock(searchIndexRef, out var c, out var ci); Assert.IsTrue(found); Assert.AreSame(block, c); Assert.AreEqual(searchIndex, ci); Assert.AreEqual(searchIndex, searchIndexRef); bc.Dispose(); }
public void Equality() { VecStorage vs1 = default; VecStorage vs2 = default; Assert.AreEqual(vs1, vs2); Assert.AreEqual(vs1.Vec.Length, 0); var count = 1000; var arr = Enumerable.Range(0, count).ToArray(); var r = ArrayMemory <int> .Create(arr); var vs = VecStorage.Create(r, 0, r.Length); Assert.AreNotEqual(vs1, vs); var vsCopy = vs.Slice(0, vs.Vec.Length, true); var vsSlice = vs.Slice(0, vs.Vec.Length - 1, true); Assert.AreEqual(vs, vsCopy); Assert.AreNotEqual(vs, vsSlice); vs.Dispose(); }
internal int SeriesIncreaseCapacity <TKey, TValue>(int newCapacity = -1) { EnsureSeriesLayout(); EnsureNotSentinel(); // TODO handle OutOfMemory in RentMemory, operation must be atomic in a sense that any error does not change existing data, no partial updates. // TODO _rowIndex.Vec.Length could be already 2x larger because array pool could have returned larger array on previous doubling // TODO (!, new) VS now hides total capacity of RM, we could get RM by casting and here we have types. // But is it always true that unused part of RM is always free and we could just expand to it without copying? // We ignore this now //if (_rowIndex.Vec.Length != _rowIndex.Length) //{ // Console.WriteLine($"_rowIndex.Vec.Length {_rowIndex.Vec.Length} != _rowIndex.Length {_rowIndex.Length}"); //} var ri = _rowKeys; var vals = _values; var minCapacity = Math.Max(newCapacity, Settings.MIN_POOLED_BUFFER_LEN); var newLen = Math.Max(minCapacity, BitUtil.FindNextPositivePowerOfTwo(ri.Vec.Length + 1)); RetainableMemory <TKey>?newRiBuffer = null; VecStorage newRi = default; RetainableMemory <TValue>?newValsBuffer = null; VecStorage newVals = default; try { newRiBuffer = BufferPool <TKey> .MemoryPool.RentMemory(newLen); newRi = VecStorage.Create(newRiBuffer, 0, newRiBuffer.Length); // new buffer could be larger if (ri.Vec.Length > 0) { ri.Vec.AsSpan <TKey>().CopyTo(newRi.Vec.AsSpan <TKey>()); } newValsBuffer = BufferPool <TValue> .MemoryPool.RentMemory(newLen); newVals = VecStorage.Create(newValsBuffer, 0, newValsBuffer.Length); if (vals.Vec.Length > 0) { vals.Vec.AsSpan <TValue>().CopyTo(newVals.Vec.AsSpan <TValue>()); } } catch (OutOfMemoryException) { if (newRi != default) { newRi.Dispose(); } else { newRiBuffer?.DecrementIfOne(); } if (newVals != default) { newVals.Dispose(); } else { newValsBuffer?.DecrementIfOne(); } // TODO log this event return(-1); } try { try { } finally { // we have all needed buffers, must switch in one operation _rowKeys = newRi; _values = newVals; ri.Dispose(); vals.Dispose(); } return(_rowKeys.Vec.Length); } catch { return(-1); } }
public void CouldSerializeVectorStorage() { var rng = new Random(42); var count = 100_000; var arr = new SmallDecimal[count]; arr[0] = new SmallDecimal(1000 * 1.0, 4); for (int i = 1; i < count; i++) { arr[i] = arr[i - 1] + new SmallDecimal((double)arr[i - 1] * (0.02 + -0.04 * rng.NextDouble()), 4); } // arr = Enumerable.Range(0, count).Select(x => new SmallDecimal(1000 + (double)x + (double)Math.Round(0.1 * rng.NextDouble(), 5), precision:3)).ToArray(); var r = ArrayMemory <SmallDecimal> .Create(arr); var vs = VecStorage.Create(r, 0, r.Length); var vsT = new VecStorage <SmallDecimal>(vs); var payload = count * Unsafe.SizeOf <double>() + 4; foreach (SerializationFormat format in ((SerializationFormat[])Enum.GetValues(typeof(SerializationFormat))).OrderBy(e => e.ToString())) { var len = BinarySerializer.SizeOf(in vsT, out var rm, format); var destination = BufferPool.Retain(len); var destinationDb = new DirectBuffer(destination); var len1 = BinarySerializer.Write(in vsT, destinationDb, rm, format); Assert.AreEqual(destination.Length, destinationDb.Length); Assert.AreEqual(len, len1); var flags = destinationDb.Read <VersionAndFlags>(0); Assert.AreEqual(format, flags.SerializationFormat); var header = destinationDb.Read <DataTypeHeader>(0); Assert.AreEqual(TypeEnum.Array, header.TEOFS.TypeEnum); Assert.AreEqual(TypeEnum.SmallDecimal, header.TEOFS1.TypeEnum); Assert.AreEqual(Unsafe.SizeOf <SmallDecimal>(), header.TEOFS1.Size); var len2 = BinarySerializer.Read(destinationDb, out VecStorage <SmallDecimal> value); Assert.AreEqual(destination.Length, destinationDb.Length); Assert.AreEqual(len, len2); Assert.AreEqual(vs.Vec.Length, value.Storage.Vec.Length); for (int i = 0; i < count; i++) { SmallDecimal left; SmallDecimal right; if ((left = vs.Vec.DangerousGetUnaligned <SmallDecimal>(i)) != (right = value.Storage.Vec.DangerousGetUnaligned <SmallDecimal>(i))) { Console.WriteLine("Not equals"); } } Assert.IsTrue(vs.Vec.Slice(0, vs.Vec.Length).AsSpan <SmallDecimal>().SequenceEqual(value.Storage.Vec.Slice(0, value.Storage.Vec.Length).AsSpan <SmallDecimal>())); Console.WriteLine($"{format} len: {len:N0} x{Math.Round((double)payload/len, 2)}"); destination.Dispose(); value.Storage.Dispose(); } vs.Dispose(); }