internal unsafe SharedMemory RentNative(int bucketSize) { var bucketIndex = SelectBucketIndex(bucketSize); BufferRef.EnsureBucketIndexInRange(bucketIndex); // We must init the buffer header before allocating. // We know from inside BRA.Allocate txn the value or BufferRef // before committing. For new we must init the header, // for one from the free list we must assert the buffer is disposed // and not owned (it may be in Releasing state though). var br = _bra.Allocate((byte)bucketIndex, out _, _buckets); var db = _buckets.DangerousGet(br); var sm = SharedMemory.Create(db, br, this); Debug.Assert(sm.IsDisposed); Debug.Assert(Unsafe.Read <uint>(sm.HeaderPointer) == (HeaderFlags.Releasing | HeaderFlags.IsDisposed)); sm.FromReleasingDisposedToOwned(); Debug.Assert(!sm.IsDisposed); Debug.Assert(sm.ReferenceCount == 0); Debug.Assert(Unsafe.Read <uint>(sm.HeaderPointer) == HeaderFlags.IsOwned); return(sm); }
public BufferRef Allocate(byte bucketIndex, out bool fromFreeList, SharedMemoryBuckets buckets = null) { BufferRef.EnsureBucketIndexInRange(bucketIndex); using (var txn = _env.BeginTransaction()) { try { BufferRef freeRef = Allocate(txn, bucketIndex, out fromFreeList, buckets); txn.Commit(); // Without packing and returning buffers to pool this kills performance // But it must be done. if (!fromFreeList) { _env.Sync(true); } return(freeRef); } catch { txn.Abort(); throw; } } }
public SharedMemoryBucket(DirectFile directFile, int bucketIndex, int pageSize = DefaultPageSize) { BufferRef.EnsureBucketIndexInRange(bucketIndex); DirectFile = directFile; var directBuffer = directFile.DirectBuffer; var bufferSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndex, pageSize); BufferCount = directBuffer.Length / bufferSize; DirectBuffer = directBuffer; BucketIndex = bucketIndex; PageSize = pageSize; BufferSize = bufferSize; }
public long GetTotalAllocated(int bucketIndex = -1) { using (var txn = _env.BeginReadOnlyTransaction()) using (var aC = _allocatedDb.OpenReadOnlyCursor(txn)) { BufferRef lastAllocated = default; var totalAllocated = 0L; if (bucketIndex == -1) { byte bucketIndexByte = 0; if (aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.First)) { var dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndexByte, _pageSize); totalAllocated += dataSize * (long)aC.Count(); while (aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.NextNoDuplicate)) { dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndex, _pageSize); totalAllocated += dataSize * (long)aC.Count(); } } } else { BufferRef.EnsureBucketIndexInRange(bucketIndex); var bucketIndexByte = (byte)bucketIndex; if (aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.Set)) { var dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndex, _pageSize); totalAllocated += dataSize * (long)aC.Count(); } } _lastTotalSize = totalAllocated; return(totalAllocated); } }
public void PrintFree(int bucketIndex = -1, bool printBuffers = false, Func <BufferRef, string> additionalInfo = null) { using (var txn = _env.BeginReadOnlyTransaction()) using (var fC = _freeDb.OpenReadOnlyCursor(txn)) { BufferRef lastFree = default; var totalFree = 0L; if (bucketIndex == -1) { Console.WriteLine("------------------- FREE -------------------"); byte bucketIndexByte = 0; if (fC.TryGet(ref bucketIndexByte, ref lastFree, CursorGetOption.First)) { var dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndexByte, _pageSize); var count = (long)fC.Count(); var bucketSize = dataSize * count; totalFree += bucketSize; Console.WriteLine( $"[{bucketIndexByte}]: count [{count:N0}], buffer size [{dataSize:N0}], bucket size: [{bucketSize:N0}]"); if (printBuffers) { PrintBuffers(bucketIndexByte); } while (fC.TryGet(ref bucketIndexByte, ref lastFree, CursorGetOption.NextNoDuplicate)) { dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndexByte, _pageSize); count = (long)fC.Count(); bucketSize = dataSize * count; totalFree += bucketSize; Console.WriteLine( $"[{bucketIndexByte}]: count [{count:N0}], buffer size [{dataSize:N0}], bucket size: [{bucketSize:N0}]"); if (printBuffers) { PrintBuffers(bucketIndexByte); } } } Console.WriteLine($"------ FREE {totalFree:N0}, {Math.Round(100 * (totalFree / (double)MaxTotalSize), 2)}% of Max {MaxTotalSize:N0} ------"); } else { BufferRef.EnsureBucketIndexInRange(bucketIndex); Console.WriteLine($"------------------- FREE [{bucketIndex}] -------------------"); var bucketIndexByte = (byte)bucketIndex; if (fC.TryGet(ref bucketIndexByte, ref lastFree, CursorGetOption.Set)) { var dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndexByte, _pageSize); var count = (long)fC.Count(); var bucketSize = dataSize * count; totalFree += bucketSize; Console.WriteLine( $"[{bucketIndexByte}]: count [{count}], buffer size [{dataSize}], bucket size: [{bucketSize}]"); if (printBuffers) { PrintBuffers(bucketIndexByte); } } Console.WriteLine($"------ FREE [{bucketIndex}] {totalFree:N0}, {Math.Round(100 * (totalFree / (double)MaxTotalSize), 2)}% of Max {MaxTotalSize:N0} ------"); } void PrintBuffers(byte bucketIndexByte) { if (fC.TryGet(ref bucketIndexByte, ref lastFree, CursorGetOption.SetKey) && fC.TryGet(ref bucketIndexByte, ref lastFree, CursorGetOption.FirstDuplicate)) { var info = String.Empty; if (additionalInfo != null) { info = ": " + additionalInfo(lastFree); } Console.WriteLine($" [{lastFree.BufferIndex}]{info}"); while (fC.TryGet(ref bucketIndexByte, ref lastFree, CursorGetOption.NextDuplicate)) { info = String.Empty; if (additionalInfo != null) { info = ": " + additionalInfo(lastFree); } Console.WriteLine($" [{lastFree.BufferIndex}]{info}"); } } } } }
public void PrintAllocated(int bucketIndex = -1, bool printBuffers = false, Func <BufferRef, string> additionalInfo = null) { using (var txn = _env.BeginReadOnlyTransaction()) using (var aC = _allocatedDb.OpenReadOnlyCursor(txn)) { BufferRef lastAllocated = default; var totalAllocated = 0L; if (bucketIndex == -1) { Console.WriteLine("------------------- ALLOCATED -------------------"); byte bucketIndexByte = 0; if (aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.First)) { var dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndexByte, _pageSize); var count = (long)aC.Count(); var bucketSize = dataSize * count; totalAllocated += bucketSize; Console.WriteLine( $"[{bucketIndexByte}]: count [{count:N0}], buffer size [{dataSize:N0}], bucket size: [{bucketSize:N0}]"); if (printBuffers) { PrintBuffers(bucketIndexByte); } while (aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.NextNoDuplicate)) { dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndexByte, _pageSize); count = (long)aC.Count(); bucketSize = dataSize * count; totalAllocated += bucketSize; Console.WriteLine( $"[{bucketIndexByte}]: count [{count:N0}], buffer size [{dataSize:N0}], bucket size: [{bucketSize:N0}]"); if (printBuffers) { PrintBuffers(bucketIndexByte); } } } Console.WriteLine($"------ ALLOCATED {totalAllocated:N0}, {Utilization}% of Max {MaxTotalSize:N0} ------"); } else { BufferRef.EnsureBucketIndexInRange(bucketIndex); Console.WriteLine($"------------------- ALLOCATED [{bucketIndex}] -------------------"); var bucketIndexByte = (byte)bucketIndex; if (aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.Set)) { var dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndexByte, _pageSize); var count = (long)aC.Count(); var bucketSize = dataSize * count; totalAllocated += bucketSize; Console.WriteLine( $"[{bucketIndexByte}]: count [{count:N0}], buffer size [{dataSize:N0}], bucket size: [{bucketSize:N0}]"); if (printBuffers) { PrintBuffers(bucketIndexByte); } } Console.WriteLine($"------ ALLOCATED [{bucketIndex}] {totalAllocated:N0}, {Utilization}% of Max {MaxTotalSize:N0} ------"); } void PrintBuffers(byte bucketIndexByte) { if (aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.SetKey) && aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.FirstDuplicate)) { var info = string.Empty; if (additionalInfo != null) { info = ": " + additionalInfo(lastAllocated); } Console.WriteLine($" [{lastAllocated.BufferIndex}]{info}"); while (aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.NextDuplicate)) { info = string.Empty; if (additionalInfo != null) { info = ": " + additionalInfo(lastAllocated); } Console.WriteLine($" [{lastAllocated.BufferIndex}]{info}"); } } } _lastTotalSize = totalAllocated; } }
protected BufferRef Allocate(Transaction txn, byte bucketIndex, out bool fromFreeList, SharedMemoryBuckets buckets = null) { var originalBucketIndex = bucketIndex; fromFreeList = false; BufferRef.EnsureBucketIndexInRange(bucketIndex); BufferRef freeRef = default; using (var fC = _freeDb.OpenCursor(txn)) using (var aC = _allocatedDb.OpenCursor(txn)) { // If free list is not empty then everything is simple - just move an item from FL to AL if (fC.TryGet(ref bucketIndex, ref freeRef, CursorGetOption.Set) && fC.TryGet(ref bucketIndex, ref freeRef, CursorGetOption.FirstDuplicate) ) // prefer closer to the bucket start, DRs are sorted by index // TODO test with first with high churn & packer { ThrowHelper.AssertFailFast(originalBucketIndex == bucketIndex, $"originalBucketIndex {originalBucketIndex} == bucketIndex {bucketIndex}"); _freeDb.Delete(txn, bucketIndex, freeRef); var ar = freeRef; aC.Put(ref bucketIndex, ref ar, CursorPutOptions.None); fromFreeList = true; if (buckets != null) { #pragma warning disable 618 var directBuffer = buckets.DangerousGet(freeRef); #pragma warning restore 618 var header = directBuffer.Read <SharedMemory.BufferHeader>(0); unchecked { if (header.FlagsCounter != (HeaderFlags.Releasing | HeaderFlags.IsDisposed)) { ThrowHelper.ThrowInvalidOperationException( "Buffer in free list must be (Releasing | IsDisposed)."); } // TODO review, we require (Releasing | IsDisposed) in FL. Remove commented code below if that is OK //if (((int)header.FlagsCounter & (int)AtomicCounter.CountMask) != (int)AtomicCounter.CountMask) //{ // ThrowHelper.ThrowInvalidOperationException("Buffer in free list must be disposed."); //} //var ignoreCountAndReleasing = ((int)header.FlagsCounter & ~(int)AtomicCounter.CountMask) & ~((int)SharedMemory.BufferHeader.HeaderFlags.Releasing); //if (ignoreCountAndReleasing != 0) //{ // ThrowHelper.ThrowInvalidOperationException("Non-Releasing flags in free buffer header."); //} var newHeader = header; // newHeader.FlagsCounter = (SharedMemory.BufferHeader.HeaderFlags)(uint)(AtomicCounter.CountMask | (int)SharedMemory.BufferHeader.HeaderFlags.Releasing); newHeader.AllocatorInstanceId = _wpid.InstanceId; // TODO check existing header values // It must be zeros if the buffer was disposed normally //if ((long)header != 0) //{ // Trace.TraceWarning($"Header of free list buffer is not zero: FlagsCounter {header.FlagsCounter}, InstanceId: {header.AllocatorInstanceId}"); //} if ((long)header != directBuffer.InterlockedCompareExchangeInt64(0, (long)newHeader, (long)header)) { // This should only happen if someone if touching the header concurrently // But we are inside write txn now so this should never happen. // TODO could remove expensive CAS for simple write. ThrowHelper.FailFast("Header changed during allocating from free list."); } } } } else { // Free list is empty. // Allocated must be sorted by BufferRef as uint. Since in DUP_SORTED bucket bites are // the same then the last allocated value has the maximum buffer index. int bufferIndex = 1; // if is empty then start from 1 BufferRef lastAllocated = default; if (aC.TryGet(ref bucketIndex, ref lastAllocated, CursorGetOption.Set) && aC.TryGet(ref bucketIndex, ref lastAllocated, CursorGetOption.LastDuplicate)) { ThrowHelper.AssertFailFast(lastAllocated.BufferIndex > 0, "BufferRef.BufferIndex is one-based"); bufferIndex += lastAllocated.BufferIndex; } ThrowHelper.AssertFailFast(originalBucketIndex == bucketIndex, $"originalBucketIndex {originalBucketIndex} == bucketIndex {bucketIndex}"); freeRef = BufferRef.Create(bucketIndex, bufferIndex); lastAllocated = freeRef; // _allocatedDb.Put(txn, bucketIndex, lastAllocated, TransactionPutOptions.AppendDuplicateData); aC.Put(ref bucketIndex, ref lastAllocated, CursorPutOptions.AppendDuplicateData); // We are done, freeRef is set. Reuse cursors and locals. var dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndex, _pageSize); _allocationsWithoutSizeCheckCount++; if (_allocationsWithoutSizeCheckCount == CheckTotalSizeEveryNAllocation && CheckTotalSizeEveryNAllocation > 0 || _lastTotalSize / (double)_maxTotalSize > 0.9 || dataSize / (double)MaxTotalSize > 0.01 // 1% ) { _allocationsWithoutSizeCheckCount = 0; var totalAllocated = 0L; if (aC.TryGet(ref bucketIndex, ref lastAllocated, CursorGetOption.First)) { totalAllocated += dataSize * (long)aC.Count(); } while (aC.TryGet(ref bucketIndex, ref lastAllocated, CursorGetOption.NextNoDuplicate)) { totalAllocated += dataSize * (long)aC.Count(); } _lastTotalSize = totalAllocated; if (totalAllocated > _maxTotalSize) { ThrowBufferRefAllocatorFullException(); } } if (buckets != null) { unchecked { #pragma warning disable 618 var directBuffer = buckets.DangerousGet(freeRef); #pragma warning restore 618 var header = directBuffer.Read <SharedMemory.BufferHeader>(0); if (header.FlagsCounter != default) { throw new InvalidOperationException($"Allocated buffer for {freeRef} has non-empty header.FlagsCounter: {header.FlagsCounter}"); } if (header.AllocatorInstanceId != default) { throw new InvalidOperationException($"Allocated buffer has non-empry header.AllocatorInstanceId: {header.AllocatorInstanceId}"); } header.FlagsCounter = HeaderFlags.Releasing | HeaderFlags.IsDisposed; header.AllocatorInstanceId = _wpid.InstanceId; // No need to check previous value because it is a new buffer // that was never touched from this BRA perspective. directBuffer.VolatileWriteInt64(0, (long)header); } } } } return(freeRef); }