internal unsafe SharedMemory RentNative(int bucketSize) { var bucketIndex = SelectBucketIndex(bucketSize); BufferRef.EnsureBucketIndexInRange(bucketIndex); // We must init the buffer header before allocating. // We know from inside BRA.Allocate txn the value or BufferRef // before committing. For new we must init the header, // for one from the free list we must assert the buffer is disposed // and not owned (it may be in Releasing state though). var br = _bra.Allocate((byte)bucketIndex, out _, _buckets); var db = _buckets.DangerousGet(br); var sm = SharedMemory.Create(db, br, this); Debug.Assert(sm.IsDisposed); Debug.Assert(Unsafe.Read <uint>(sm.HeaderPointer) == (HeaderFlags.Releasing | HeaderFlags.IsDisposed)); sm.FromReleasingDisposedToOwned(); Debug.Assert(!sm.IsDisposed); Debug.Assert(sm.ReferenceCount == 0); Debug.Assert(Unsafe.Read <uint>(sm.HeaderPointer) == HeaderFlags.IsOwned); return(sm); }
protected BufferRef Allocate(Transaction txn, byte bucketIndex, out bool fromFreeList, SharedMemoryBuckets buckets = null) { var originalBucketIndex = bucketIndex; fromFreeList = false; BufferRef.EnsureBucketIndexInRange(bucketIndex); BufferRef freeRef = default; using (var fC = _freeDb.OpenCursor(txn)) using (var aC = _allocatedDb.OpenCursor(txn)) { // If free list is not empty then everything is simple - just move an item from FL to AL if (fC.TryGet(ref bucketIndex, ref freeRef, CursorGetOption.Set) && fC.TryGet(ref bucketIndex, ref freeRef, CursorGetOption.FirstDuplicate) ) // prefer closer to the bucket start, DRs are sorted by index // TODO test with first with high churn & packer { ThrowHelper.AssertFailFast(originalBucketIndex == bucketIndex, $"originalBucketIndex {originalBucketIndex} == bucketIndex {bucketIndex}"); _freeDb.Delete(txn, bucketIndex, freeRef); var ar = freeRef; aC.Put(ref bucketIndex, ref ar, CursorPutOptions.None); fromFreeList = true; if (buckets != null) { #pragma warning disable 618 var directBuffer = buckets.DangerousGet(freeRef); #pragma warning restore 618 var header = directBuffer.Read <SharedMemory.BufferHeader>(0); unchecked { if (header.FlagsCounter != (HeaderFlags.Releasing | HeaderFlags.IsDisposed)) { ThrowHelper.ThrowInvalidOperationException( "Buffer in free list must be (Releasing | IsDisposed)."); } // TODO review, we require (Releasing | IsDisposed) in FL. Remove commented code below if that is OK //if (((int)header.FlagsCounter & (int)AtomicCounter.CountMask) != (int)AtomicCounter.CountMask) //{ // ThrowHelper.ThrowInvalidOperationException("Buffer in free list must be disposed."); //} //var ignoreCountAndReleasing = ((int)header.FlagsCounter & ~(int)AtomicCounter.CountMask) & ~((int)SharedMemory.BufferHeader.HeaderFlags.Releasing); //if (ignoreCountAndReleasing != 0) //{ // ThrowHelper.ThrowInvalidOperationException("Non-Releasing flags in free buffer header."); //} var newHeader = header; // newHeader.FlagsCounter = (SharedMemory.BufferHeader.HeaderFlags)(uint)(AtomicCounter.CountMask | (int)SharedMemory.BufferHeader.HeaderFlags.Releasing); newHeader.AllocatorInstanceId = _wpid.InstanceId; // TODO check existing header values // It must be zeros if the buffer was disposed normally //if ((long)header != 0) //{ // Trace.TraceWarning($"Header of free list buffer is not zero: FlagsCounter {header.FlagsCounter}, InstanceId: {header.AllocatorInstanceId}"); //} if ((long)header != directBuffer.InterlockedCompareExchangeInt64(0, (long)newHeader, (long)header)) { // This should only happen if someone if touching the header concurrently // But we are inside write txn now so this should never happen. // TODO could remove expensive CAS for simple write. ThrowHelper.FailFast("Header changed during allocating from free list."); } } } } else { // Free list is empty. // Allocated must be sorted by BufferRef as uint. Since in DUP_SORTED bucket bites are // the same then the last allocated value has the maximum buffer index. int bufferIndex = 1; // if is empty then start from 1 BufferRef lastAllocated = default; if (aC.TryGet(ref bucketIndex, ref lastAllocated, CursorGetOption.Set) && aC.TryGet(ref bucketIndex, ref lastAllocated, CursorGetOption.LastDuplicate)) { ThrowHelper.AssertFailFast(lastAllocated.BufferIndex > 0, "BufferRef.BufferIndex is one-based"); bufferIndex += lastAllocated.BufferIndex; } ThrowHelper.AssertFailFast(originalBucketIndex == bucketIndex, $"originalBucketIndex {originalBucketIndex} == bucketIndex {bucketIndex}"); freeRef = BufferRef.Create(bucketIndex, bufferIndex); lastAllocated = freeRef; // _allocatedDb.Put(txn, bucketIndex, lastAllocated, TransactionPutOptions.AppendDuplicateData); aC.Put(ref bucketIndex, ref lastAllocated, CursorPutOptions.AppendDuplicateData); // We are done, freeRef is set. Reuse cursors and locals. var dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndex, _pageSize); _allocationsWithoutSizeCheckCount++; if (_allocationsWithoutSizeCheckCount == CheckTotalSizeEveryNAllocation && CheckTotalSizeEveryNAllocation > 0 || _lastTotalSize / (double)_maxTotalSize > 0.9 || dataSize / (double)MaxTotalSize > 0.01 // 1% ) { _allocationsWithoutSizeCheckCount = 0; var totalAllocated = 0L; if (aC.TryGet(ref bucketIndex, ref lastAllocated, CursorGetOption.First)) { totalAllocated += dataSize * (long)aC.Count(); } while (aC.TryGet(ref bucketIndex, ref lastAllocated, CursorGetOption.NextNoDuplicate)) { totalAllocated += dataSize * (long)aC.Count(); } _lastTotalSize = totalAllocated; if (totalAllocated > _maxTotalSize) { ThrowBufferRefAllocatorFullException(); } } if (buckets != null) { unchecked { #pragma warning disable 618 var directBuffer = buckets.DangerousGet(freeRef); #pragma warning restore 618 var header = directBuffer.Read <SharedMemory.BufferHeader>(0); if (header.FlagsCounter != default) { throw new InvalidOperationException($"Allocated buffer for {freeRef} has non-empty header.FlagsCounter: {header.FlagsCounter}"); } if (header.AllocatorInstanceId != default) { throw new InvalidOperationException($"Allocated buffer has non-empry header.AllocatorInstanceId: {header.AllocatorInstanceId}"); } header.FlagsCounter = HeaderFlags.Releasing | HeaderFlags.IsDisposed; header.AllocatorInstanceId = _wpid.InstanceId; // No need to check previous value because it is a new buffer // that was never touched from this BRA perspective. directBuffer.VolatileWriteInt64(0, (long)header); } } } } return(freeRef); }