internal unsafe SharedMemory RentNative(int bucketSize)
{
var bucketIndex = SelectBucketIndex(bucketSize);
BufferRef.EnsureBucketIndexInRange(bucketIndex);
// We must init the buffer header before allocating.
// We know from inside BRA.Allocate txn the value or BufferRef
// before committing. For new we must init the header,
// for one from the free list we must assert the buffer is disposed
// and not owned (it may be in Releasing state though).
var br = _bra.Allocate((byte)bucketIndex, out _, _buckets);
var db = _buckets.DangerousGet(br);
var sm = SharedMemory.Create(db, br, this);
Debug.Assert(sm.IsDisposed);
Debug.Assert(Unsafe.Read <uint>(sm.HeaderPointer) == (HeaderFlags.Releasing | HeaderFlags.IsDisposed));
sm.FromReleasingDisposedToOwned();
Debug.Assert(!sm.IsDisposed);
Debug.Assert(sm.ReferenceCount == 0);
Debug.Assert(Unsafe.Read <uint>(sm.HeaderPointer) == HeaderFlags.IsOwned);
return(sm);
}
protected BufferRef Allocate(Transaction txn, byte bucketIndex, out bool fromFreeList, SharedMemoryBuckets buckets = null)
{
var originalBucketIndex = bucketIndex;
fromFreeList = false;
BufferRef.EnsureBucketIndexInRange(bucketIndex);
BufferRef freeRef = default;
using (var fC = _freeDb.OpenCursor(txn))
using (var aC = _allocatedDb.OpenCursor(txn))
{
// If free list is not empty then everything is simple - just move an item from FL to AL
if (fC.TryGet(ref bucketIndex, ref freeRef, CursorGetOption.Set) &&
fC.TryGet(ref bucketIndex, ref freeRef, CursorGetOption.FirstDuplicate)
) // prefer closer to the bucket start, DRs are sorted by index // TODO test with first with high churn & packer
{
ThrowHelper.AssertFailFast(originalBucketIndex == bucketIndex, $"originalBucketIndex {originalBucketIndex} == bucketIndex {bucketIndex}");
_freeDb.Delete(txn, bucketIndex, freeRef);
var ar = freeRef;
aC.Put(ref bucketIndex, ref ar, CursorPutOptions.None);
fromFreeList = true;
if (buckets != null)
{
#pragma warning disable 618
var directBuffer = buckets.DangerousGet(freeRef);
#pragma warning restore 618
var header = directBuffer.Read <SharedMemory.BufferHeader>(0);
unchecked
{
if (header.FlagsCounter != (HeaderFlags.Releasing | HeaderFlags.IsDisposed))
{
ThrowHelper.ThrowInvalidOperationException(
"Buffer in free list must be (Releasing | IsDisposed).");
}
// TODO review, we require (Releasing | IsDisposed) in FL. Remove commented code below if that is OK
//if (((int)header.FlagsCounter & (int)AtomicCounter.CountMask) != (int)AtomicCounter.CountMask)
//{
// ThrowHelper.ThrowInvalidOperationException("Buffer in free list must be disposed.");
//}
//var ignoreCountAndReleasing = ((int)header.FlagsCounter & ~(int)AtomicCounter.CountMask) & ~((int)SharedMemory.BufferHeader.HeaderFlags.Releasing);
//if (ignoreCountAndReleasing != 0)
//{
// ThrowHelper.ThrowInvalidOperationException("Non-Releasing flags in free buffer header.");
//}
var newHeader = header;
// newHeader.FlagsCounter = (SharedMemory.BufferHeader.HeaderFlags)(uint)(AtomicCounter.CountMask | (int)SharedMemory.BufferHeader.HeaderFlags.Releasing);
newHeader.AllocatorInstanceId = _wpid.InstanceId;
// TODO check existing header values
// It must be zeros if the buffer was disposed normally
//if ((long)header != 0)
//{
// Trace.TraceWarning($"Header of free list buffer is not zero: FlagsCounter {header.FlagsCounter}, InstanceId: {header.AllocatorInstanceId}");
//}
if ((long)header !=
directBuffer.InterlockedCompareExchangeInt64(0, (long)newHeader,
(long)header))
{
// This should only happen if someone if touching the header concurrently
// But we are inside write txn now so this should never happen.
// TODO could remove expensive CAS for simple write.
ThrowHelper.FailFast("Header changed during allocating from free list.");
}
}
}
}
else
{
// Free list is empty.
// Allocated must be sorted by BufferRef as uint. Since in DUP_SORTED bucket bites are
// the same then the last allocated value has the maximum buffer index.
int bufferIndex = 1; // if is empty then start from 1
BufferRef lastAllocated = default;
if (aC.TryGet(ref bucketIndex, ref lastAllocated, CursorGetOption.Set) &&
aC.TryGet(ref bucketIndex, ref lastAllocated, CursorGetOption.LastDuplicate))
{
ThrowHelper.AssertFailFast(lastAllocated.BufferIndex > 0, "BufferRef.BufferIndex is one-based");
bufferIndex += lastAllocated.BufferIndex;
}
ThrowHelper.AssertFailFast(originalBucketIndex == bucketIndex, $"originalBucketIndex {originalBucketIndex} == bucketIndex {bucketIndex}");
freeRef = BufferRef.Create(bucketIndex, bufferIndex);
lastAllocated = freeRef;
// _allocatedDb.Put(txn, bucketIndex, lastAllocated, TransactionPutOptions.AppendDuplicateData);
aC.Put(ref bucketIndex, ref lastAllocated, CursorPutOptions.AppendDuplicateData);
// We are done, freeRef is set. Reuse cursors and locals.
var dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndex, _pageSize);
_allocationsWithoutSizeCheckCount++;
if (_allocationsWithoutSizeCheckCount == CheckTotalSizeEveryNAllocation &&
CheckTotalSizeEveryNAllocation > 0
||
_lastTotalSize / (double)_maxTotalSize > 0.9
||
dataSize / (double)MaxTotalSize > 0.01 // 1%
)
{
_allocationsWithoutSizeCheckCount = 0;
var totalAllocated = 0L;
if (aC.TryGet(ref bucketIndex, ref lastAllocated, CursorGetOption.First))
{
totalAllocated += dataSize * (long)aC.Count();
}
while (aC.TryGet(ref bucketIndex, ref lastAllocated, CursorGetOption.NextNoDuplicate))
{
totalAllocated += dataSize * (long)aC.Count();
}
_lastTotalSize = totalAllocated;
if (totalAllocated > _maxTotalSize)
{
ThrowBufferRefAllocatorFullException();
}
}
if (buckets != null)
{
unchecked
{
#pragma warning disable 618
var directBuffer = buckets.DangerousGet(freeRef);
#pragma warning restore 618
var header = directBuffer.Read <SharedMemory.BufferHeader>(0);
if (header.FlagsCounter != default)
{
throw new InvalidOperationException($"Allocated buffer for {freeRef} has non-empty header.FlagsCounter: {header.FlagsCounter}");
}
if (header.AllocatorInstanceId != default)
{
throw new InvalidOperationException($"Allocated buffer has non-empry header.AllocatorInstanceId: {header.AllocatorInstanceId}");
}
header.FlagsCounter = HeaderFlags.Releasing | HeaderFlags.IsDisposed;
header.AllocatorInstanceId = _wpid.InstanceId;
// No need to check previous value because it is a new buffer
// that was never touched from this BRA perspective.
directBuffer.VolatileWriteInt64(0, (long)header);
}
}
}
}
return(freeRef);
}
