public BufferRef Allocate(byte bucketIndex, out bool fromFreeList, SharedMemoryBuckets buckets = null)
{
BufferRef.EnsureBucketIndexInRange(bucketIndex);
using (var txn = _env.BeginTransaction())
{
try
{
BufferRef freeRef = Allocate(txn, bucketIndex, out fromFreeList, buckets);
txn.Commit();
// Without packing and returning buffers to pool this kills performance
// But it must be done.
if (!fromFreeList)
{
_env.Sync(true);
}
return(freeRef);
}
catch
{
txn.Abort();
throw;
}
}
}
internal unsafe SharedMemoryPool(string path, uint maxLogSizeMb,
LMDBEnvironmentFlags envFlags,
Wpid ownerId,
int rmMaxBufferLength = RmDefaultMaxBufferLength,
int rmMaxBuffersPerBucket = RmDefaultMaxBuffersPerBucket, bool rentAlwaysClean = false)
: base(
(pool, bucketSize) =>
{
if (bucketSize > pool.MaxBufferSize)
{
BuffersThrowHelper.ThrowBadLength();
}
var smbp = (SharedMemoryPool)pool;
#pragma warning disable 618
var sm = smbp.RentNative(bucketSize);
Debug.Assert(!sm.IsDisposed);
Debug.Assert(sm.ReferenceCount == 0);
Debug.Assert(Unsafe.ReadUnaligned <uint>(sm.HeaderPointer) == HeaderFlags.IsOwned);
#pragma warning restore 618
// TODO review if Releasing -> IsOwned should keep disposed state?
// RMP calls CreateNew outside lock, so if we call RentNative only here
// then it could return IsOwned without IsDisposed. But we a technically
// inside the pool until this factory returns. Buffers from RentNative
// should be unusable without explicit un-dispose action.
// Set counter to zero
sm.CounterRef &= ~AtomicCounter.CountMask;
return(sm);
},
RmMinPoolBufferLength,
Math.Max(RmDefaultMaxBufferLength, Math.Min(RmMaxPoolBufferLength, BitUtil.FindNextPositivePowerOfTwo(rmMaxBufferLength))),
// from ProcCount to DefaultMaxNumberOfBuffersPerBucket x 2
Math.Max(Environment.ProcessorCount, Math.Min(RmDefaultMaxBuffersPerBucket * 2, rmMaxBuffersPerBucket)),
MaxBucketsToTry, rentAlwaysClean: rentAlwaysClean)
{
if (ownerId <= 0)
{
ThrowHelper.ThrowArgumentOutOfRangeException("ownerId <= 0");
}
Directory.CreateDirectory(path);
_bra = new BufferRefAllocator(path, envFlags, ownerId, PageSize, maxLogSizeMb * 1024 * 1024L);
_buckets = new SharedMemoryBuckets(Path.Combine(path, "buckets"), pageSize: PageSize, maxBucketIndex: BufferRef.MaxBucketIdx);
StartMonitoringTask();
}
protected BufferRef Allocate(Transaction txn, byte bucketIndex, out bool fromFreeList, SharedMemoryBuckets buckets = null)
{
var originalBucketIndex = bucketIndex;
fromFreeList = false;
BufferRef.EnsureBucketIndexInRange(bucketIndex);
BufferRef freeRef = default;
using (var fC = _freeDb.OpenCursor(txn))
using (var aC = _allocatedDb.OpenCursor(txn))
{
// If free list is not empty then everything is simple - just move an item from FL to AL
if (fC.TryGet(ref bucketIndex, ref freeRef, CursorGetOption.Set) &&
fC.TryGet(ref bucketIndex, ref freeRef, CursorGetOption.FirstDuplicate)
) // prefer closer to the bucket start, DRs are sorted by index // TODO test with first with high churn & packer
{
ThrowHelper.AssertFailFast(originalBucketIndex == bucketIndex, $"originalBucketIndex {originalBucketIndex} == bucketIndex {bucketIndex}");
_freeDb.Delete(txn, bucketIndex, freeRef);
var ar = freeRef;
aC.Put(ref bucketIndex, ref ar, CursorPutOptions.None);
fromFreeList = true;
if (buckets != null)
{
#pragma warning disable 618
var directBuffer = buckets.DangerousGet(freeRef);
#pragma warning restore 618
var header = directBuffer.Read <SharedMemory.BufferHeader>(0);
unchecked
{
if (header.FlagsCounter != (HeaderFlags.Releasing | HeaderFlags.IsDisposed))
{
ThrowHelper.ThrowInvalidOperationException(
"Buffer in free list must be (Releasing | IsDisposed).");
}
// TODO review, we require (Releasing | IsDisposed) in FL. Remove commented code below if that is OK
//if (((int)header.FlagsCounter & (int)AtomicCounter.CountMask) != (int)AtomicCounter.CountMask)
//{
// ThrowHelper.ThrowInvalidOperationException("Buffer in free list must be disposed.");
//}
//var ignoreCountAndReleasing = ((int)header.FlagsCounter & ~(int)AtomicCounter.CountMask) & ~((int)SharedMemory.BufferHeader.HeaderFlags.Releasing);
//if (ignoreCountAndReleasing != 0)
//{
// ThrowHelper.ThrowInvalidOperationException("Non-Releasing flags in free buffer header.");
//}
var newHeader = header;
// newHeader.FlagsCounter = (SharedMemory.BufferHeader.HeaderFlags)(uint)(AtomicCounter.CountMask | (int)SharedMemory.BufferHeader.HeaderFlags.Releasing);
newHeader.AllocatorInstanceId = _wpid.InstanceId;
// TODO check existing header values
// It must be zeros if the buffer was disposed normally
//if ((long)header != 0)
//{
// Trace.TraceWarning($"Header of free list buffer is not zero: FlagsCounter {header.FlagsCounter}, InstanceId: {header.AllocatorInstanceId}");
//}
if ((long)header !=
directBuffer.InterlockedCompareExchangeInt64(0, (long)newHeader,
(long)header))
{
// This should only happen if someone if touching the header concurrently
// But we are inside write txn now so this should never happen.
// TODO could remove expensive CAS for simple write.
ThrowHelper.FailFast("Header changed during allocating from free list.");
}
}
}
}
else
{
// Free list is empty.
// Allocated must be sorted by BufferRef as uint. Since in DUP_SORTED bucket bites are
// the same then the last allocated value has the maximum buffer index.
int bufferIndex = 1; // if is empty then start from 1
BufferRef lastAllocated = default;
if (aC.TryGet(ref bucketIndex, ref lastAllocated, CursorGetOption.Set) &&
aC.TryGet(ref bucketIndex, ref lastAllocated, CursorGetOption.LastDuplicate))
{
ThrowHelper.AssertFailFast(lastAllocated.BufferIndex > 0, "BufferRef.BufferIndex is one-based");
bufferIndex += lastAllocated.BufferIndex;
}
ThrowHelper.AssertFailFast(originalBucketIndex == bucketIndex, $"originalBucketIndex {originalBucketIndex} == bucketIndex {bucketIndex}");
freeRef = BufferRef.Create(bucketIndex, bufferIndex);
lastAllocated = freeRef;
// _allocatedDb.Put(txn, bucketIndex, lastAllocated, TransactionPutOptions.AppendDuplicateData);
aC.Put(ref bucketIndex, ref lastAllocated, CursorPutOptions.AppendDuplicateData);
// We are done, freeRef is set. Reuse cursors and locals.
var dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndex, _pageSize);
_allocationsWithoutSizeCheckCount++;
if (_allocationsWithoutSizeCheckCount == CheckTotalSizeEveryNAllocation &&
CheckTotalSizeEveryNAllocation > 0
||
_lastTotalSize / (double)_maxTotalSize > 0.9
||
dataSize / (double)MaxTotalSize > 0.01 // 1%
)
{
_allocationsWithoutSizeCheckCount = 0;
var totalAllocated = 0L;
if (aC.TryGet(ref bucketIndex, ref lastAllocated, CursorGetOption.First))
{
totalAllocated += dataSize * (long)aC.Count();
}
while (aC.TryGet(ref bucketIndex, ref lastAllocated, CursorGetOption.NextNoDuplicate))
{
totalAllocated += dataSize * (long)aC.Count();
}
_lastTotalSize = totalAllocated;
if (totalAllocated > _maxTotalSize)
{
ThrowBufferRefAllocatorFullException();
}
}
if (buckets != null)
{
unchecked
{
#pragma warning disable 618
var directBuffer = buckets.DangerousGet(freeRef);
#pragma warning restore 618
var header = directBuffer.Read <SharedMemory.BufferHeader>(0);
if (header.FlagsCounter != default)
{
throw new InvalidOperationException($"Allocated buffer for {freeRef} has non-empty header.FlagsCounter: {header.FlagsCounter}");
}
if (header.AllocatorInstanceId != default)
{
throw new InvalidOperationException($"Allocated buffer has non-empry header.AllocatorInstanceId: {header.AllocatorInstanceId}");
}
header.FlagsCounter = HeaderFlags.Releasing | HeaderFlags.IsDisposed;
header.AllocatorInstanceId = _wpid.InstanceId;
// No need to check previous value because it is a new buffer
// that was never touched from this BRA perspective.
directBuffer.VolatileWriteInt64(0, (long)header);
}
}
}
}
return(freeRef);
}