private SharedMemoryBucket CreateBucket(ref SharedMemoryBucket[] bucketFiles, BufferRef bufferRef, int bufferFileIndex, int bucketIndex)
{
SharedMemoryBucket bucket;
lock (_buckets)
{
if (bucketFiles == null)
{
bucketFiles = new SharedMemoryBucket[4];
_buckets[bufferRef.BucketIndex] = bucketFiles;
}
if (bufferFileIndex >= bucketFiles.Length)
{
var newSize = BitUtil.FindNextPositivePowerOfTwo(bufferFileIndex + 1);
var newbucketFiles = new SharedMemoryBucket[newSize];
Array.Copy(bucketFiles, 0, newbucketFiles, 0, bucketFiles.Length);
bucketFiles = newbucketFiles;
_buckets[bufferRef.BucketIndex] = newbucketFiles;
}
bucket = bucketFiles[bufferFileIndex];
if (bucket.DirectFile == null)
{
var bucketDir = Path.Combine(_directoryPath, bucketIndex.ToString());
Directory.CreateDirectory(bucketDir);
var buffersPerBucketFile = (1 << 15) >> bucketIndex;
var bufferSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndex, _pageSize);
var allocationSize = buffersPerBucketFile * bufferSize;
// Since we are using sparse files check available free space
var available = GetAvailableFreeSpace();
// 2x just in case, maybe we need to monitor this and stop any allocations everywhere when below c.200MB or 1GB to be sure
if (available >= 0 && allocationSize * 2 > available)
{
throw new NotEnoughSpaceException(true);
}
// Console.WriteLine($"allocationSize for bucket {bucketIndex}: {allocationSize}");
var df = new DirectFile(Path.Combine(bucketDir, bufferFileIndex + ".smbkt"), allocationSize, true,
FileOptions.RandomAccess // Note that we cannot use async with sparse without using valid NativeOverlapped struct in DeviceIoControl function
| FileOptions.WriteThrough, // TODO review if we need this
true);
DirectFile.PrefetchMemory(df.DirectBuffer);
Trace.TraceInformation($"Allocated new file in a bucket {bucketIndex}. Total # of files: {bucketFiles.Count(x => x.DirectFile != null)}");
bucket = new SharedMemoryBucket(df, bucketIndex, _pageSize);
// Console.WriteLine($"File capacity for bucket {bucketIndex}: {bucket.BufferCount}");
bucketFiles[bufferFileIndex] = bucket;
}
}
return(bucket);
}
public SharedMemoryBucket(DirectFile directFile, int bucketIndex, int pageSize = DefaultPageSize)
{
BufferRef.EnsureBucketIndexInRange(bucketIndex);
DirectFile = directFile;
var directBuffer = directFile.DirectBuffer;
var bufferSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndex, pageSize);
BufferCount = directBuffer.Length / bufferSize;
DirectBuffer = directBuffer;
BucketIndex = bucketIndex;
PageSize = pageSize;
BufferSize = bufferSize;
}
public long GetTotalAllocated(int bucketIndex = -1)
{
using (var txn = _env.BeginReadOnlyTransaction())
using (var aC = _allocatedDb.OpenReadOnlyCursor(txn))
{
BufferRef lastAllocated = default;
var totalAllocated = 0L;
if (bucketIndex == -1)
{
byte bucketIndexByte = 0;
if (aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.First))
{
var dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndexByte, _pageSize);
totalAllocated += dataSize * (long)aC.Count();
while (aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.NextNoDuplicate))
{
dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndex, _pageSize);
totalAllocated += dataSize * (long)aC.Count();
}
}
}
else
{
BufferRef.EnsureBucketIndexInRange(bucketIndex);
var bucketIndexByte = (byte)bucketIndex;
if (aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.Set))
{
var dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndex, _pageSize);
totalAllocated += dataSize * (long)aC.Count();
}
}
_lastTotalSize = totalAllocated;
return(totalAllocated);
}
}
public void PrintFree(int bucketIndex = -1, bool printBuffers = false,
Func <BufferRef, string> additionalInfo = null)
{
using (var txn = _env.BeginReadOnlyTransaction())
using (var fC = _freeDb.OpenReadOnlyCursor(txn))
{
BufferRef lastFree = default;
var totalFree = 0L;
if (bucketIndex == -1)
{
Console.WriteLine("------------------- FREE -------------------");
byte bucketIndexByte = 0;
if (fC.TryGet(ref bucketIndexByte, ref lastFree, CursorGetOption.First))
{
var dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndexByte, _pageSize);
var count = (long)fC.Count();
var bucketSize = dataSize * count;
totalFree += bucketSize;
Console.WriteLine(
$"[{bucketIndexByte}]: count [{count:N0}], buffer size [{dataSize:N0}], bucket size: [{bucketSize:N0}]");
if (printBuffers)
{
PrintBuffers(bucketIndexByte);
}
while (fC.TryGet(ref bucketIndexByte, ref lastFree, CursorGetOption.NextNoDuplicate))
{
dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndexByte, _pageSize);
count = (long)fC.Count();
bucketSize = dataSize * count;
totalFree += bucketSize;
Console.WriteLine(
$"[{bucketIndexByte}]: count [{count:N0}], buffer size [{dataSize:N0}], bucket size: [{bucketSize:N0}]");
if (printBuffers)
{
PrintBuffers(bucketIndexByte);
}
}
}
Console.WriteLine($"------ FREE {totalFree:N0}, {Math.Round(100 * (totalFree / (double)MaxTotalSize), 2)}% of Max {MaxTotalSize:N0} ------");
}
else
{
BufferRef.EnsureBucketIndexInRange(bucketIndex);
Console.WriteLine($"------------------- FREE [{bucketIndex}] -------------------");
var bucketIndexByte = (byte)bucketIndex;
if (fC.TryGet(ref bucketIndexByte, ref lastFree, CursorGetOption.Set))
{
var dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndexByte, _pageSize);
var count = (long)fC.Count();
var bucketSize = dataSize * count;
totalFree += bucketSize;
Console.WriteLine(
$"[{bucketIndexByte}]: count [{count}], buffer size [{dataSize}], bucket size: [{bucketSize}]");
if (printBuffers)
{
PrintBuffers(bucketIndexByte);
}
}
Console.WriteLine($"------ FREE [{bucketIndex}] {totalFree:N0}, {Math.Round(100 * (totalFree / (double)MaxTotalSize), 2)}% of Max {MaxTotalSize:N0} ------");
}
void PrintBuffers(byte bucketIndexByte)
{
if (fC.TryGet(ref bucketIndexByte, ref lastFree, CursorGetOption.SetKey) &&
fC.TryGet(ref bucketIndexByte, ref lastFree, CursorGetOption.FirstDuplicate))
{
var info = String.Empty;
if (additionalInfo != null)
{
info = ": " + additionalInfo(lastFree);
}
Console.WriteLine($" [{lastFree.BufferIndex}]{info}");
while (fC.TryGet(ref bucketIndexByte, ref lastFree, CursorGetOption.NextDuplicate))
{
info = String.Empty;
if (additionalInfo != null)
{
info = ": " + additionalInfo(lastFree);
}
Console.WriteLine($" [{lastFree.BufferIndex}]{info}");
}
}
}
}
}
public void PrintAllocated(int bucketIndex = -1, bool printBuffers = false,
Func <BufferRef, string> additionalInfo = null)
{
using (var txn = _env.BeginReadOnlyTransaction())
using (var aC = _allocatedDb.OpenReadOnlyCursor(txn))
{
BufferRef lastAllocated = default;
var totalAllocated = 0L;
if (bucketIndex == -1)
{
Console.WriteLine("------------------- ALLOCATED -------------------");
byte bucketIndexByte = 0;
if (aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.First))
{
var dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndexByte, _pageSize);
var count = (long)aC.Count();
var bucketSize = dataSize * count;
totalAllocated += bucketSize;
Console.WriteLine(
$"[{bucketIndexByte}]: count [{count:N0}], buffer size [{dataSize:N0}], bucket size: [{bucketSize:N0}]");
if (printBuffers)
{
PrintBuffers(bucketIndexByte);
}
while (aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.NextNoDuplicate))
{
dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndexByte, _pageSize);
count = (long)aC.Count();
bucketSize = dataSize * count;
totalAllocated += bucketSize;
Console.WriteLine(
$"[{bucketIndexByte}]: count [{count:N0}], buffer size [{dataSize:N0}], bucket size: [{bucketSize:N0}]");
if (printBuffers)
{
PrintBuffers(bucketIndexByte);
}
}
}
Console.WriteLine($"------ ALLOCATED {totalAllocated:N0}, {Utilization}% of Max {MaxTotalSize:N0} ------");
}
else
{
BufferRef.EnsureBucketIndexInRange(bucketIndex);
Console.WriteLine($"------------------- ALLOCATED [{bucketIndex}] -------------------");
var bucketIndexByte = (byte)bucketIndex;
if (aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.Set))
{
var dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndexByte, _pageSize);
var count = (long)aC.Count();
var bucketSize = dataSize * count;
totalAllocated += bucketSize;
Console.WriteLine(
$"[{bucketIndexByte}]: count [{count:N0}], buffer size [{dataSize:N0}], bucket size: [{bucketSize:N0}]");
if (printBuffers)
{
PrintBuffers(bucketIndexByte);
}
}
Console.WriteLine($"------ ALLOCATED [{bucketIndex}] {totalAllocated:N0}, {Utilization}% of Max {MaxTotalSize:N0} ------");
}
void PrintBuffers(byte bucketIndexByte)
{
if (aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.SetKey) &&
aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.FirstDuplicate))
{
var info = string.Empty;
if (additionalInfo != null)
{
info = ": " + additionalInfo(lastAllocated);
}
Console.WriteLine($" [{lastAllocated.BufferIndex}]{info}");
while (aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.NextDuplicate))
{
info = string.Empty;
if (additionalInfo != null)
{
info = ": " + additionalInfo(lastAllocated);
}
Console.WriteLine($" [{lastAllocated.BufferIndex}]{info}");
}
}
}
_lastTotalSize = totalAllocated;
}
}
protected BufferRef Allocate(Transaction txn, byte bucketIndex, out bool fromFreeList, SharedMemoryBuckets buckets = null)
{
var originalBucketIndex = bucketIndex;
fromFreeList = false;
BufferRef.EnsureBucketIndexInRange(bucketIndex);
BufferRef freeRef = default;
using (var fC = _freeDb.OpenCursor(txn))
using (var aC = _allocatedDb.OpenCursor(txn))
{
// If free list is not empty then everything is simple - just move an item from FL to AL
if (fC.TryGet(ref bucketIndex, ref freeRef, CursorGetOption.Set) &&
fC.TryGet(ref bucketIndex, ref freeRef, CursorGetOption.FirstDuplicate)
) // prefer closer to the bucket start, DRs are sorted by index // TODO test with first with high churn & packer
{
ThrowHelper.AssertFailFast(originalBucketIndex == bucketIndex, $"originalBucketIndex {originalBucketIndex} == bucketIndex {bucketIndex}");
_freeDb.Delete(txn, bucketIndex, freeRef);
var ar = freeRef;
aC.Put(ref bucketIndex, ref ar, CursorPutOptions.None);
fromFreeList = true;
if (buckets != null)
{
#pragma warning disable 618
var directBuffer = buckets.DangerousGet(freeRef);
#pragma warning restore 618
var header = directBuffer.Read <SharedMemory.BufferHeader>(0);
unchecked
{
if (header.FlagsCounter != (HeaderFlags.Releasing | HeaderFlags.IsDisposed))
{
ThrowHelper.ThrowInvalidOperationException(
"Buffer in free list must be (Releasing | IsDisposed).");
}
// TODO review, we require (Releasing | IsDisposed) in FL. Remove commented code below if that is OK
//if (((int)header.FlagsCounter & (int)AtomicCounter.CountMask) != (int)AtomicCounter.CountMask)
//{
// ThrowHelper.ThrowInvalidOperationException("Buffer in free list must be disposed.");
//}
//var ignoreCountAndReleasing = ((int)header.FlagsCounter & ~(int)AtomicCounter.CountMask) & ~((int)SharedMemory.BufferHeader.HeaderFlags.Releasing);
//if (ignoreCountAndReleasing != 0)
//{
// ThrowHelper.ThrowInvalidOperationException("Non-Releasing flags in free buffer header.");
//}
var newHeader = header;
// newHeader.FlagsCounter = (SharedMemory.BufferHeader.HeaderFlags)(uint)(AtomicCounter.CountMask | (int)SharedMemory.BufferHeader.HeaderFlags.Releasing);
newHeader.AllocatorInstanceId = _wpid.InstanceId;
// TODO check existing header values
// It must be zeros if the buffer was disposed normally
//if ((long)header != 0)
//{
// Trace.TraceWarning($"Header of free list buffer is not zero: FlagsCounter {header.FlagsCounter}, InstanceId: {header.AllocatorInstanceId}");
//}
if ((long)header !=
directBuffer.InterlockedCompareExchangeInt64(0, (long)newHeader,
(long)header))
{
// This should only happen if someone if touching the header concurrently
// But we are inside write txn now so this should never happen.
// TODO could remove expensive CAS for simple write.
ThrowHelper.FailFast("Header changed during allocating from free list.");
}
}
}
}
else
{
// Free list is empty.
// Allocated must be sorted by BufferRef as uint. Since in DUP_SORTED bucket bites are
// the same then the last allocated value has the maximum buffer index.
int bufferIndex = 1; // if is empty then start from 1
BufferRef lastAllocated = default;
if (aC.TryGet(ref bucketIndex, ref lastAllocated, CursorGetOption.Set) &&
aC.TryGet(ref bucketIndex, ref lastAllocated, CursorGetOption.LastDuplicate))
{
ThrowHelper.AssertFailFast(lastAllocated.BufferIndex > 0, "BufferRef.BufferIndex is one-based");
bufferIndex += lastAllocated.BufferIndex;
}
ThrowHelper.AssertFailFast(originalBucketIndex == bucketIndex, $"originalBucketIndex {originalBucketIndex} == bucketIndex {bucketIndex}");
freeRef = BufferRef.Create(bucketIndex, bufferIndex);
lastAllocated = freeRef;
// _allocatedDb.Put(txn, bucketIndex, lastAllocated, TransactionPutOptions.AppendDuplicateData);
aC.Put(ref bucketIndex, ref lastAllocated, CursorPutOptions.AppendDuplicateData);
// We are done, freeRef is set. Reuse cursors and locals.
var dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndex, _pageSize);
_allocationsWithoutSizeCheckCount++;
if (_allocationsWithoutSizeCheckCount == CheckTotalSizeEveryNAllocation &&
CheckTotalSizeEveryNAllocation > 0
||
_lastTotalSize / (double)_maxTotalSize > 0.9
||
dataSize / (double)MaxTotalSize > 0.01 // 1%
)
{
_allocationsWithoutSizeCheckCount = 0;
var totalAllocated = 0L;
if (aC.TryGet(ref bucketIndex, ref lastAllocated, CursorGetOption.First))
{
totalAllocated += dataSize * (long)aC.Count();
}
while (aC.TryGet(ref bucketIndex, ref lastAllocated, CursorGetOption.NextNoDuplicate))
{
totalAllocated += dataSize * (long)aC.Count();
}
_lastTotalSize = totalAllocated;
if (totalAllocated > _maxTotalSize)
{
ThrowBufferRefAllocatorFullException();
}
}
if (buckets != null)
{
unchecked
{
#pragma warning disable 618
var directBuffer = buckets.DangerousGet(freeRef);
#pragma warning restore 618
var header = directBuffer.Read <SharedMemory.BufferHeader>(0);
if (header.FlagsCounter != default)
{
throw new InvalidOperationException($"Allocated buffer for {freeRef} has non-empty header.FlagsCounter: {header.FlagsCounter}");
}
if (header.AllocatorInstanceId != default)
{
throw new InvalidOperationException($"Allocated buffer has non-empry header.AllocatorInstanceId: {header.AllocatorInstanceId}");
}
header.FlagsCounter = HeaderFlags.Releasing | HeaderFlags.IsDisposed;
header.AllocatorInstanceId = _wpid.InstanceId;
// No need to check previous value because it is a new buffer
// that was never touched from this BRA perspective.
directBuffer.VolatileWriteInt64(0, (long)header);
}
}
}
}
return(freeRef);
}