public unsafe void Return(ref RawBuffer buffer)
{
#if DEBUG
DecomposeId(buffer.Id, out var powerIndex, out var slotIndex);
pools[powerIndex].ValidateBufferIsContained(ref buffer);
#endif
ReturnUnsafely(buffer.Id);
buffer = default;
}
internal unsafe void ValidateBufferIsContained(ref RawBuffer buffer)
{
//There are a lot of ways to screw this up. Try to catch as many as possible!
var slotIndex = buffer.Id & ((1 << IdPowerShift) - 1);
var blockIndex = slotIndex >> SuballocationsPerBlockShift;
var indexInAllocatorBlock = slotIndex & SuballocationsPerBlockMask;
Debug.Assert(buffer.Length == SuballocationSize,
"A buffer taken from a pool should have a specific size.");
Debug.Assert(blockIndex >= 0 && blockIndex < BlockCount,
"The block pointed to by a returned buffer should actually exist within the pool.");
var memoryOffset = buffer.Memory - Blocks[blockIndex].Pointer;
Debug.Assert(memoryOffset >= 0 && memoryOffset < Blocks[blockIndex].Array.Length,
"If a raw buffer points to a given block as its source, the address should be within the block's memory region.");
Debug.Assert(Blocks[blockIndex].Pointer + indexInAllocatorBlock * SuballocationSize == buffer.Memory,
"The implied address of a buffer in its block should match its actual address.");
Debug.Assert(buffer.Length + indexInAllocatorBlock * SuballocationSize <= Blocks[blockIndex].Array.Length,
"The extent of the buffer should fit within the block.");
}
public unsafe void Take(out RawBuffer buffer)
{
var slot = Slots.Take();
var blockIndex = slot >> SuballocationsPerBlockShift;
if (blockIndex >= Blocks.Length)
{
Array.Resize(ref Blocks, 1 << SpanHelper.GetContainingPowerOf2(blockIndex + 1));
}
if (blockIndex >= BlockCount)
{
#if DEBUG
for (int i = 0; i < blockIndex; ++i)
{
Debug.Assert(Blocks[i].Allocated, "If a block index is found to exceed the current block count, then every block preceding the index should be allocated.");
}
#endif
BlockCount = blockIndex + 1;
Debug.Assert(!Blocks[blockIndex].Allocated);
Blocks[blockIndex] = new Block(BlockSize);
}
var indexInBlock = slot & SuballocationsPerBlockMask;
buffer = new RawBuffer(Blocks[blockIndex].Allocate(indexInBlock, SuballocationSize), SuballocationSize, (Power << IdPowerShift) | slot);
Debug.Assert(buffer.Id >= 0 && Power >= 0 && Power < 32, "Slot/power should be safely encoded in a 32 bit integer.");
#if DEBUG
const int maximumOutstandingCapacity = 1 << 29;
Debug.Assert(outstandingIds.Count * SuballocationSize <= maximumOutstandingCapacity,
$"Do you actually truly really need to have {maximumOutstandingCapacity} bytes taken from this power pool, or is this a memory leak?");
Debug.Assert(outstandingIds.Add(slot), "Should not be able to request the same slot twice.");
#if LEAKDEBUG
var allocator = new StackTrace().ToString();
if (!outstandingAllocators.TryGetValue(allocator, out var idsForAllocator))
{
idsForAllocator = new HashSet <int>();
outstandingAllocators.Add(allocator, idsForAllocator);
}
Debug.Assert(idsForAllocator.Count < (1 << 25), "Do you actually have that many allocations for this one allocator?");
idsForAllocator.Add(slot);
#endif
#endif
}
public unsafe void Resize(ref RawBuffer buffer, int targetSize, int copyCount)
{
//Only do anything if the new size is actually different from the current size.
targetSize = 1 << (SpanHelper.GetContainingPowerOf2(targetSize));
if (buffer.Length != targetSize) //Note that we don't check for allocated status- for buffers, a length of 0 is the same as being unallocated.
{
Take(targetSize, out var newBuffer);
if (buffer.Length > 0)
{
//Don't bother copying from or re-pooling empty buffers. They're uninitialized.
Debug.Assert(copyCount <= targetSize);
Unsafe.CopyBlockUnaligned(newBuffer.Memory, buffer.Memory, (uint)copyCount);
ReturnUnsafely(ref buffer);
}
else
{
Debug.Assert(copyCount == 0, "Should not be trying to copy elements from an empty span.");
}
buffer = newBuffer;
}
}
public unsafe void Return(ref RawBuffer buffer)
{
#if DEBUG
//There are a lot of ways to screw this up. Try to catch as many as possible!
var blockIndex = buffer.Id >> SuballocationsPerBlockShift;
var indexInAllocatorBlock = buffer.Id & SuballocationsPerBlockMask;
Debug.Assert(outstandingIds.Remove(buffer.Id),
"This buffer id must have been taken from the pool previously.");
#if LEAKDEBUG
bool found = false;
foreach (var pair in outstandingAllocators)
{
if (pair.Value.Remove(buffer.Id))
{
found = true;
if (pair.Value.Count == 0)
{
outstandingAllocators.Remove(pair.Key);
break;
}
}
}
Debug.Assert(found, "Allocator set must contain the buffer id.");
#endif
Debug.Assert(buffer.Length == SuballocationSize,
"A buffer taken from a pool should have a specific size.");
Debug.Assert(blockIndex >= 0 && blockIndex < BlockCount,
"The block pointed to by a returned buffer should actually exist within the pool.");
var memoryOffset = buffer.Memory - Blocks[blockIndex].Pointer;
Debug.Assert(memoryOffset >= 0 && memoryOffset < Blocks[blockIndex].Array.Length,
"If a raw buffer points to a given block as its source, the address should be within the block's memory region.");
Debug.Assert(Blocks[blockIndex].Pointer + indexInAllocatorBlock * SuballocationSize == buffer.Memory,
"The implied address of a buffer in its block should match its actual address.");
Debug.Assert(buffer.Length + indexInAllocatorBlock * SuballocationSize <= Blocks[blockIndex].Array.Length,
"The extent of the buffer should fit within the block.");
#endif
Slots.Return(buffer.Id, new PassthroughArrayPool <int>());
}
public static ref T Get(ref RawBuffer buffer, int index)
{
Debug.Assert(index >= 0 && index * Unsafe.SizeOf <T>() < buffer.Length, "Index out of range.");
return(ref Get(buffer.Memory, index));
}
public static ref T Get(ref RawBuffer buffer, int index)
{
return(ref Get(buffer.Memory, index));
}
public unsafe void Return(ref RawBuffer buffer)
{
ReturnUnsafely(ref buffer);
buffer = new RawBuffer();
}
public unsafe void ReturnUnsafely(ref RawBuffer buffer)
{
ValidatePinnedState(true);
pools[SpanHelper.GetContainingPowerOf2(buffer.Length)].Return(ref buffer);
}
public void TakeForPower(int power, out RawBuffer buffer)
{
ValidatePinnedState(true);
Debug.Assert(power >= 0 && power <= SpanHelper.MaximumSpanSizePower);
pools[power].Take(out buffer);
}
public void Take(int count, out RawBuffer buffer)
{
TakeForPower(SpanHelper.GetContainingPowerOf2(count), out buffer);
}
public unsafe void Resize(ref RawBuffer buffer, int targetSize, int copyCount)
{
ResizeToAtLeast(ref buffer, targetSize, copyCount);
buffer.Length = targetSize;
}
public void Take(int count, out RawBuffer buffer)
{
TakeAtLeast(count, out buffer);
buffer.Length = count;
}
