// Mark the given block as free. The given block must have been
// wholly allocated before, but it's legal to release big
// allocations in smaller non-overlapping sub-blocks.
public void Release(HeapBlock block)
{
// Merge the newly released block with any free blocks on either
// side of it. Remove those blocks from the list of free blocks,
// as they no longer exist as separate blocks.
block = Coalesce(block);
SizeBin bin = new SizeBin(block);
int index = FindSmallestSufficientBin(bin);
// If the exact bin doesn't exist, add it.
if (index >= m_SizeBins.Length || bin.CompareTo(m_SizeBins[index]) != 0)
{
index = AddNewBin(ref bin, index);
}
m_Blocks[m_SizeBins[index].blocksId].Push(block);
m_Free += block.Length;
#if DEBUG_ASSERTS
Assert.IsFalse(m_FreeEndpoints.ContainsKey(block.begin));
Assert.IsFalse(m_FreeEndpoints.ContainsKey(block.end));
#endif
// Store both endpoints of the free block to the hashmap for
// easy coalescing.
m_FreeEndpoints[block.begin] = block.end;
m_FreeEndpoints[block.end] = block.begin;
}
private HeapBlock CutAllocationFromBlock(SizeBin allocation, HeapBlock block)
{
#if DEBUG_ASSERTS
Assert.IsTrue(block.Length >= allocation.Size, "Block is not large enough.");
#endif
// If the match is exact, no need to cut.
if (allocation.Size == block.Length)
{
return(block);
}
// Otherwise, round the begin to next multiple of alignment, and then cut away the required size,
// potentially leaving empty space on both ends.
ulong alignedBegin = NextAligned(block.begin, allocation.AlignmentLog2);
ulong alignedEnd = alignedBegin + allocation.Size;
if (alignedBegin > block.begin)
{
Release(new HeapBlock(block.begin, alignedBegin));
}
if (alignedEnd < block.end)
{
Release(new HeapBlock(alignedEnd, block.end));
}
return(new HeapBlock(alignedBegin, alignedEnd));
}
private unsafe int AddNewBin(ref SizeBin bin, int index)
{
// If there are no free block lists, make a new one
if (m_BlocksFreelist.IsEmpty)
{
bin.blocksId = m_Blocks.Length;
m_Blocks.Add(new BlocksOfSize(0));
}
else
{
int last = m_BlocksFreelist.Length - 1;
bin.blocksId = m_BlocksFreelist[last];
m_BlocksFreelist.ResizeUninitialized(last);
}
#if DEBUG_ASSERTS
Assert.IsTrue(m_Blocks[bin.blocksId].Empty);
#endif
int tail = m_SizeBins.Length - index;
m_SizeBins.ResizeUninitialized(m_SizeBins.Length + 1);
SizeBin *p = (SizeBin *)m_SizeBins.GetUnsafePtr();
UnsafeUtility.MemMove(
p + (index + 1),
p + index,
tail * UnsafeUtility.SizeOf <SizeBin>());
p[index] = bin;
return(index);
}
// Attempt to allocate a block from the heap with at least the given
// size and alignment. The allocated block might be bigger than the
// requested size, but will never be smaller.
// If the allocation fails, an empty block is returned.
public HeapBlock Allocate(ulong size, uint alignment = 1)
{
// Always use at least the minimum alignment, and round all sizes
// to multiples of the minimum alignment.
size = NextAligned(size, m_MinimumAlignmentLog2);
alignment = math.max(alignment, MinimumAlignment);
SizeBin allocBin = new SizeBin(size, alignment);
int index = FindSmallestSufficientBin(allocBin);
while (index < m_SizeBins.Length)
{
SizeBin bin = m_SizeBins[index];
if (CanFitAllocation(allocBin, bin))
{
HeapBlock block = PopBlockFromBin(bin, index);
return(CutAllocationFromBlock(allocBin, block));
}
else
{
++index;
}
}
return(new HeapBlock());
}
private unsafe HeapBlock PopBlockFromBin(SizeBin bin, int index)
{
HeapBlock block = m_Blocks[bin.blocksId].Pop();
RemoveEndpoints(block);
m_Free -= block.Length;
RemoveBinIfEmpty(bin, index);
return(block);
}
private unsafe int AddNewBin(SizeBin bin, int index)
{
int tail = m_SizeBins.Length - index;
m_SizeBins.ResizeUninitialized(m_SizeBins.Length + 1);
SizeBin *p = (SizeBin *)m_SizeBins.GetUnsafePtr();
UnsafeUtility.MemMove(
p + (index + 1),
p + index,
tail * UnsafeUtility.SizeOf <SizeBin>());
p[index] = bin;
return(index);
}
private unsafe void RemoveEmptyBins(SizeBin bin, int index)
{
if (!m_Blocks[bin.blocksId].Empty)
{
return;
}
int tail = m_SizeBins.Length - (index + 1);
SizeBin *p = (SizeBin *)m_SizeBins.GetUnsafePtr();
UnsafeUtility.MemMove(
p + index,
p + (index + 1),
tail * UnsafeUtility.SizeOf <SizeBin>());
m_SizeBins.ResizeUninitialized(m_SizeBins.Length - 1);
}
private int FindSmallestSufficientBin(SizeBin needle)
{
if (m_SizeBins.Length == 0)
{
return(0);
}
int lo = 0; // Low endpoint of search, inclusive
int hi = m_SizeBins.Length; // High endpoint of search, exclusive
for (;;)
{
int d2 = (hi - lo) / 2;
// Search has terminated. If lo is large enough, return it.
if (d2 == 0)
{
if (needle.CompareTo(m_SizeBins[lo]) <= 0)
{
return(lo);
}
else
{
return(lo + 1);
}
}
int probe = lo + d2;
int cmp = needle.CompareTo(m_SizeBins[probe]);
// Needle is smaller than probe?
if (cmp < 0)
{
hi = probe;
}
// Needle is greater than probe?
else if (cmp > 0)
{
lo = probe;
}
// Found needle exactly.
else
{
return(probe);
}
}
}
private void RemoveFreeBlock(HeapBlock block)
{
RemoveEndpoints(block);
SizeBin bin = new SizeBin(block);
int index = FindSmallestSufficientBin(bin);
#if DEBUG_ASSERTS
Assert.IsTrue(index >= 0 && m_SizeBins[index].sizeClass == bin.sizeClass,
"Expected to find exact match for size bin since block was supposed to exist");
#endif
bool removed = m_Blocks[m_SizeBins[index].blocksId].Remove(block);
RemoveBinIfEmpty(m_SizeBins[index], index);
#if DEBUG_ASSERTS
Assert.IsTrue(removed, "Block was supposed to exist");
#endif
m_Free -= block.Length;
}
private bool CanFitAllocation(SizeBin allocation, SizeBin bin)
{
#if DEBUG_ASSERTS
Assert.IsTrue(bin.sizeClass >= allocation.sizeClass, "Should have compatible size classes to begin with");
#endif
// Check that the bin is not empty.
if (m_Blocks[bin.blocksId].Empty)
{
return(false);
}
// If the bin meets alignment restrictions, it is usable.
if (bin.HasCompatibleAlignment(allocation))
{
return(true);
}
// Else, require one alignment worth of extra space so we can guarantee space.
else
{
return(bin.Size >= (allocation.Size + allocation.Alignment));
}
}
// Do a slow exhaustive test of the allocator's internal state to verify
// that its invariants have not been broken. Should be only used for debugging.
public void DebugValidateInternalState()
{
// Amount of size bins should be the same as the amount of block lists
// for those size bins.
int numBins = m_SizeBins.Length;
int numFreeBlockLists = m_BlocksFreelist.Length;
int numEmptyBlocks = 0;
int numNonEmptyBlocks = 0;
for (int i = 0; i < m_Blocks.Length; ++i)
{
if (m_Blocks[i].Empty)
{
++numEmptyBlocks;
}
else
{
++numNonEmptyBlocks;
}
}
Assert.AreEqual(numBins, numNonEmptyBlocks, "There should be exactly one non-empty block list per size bin");
Assert.AreEqual(numEmptyBlocks, numFreeBlockLists, "All empty block lists should be in the free list");
for (int i = 0; i < m_BlocksFreelist.Length; ++i)
{
int freeBlock = m_BlocksFreelist[i];
Assert.IsTrue(m_Blocks[freeBlock].Empty, "There should be only empty block lists in the free list");
}
ulong totalFreeSize = 0;
int totalFreeBlocks = 0;
for (int i = 0; i < m_SizeBins.Length; ++i)
{
var sizeBin = m_SizeBins[i];
var size = sizeBin.Size;
var align = sizeBin.Alignment;
var blocks = m_Blocks[sizeBin.blocksId];
Assert.IsFalse(blocks.Empty, "All block lists should be non-empty, empty lists should be removed");
int count = blocks.Length;
for (int j = 0; j < count; ++j)
{
var b = blocks.Block(j);
var bin = new SizeBin(b);
Assert.AreEqual(size, bin.Size, "Block size should match its bin");
Assert.AreEqual(align, bin.Alignment, "Block alignment should match its bin");
totalFreeSize += b.Length;
if (m_FreeEndpoints.TryGetValue(b.begin, out var foundEnd))
{
Assert.AreEqual(b.end, foundEnd, "Free block end does not match stored endpoint");
}
else
{
Assert.IsTrue(false, "No end endpoint found for free block");
}
if (m_FreeEndpoints.TryGetValue(b.end, out var foundBegin))
{
Assert.AreEqual(b.begin, foundBegin, "Free block begin does not match stored endpoint");
}
else
{
Assert.IsTrue(false, "No begin endpoint found for free block");
}
++totalFreeBlocks;
}
}
// Reported free space should be equal to the total size of the free blocks
Assert.AreEqual(totalFreeSize, FreeSpace, "Free size reported incorrectly");
Assert.IsTrue(totalFreeSize <= Size, "Amount of free size larger than maximum");
Assert.AreEqual(2 * totalFreeBlocks, m_FreeEndpoints.Count(),
"Each free block should have exactly 2 stored endpoints");
}
