static void AppendPoolSubPages(StringBuilder buf, PoolSubpage <T>[] subpages)
{
for (int i = 0; i < subpages.Length; i++)
{
PoolSubpage <T> head = subpages[i];
if (head.Next == head)
{
continue;
}
buf.Append(StringUtil.Newline)
.Append(i)
.Append(": ");
PoolSubpage <T> s = head.Next;
for (; ;)
{
buf.Append(s);
s = s.Next;
if (s == head)
{
break;
}
}
}
}
/**
* Returns the bitmap index of the subpage allocation.
*/
internal long Allocate()
{
if (this.ElemSize == 0)
{
return(this.ToHandle(0));
}
/**
* Synchronize on the head of the SubpagePool stored in the {@link PoolArena. This is needed as we synchronize
* on it when calling {@link PoolArena#allocate(PoolThreadCache, int, int)} und try to allocate out of the
* {@link PoolSubpage} pool for a given size.
*/
PoolSubpage <T> head = this.Chunk.Arena.FindSubpagePoolHead(this.ElemSize);
lock (head)
{
if (this.numAvail == 0 || !this.DoNotDestroy)
{
return(-1);
}
int bitmapIdx = this.GetNextAvail();
int q = bitmapIdx.RightUShift(6);
int r = bitmapIdx & 63;
Contract.Assert((this.bitmap[q].RightUShift(r) & 1) == 0);
this.bitmap[q] |= 1L << r;
if (--this.numAvail == 0)
{
this.RemoveFromPool();
}
return(this.ToHandle(bitmapIdx));
}
}
/**
* Create/ initialize a new PoolSubpage of normCapacity
* Any PoolSubpage created/ initialized here is added to subpage pool in the PoolArena that owns this PoolChunk
*
* @param normCapacity normalized capacity
* @return index in memoryMap
*/
long AllocateSubpage(int normCapacity)
{
int d = this.maxOrder; // subpages are only be allocated from pages i.e., leaves
int id = this.AllocateNode(d);
if (id < 0)
{
return(id);
}
PoolSubpage <T>[] subPages = this.subpages;
int size = this.pageSize;
this.freeBytes -= size;
int subpageIdx = this.SubpageIdx(id);
PoolSubpage <T> subpage = subPages[subpageIdx];
if (subpage == null)
{
subpage = new PoolSubpage <T>(this, id, this.RunOffset(id), size, normCapacity);
subPages[subpageIdx] = subpage;
}
else
{
subpage.Init(normCapacity);
}
return(subpage.Allocate());
}
/**
* Free a subpage or a run of pages
* When a subpage is freed from PoolSubpage, it might be added back to subpage pool of the owning PoolArena
* If the subpage pool in PoolArena has at least one other PoolSubpage of given elemSize, we can
* completely free the owning Page so it is available for subsequent allocations
*
* @param handle handle to free
*/
internal void Free(long handle)
{
int memoryMapIdx = MemoryMapIdx(handle);
int bitmapIdx = BitmapIdx(handle);
if (bitmapIdx != 0)
{
// free a subpage
PoolSubpage <T> subpage = this.subpages[this.SubpageIdx(memoryMapIdx)];
Debug.Assert(subpage != null && subpage.DoNotDestroy);
// Obtain the head of the PoolSubPage pool that is owned by the PoolArena and synchronize on it.
// This is need as we may add it back and so alter the linked-list structure.
PoolSubpage <T> head = this.Arena.FindSubpagePoolHead(subpage.ElemSize);
lock (head)
{
if (subpage.Free(head, bitmapIdx & 0x3FFFFFFF))
{
return;
}
}
}
this.freeBytes += this.RunLength(memoryMapIdx);
this.SetValue(memoryMapIdx, this.Depth(memoryMapIdx));
this.UpdateParentsFree(memoryMapIdx);
}
static IReadOnlyCollection <IPoolSubpageMetric> SubPageMetricList(IReadOnlyList <PoolSubpage <T> > pages)
{
var metrics = new List <IPoolSubpageMetric>();
for (int i = 1; i < pages.Count; i++)
{
PoolSubpage <T> head = pages[i];
if (head.Next == head)
{
continue;
}
PoolSubpage <T> s = head.Next;
for (;;)
{
metrics.Add(s);
s = s.Next;
if (s == head)
{
break;
}
}
}
return(metrics);
}
/**
* Create/ initialize a new PoolSubpage of normCapacity
* Any PoolSubpage created/ initialized here is added to subpage pool in the PoolArena that owns this PoolChunk
*
* @param normCapacity normalized capacity
* @return index in memoryMap
*/
long AllocateSubpage(int normCapacity)
{
// Obtain the head of the PoolSubPage pool that is owned by the PoolArena and synchronize on it.
// This is need as we may add it back and so alter the linked-list structure.
PoolSubpage <T> head = this.Arena.FindSubpagePoolHead(normCapacity);
lock (head)
{
int d = this.maxOrder; // subpages are only be allocated from pages i.e., leaves
int id = this.AllocateNode(d);
if (id < 0)
{
return(id);
}
PoolSubpage <T>[] subpages = this.subpages;
int pageSize = this.pageSize;
this.freeBytes -= pageSize;
int subpageIdx = this.SubpageIdx(id);
PoolSubpage <T> subpage = subpages[subpageIdx];
if (subpage == null)
{
subpage = new PoolSubpage <T>(head, this, id, this.RunOffset(id), pageSize, normCapacity);
subpages[subpageIdx] = subpage;
}
else
{
subpage.Init(head, normCapacity);
}
return(subpage.Allocate());
}
}
public void Init(int elemSize)
{
this.DoNotDestroy = true;
this.ElemSize = elemSize;
if (elemSize != 0)
{
this.maxNumElems = this.numAvail = this.pageSize / elemSize;
this.nextAvail = 0;
this.bitmapLength = this.maxNumElems.RightUShift(6);
if ((this.maxNumElems & 63) != 0)
{
this.bitmapLength++;
}
for (int i = 0; i < this.bitmapLength; i++)
{
this.bitmap[i] = 0;
}
}
PoolSubpage <T> head = this.Chunk.Arena.FindSubpagePoolHead(elemSize);
lock (head)
{
this.AddToPool(head);
}
}
static PoolSubpage <T> NewSubpagePoolHead(int pageSize)
{
var head = new PoolSubpage <T>(pageSize);
head.Prev = head;
head.Next = head;
return(head);
}
void AddToPool(PoolSubpage <T> head)
{
Contract.Assert(this.Prev == null && this.Next == null);
this.Prev = head;
this.Next = head.Next;
this.Next.Prev = this;
head.Next = this;
}
void RemoveFromPool()
{
Contract.Assert(this.Prev != null && this.Next != null);
this.Prev.Next = this.Next;
this.Next.Prev = this.Prev;
this.Next = null;
this.Prev = null;
}
public PoolSubpage(PoolSubpage <T> head, PoolChunk <T> chunk, int memoryMapIdx, int runOffset, int pageSize, int elemSize)
{
this.Chunk = chunk;
this.memoryMapIdx = memoryMapIdx;
this.runOffset = runOffset;
this.pageSize = pageSize;
this.bitmap = new long[pageSize.RightUShift(10)]; // pageSize / 16 / 64
this.Init(head, elemSize);
}
void RemoveFromPool()
{
Contract.Assert(this.Prev != null && this.Next != null);
// ReSharper disable PossibleNullReferenceException
this.Prev.Next = this.Next;
this.Next.Prev = this.Prev;
// ReSharper restore PossibleNullReferenceException
this.Next = null;
this.Prev = null;
}
/**
* @return {@code true} if this subpage is in use.
* {@code false} if this subpage is not used by its chunk and thus it's OK to be released.
*/
internal bool Free(int bitmapIdx)
{
if (this.ElemSize == 0)
{
return(true);
}
/**
* Synchronize on the head of the SubpagePool stored in the {@link PoolArena. This is needed as we synchronize
* on it when calling {@link PoolArena#allocate(PoolThreadCache, int, int)} und try to allocate out of the
* {@link PoolSubpage} pool for a given size.
*/
PoolSubpage <T> head = this.Chunk.Arena.FindSubpagePoolHead(this.ElemSize);
lock (head)
{
int q = bitmapIdx.RightUShift(6);
int r = bitmapIdx & 63;
Contract.Assert((this.bitmap[q].RightUShift(r) & 1) != 0);
this.bitmap[q] ^= 1L << r;
this.SetNextAvail(bitmapIdx);
if (this.numAvail++ == 0)
{
this.AddToPool(head);
return(true);
}
if (this.numAvail != this.maxNumElems)
{
return(true);
}
else
{
// Subpage not in use (numAvail == maxNumElems)
if (this.Prev == this.Next)
{
// Do not remove if this subpage is the only one left in the pool.
return(true);
}
// Remove this subpage from the pool if there are other subpages left in the pool.
this.DoNotDestroy = false;
this.RemoveFromPool();
return(false);
}
}
}
void InitBufWithSubpage(PooledByteBuffer <T> buf, long handle, int bitmapIdx, int reqCapacity)
{
Contract.Assert(bitmapIdx != 0);
int memoryMapIdx = MemoryMapIdx(handle);
PoolSubpage <T> subpage = this.subpages[this.SubpageIdx(memoryMapIdx)];
Contract.Assert(subpage.DoNotDestroy);
Contract.Assert(reqCapacity <= subpage.ElemSize);
buf.Init(
this, handle,
this.RunOffset(memoryMapIdx) + (bitmapIdx & 0x3FFFFFFF) * subpage.ElemSize + this.Offset,
reqCapacity, subpage.ElemSize, this.Arena.Parent.ThreadCache <T>());
}
/**
* @return {@code true} if this subpage is in use.
* {@code false} if this subpage is not used by its chunk and thus it's OK to be released.
*/
internal bool Free(PoolSubpage <T> head, int bitmapIdx)
{
if (this.ElemSize == 0)
{
return(true);
}
int q = bitmapIdx.RightUShift(6);
int r = bitmapIdx & 63;
Contract.Assert((this.bitmap[q].RightUShift(r) & 1) != 0);
this.bitmap[q] ^= 1L << r;
this.SetNextAvail(bitmapIdx);
if (this.NumAvailable++ == 0)
{
this.AddToPool(head);
return(true);
}
if (this.NumAvailable != this.MaxNumElements)
{
return(true);
}
else
{
// Subpage not in use (numAvail == maxNumElems)
if (this.Prev == this.Next)
{
// Do not remove if this subpage is the only one left in the pool.
return(true);
}
// Remove this subpage from the pool if there are other subpages left in the pool.
this.DoNotDestroy = false;
this.RemoveFromPool();
return(false);
}
}
public void Init(PoolSubpage <T> head, int elemSize)
{
this.DoNotDestroy = true;
this.ElemSize = elemSize;
if (elemSize != 0)
{
this.MaxNumElements = this.NumAvailable = this.pageSize / elemSize;
this.nextAvail = 0;
this.bitmapLength = this.MaxNumElements.RightUShift(6);
if ((this.MaxNumElements & 63) != 0)
{
this.bitmapLength++;
}
for (int i = 0; i < this.bitmapLength; i++)
{
this.bitmap[i] = 0;
}
}
this.AddToPool(head);
}
static List <IPoolSubpageMetric> SubPageMetricList(PoolSubpage <T>[] pages)
{
var metrics = new List <IPoolSubpageMetric>();
foreach (PoolSubpage <T> head in pages)
{
if (head.Next == head)
{
continue;
}
PoolSubpage <T> s = head.Next;
for (; ;)
{
metrics.Add(s);
s = s.Next;
if (s == head)
{
break;
}
}
}
return(metrics);
}
static List <IPoolSubpageMetric> SubPageMetricList(PoolSubpage <T>[] pages)
{
var metrics = new List <IPoolSubpageMetric>();
for (int i = 1; i < pages.Length; i++)
{
PoolSubpage <T> head = pages[i];
if (head.Next == head)
{
continue;
}
PoolSubpage <T> s = head.Next;
for (;;)
{
metrics.Add(s);
s = s.Next;
if (s == head)
{
break;
}
}
}
return(metrics);
}
void Allocate(PoolThreadCache <T> cache, PooledArrayBuffer <T> buf, int reqCapacity)
{
int normCapacity = this.NormalizeCapacity(reqCapacity);
if (this.IsTinyOrSmall(normCapacity))
{
// capacity < pageSize
int tableIdx;
PoolSubpage <T>[] table;
bool tiny = IsTiny(normCapacity);
if (tiny)
{
// < 512
if (cache.AllocateTiny(this, buf, reqCapacity, normCapacity))
{
// was able to allocate out of the cache so move on
return;
}
tableIdx = TinyIdx(normCapacity);
table = this.tinySubpagePools;
}
else
{
if (cache.AllocateSmall(this, buf, reqCapacity, normCapacity))
{
// was able to allocate out of the cache so move on
return;
}
tableIdx = SmallIdx(normCapacity);
table = this.smallSubpagePools;
}
PoolSubpage <T> head = table[tableIdx];
/**
* Synchronize on the head. This is needed as {@link PoolSubpage#allocate()} and
* {@link PoolSubpage#free(int)} may modify the doubly linked list as well.
*/
lock (head)
{
PoolSubpage <T> s = head.Next;
if (s != head)
{
Contract.Assert(s.DoNotDestroy && s.ElemSize == normCapacity);
long handle = s.Allocate();
Contract.Assert(handle >= 0);
s.Chunk.InitBufWithSubpage(buf, handle, reqCapacity);
if (tiny)
{
++this.NumTinyAllocations;
}
else
{
++this.NumSmallAllocations;
}
return;
}
}
this.AllocateNormal(buf, reqCapacity, normCapacity);
return;
}
if (normCapacity <= this.ChunkSize)
{
if (cache.AllocateNormal(this, buf, reqCapacity, normCapacity))
{
// was able to allocate out of the cache so move on
return;
}
this.AllocateNormal(buf, reqCapacity, normCapacity);
}
else
{
// Huge allocations are never served via the cache so just call allocateHuge
this.AllocateHuge(buf, reqCapacity);
}
}
public override string ToString()
{
lock (this)
{
StringBuilder buf = new StringBuilder()
.Append("Chunk(s) at 0~25%:")
.Append(Environment.NewLine)
.Append(this.qInit)
.Append(Environment.NewLine)
.Append("Chunk(s) at 0~50%:")
.Append(Environment.NewLine)
.Append(this.q000)
.Append(Environment.NewLine)
.Append("Chunk(s) at 25~75%:")
.Append(Environment.NewLine)
.Append(this.q025)
.Append(Environment.NewLine)
.Append("Chunk(s) at 50~100%:")
.Append(Environment.NewLine)
.Append(this.q050)
.Append(Environment.NewLine)
.Append("Chunk(s) at 75~100%:")
.Append(Environment.NewLine)
.Append(this.q075)
.Append(Environment.NewLine)
.Append("Chunk(s) at 100%:")
.Append(Environment.NewLine)
.Append(this.q100)
.Append(Environment.NewLine)
.Append("tiny subpages:");
for (int i = 1; i < this.tinySubpagePools.Length; i++)
{
PoolSubpage <T> head = this.tinySubpagePools[i];
if (head.Next == head)
{
continue;
}
buf.Append(Environment.NewLine)
.Append(i)
.Append(": ");
PoolSubpage <T> s = head.Next;
for (;;)
{
buf.Append(s);
s = s.Next;
if (s == head)
{
break;
}
}
}
buf.Append(Environment.NewLine)
.Append("small subpages:");
for (int i = 1; i < this.smallSubpagePools.Length; i++)
{
PoolSubpage <T> head = this.smallSubpagePools[i];
if (head.Next == head)
{
continue;
}
buf.Append(Environment.NewLine)
.Append(i)
.Append(": ");
PoolSubpage <T> s = head.Next;
for (;;)
{
buf.Append(s);
s = s.Next;
if (s == head)
{
break;
}
}
}
buf.Append(Environment.NewLine);
return(buf.ToString());
}
}
