internal PooledByteBuffer <T> Allocate(PoolThreadCache <T> cache, int reqCapacity, int maxCapacity)
{
PooledByteBuffer <T> buf = this.NewByteBuf(maxCapacity);
this.Allocate(cache, buf, reqCapacity);
return(buf);
}
internal bool Allocate(PooledByteBuffer <T> buf, int reqCapacity, int normCapacity)
{
if (this.head == null || normCapacity > this.maxCapacity)
{
// Either this PoolChunkList is empty or the requested capacity is larger then the capacity which can
// be handled by the PoolChunks that are contained in this PoolChunkList.
return(false);
}
for (PoolChunk <T> cur = this.head;;)
{
long handle = cur.Allocate(normCapacity);
if (handle < 0)
{
cur = cur.Next;
if (cur == null)
{
return(false);
}
}
else
{
cur.InitBuf(buf, handle, reqCapacity);
if (cur.Usage >= this.maxUsage)
{
this.Remove(cur);
this.nextList.Add(cur);
}
return(true);
}
}
}
void AllocateHuge(PooledByteBuffer <T> buf, int reqCapacity)
{
PoolChunk <T> chunk = this.NewUnpooledChunk(reqCapacity);
Interlocked.Add(ref this.activeBytesHuge, chunk.ChunkSize);
buf.InitUnpooled(chunk, reqCapacity);
Interlocked.Increment(ref this.allocationsHuge);
}
/**
* Allocate something out of the cache if possible and remove the entry from the cache.
*/
public bool Allocate(PooledByteBuffer <T> buf, int reqCapacity)
{
if (!this.queue.TryDequeue(out Entry entry))
{
return(false);
}
this.InitBuf(entry.Chunk, entry.Handle, buf, reqCapacity);
entry.Recycle();
// allocations is not thread-safe which is fine as this is only called from the same thread all time.
++this.allocations;
return(true);
}
internal void Reallocate(PooledByteBuffer <T> buf, int newCapacity, bool freeOldMemory)
{
Contract.Requires(newCapacity >= 0 && newCapacity <= buf.MaxCapacity);
int oldCapacity = buf.Length;
if (oldCapacity == newCapacity)
{
return;
}
PoolChunk <T> oldChunk = buf.Chunk;
long oldHandle = buf.Handle;
T oldMemory = buf.Memory;
int oldOffset = buf.Offset;
int oldMaxLength = buf.MaxLength;
int readerIndex = buf.ReaderIndex;
int writerIndex = buf.WriterIndex;
this.Allocate(this.Parent.ThreadCache <T>(), buf, newCapacity);
if (newCapacity > oldCapacity)
{
this.MemoryCopy(
oldMemory, oldOffset,
buf.Memory, buf.Offset, oldCapacity);
}
else if (newCapacity < oldCapacity)
{
if (readerIndex < newCapacity)
{
if (writerIndex > newCapacity)
{
writerIndex = newCapacity;
}
this.MemoryCopy(
oldMemory, oldOffset + readerIndex,
buf.Memory, buf.Offset + readerIndex, writerIndex - readerIndex);
}
else
{
readerIndex = writerIndex = newCapacity;
}
}
buf.SetIndex(readerIndex, writerIndex);
if (freeOldMemory)
{
this.Free(oldChunk, oldHandle, oldMaxLength, buf.Cache);
}
}
bool Allocate(MemoryRegionCache cache, PooledByteBuffer <T> buf, int reqCapacity)
{
if (cache == null)
{
// no cache found so just return false here
return(false);
}
bool allocated = cache.Allocate(buf, reqCapacity);
if (++this.allocations >= this.freeSweepAllocationThreshold)
{
this.allocations = 0;
this.Trim();
}
return(allocated);
}
void InitBufWithSubpage(PooledByteBuffer <T> buf, long handle, int bitmapIdx, int reqCapacity)
{
Contract.Assert(bitmapIdx != 0);
int memoryMapIdx = MemoryMapIdx(handle);
PoolSubpage <T> subpage = this.subpages[this.SubpageIdx(memoryMapIdx)];
Contract.Assert(subpage.DoNotDestroy);
Contract.Assert(reqCapacity <= subpage.ElemSize);
buf.Init(
this, handle,
this.RunOffset(memoryMapIdx) + (bitmapIdx & 0x3FFFFFFF) * subpage.ElemSize + this.Offset,
reqCapacity, subpage.ElemSize, this.Arena.Parent.ThreadCache <T>());
}
void AllocateNormal(PooledByteBuffer <T> buf, int reqCapacity, int normCapacity)
{
if (this.q050.Allocate(buf, reqCapacity, normCapacity) || this.q025.Allocate(buf, reqCapacity, normCapacity) ||
this.q000.Allocate(buf, reqCapacity, normCapacity) || this.qInit.Allocate(buf, reqCapacity, normCapacity) ||
this.q075.Allocate(buf, reqCapacity, normCapacity))
{
return;
}
// Add a new chunk.
PoolChunk <T> c = this.NewChunk(this.PageSize, this.maxOrder, this.PageShifts, this.ChunkSize);
long handle = c.Allocate(normCapacity);
Debug.Assert(handle > 0);
c.InitBuf(buf, handle, reqCapacity);
this.qInit.Add(c);
}
internal void InitBuf(PooledByteBuffer <T> buf, long handle, int reqCapacity)
{
int memoryMapIdx = MemoryMapIdx(handle);
int bitmapIdx = BitmapIdx(handle);
if (bitmapIdx == 0)
{
sbyte val = this.Value(memoryMapIdx);
Debug.Assert(val == this.unusable, val.ToString());
buf.Init(this, handle, this.RunOffset(memoryMapIdx) + this.Offset, reqCapacity, this.RunLength(memoryMapIdx),
this.Arena.Parent.ThreadCache <T>());
}
else
{
this.InitBufWithSubpage(buf, handle, bitmapIdx, reqCapacity);
}
}
/**
* Init the {@link PooledByteBuffer} using the provided chunk and handle with the capacity restrictions.
*/
protected abstract void InitBuf(PoolChunk <T> chunk, long handle,
PooledByteBuffer <T> buf, int reqCapacity);
protected override void InitBuf(
PoolChunk <T> chunk, long handle, PooledByteBuffer <T> buf, int reqCapacity) =>
chunk.InitBuf(buf, handle, reqCapacity);
/**
* Try to allocate a small buffer out of the cache. Returns {@code true} if successful {@code false} otherwise
*/
internal bool AllocateNormal(PoolArena <T> area, PooledByteBuffer <T> buf, int reqCapacity, int normCapacity) =>
this.Allocate(this.CacheForNormal(area, normCapacity), buf, reqCapacity);
void Allocate(PoolThreadCache <T> cache, PooledByteBuffer <T> buf, int reqCapacity)
{
int normCapacity = this.NormalizeCapacity(reqCapacity);
if (this.IsTinyOrSmall(normCapacity))
{
// capacity < pageSize
int tableIdx;
PoolSubpage <T>[] table;
bool tiny = IsTiny(normCapacity);
if (tiny)
{
// < 512
if (cache.AllocateTiny(this, buf, reqCapacity, normCapacity))
{
// was able to allocate out of the cache so move on
return;
}
tableIdx = TinyIdx(normCapacity);
table = this.tinySubpagePools;
}
else
{
if (cache.AllocateSmall(this, buf, reqCapacity, normCapacity))
{
// was able to allocate out of the cache so move on
return;
}
tableIdx = SmallIdx(normCapacity);
table = this.smallSubpagePools;
}
PoolSubpage <T> head = table[tableIdx];
//
// Synchronize on the head. This is needed as {@link PoolSubpage#allocate()} and
// {@link PoolSubpage#free(int)} may modify the doubly linked list as well.
//
lock (head)
{
PoolSubpage <T> s = head.Next;
if (s != head)
{
Debug.Assert(s.DoNotDestroy && s.ElemSize == normCapacity);
long handle = s.Allocate();
Debug.Assert(handle >= 0);
s.Chunk.InitBufWithSubpage(buf, handle, reqCapacity);
this.IncTinySmallAllocation(tiny);
return;
}
}
lock (this)
{
this.AllocateNormal(buf, reqCapacity, normCapacity);
}
this.IncTinySmallAllocation(tiny);
return;
}
if (normCapacity <= this.ChunkSize)
{
if (cache.AllocateNormal(this, buf, reqCapacity, normCapacity))
{
// was able to allocate out of the cache so move on
return;
}
lock (this)
{
this.AllocateNormal(buf, reqCapacity, normCapacity);
this.allocationsNormal++;
}
}
else
{
// Huge allocations are never served via the cache so just call allocateHuge
this.AllocateHuge(buf, reqCapacity);
}
}
internal void InitBufWithSubpage(PooledByteBuffer <T> buf, long handle, int reqCapacity) => this.InitBufWithSubpage(buf, handle, BitmapIdx(handle), reqCapacity);