internal PooledByteBuffer <T> Allocate(PoolThreadCache <T> cache, int reqCapacity, int maxCapacity)
{
PooledByteBuffer <T> buf = NewByteBuf(maxCapacity);
Allocate(cache, buf, reqCapacity);
return(buf);
}
internal bool Allocate(PooledByteBuffer <T> buf, int reqCapacity, int normCapacity)
{
if (this.head == null || normCapacity > this.maxCapacity)
{
// Either this PoolChunkList is empty or the requested capacity is larger then the capacity which can
// be handled by the PoolChunks that are contained in this PoolChunkList.
return(false);
}
for (PoolChunk <T> cur = this.head;;)
{
long handle = cur.Allocate(normCapacity);
if (handle < 0)
{
cur = cur.Next;
if (cur == null)
{
return(false);
}
}
else
{
cur.InitBuf(buf, handle, reqCapacity);
if (cur.Usage >= this.maxUsage)
{
this.Remove(cur);
this.nextList.Add(cur);
}
return(true);
}
}
}
void AllocateNormal(PooledByteBuffer <T> buf, int reqCapacity, int normCapacity)
{
lock (this)
{
if (this.q050.Allocate(buf, reqCapacity, normCapacity) || this.q025.Allocate(buf, reqCapacity, normCapacity) ||
this.q000.Allocate(buf, reqCapacity, normCapacity) || this.qInit.Allocate(buf, reqCapacity, normCapacity) ||
this.q075.Allocate(buf, reqCapacity, normCapacity))
{
++this.allocationsNormal;
return;
}
if (this.chunkCount < this.maxChunkCount)
{
// Add a new chunk.
PoolChunk <T> c = this.NewChunk(this.PageSize, this.maxOrder, this.PageShifts, this.ChunkSize);
this.chunkCount++;
long handle = c.Allocate(normCapacity);
++this.allocationsNormal;
Contract.Assert(handle > 0);
c.InitBuf(buf, handle, reqCapacity);
this.qInit.Add(c);
return;
}
}
PoolChunk <T> chunk = this.NewUnpooledChunk(reqCapacity, false);
buf.InitUnpooled(chunk, reqCapacity);
Interlocked.Increment(ref this.allocationsNormal);
}
void AllocateHuge(PooledByteBuffer <T> buf, int reqCapacity)
{
PoolChunk <T> chunk = NewUnpooledChunk(reqCapacity);
_ = Interlocked.Add(ref _activeBytesHuge, chunk.ChunkSize);
buf.InitUnpooled(chunk, reqCapacity);
_ = Interlocked.Increment(ref _allocationsHuge);
}
protected override IByteBuffer NewBuffer(int initialCapacity, int maxCapacity)
{
if (initialCapacity > this.MaxPooledBufSize)
{
return(new UnpooledHeapByteBuffer(this, initialCapacity, maxCapacity));
}
PooledByteBuffer buffer = this.pool.Take();
buffer.Init();
return(buffer);
}
/// <summary>
/// Allocate something out of the cache if possible and remove the entry from the cache.
/// </summary>
public bool Allocate(PooledByteBuffer <T> buf, int reqCapacity, PoolThreadCache <T> threadCache)
{
if (!_queue.TryDequeue(out Entry entry))
{
return(false);
}
InitBuf(entry.Chunk, entry.Handle, buf, reqCapacity, threadCache);
entry.Recycle();
// allocations is not thread-safe which is fine as this is only called from the same thread all time.
++_allocations;
return(true);
}
internal void Reallocate(PooledByteBuffer <T> buf, int newCapacity, bool freeOldMemory)
{
Contract.Requires(newCapacity >= 0 && newCapacity <= buf.MaxCapacity);
int oldCapacity = buf.Length;
if (oldCapacity == newCapacity)
{
return;
}
PoolChunk <T> oldChunk = buf.Chunk;
long oldHandle = buf.Handle;
T oldMemory = buf.Memory;
int oldOffset = buf.Offset;
int oldMaxLength = buf.MaxLength;
int readerIndex = buf.ReaderIndex;
int writerIndex = buf.WriterIndex;
this.Allocate(this.Parent.ThreadCache <T>(), buf, newCapacity);
if (newCapacity > oldCapacity)
{
this.MemoryCopy(
oldMemory, oldOffset,
buf.Memory, buf.Offset, oldCapacity);
}
else if (newCapacity < oldCapacity)
{
if (readerIndex < newCapacity)
{
if (writerIndex > newCapacity)
{
writerIndex = newCapacity;
}
this.MemoryCopy(
oldMemory, oldOffset + readerIndex,
buf.Memory, buf.Offset + readerIndex, writerIndex - readerIndex);
}
else
{
readerIndex = writerIndex = newCapacity;
}
}
buf.SetIndex(readerIndex, writerIndex);
if (freeOldMemory)
{
this.Free(oldChunk, oldHandle, oldMaxLength, buf.Cache);
}
}
/**
* Allocate something out of the cache if possible and remove the entry from the cache.
*/
public bool Allocate(PooledByteBuffer <T> buf, int reqCapacity)
{
Entry entry = this.queue.Dequeue();
if (entry == null)
{
return(false);
}
this.InitBuf(entry.Chunk, entry.Handle, buf, reqCapacity);
entry.Recycle();
// allocations is not thread-safe which is fine as this is only called from the same thread all time.
++this.allocations;
return(true);
}
internal void InitBuf(PooledByteBuffer <T> buf, long handle, int reqCapacity, PoolThreadCache <T> threadCache)
{
int memoryMapIdx = MemoryMapIdx(handle);
int bitmapIdx = BitmapIdx(handle);
if (0u >= (uint)bitmapIdx)
{
sbyte val = Value(memoryMapIdx);
Debug.Assert(val == _unusable, val.ToString());
buf.Init(this, handle, RunOffset(memoryMapIdx) + Offset, reqCapacity, RunLength(memoryMapIdx), threadCache);
}
else
{
InitBufWithSubpage(buf, handle, bitmapIdx, reqCapacity, threadCache);
}
}
void InitBufWithSubpage(PooledByteBuffer <T> buf, long handle, int bitmapIdx, int reqCapacity)
{
Contract.Assert(bitmapIdx != 0);
int memoryMapIdx = MemoryMapIdx(handle);
PoolSubpage <T> subpage = this.subpages[this.SubpageIdx(memoryMapIdx)];
Contract.Assert(subpage.DoNotDestroy);
Contract.Assert(reqCapacity <= subpage.ElemSize);
buf.Init(
this, handle,
this.RunOffset(memoryMapIdx) + (bitmapIdx & 0x3FFFFFFF) * subpage.ElemSize, reqCapacity, subpage.ElemSize,
this.Arena.Parent.ThreadCache <T>());
}
bool Allocate(MemoryRegionCache cache, PooledByteBuffer <T> buf, int reqCapacity)
{
if (cache is null)
{
// no cache found so just return false here
return(false);
}
bool allocated = cache.Allocate(buf, reqCapacity, this);
if (++_allocations >= _freeSweepAllocationThreshold)
{
_allocations = 0;
Trim();
}
return(allocated);
}
void InitBufWithSubpage(PooledByteBuffer <T> buf, long handle, int bitmapIdx, int reqCapacity, PoolThreadCache <T> threadCache)
{
Debug.Assert(bitmapIdx != 0);
int memoryMapIdx = MemoryMapIdx(handle);
PoolSubpage <T> subpage = _subpages[SubpageIdx(memoryMapIdx)];
Debug.Assert(subpage.DoNotDestroy);
Debug.Assert(reqCapacity <= subpage.ElemSize);
buf.Init(
this, handle,
RunOffset(memoryMapIdx) + (bitmapIdx & 0x3FFFFFFF) * subpage.ElemSize + Offset,
reqCapacity, subpage.ElemSize, threadCache);
}
void AllocateNormal(PooledByteBuffer <T> buf, int reqCapacity, int normCapacity)
{
if (this.q050.Allocate(buf, reqCapacity, normCapacity) || this.q025.Allocate(buf, reqCapacity, normCapacity) ||
this.q000.Allocate(buf, reqCapacity, normCapacity) || this.qInit.Allocate(buf, reqCapacity, normCapacity) ||
this.q075.Allocate(buf, reqCapacity, normCapacity))
{
return;
}
// Add a new chunk.
PoolChunk <T> c = this.NewChunk(this.PageSize, this.maxOrder, this.PageShifts, this.ChunkSize);
long handle = c.Allocate(normCapacity);
Debug.Assert(handle > 0);
c.InitBuf(buf, handle, reqCapacity);
this.qInit.Add(c);
}
internal void InitBuf(PooledByteBuffer <T> buf, long handle, int reqCapacity)
{
int memoryMapIdx = MemoryMapIdx(handle);
int bitmapIdx = BitmapIdx(handle);
if (bitmapIdx == 0)
{
sbyte val = this.Value(memoryMapIdx);
Contract.Assert(val == this.unusable, val.ToString());
buf.Init(this, handle, this.RunOffset(memoryMapIdx), reqCapacity, this.RunLength(memoryMapIdx),
this.Arena.Parent.ThreadCache <T>());
}
else
{
this.InitBufWithSubpage(buf, handle, bitmapIdx, reqCapacity);
}
}
void AllocateNormal(PooledByteBuffer <T> buf, int reqCapacity, int normCapacity, PoolThreadCache <T> threadCache)
{
if (_q050.Allocate(buf, reqCapacity, normCapacity, threadCache) ||
_q025.Allocate(buf, reqCapacity, normCapacity, threadCache) ||
_q000.Allocate(buf, reqCapacity, normCapacity, threadCache) ||
_qInit.Allocate(buf, reqCapacity, normCapacity, threadCache) ||
_q075.Allocate(buf, reqCapacity, normCapacity, threadCache))
{
return;
}
// Add a new chunk.
PoolChunk <T> c = NewChunk(PageSize, _maxOrder, PageShifts, ChunkSize);
bool success = c.Allocate(buf, reqCapacity, normCapacity, threadCache);
Debug.Assert(success);
_qInit.Add(c);
}
internal void Reallocate(PooledByteBuffer <T> buf, int newCapacity, bool freeOldMemory)
{
if (/*newCapacity < 0 || */ (uint)newCapacity > (uint)buf.MaxCapacity)
{
ThrowHelper.ThrowIndexOutOfRangeException();
}
int oldCapacity = buf.Length;
if (oldCapacity == newCapacity)
{
return;
}
PoolChunk <T> oldChunk = buf.Chunk;
long oldHandle = buf.Handle;
T oldMemory = buf.Memory;
int oldOffset = buf.Offset;
int oldMaxLength = buf.MaxLength;
// This does not touch buf's reader/writer indices
Allocate(Parent.ThreadCache <T>(), buf, newCapacity);
int bytesToCopy;
if (newCapacity > oldCapacity)
{
bytesToCopy = oldCapacity;
}
else
{
buf.TrimIndicesToCapacity(newCapacity);
bytesToCopy = newCapacity;
}
MemoryCopy(oldMemory, oldOffset, buf.Memory, buf.Offset, bytesToCopy);
if (freeOldMemory)
{
Free(oldChunk, oldHandle, oldMaxLength, buf.Cache);
}
}
internal bool Allocate(PooledByteBuffer <T> buf, int reqCapacity, int normCapacity, PoolThreadCache <T> threadCache)
{
long handle;
if ((normCapacity & _subpageOverflowMask) != 0)
{
// >= pageSize
handle = AllocateRun(normCapacity);
}
else
{
handle = AllocateSubpage(normCapacity);
}
if (handle < 0)
{
return(false);
}
InitBuf(buf, handle, reqCapacity, threadCache);
return(true);
}
internal bool Allocate(PooledByteBuffer <T> buf, int reqCapacity, int normCapacity, PoolThreadCache <T> threadCache)
{
if (_head is null || normCapacity > _maxCapacity)
{
// Either this PoolChunkList is empty or the requested capacity is larger then the capacity which can
// be handled by the PoolChunks that are contained in this PoolChunkList.
return(false);
}
for (PoolChunk <T> cur = _head; cur is object; cur = cur.Next)
{
if (cur.Allocate(buf, reqCapacity, normCapacity, threadCache))
{
if (cur._freeBytes <= _freeMinThreshold)
{
Remove(cur);
_nextList.Add(cur);
}
return(true);
}
}
return(false);
}
protected override void InitBuf(
PoolChunk <T> chunk, long handle, PooledByteBuffer <T> buf, int reqCapacity, PoolThreadCache <T> threadCache) =>
chunk.InitBuf(buf, handle, reqCapacity, threadCache);
/// <summary> /// Try to allocate a small buffer out of the cache /// </summary> /// <returns><c>true</c> if successful <c>false</c> otherwise</returns> internal bool AllocateNormal(PoolArena <T> area, PooledByteBuffer <T> buf, int reqCapacity, int normCapacity) => Allocate(CacheForNormal(area, normCapacity), buf, reqCapacity);
/**
* Init the {@link PooledByteBuffer} using the provided chunk and handle with the capacity restrictions.
*/
protected abstract void InitBuf(PoolChunk <T> chunk, long handle,
PooledByteBuffer <T> buf, int reqCapacity);
protected override void InitBuf(
PoolChunk <T> chunk, long handle, PooledByteBuffer <T> buf, int reqCapacity) =>
chunk.InitBuf(buf, handle, reqCapacity);
/**
* Try to allocate a small buffer out of the cache. Returns {@code true} if successful {@code false} otherwise
*/
internal bool AllocateSmall(PoolArena <T> area, PooledByteBuffer <T> buf, int reqCapacity, int normCapacity) =>
this.Allocate(this.CacheForSmall(area, normCapacity), buf, reqCapacity);
void Allocate(PoolThreadCache <T> cache, PooledByteBuffer <T> buf, int reqCapacity)
{
int normCapacity = NormalizeCapacity(reqCapacity);
if (IsTinyOrSmall(normCapacity))
{
// capacity < pageSize
int tableIdx;
PoolSubpage <T>[] table;
bool tiny = IsTiny(normCapacity);
if (tiny)
{
// < 512
if (cache.AllocateTiny(this, buf, reqCapacity, normCapacity))
{
// was able to allocate out of the cache so move on
return;
}
tableIdx = TinyIdx(normCapacity);
table = _tinySubpagePools;
}
else
{
if (cache.AllocateSmall(this, buf, reqCapacity, normCapacity))
{
// was able to allocate out of the cache so move on
return;
}
tableIdx = SmallIdx(normCapacity);
table = _smallSubpagePools;
}
PoolSubpage <T> head = table[tableIdx];
//
// Synchronize on the head. This is needed as {@link PoolSubpage#allocate()} and
// {@link PoolSubpage#free(int)} may modify the doubly linked list as well.
//
lock (head)
{
PoolSubpage <T> s = head.Next;
if (s != head)
{
Debug.Assert(s.DoNotDestroy && s.ElemSize == normCapacity);
long handle = s.Allocate();
Debug.Assert(handle >= 0);
s.Chunk.InitBufWithSubpage(buf, handle, reqCapacity, cache);
IncTinySmallAllocation(tiny);
return;
}
}
lock (this)
{
AllocateNormal(buf, reqCapacity, normCapacity, cache);
}
IncTinySmallAllocation(tiny);
return;
}
if (normCapacity <= ChunkSize)
{
if (cache.AllocateNormal(this, buf, reqCapacity, normCapacity))
{
// was able to allocate out of the cache so move on
return;
}
lock (this)
{
AllocateNormal(buf, reqCapacity, normCapacity, cache);
_allocationsNormal++;
}
}
else
{
// Huge allocations are never served via the cache so just call allocateHuge
AllocateHuge(buf, reqCapacity);
}
}
/// <summary>
/// Init the <see cref="PooledByteBuffer{T}"/> using the provided chunk and handle with the capacity restrictions.
/// </summary>
protected abstract void InitBuf(PoolChunk <T> chunk, long handle,
PooledByteBuffer <T> buf, int reqCapacity, PoolThreadCache <T> threadCache);
internal void InitBufWithSubpage(PooledByteBuffer <T> buf, long handle, int reqCapacity, PoolThreadCache <T> threadCache) => InitBufWithSubpage(buf, handle, BitmapIdx(handle), reqCapacity, threadCache);
internal void InitBufWithSubpage(PooledByteBuffer <T> buf, long handle, int reqCapacity) => this.InitBufWithSubpage(buf, handle, BitmapIdx(handle), reqCapacity);