internal void Free(PoolChunk <T> chunk, long handle, int normCapacity, PoolThreadCache <T> cache)
{
if (chunk.Unpooled)
{
int size = chunk.ChunkSize;
this.DestroyChunk(chunk);
switch (chunk.Origin)
{
case PoolChunk <T> .PoolChunkOrigin.UnpooledNormal:
Interlocked.Decrement(ref this.deallocationsNormal);
break;
case PoolChunk <T> .PoolChunkOrigin.UnpooledHuge:
Interlocked.Add(ref this.activeBytesHuge, -size);
Interlocked.Decrement(ref this.deallocationsHuge);
break;
default:
throw new InvalidOperationException("Unsupported PoolChunk.Origin: " + chunk.Origin);
}
}
else
{
SizeClass sc = this.SizeClass(normCapacity);
if (cache != null && cache.Add(this, chunk, handle, normCapacity, sc))
{
// cached so not free it.
return;
}
this.FreeChunk(chunk, handle, sc);
}
}
internal PooledByteBuffer <T> Allocate(PoolThreadCache <T> cache, int reqCapacity, int maxCapacity)
{
PooledByteBuffer <T> buf = NewByteBuf(maxCapacity);
Allocate(cache, buf, reqCapacity);
return(buf);
}
internal void InitUnpooled(PoolChunk <T> chunk, int length)
{
Contract.Assert(chunk != null);
this.Chunk = chunk;
this.Handle = 0;
this.Memory = chunk.Memory;
this.Offset = 0;
this.Length = this.MaxLength = length;
this.SetIndex(0, 0);
//tmpNioBuf = null;
this.Cache = null;
}
/// <summary>
/// Allocate something out of the cache if possible and remove the entry from the cache.
/// </summary>
public bool Allocate(PooledByteBuffer <T> buf, int reqCapacity, PoolThreadCache <T> threadCache)
{
if (!_queue.TryDequeue(out Entry entry))
{
return(false);
}
InitBuf(entry.Chunk, entry.Handle, buf, reqCapacity, threadCache);
entry.Recycle();
// allocations is not thread-safe which is fine as this is only called from the same thread all time.
++_allocations;
return(true);
}
void Init0(PoolChunk <T> chunk, long handle, int offset, int length, int maxLength, PoolThreadCache <T> cache)
{
Debug.Assert(handle >= 0);
Debug.Assert(chunk != null);
this.Chunk = chunk;
this.Memory = chunk.Memory;
this.allocator = chunk.Arena.Parent;
this.Cache = cache;
this.Handle = handle;
this.Offset = offset;
this.Length = length;
this.MaxLength = maxLength;
}
unsafe void Init0(PoolChunk <T> chunk, long handle, int offset, int length, int maxLength, PoolThreadCache <T> cache)
{
Debug.Assert(handle >= 0);
Debug.Assert(chunk is object);
Chunk = chunk;
Memory = chunk.Memory;
_allocator = chunk.Arena.Parent;
Origin = chunk.NativePointer;
Cache = cache;
Handle = handle;
Offset = offset;
Length = length;
MaxLength = maxLength;
}
internal void Init(PoolChunk <T> chunk, long handle, int offset, int length, int maxLength, PoolThreadCache <T> cache)
{
Contract.Assert(handle >= 0);
Contract.Assert(chunk != null);
this.Chunk = chunk;
this.Handle = handle;
this.Memory = chunk.Memory;
this.Offset = offset;
this.Length = length;
this.MaxLength = maxLength;
this.SetIndex(0, 0);
this.DiscardMarkers();
//tmpNioBuf = null;
this.Cache = cache;
}
protected override IByteBuffer NewHeapBuffer(int initialCapacity, int maxCapacity)
{
PoolThreadCache <byte[]> cache = this.threadCache.Value;
PoolArena <byte[]> heapArena = cache.HeapArena;
IByteBuffer buf;
if (heapArena != null)
{
buf = heapArena.Allocate(cache, initialCapacity, maxCapacity);
}
else
{
buf = new UnpooledHeapByteBuffer(this, initialCapacity, maxCapacity);
}
return(ToLeakAwareBuffer(buf));
}
protected unsafe override IByteBuffer NewDirectBuffer(int initialCapacity, int maxCapacity)
{
PoolThreadCache <byte[]> cache = this.threadCache.Value;
PoolArena <byte[]> directArena = cache.DirectArena;
IByteBuffer buf;
if (directArena != null)
{
buf = directArena.Allocate(cache, initialCapacity, maxCapacity);
}
else
{
buf = UnsafeByteBufferUtil.NewUnsafeDirectByteBuffer(this, initialCapacity, maxCapacity);
}
return(ToLeakAwareBuffer(buf));
}
internal void Free(PoolChunk <T> chunk, long handle, int normCapacity, PoolThreadCache <T> cache)
{
if (chunk.Unpooled)
{
int size = chunk.ChunkSize;
DestroyChunk(chunk);
_ = Interlocked.Add(ref _activeBytesHuge, -size);
_ = Interlocked.Increment(ref _deallocationsHuge);
}
else
{
SizeClass sizeClass = SizeClass(normCapacity);
if (cache is object && cache.Add(this, chunk, handle, normCapacity, sizeClass))
{
// cached so not free it.
return;
}
FreeChunk(chunk, handle, sizeClass, false);
}
}
internal void Free(PoolChunk <T> chunk, long handle, int normCapacity, PoolThreadCache <T> cache)
{
if (chunk.Unpooled)
{
int size = chunk.ChunkSize;
this.DestroyChunk(chunk);
Interlocked.Add(ref this.activeBytesHuge, -size);
Interlocked.Decrement(ref this.deallocationsHuge);
}
else
{
SizeClass sc = this.SizeClass(normCapacity);
if (cache != null && cache.Add(this, chunk, handle, normCapacity, sc))
{
// cached so not free it.
return;
}
this.FreeChunk(chunk, handle, sc);
}
}
/// <summary>
/// Init the <see cref="PooledByteBuffer{T}"/> using the provided chunk and handle with the capacity restrictions.
/// </summary>
protected abstract void InitBuf(PoolChunk <T> chunk, long handle,
PooledByteBuffer <T> buf, int reqCapacity, PoolThreadCache <T> threadCache);
internal void InitBuf(PooledByteBuffer <T> buf, long handle, int reqCapacity, PoolThreadCache <T> threadCache)
{
int memoryMapIdx = MemoryMapIdx(handle);
int bitmapIdx = BitmapIdx(handle);
if (0u >= (uint)bitmapIdx)
{
sbyte val = Value(memoryMapIdx);
Debug.Assert(val == _unusable, val.ToString());
buf.Init(this, handle, RunOffset(memoryMapIdx) + Offset, reqCapacity, RunLength(memoryMapIdx), threadCache);
}
else
{
InitBufWithSubpage(buf, handle, bitmapIdx, reqCapacity, threadCache);
}
}
internal void Init(PoolChunk <T> chunk, long handle, int offset, int length, int maxLength, PoolThreadCache <T> cache)
{
this.Init0(chunk, handle, offset, length, maxLength, cache);
this.DiscardMarkers();
}
protected override void InitBuf(
PoolChunk <T> chunk, long handle, PooledByteBuffer <T> buf, int reqCapacity, PoolThreadCache <T> threadCache) =>
chunk.InitBuf(buf, handle, reqCapacity, threadCache);
internal override void Init(PoolChunk <byte[]> chunk, long handle, int offset, int length, int maxLength,
PoolThreadCache <byte[]> cache)
{
base.Init(chunk, handle, offset, length, maxLength, cache);
this.InitMemoryAddress();
}
internal void Init(PoolChunk <T> chunk, long handle, int offset, int length, int maxLength, PoolThreadCache <T> cache) => this.Init0(chunk, handle, offset, length, maxLength, cache);
void Allocate(PoolThreadCache <T> cache, PooledByteBuffer <T> buf, int reqCapacity)
{
int normCapacity = NormalizeCapacity(reqCapacity);
if (IsTinyOrSmall(normCapacity))
{
// capacity < pageSize
int tableIdx;
PoolSubpage <T>[] table;
bool tiny = IsTiny(normCapacity);
if (tiny)
{
// < 512
if (cache.AllocateTiny(this, buf, reqCapacity, normCapacity))
{
// was able to allocate out of the cache so move on
return;
}
tableIdx = TinyIdx(normCapacity);
table = _tinySubpagePools;
}
else
{
if (cache.AllocateSmall(this, buf, reqCapacity, normCapacity))
{
// was able to allocate out of the cache so move on
return;
}
tableIdx = SmallIdx(normCapacity);
table = _smallSubpagePools;
}
PoolSubpage <T> head = table[tableIdx];
//
// Synchronize on the head. This is needed as {@link PoolSubpage#allocate()} and
// {@link PoolSubpage#free(int)} may modify the doubly linked list as well.
//
lock (head)
{
PoolSubpage <T> s = head.Next;
if (s != head)
{
Debug.Assert(s.DoNotDestroy && s.ElemSize == normCapacity);
long handle = s.Allocate();
Debug.Assert(handle >= 0);
s.Chunk.InitBufWithSubpage(buf, handle, reqCapacity, cache);
IncTinySmallAllocation(tiny);
return;
}
}
lock (this)
{
AllocateNormal(buf, reqCapacity, normCapacity, cache);
}
IncTinySmallAllocation(tiny);
return;
}
if (normCapacity <= ChunkSize)
{
if (cache.AllocateNormal(this, buf, reqCapacity, normCapacity))
{
// was able to allocate out of the cache so move on
return;
}
lock (this)
{
AllocateNormal(buf, reqCapacity, normCapacity, cache);
_allocationsNormal++;
}
}
else
{
// Huge allocations are never served via the cache so just call allocateHuge
AllocateHuge(buf, reqCapacity);
}
}
protected override void OnRemoval(PoolThreadCache <byte[]> threadCache) => threadCache.Free();
void AllocateNormal(PooledByteBuffer <T> buf, int reqCapacity, int normCapacity, PoolThreadCache <T> threadCache)
{
if (_q050.Allocate(buf, reqCapacity, normCapacity, threadCache) ||
_q025.Allocate(buf, reqCapacity, normCapacity, threadCache) ||
_q000.Allocate(buf, reqCapacity, normCapacity, threadCache) ||
_qInit.Allocate(buf, reqCapacity, normCapacity, threadCache) ||
_q075.Allocate(buf, reqCapacity, normCapacity, threadCache))
{
return;
}
// Add a new chunk.
PoolChunk <T> c = NewChunk(PageSize, _maxOrder, PageShifts, ChunkSize);
bool success = c.Allocate(buf, reqCapacity, normCapacity, threadCache);
Debug.Assert(success);
_qInit.Add(c);
}
void InitBufWithSubpage(PooledByteBuffer <T> buf, long handle, int bitmapIdx, int reqCapacity, PoolThreadCache <T> threadCache)
{
Debug.Assert(bitmapIdx != 0);
int memoryMapIdx = MemoryMapIdx(handle);
PoolSubpage <T> subpage = _subpages[SubpageIdx(memoryMapIdx)];
Debug.Assert(subpage.DoNotDestroy);
Debug.Assert(reqCapacity <= subpage.ElemSize);
buf.Init(
this, handle,
RunOffset(memoryMapIdx) + (bitmapIdx & 0x3FFFFFFF) * subpage.ElemSize + Offset,
reqCapacity, subpage.ElemSize, threadCache);
}
internal void InitBufWithSubpage(PooledByteBuffer <T> buf, long handle, int reqCapacity, PoolThreadCache <T> threadCache) => InitBufWithSubpage(buf, handle, BitmapIdx(handle), reqCapacity, threadCache);
internal bool Allocate(PooledByteBuffer <T> buf, int reqCapacity, int normCapacity, PoolThreadCache <T> threadCache)
{
if (_head is null || normCapacity > _maxCapacity)
{
// Either this PoolChunkList is empty or the requested capacity is larger then the capacity which can
// be handled by the PoolChunks that are contained in this PoolChunkList.
return(false);
}
for (PoolChunk <T> cur = _head; cur is object; cur = cur.Next)
{
if (cur.Allocate(buf, reqCapacity, normCapacity, threadCache))
{
if (cur._freeBytes <= _freeMinThreshold)
{
Remove(cur);
_nextList.Add(cur);
}
return(true);
}
}
return(false);
}
internal bool Allocate(PooledByteBuffer <T> buf, int reqCapacity, int normCapacity, PoolThreadCache <T> threadCache)
{
long handle;
if ((normCapacity & _subpageOverflowMask) != 0)
{
// >= pageSize
handle = AllocateRun(normCapacity);
}
else
{
handle = AllocateSubpage(normCapacity);
}
if (handle < 0)
{
return(false);
}
InitBuf(buf, handle, reqCapacity, threadCache);
return(true);
}
