void AllocateNormal(PooledArrayBuffer <T> buf, int reqCapacity, int normCapacity)
{
lock (this)
{
if (this.q050.Allocate(buf, reqCapacity, normCapacity) || this.q025.Allocate(buf, reqCapacity, normCapacity) ||
this.q000.Allocate(buf, reqCapacity, normCapacity) || this.qInit.Allocate(buf, reqCapacity, normCapacity) ||
this.q075.Allocate(buf, reqCapacity, normCapacity))
{
++this.allocationsNormal;
return;
}
if (this.chunkCount < this.maxChunkCount)
{
// Add a new chunk.
PoolChunk <T> c = this.NewChunk(this.PageSize, this.maxOrder, this.PageShifts, this.ChunkSize);
this.chunkCount++;
long handle = c.Allocate(normCapacity);
++this.allocationsNormal;
Contract.Assert(handle > 0);
c.InitBuf(buf, handle, reqCapacity);
this.qInit.Add(c);
return;
}
}
PoolChunk <T> chunk = this.NewUnpooledChunk(reqCapacity, false);
buf.InitUnpooled(chunk, reqCapacity);
Interlocked.Increment(ref this.allocationsNormal);
}
internal PooledArrayBuffer <T> Allocate(PoolThreadCache <T> cache, int reqCapacity, int maxCapacity)
{
PooledArrayBuffer <T> buf = this.NewByteBuf(maxCapacity);
this.Allocate(cache, buf, reqCapacity);
return(buf);
}
internal bool Allocate(PooledArrayBuffer <T> buf, int reqCapacity, int normCapacity)
{
if (this.head == null || normCapacity > this.maxCapacity)
{
// Either this PoolChunkList is empty or the requested capacity is larger then the capacity which can
// be handled by the PoolChunks that are contained in this PoolChunkList.
return(false);
}
for (PoolChunk <T> cur = this.head; ;)
{
long handle = cur.Allocate(normCapacity);
if (handle < 0)
{
cur = cur.Next;
if (cur == null)
{
return(false);
}
}
else
{
cur.InitBuf(buf, handle, reqCapacity);
if (cur.Usage >= this.maxUsage)
{
this.Remove(cur);
this.nextList.Add(cur);
}
return(true);
}
}
}
void AllocateHuge(PooledArrayBuffer <T> buf, int reqCapacity)
{
PoolChunk <T> chunk = this.NewUnpooledChunk(reqCapacity);
Interlocked.Add(ref this.activeBytesHuge, chunk.ChunkSize);
buf.InitUnpooled(chunk, reqCapacity);
Interlocked.Increment(ref this.allocationsHuge);
}
internal static PooledArrayBuffer <T> NewInstance(int capacity)
{
PooledArrayBuffer <T> buf = Recycler.Take();
buf.Recycle(capacity);
return(buf);
}
internal static PooledArrayBuffer <T> NewInstance(int maxCapacity)
{
PooledArrayBuffer <T> buf = Recycler.Take();
buf.SetReferenceCount(1);
buf.MaxCapacity = maxCapacity;
buf.SetIndex(0, 0);
buf.DiscardMarkers();
return(buf);
}
internal void Reallocate(PooledArrayBuffer <T> buf, int newCapacity, bool freeOldMemory)
{
Contract.Requires(newCapacity >= 0 && newCapacity <= buf.MaxCapacity);
int oldCapacity = buf.Length;
if (oldCapacity == newCapacity)
{
return;
}
PoolChunk <T> oldChunk = buf.Chunk;
long oldHandle = buf.Handle;
T[] oldMemory = buf.Memory;
int oldOffset = buf.Offset;
int oldMaxLength = buf.MaxLength;
int readerIndex = buf.ReaderIndex;
int writerIndex = buf.WriterIndex;
this.Allocate(this.Parent.ThreadCache(), buf, newCapacity);
if (newCapacity > oldCapacity)
{
MemoryCopy(
oldMemory, oldOffset,
buf.Memory, buf.Offset, oldCapacity);
}
else if (newCapacity < oldCapacity)
{
if (readerIndex < newCapacity)
{
if (writerIndex > newCapacity)
{
writerIndex = newCapacity;
}
MemoryCopy(
oldMemory, oldOffset + readerIndex,
buf.Memory, buf.Offset + readerIndex, writerIndex - readerIndex);
}
else
{
readerIndex = writerIndex = newCapacity;
}
}
buf.SetIndex(readerIndex, writerIndex);
if (freeOldMemory)
{
this.Free(oldChunk, oldHandle, oldMaxLength, buf.Cache);
}
}
public bool Allocate(PooledArrayBuffer <T> buf, int reqCapacity)
{
if (!this.queue.TryDequeue(out Entry entry))
{
return(false);
}
this.InitBuf(entry.Chunk, entry.Handle, buf, reqCapacity);
entry.Recycle();
// allocations is not thread-safe which is fine as this is only called from the same thread all time.
++this.allocations;
return(true);
}
void InitBufWithSubpage(PooledArrayBuffer <T> buf, long handle, int bitmapIdx, int reqCapacity)
{
Contract.Assert(bitmapIdx != 0);
int memoryMapIdx = MemoryMapIdx(handle);
PoolSubpage <T> subpage = this.subpages[this.SubpageIdx(memoryMapIdx)];
Contract.Assert(subpage.DoNotDestroy);
Contract.Assert(reqCapacity <= subpage.ElemSize);
buf.Init(
this, handle,
this.RunOffset(memoryMapIdx) + (bitmapIdx & 0x3FFFFFFF) * subpage.ElemSize, reqCapacity, subpage.ElemSize,
this.Arena.Parent.ThreadCache());
}
bool Allocate(MemoryRegionCache cache, PooledArrayBuffer <T> buf, int reqCapacity)
{
if (cache == null)
{
// no cache found so just return false here
return(false);
}
bool allocated = cache.Allocate(buf, reqCapacity);
if (++this.allocations >= this.freeSweepAllocationThreshold)
{
this.allocations = 0;
this.Trim();
}
return(allocated);
}
internal void InitBuf(PooledArrayBuffer <T> buf, long handle, int reqCapacity)
{
int memoryMapIdx = MemoryMapIdx(handle);
int bitmapIdx = BitmapIdx(handle);
if (bitmapIdx == 0)
{
sbyte val = this.Value(memoryMapIdx);
Contract.Assert(val == this.unusable, val.ToString());
buf.Init(this, handle, this.RunOffset(memoryMapIdx), reqCapacity, this.RunLength(memoryMapIdx),
this.Arena.Parent.ThreadCache());
}
else
{
this.InitBufWithSubpage(buf, handle, bitmapIdx, reqCapacity);
}
}
void AllocateNormal(PooledArrayBuffer <T> buf, int reqCapacity, int normCapacity)
{
lock (this)
{
if (this.q050.Allocate(buf, reqCapacity, normCapacity) ||
this.q025.Allocate(buf, reqCapacity, normCapacity) ||
this.q000.Allocate(buf, reqCapacity, normCapacity) ||
this.qInit.Allocate(buf, reqCapacity, normCapacity) ||
this.q075.Allocate(buf, reqCapacity, normCapacity))
{
++this.NumNormalAllocations;
return;
}
// Add a new chunk.
PoolChunk <T> c = this.NewChunk(this.PageSize, this.maxOrder, this.PageShifts, this.ChunkSize);
long handle = c.Allocate(normCapacity);
++this.NumNormalAllocations;
Contract.Assert(handle > 0);
c.InitBuf(buf, handle, reqCapacity);
this.qInit.Add(c);
}
}
internal void Reallocate(PooledArrayBuffer <T> buf, int newCapacity, bool freeOldMemory)
{
Contract.Requires(newCapacity >= 0 && newCapacity <= buf.Capacity);
int oldCapacity = buf.Capacity;
if (oldCapacity == newCapacity)
{
return;
}
PoolChunk <T> oldChunk = buf.Chunk;
long oldHandle = buf.Handle;
T[] oldMemory = buf.Array;
int oldOffset = buf.Offset;
int oldMaxLength = buf.Length;
this.Allocate(this.Parent.ThreadCache(), buf, newCapacity);
if (newCapacity > oldCapacity)
{
this.MemoryCopy(
oldMemory, oldOffset,
buf.Array, buf.Offset, oldCapacity);
}
else if (newCapacity < oldCapacity)
{
this.MemoryCopy(
oldMemory, oldOffset,
buf.Array, buf.Offset, newCapacity);
}
if (freeOldMemory)
{
this.Free(oldChunk, oldHandle, oldMaxLength, buf.Cache);
}
}
protected override void InitBuf(
PoolChunk <T> chunk, long handle, PooledArrayBuffer <T> buf, int reqCapacity) => chunk.InitBuf(buf, handle, reqCapacity);
protected PooledArrayBuffer <T> NewByteBuf(int maxCapacity) => PooledArrayBuffer <T> .NewInstance(maxCapacity);
void Allocate(PoolThreadCache <T> cache, PooledArrayBuffer <T> buf, int reqCapacity)
{
int normCapacity = this.NormalizeCapacity(reqCapacity);
if (this.IsTinyOrSmall(normCapacity))
{
// capacity < pageSize
int tableIdx;
PoolSubpage <T>[] table;
bool tiny = IsTiny(normCapacity);
if (tiny)
{
// < 512
if (cache.AllocateTiny(this, buf, reqCapacity, normCapacity))
{
// was able to allocate out of the cache so move on
return;
}
tableIdx = TinyIdx(normCapacity);
table = this.tinySubpagePools;
}
else
{
if (cache.AllocateSmall(this, buf, reqCapacity, normCapacity))
{
// was able to allocate out of the cache so move on
return;
}
tableIdx = SmallIdx(normCapacity);
table = this.smallSubpagePools;
}
PoolSubpage <T> head = table[tableIdx];
/**
* Synchronize on the head. This is needed as {@link PoolSubpage#allocate()} and
* {@link PoolSubpage#free(int)} may modify the doubly linked list as well.
*/
lock (head)
{
PoolSubpage <T> s = head.Next;
if (s != head)
{
Contract.Assert(s.DoNotDestroy && s.ElemSize == normCapacity);
long handle = s.Allocate();
Contract.Assert(handle >= 0);
s.Chunk.InitBufWithSubpage(buf, handle, reqCapacity);
if (tiny)
{
++this.NumTinyAllocations;
}
else
{
++this.NumSmallAllocations;
}
return;
}
}
this.AllocateNormal(buf, reqCapacity, normCapacity);
return;
}
if (normCapacity <= this.ChunkSize)
{
if (cache.AllocateNormal(this, buf, reqCapacity, normCapacity))
{
// was able to allocate out of the cache so move on
return;
}
this.AllocateNormal(buf, reqCapacity, normCapacity);
}
else
{
// Huge allocations are never served via the cache so just call allocateHuge
this.AllocateHuge(buf, reqCapacity);
}
}
internal void InitBufWithSubpage(PooledArrayBuffer <T> buf, long handle, int reqCapacity) => this.InitBufWithSubpage(buf, handle, BitmapIdx(handle), reqCapacity);
protected abstract void InitBuf(PoolChunk <T> chunk, long handle, PooledArrayBuffer <T> buf, int reqCapacity);
internal bool AllocateNormal(PoolArena <T> area, PooledArrayBuffer <T> buf, int reqCapacity, int normCapacity) => this.Allocate(this.CacheForNormal(area, normCapacity), buf, reqCapacity);
