public override void Return(T[] array, bool clearArray = false)
{
if (array == null)
{
throw new ArgumentNullException(nameof(array));
}
// Determine with what bucket this array length is associated
int bucketIndex = Utilities.SelectBucketIndex(array.Length);
// If we can tell that the buffer was allocated (or empty), drop it. Otherwise, check if we have space in the pool.
if (bucketIndex < _buckets.Length)
{
// Clear the array if the user requests.
if (clearArray)
{
Array.Clear(array, 0, array.Length);
}
// Check to see if the buffer is the correct size for this bucket
if (array.Length != _bucketArraySizes[bucketIndex])
{
throw new ArgumentException(SR.ArgumentException_BufferNotFromPool, nameof(array));
}
// Write through the TLS bucket. If there weren't any buckets, create them
// and store this array into it. If there were, store this into it, and
// if there was a previous one there, push that to the global stack. This
// helps to keep LIFO access such that the most recently pushed stack will
// be in TLS and the first to be popped next.
T[][] tlsBuckets = t_tlsBuckets;
if (tlsBuckets == null)
{
t_tlsBuckets = tlsBuckets = new T[NumBuckets][];
tlsBuckets[bucketIndex] = array;
}
else
{
T[] prev = tlsBuckets[bucketIndex];
tlsBuckets[bucketIndex] = array;
if (prev != null)
{
PerCoreLockedStacks bucket = _buckets[bucketIndex] ?? CreatePerCoreLockedStacks(bucketIndex);
bucket.TryPush(prev);
}
}
}
// Log that the buffer was returned
ArrayPoolEventSource log = ArrayPoolEventSource.Log;
if (log.IsEnabled())
{
log.BufferReturned(array.GetHashCode(), array.Length, Id);
}
}
public bool Trim()
{
Debug.Assert(s_trimBuffers);
Debug.Assert(s_allTlsBuckets != null);
int milliseconds = Environment.TickCount;
MemoryPressure pressure = GetMemoryPressure();
ArrayPoolEventSource log = ArrayPoolEventSource.Log;
if (log.IsEnabled())
{
log.BufferTrimPoll(milliseconds, (int)pressure);
}
PerCoreLockedStacks?[] perCoreBuckets = _buckets;
for (int i = 0; i < perCoreBuckets.Length; i++)
{
perCoreBuckets[i]?.Trim((uint)milliseconds, Id, pressure, _bucketArraySizes[i]);
}
if (pressure == MemoryPressure.High)
{
// Under high pressure, release all thread locals
if (log.IsEnabled())
{
foreach (KeyValuePair <T[]?[], object?> tlsBuckets in s_allTlsBuckets)
{
T[]?[] buckets = tlsBuckets.Key;
for (int i = 0; i < buckets.Length; i++)
{
T[]? buffer = Interlocked.Exchange(ref buckets[i], null);
if (buffer != null)
{
// As we don't want to take a perf hit in the rent path it
// is possible that a buffer could be rented as we "free" it.
log.BufferTrimmed(buffer.GetHashCode(), buffer.Length, Id);
}
}
}
}
else
{
foreach (KeyValuePair <T[]?[], object?> tlsBuckets in s_allTlsBuckets)
{
T[]?[] buckets = tlsBuckets.Key;
Array.Clear(buckets, 0, buckets.Length);
}
}
}
return(true);
}
/// <summary>Takes an array from the bucket. If the bucket is empty, returns null.</summary>
internal T[]? Rent()
{
T[]?[] buffers = _buffers;
T[]? buffer = null;
// While holding the lock, grab whatever is at the next available index and
// update the index. We do as little work as possible while holding the spin
// lock to minimize contention with other threads. The try/finally is
// necessary to properly handle thread aborts on platforms which have them.
bool lockTaken = false, allocateBuffer = false;
try
{
_lock.Enter(ref lockTaken);
if (_index < buffers.Length)
{
buffer = buffers[_index];
buffers[_index++] = null;
allocateBuffer = buffer == null;
}
}
finally
{
if (lockTaken)
{
_lock.Exit(false);
}
}
// While we were holding the lock, we grabbed whatever was at the next available index, if
// there was one. If we tried and if we got back null, that means we hadn't yet allocated
// for that slot, in which case we should do so now.
if (allocateBuffer)
{
buffer = new T[_bufferLength];
ArrayPoolEventSource log = ArrayPoolEventSource.Log;
if (log.IsEnabled())
{
log.BufferAllocated(buffer.GetHashCode(), _bufferLength, _poolId, Id,
ArrayPoolEventSource.BufferAllocatedReason.Pooled);
}
}
return(buffer);
}
public override void Return(T[] array, bool clearArray = false)
{
if (array == null)
{
throw new ArgumentNullException(nameof(array));
}
else if (array.Length == 0)
{
// Ignore empty arrays. When a zero-length array is rented, we return a singleton
// rather than actually taking a buffer out of the lowest bucket.
return;
}
// Determine with what bucket this array length is associated
int bucket = Utilities.SelectBucketIndex(array.Length);
// If we can tell that the buffer was allocated, drop it. Otherwise, check if we have space in the pool
bool haveBucket = bucket < _buckets.Length;
if (haveBucket)
{
// Clear the array if the user requests
if (clearArray)
{
Array.Clear(array, 0, array.Length);
}
// Return the buffer to its bucket. In the future, we might consider having Return return false
// instead of dropping a bucket, in which case we could try to return to a lower-sized bucket,
// just as how in Rent we allow renting from a higher-sized bucket.
_buckets[bucket].Return(array);
}
// Log that the buffer was returned
ArrayPoolEventSource log = ArrayPoolEventSource.Log;
if (log.IsEnabled())
{
int bufferId = array.GetHashCode();
log.BufferReturned(bufferId, array.Length, Id);
if (!haveBucket)
{
log.BufferDropped(bufferId, array.Length, Id, ArrayPoolEventSource.NoBucketId, ArrayPoolEventSource.BufferDroppedReason.Full);
}
}
}
/// <summary>
/// Attempts to return the buffer to the bucket. If successful, the buffer will be stored
/// in the bucket and true will be returned; otherwise, the buffer won't be stored, and false
/// will be returned.
/// </summary>
internal void Return(T[] array)
{
// Check to see if the buffer is the correct size for this bucket
if (array.Length != _bufferLength)
{
throw new ArgumentException(SR.ArgumentException_BufferNotFromPool, nameof(array));
}
bool returned;
// While holding the spin lock, if there's room available in the bucket,
// put the buffer into the next available slot. Otherwise, we just drop it.
// The try/finally is necessary to properly handle thread aborts on platforms
// which have them.
bool lockTaken = false;
try
{
_lock.Enter(ref lockTaken);
returned = _index != 0;
if (returned)
{
_buffers[--_index] = array;
}
}
finally
{
if (lockTaken)
{
_lock.Exit(false);
}
}
if (!returned)
{
ArrayPoolEventSource log = ArrayPoolEventSource.Log;
if (log.IsEnabled())
{
log.BufferDropped(array.GetHashCode(), _bufferLength, _poolId, Id, ArrayPoolEventSource.BufferDroppedReason.Full);
}
}
}
public bool Trim()
{
int milliseconds = Environment.TickCount;
MemoryPressure pressure = GetMemoryPressure();
ArrayPoolEventSource log = ArrayPoolEventSource.Log;
if (log.IsEnabled())
{
log.BufferTrimPoll(milliseconds, (int)pressure);
}
foreach (PerCoreLockedStacks bucket in _buckets)
{
bucket?.Trim((uint)milliseconds, Id, pressure, _bucketArraySizes);
}
if (pressure == MemoryPressure.High)
{
// Under high pressure, release all thread locals
foreach (KeyValuePair <T[][], object> tlsBuckets in s_AllTlsBuckets)
{
T[][] buckets = tlsBuckets.Key;
for (int i = 0; i < NumBuckets; i++)
{
T[] buffer = buckets[i];
buckets[i] = null;
if (log.IsEnabled() && buffer != null)
{
log.BufferTrimmed(buffer.GetHashCode(), buffer.Length, Id);
}
}
}
}
return(true);
}
public override T[] Rent(int minimumLength)
{
// Arrays can't be smaller than zero. We allow requesting zero-length arrays (even though
// pooling such an array isn't valuable) as it's a valid length array, and we want the pool
// to be usable in general instead of using `new`, even for computed lengths.
if (minimumLength < 0)
{
throw new ArgumentOutOfRangeException(nameof(minimumLength));
}
else if (minimumLength == 0)
{
// No need to log the empty array. Our pool is effectively infinite
// and we'll never allocate for rents and never store for returns.
return(Array.Empty <T>());
}
ArrayPoolEventSource log = ArrayPoolEventSource.Log;
T[]? buffer;
// Get the bucket number for the array length
int bucketIndex = Utilities.SelectBucketIndex(minimumLength);
// If the array could come from a bucket...
if (bucketIndex < _buckets.Length)
{
// First try to get it from TLS if possible.
T[]?[]? tlsBuckets = t_tlsBuckets;
if (tlsBuckets != null)
{
buffer = tlsBuckets[bucketIndex];
if (buffer != null)
{
tlsBuckets[bucketIndex] = null;
if (log.IsEnabled())
{
log.BufferRented(buffer.GetHashCode(), buffer.Length, Id, bucketIndex);
}
return(buffer);
}
}
// We couldn't get a buffer from TLS, so try the global stack.
PerCoreLockedStacks?b = _buckets[bucketIndex];
if (b != null)
{
buffer = b.TryPop();
if (buffer != null)
{
if (log.IsEnabled())
{
log.BufferRented(buffer.GetHashCode(), buffer.Length, Id, bucketIndex);
}
return(buffer);
}
}
// No buffer available. Allocate a new buffer with a size corresponding to the appropriate bucket.
buffer = GC.AllocateUninitializedArray <T>(_bucketArraySizes[bucketIndex]);
}
else
{
// The request was for a size too large for the pool. Allocate an array of exactly the requested length.
// When it's returned to the pool, we'll simply throw it away.
buffer = GC.AllocateUninitializedArray <T>(minimumLength);
}
if (log.IsEnabled())
{
int bufferId = buffer.GetHashCode(), bucketId = -1; // no bucket for an on-demand allocated buffer
log.BufferRented(bufferId, buffer.Length, Id, bucketId);
log.BufferAllocated(bufferId, buffer.Length, Id, bucketId, bucketIndex >= _buckets.Length ?
ArrayPoolEventSource.BufferAllocatedReason.OverMaximumSize :
ArrayPoolEventSource.BufferAllocatedReason.PoolExhausted);
}
return(buffer);
}
public void Trim(uint tickCount, int id, MemoryPressure pressure, int bucketSize)
{
const uint StackTrimAfterMS = 60 * 1000; // Trim after 60 seconds for low/moderate pressure
const uint StackHighTrimAfterMS = 10 * 1000; // Trim after 10 seconds for high pressure
const uint StackRefreshMS = StackTrimAfterMS / 4; // Time bump after trimming (1/4 trim time)
const int StackLowTrimCount = 1; // Trim one item when pressure is low
const int StackMediumTrimCount = 2; // Trim two items when pressure is moderate
const int StackHighTrimCount = MaxBuffersPerArraySizePerCore; // Trim all items when pressure is high
const int StackLargeBucket = 16384; // If the bucket is larger than this we'll trim an extra when under high pressure
const int StackModerateTypeSize = 16; // If T is larger than this we'll trim an extra when under high pressure
const int StackLargeTypeSize = 32; // If T is larger than this we'll trim an extra (additional) when under high pressure
if (_count == 0)
{
return;
}
uint trimTicks = pressure == MemoryPressure.High ? StackHighTrimAfterMS : StackTrimAfterMS;
lock (this)
{
if (_count > 0 && _firstStackItemMS > tickCount || (tickCount - _firstStackItemMS) > trimTicks)
{
// We've wrapped the tick count or elapsed enough time since the
// first item went into the stack. Drop the top item so it can
// be collected and make the stack look a little newer.
ArrayPoolEventSource log = ArrayPoolEventSource.Log;
int trimCount = StackLowTrimCount;
switch (pressure)
{
case MemoryPressure.High:
trimCount = StackHighTrimCount;
// When pressure is high, aggressively trim larger arrays.
if (bucketSize > StackLargeBucket)
{
trimCount++;
}
if (Unsafe.SizeOf <T>() > StackModerateTypeSize)
{
trimCount++;
}
if (Unsafe.SizeOf <T>() > StackLargeTypeSize)
{
trimCount++;
}
break;
case MemoryPressure.Medium:
trimCount = StackMediumTrimCount;
break;
}
while (_count > 0 && trimCount-- > 0)
{
T[]? array = _arrays[--_count];
Debug.Assert(array != null, "No nulls should have been present in slots < _count.");
_arrays[_count] = null;
if (log.IsEnabled())
{
log.BufferTrimmed(array.GetHashCode(), array.Length, id);
}
}
if (_count > 0 && _firstStackItemMS < uint.MaxValue - StackRefreshMS)
{
// Give the remaining items a bit more time
_firstStackItemMS += StackRefreshMS;
}
}
}
}
public override T[] Rent(int minimumLength)
{
ArrayPoolEventSource log = ArrayPoolEventSource.Log;
T[]? buffer;
// Get the bucket number for the array length. The result may be out of range of buckets,
// either for too large a value or for 0 and negative values.
int bucketIndex = Utilities.SelectBucketIndex(minimumLength);
// First, try to get an array from TLS if possible.
ThreadLocalArray[]? tlsBuckets = t_tlsBuckets;
if (tlsBuckets is not null && (uint)bucketIndex < (uint)tlsBuckets.Length)
{
buffer = tlsBuckets[bucketIndex].Array;
if (buffer is not null)
{
tlsBuckets[bucketIndex].Array = null;
if (log.IsEnabled())
{
log.BufferRented(buffer.GetHashCode(), buffer.Length, Id, bucketIndex);
}
return(buffer);
}
}
// Next, try to get an array from one of the per-core stacks.
PerCoreLockedStacks?[] perCoreBuckets = _buckets;
if ((uint)bucketIndex < (uint)perCoreBuckets.Length)
{
PerCoreLockedStacks?b = perCoreBuckets[bucketIndex];
if (b is not null)
{
buffer = b.TryPop();
if (buffer is not null)
{
if (log.IsEnabled())
{
log.BufferRented(buffer.GetHashCode(), buffer.Length, Id, bucketIndex);
}
return(buffer);
}
}
// No buffer available. Ensure the length we'll allocate matches that of a bucket
// so we can later return it.
minimumLength = Utilities.GetMaxSizeForBucket(bucketIndex);
}
else if (minimumLength == 0)
{
// We allow requesting zero-length arrays (even though pooling such an array isn't valuable)
// as it's a valid length array, and we want the pool to be usable in general instead of using
// `new`, even for computed lengths. But, there's no need to log the empty array. Our pool is
// effectively infinite for empty arrays and we'll never allocate for rents and never store for returns.
return(Array.Empty <T>());
}
else if (minimumLength < 0)
{
throw new ArgumentOutOfRangeException(nameof(minimumLength));
}
buffer = GC.AllocateUninitializedArray <T>(minimumLength);
if (log.IsEnabled())
{
int bufferId = buffer.GetHashCode();
log.BufferRented(bufferId, buffer.Length, Id, ArrayPoolEventSource.NoBucketId);
log.BufferAllocated(bufferId, buffer.Length, Id, ArrayPoolEventSource.NoBucketId, bucketIndex >= _buckets.Length ?
ArrayPoolEventSource.BufferAllocatedReason.OverMaximumSize :
ArrayPoolEventSource.BufferAllocatedReason.PoolExhausted);
}
return(buffer);
}
public override T[] Rent(int minimumLength)
{
// Arrays can't be smaller than zero. We allow requesting zero-length arrays (even though
// pooling such an array isn't valuable) as it's a valid length array, and we want the pool
// to be usable in general instead of using `new`, even for computed lengths.
if (minimumLength < 0)
{
throw new ArgumentOutOfRangeException(nameof(minimumLength));
}
else if (minimumLength == 0)
{
// No need for events with the empty array. Our pool is effectively infinite
// and we'll never allocate for rents and never store for returns.
return(Array.Empty <T>());
}
ArrayPoolEventSource log = ArrayPoolEventSource.Log;
T[]? buffer;
int index = Utilities.SelectBucketIndex(minimumLength);
if (index < _buckets.Length)
{
// Search for an array starting at the 'index' bucket. If the bucket is empty, bump up to the
// next higher bucket and try that one, but only try at most a few buckets.
const int MaxBucketsToTry = 2;
int i = index;
do
{
// Attempt to rent from the bucket. If we get a buffer from it, return it.
buffer = _buckets[i].Rent();
if (buffer != null)
{
if (log.IsEnabled())
{
log.BufferRented(buffer.GetHashCode(), buffer.Length, Id, _buckets[i].Id);
}
return(buffer);
}
}while (++i < _buckets.Length && i != index + MaxBucketsToTry);
// The pool was exhausted for this buffer size. Allocate a new buffer with a size corresponding
// to the appropriate bucket.
buffer = new T[_buckets[index]._bufferLength];
}
else
{
// The request was for a size too large for the pool. Allocate an array of exactly the requested length.
// When it's returned to the pool, we'll simply throw it away.
buffer = new T[minimumLength];
}
if (log.IsEnabled())
{
int bufferId = buffer.GetHashCode(), bucketId = -1; // no bucket for an on-demand allocated buffer
log.BufferRented(bufferId, buffer.Length, Id, bucketId);
log.BufferAllocated(bufferId, buffer.Length, Id, bucketId, index >= _buckets.Length ?
ArrayPoolEventSource.BufferAllocatedReason.OverMaximumSize : ArrayPoolEventSource.BufferAllocatedReason.PoolExhausted);
}
return(buffer);
}
public override void Return(T[] array, bool clearArray = false)
{
if (array == null)
{
throw new ArgumentNullException(nameof(array));
}
// Determine with what bucket this array length is associated
int bucketIndex = Utilities.SelectBucketIndex(array.Length);
// If we can tell that the buffer was allocated (or empty), drop it. Otherwise, check if we have space in the pool.
bool returned = true;
bool haveBucket = bucketIndex < _buckets.Length;
if (haveBucket)
{
// Clear the array if the user requests.
if (clearArray)
{
Array.Clear(array, 0, array.Length);
}
// Check to see if the buffer is the correct size for this bucket
if (array.Length != _bucketArraySizes[bucketIndex])
{
throw new ArgumentException(SR.ArgumentException_BufferNotFromPool, nameof(array));
}
// Write through the TLS bucket. If there weren't any buckets, create them
// and store this array into it. If there were, store this into it, and
// if there was a previous one there, push that to the global stack. This
// helps to keep LIFO access such that the most recently pushed stack will
// be in TLS and the first to be popped next.
T[]?[]? tlsBuckets = t_tlsBuckets;
if (tlsBuckets == null)
{
t_tlsBuckets = tlsBuckets = new T[NumBuckets][];
tlsBuckets[bucketIndex] = array;
if (s_trimBuffers)
{
Debug.Assert(s_allTlsBuckets != null, "Should be non-null iff s_trimBuffers is true");
s_allTlsBuckets.Add(tlsBuckets, null);
if (Interlocked.Exchange(ref _callbackCreated, 1) != 1)
{
Gen2GcCallback.Register(Gen2GcCallbackFunc, this);
}
}
}
else
{
T[]? prev = tlsBuckets[bucketIndex];
tlsBuckets[bucketIndex] = array;
if (prev != null)
{
PerCoreLockedStacks stackBucket = _buckets[bucketIndex] ?? CreatePerCoreLockedStacks(bucketIndex);
returned = stackBucket.TryPush(prev);
}
}
}
// Log that the buffer was returned
ArrayPoolEventSource log = ArrayPoolEventSource.Log;
if (log.IsEnabled())
{
log.BufferReturned(array.GetHashCode(), array.Length, Id);
if (!(haveBucket & returned))
{
log.BufferDropped(array.GetHashCode(), array.Length, Id,
haveBucket ? bucketIndex : ArrayPoolEventSource.NoBucketId,
haveBucket ? ArrayPoolEventSource.BufferDroppedReason.Full : ArrayPoolEventSource.BufferDroppedReason.OverMaximumSize);
}
}
}
public override void Return(T[] array, bool clearArray = false)
{
if (array is null)
{
ThrowHelper.ThrowArgumentNullException(ExceptionArgument.array);
}
// Determine with what bucket this array length is associated
int bucketIndex = Utilities.SelectBucketIndex(array.Length);
// Make sure our TLS buckets are initialized. Technically we could avoid doing
// this if the array being returned is erroneous or too large for the pool, but the
// former condition is an error we don't need to optimize for, and the latter is incredibly
// rare, given a max size of 1B elements.
T[]?[] tlsBuckets = t_tlsBuckets ?? InitializeTlsBucketsAndTrimming();
bool haveBucket = false;
bool returned = true;
if ((uint)bucketIndex < (uint)tlsBuckets.Length)
{
haveBucket = true;
// Clear the array if the user requested it.
if (clearArray)
{
Array.Clear(array);
}
// Check to see if the buffer is the correct size for this bucket.
if (array.Length != Utilities.GetMaxSizeForBucket(bucketIndex))
{
throw new ArgumentException(SR.ArgumentException_BufferNotFromPool, nameof(array));
}
// Store the array into the TLS bucket. If there's already an array in it,
// push that array down into the per-core stacks, preferring to keep the latest
// one in TLS for better locality.
T[]? prev = tlsBuckets[bucketIndex];
tlsBuckets[bucketIndex] = array;
if (prev is not null)
{
PerCoreLockedStacks stackBucket = _buckets[bucketIndex] ?? CreatePerCoreLockedStacks(bucketIndex);
returned = stackBucket.TryPush(prev);
}
}
// Log that the buffer was returned
ArrayPoolEventSource log = ArrayPoolEventSource.Log;
if (log.IsEnabled() && array.Length != 0)
{
log.BufferReturned(array.GetHashCode(), array.Length, Id);
if (!(haveBucket & returned))
{
log.BufferDropped(array.GetHashCode(), array.Length, Id,
haveBucket ? bucketIndex : ArrayPoolEventSource.NoBucketId,
haveBucket ? ArrayPoolEventSource.BufferDroppedReason.Full : ArrayPoolEventSource.BufferDroppedReason.OverMaximumSize);
}
}
}