public override void Return(T[] array, bool clearArray = false) { if (array == null) { throw new ArgumentNullException(nameof(array)); } // Determine with what bucket this array length is associated int bucketIndex = Utilities.SelectBucketIndex(array.Length); // If we can tell that the buffer was allocated (or empty), drop it. Otherwise, check if we have space in the pool. if (bucketIndex < _buckets.Length) { // Clear the array if the user requests. if (clearArray) { Array.Clear(array, 0, array.Length); } // Check to see if the buffer is the correct size for this bucket if (array.Length != _bucketArraySizes[bucketIndex]) { throw new ArgumentException(SR.ArgumentException_BufferNotFromPool, nameof(array)); } // Write through the TLS bucket. If there weren't any buckets, create them // and store this array into it. If there were, store this into it, and // if there was a previous one there, push that to the global stack. This // helps to keep LIFO access such that the most recently pushed stack will // be in TLS and the first to be popped next. T[][] tlsBuckets = t_tlsBuckets; if (tlsBuckets == null) { t_tlsBuckets = tlsBuckets = new T[NumBuckets][]; tlsBuckets[bucketIndex] = array; } else { T[] prev = tlsBuckets[bucketIndex]; tlsBuckets[bucketIndex] = array; if (prev != null) { PerCoreLockedStacks bucket = _buckets[bucketIndex] ?? CreatePerCoreLockedStacks(bucketIndex); bucket.TryPush(prev); } } } // Log that the buffer was returned ArrayPoolEventSource log = ArrayPoolEventSource.Log; if (log.IsEnabled()) { log.BufferReturned(array.GetHashCode(), array.Length, Id); } }
public bool Trim() { Debug.Assert(s_trimBuffers); Debug.Assert(s_allTlsBuckets != null); int milliseconds = Environment.TickCount; MemoryPressure pressure = GetMemoryPressure(); ArrayPoolEventSource log = ArrayPoolEventSource.Log; if (log.IsEnabled()) { log.BufferTrimPoll(milliseconds, (int)pressure); } PerCoreLockedStacks?[] perCoreBuckets = _buckets; for (int i = 0; i < perCoreBuckets.Length; i++) { perCoreBuckets[i]?.Trim((uint)milliseconds, Id, pressure, _bucketArraySizes[i]); } if (pressure == MemoryPressure.High) { // Under high pressure, release all thread locals if (log.IsEnabled()) { foreach (KeyValuePair <T[]?[], object?> tlsBuckets in s_allTlsBuckets) { T[]?[] buckets = tlsBuckets.Key; for (int i = 0; i < buckets.Length; i++) { T[]? buffer = Interlocked.Exchange(ref buckets[i], null); if (buffer != null) { // As we don't want to take a perf hit in the rent path it // is possible that a buffer could be rented as we "free" it. log.BufferTrimmed(buffer.GetHashCode(), buffer.Length, Id); } } } } else { foreach (KeyValuePair <T[]?[], object?> tlsBuckets in s_allTlsBuckets) { T[]?[] buckets = tlsBuckets.Key; Array.Clear(buckets, 0, buckets.Length); } } } return(true); }
/// <summary>Takes an array from the bucket. If the bucket is empty, returns null.</summary> internal T[]? Rent() { T[]?[] buffers = _buffers; T[]? buffer = null; // While holding the lock, grab whatever is at the next available index and // update the index. We do as little work as possible while holding the spin // lock to minimize contention with other threads. The try/finally is // necessary to properly handle thread aborts on platforms which have them. bool lockTaken = false, allocateBuffer = false; try { _lock.Enter(ref lockTaken); if (_index < buffers.Length) { buffer = buffers[_index]; buffers[_index++] = null; allocateBuffer = buffer == null; } } finally { if (lockTaken) { _lock.Exit(false); } } // While we were holding the lock, we grabbed whatever was at the next available index, if // there was one. If we tried and if we got back null, that means we hadn't yet allocated // for that slot, in which case we should do so now. if (allocateBuffer) { buffer = new T[_bufferLength]; ArrayPoolEventSource log = ArrayPoolEventSource.Log; if (log.IsEnabled()) { log.BufferAllocated(buffer.GetHashCode(), _bufferLength, _poolId, Id, ArrayPoolEventSource.BufferAllocatedReason.Pooled); } } return(buffer); }
public override void Return(T[] array, bool clearArray = false) { if (array == null) { throw new ArgumentNullException(nameof(array)); } else if (array.Length == 0) { // Ignore empty arrays. When a zero-length array is rented, we return a singleton // rather than actually taking a buffer out of the lowest bucket. return; } // Determine with what bucket this array length is associated int bucket = Utilities.SelectBucketIndex(array.Length); // If we can tell that the buffer was allocated, drop it. Otherwise, check if we have space in the pool bool haveBucket = bucket < _buckets.Length; if (haveBucket) { // Clear the array if the user requests if (clearArray) { Array.Clear(array, 0, array.Length); } // Return the buffer to its bucket. In the future, we might consider having Return return false // instead of dropping a bucket, in which case we could try to return to a lower-sized bucket, // just as how in Rent we allow renting from a higher-sized bucket. _buckets[bucket].Return(array); } // Log that the buffer was returned ArrayPoolEventSource log = ArrayPoolEventSource.Log; if (log.IsEnabled()) { int bufferId = array.GetHashCode(); log.BufferReturned(bufferId, array.Length, Id); if (!haveBucket) { log.BufferDropped(bufferId, array.Length, Id, ArrayPoolEventSource.NoBucketId, ArrayPoolEventSource.BufferDroppedReason.Full); } } }
/// <summary> /// Attempts to return the buffer to the bucket. If successful, the buffer will be stored /// in the bucket and true will be returned; otherwise, the buffer won't be stored, and false /// will be returned. /// </summary> internal void Return(T[] array) { // Check to see if the buffer is the correct size for this bucket if (array.Length != _bufferLength) { throw new ArgumentException(SR.ArgumentException_BufferNotFromPool, nameof(array)); } bool returned; // While holding the spin lock, if there's room available in the bucket, // put the buffer into the next available slot. Otherwise, we just drop it. // The try/finally is necessary to properly handle thread aborts on platforms // which have them. bool lockTaken = false; try { _lock.Enter(ref lockTaken); returned = _index != 0; if (returned) { _buffers[--_index] = array; } } finally { if (lockTaken) { _lock.Exit(false); } } if (!returned) { ArrayPoolEventSource log = ArrayPoolEventSource.Log; if (log.IsEnabled()) { log.BufferDropped(array.GetHashCode(), _bufferLength, _poolId, Id, ArrayPoolEventSource.BufferDroppedReason.Full); } } }
public bool Trim() { int milliseconds = Environment.TickCount; MemoryPressure pressure = GetMemoryPressure(); ArrayPoolEventSource log = ArrayPoolEventSource.Log; if (log.IsEnabled()) { log.BufferTrimPoll(milliseconds, (int)pressure); } foreach (PerCoreLockedStacks bucket in _buckets) { bucket?.Trim((uint)milliseconds, Id, pressure, _bucketArraySizes); } if (pressure == MemoryPressure.High) { // Under high pressure, release all thread locals foreach (KeyValuePair <T[][], object> tlsBuckets in s_AllTlsBuckets) { T[][] buckets = tlsBuckets.Key; for (int i = 0; i < NumBuckets; i++) { T[] buffer = buckets[i]; buckets[i] = null; if (log.IsEnabled() && buffer != null) { log.BufferTrimmed(buffer.GetHashCode(), buffer.Length, Id); } } } } return(true); }
public override T[] Rent(int minimumLength) { // Arrays can't be smaller than zero. We allow requesting zero-length arrays (even though // pooling such an array isn't valuable) as it's a valid length array, and we want the pool // to be usable in general instead of using `new`, even for computed lengths. if (minimumLength < 0) { throw new ArgumentOutOfRangeException(nameof(minimumLength)); } else if (minimumLength == 0) { // No need to log the empty array. Our pool is effectively infinite // and we'll never allocate for rents and never store for returns. return(Array.Empty <T>()); } ArrayPoolEventSource log = ArrayPoolEventSource.Log; T[]? buffer; // Get the bucket number for the array length int bucketIndex = Utilities.SelectBucketIndex(minimumLength); // If the array could come from a bucket... if (bucketIndex < _buckets.Length) { // First try to get it from TLS if possible. T[]?[]? tlsBuckets = t_tlsBuckets; if (tlsBuckets != null) { buffer = tlsBuckets[bucketIndex]; if (buffer != null) { tlsBuckets[bucketIndex] = null; if (log.IsEnabled()) { log.BufferRented(buffer.GetHashCode(), buffer.Length, Id, bucketIndex); } return(buffer); } } // We couldn't get a buffer from TLS, so try the global stack. PerCoreLockedStacks?b = _buckets[bucketIndex]; if (b != null) { buffer = b.TryPop(); if (buffer != null) { if (log.IsEnabled()) { log.BufferRented(buffer.GetHashCode(), buffer.Length, Id, bucketIndex); } return(buffer); } } // No buffer available. Allocate a new buffer with a size corresponding to the appropriate bucket. buffer = GC.AllocateUninitializedArray <T>(_bucketArraySizes[bucketIndex]); } else { // The request was for a size too large for the pool. Allocate an array of exactly the requested length. // When it's returned to the pool, we'll simply throw it away. buffer = GC.AllocateUninitializedArray <T>(minimumLength); } if (log.IsEnabled()) { int bufferId = buffer.GetHashCode(), bucketId = -1; // no bucket for an on-demand allocated buffer log.BufferRented(bufferId, buffer.Length, Id, bucketId); log.BufferAllocated(bufferId, buffer.Length, Id, bucketId, bucketIndex >= _buckets.Length ? ArrayPoolEventSource.BufferAllocatedReason.OverMaximumSize : ArrayPoolEventSource.BufferAllocatedReason.PoolExhausted); } return(buffer); }
public void Trim(uint tickCount, int id, MemoryPressure pressure, int bucketSize) { const uint StackTrimAfterMS = 60 * 1000; // Trim after 60 seconds for low/moderate pressure const uint StackHighTrimAfterMS = 10 * 1000; // Trim after 10 seconds for high pressure const uint StackRefreshMS = StackTrimAfterMS / 4; // Time bump after trimming (1/4 trim time) const int StackLowTrimCount = 1; // Trim one item when pressure is low const int StackMediumTrimCount = 2; // Trim two items when pressure is moderate const int StackHighTrimCount = MaxBuffersPerArraySizePerCore; // Trim all items when pressure is high const int StackLargeBucket = 16384; // If the bucket is larger than this we'll trim an extra when under high pressure const int StackModerateTypeSize = 16; // If T is larger than this we'll trim an extra when under high pressure const int StackLargeTypeSize = 32; // If T is larger than this we'll trim an extra (additional) when under high pressure if (_count == 0) { return; } uint trimTicks = pressure == MemoryPressure.High ? StackHighTrimAfterMS : StackTrimAfterMS; lock (this) { if (_count > 0 && _firstStackItemMS > tickCount || (tickCount - _firstStackItemMS) > trimTicks) { // We've wrapped the tick count or elapsed enough time since the // first item went into the stack. Drop the top item so it can // be collected and make the stack look a little newer. ArrayPoolEventSource log = ArrayPoolEventSource.Log; int trimCount = StackLowTrimCount; switch (pressure) { case MemoryPressure.High: trimCount = StackHighTrimCount; // When pressure is high, aggressively trim larger arrays. if (bucketSize > StackLargeBucket) { trimCount++; } if (Unsafe.SizeOf <T>() > StackModerateTypeSize) { trimCount++; } if (Unsafe.SizeOf <T>() > StackLargeTypeSize) { trimCount++; } break; case MemoryPressure.Medium: trimCount = StackMediumTrimCount; break; } while (_count > 0 && trimCount-- > 0) { T[]? array = _arrays[--_count]; Debug.Assert(array != null, "No nulls should have been present in slots < _count."); _arrays[_count] = null; if (log.IsEnabled()) { log.BufferTrimmed(array.GetHashCode(), array.Length, id); } } if (_count > 0 && _firstStackItemMS < uint.MaxValue - StackRefreshMS) { // Give the remaining items a bit more time _firstStackItemMS += StackRefreshMS; } } } }
public override T[] Rent(int minimumLength) { ArrayPoolEventSource log = ArrayPoolEventSource.Log; T[]? buffer; // Get the bucket number for the array length. The result may be out of range of buckets, // either for too large a value or for 0 and negative values. int bucketIndex = Utilities.SelectBucketIndex(minimumLength); // First, try to get an array from TLS if possible. ThreadLocalArray[]? tlsBuckets = t_tlsBuckets; if (tlsBuckets is not null && (uint)bucketIndex < (uint)tlsBuckets.Length) { buffer = tlsBuckets[bucketIndex].Array; if (buffer is not null) { tlsBuckets[bucketIndex].Array = null; if (log.IsEnabled()) { log.BufferRented(buffer.GetHashCode(), buffer.Length, Id, bucketIndex); } return(buffer); } } // Next, try to get an array from one of the per-core stacks. PerCoreLockedStacks?[] perCoreBuckets = _buckets; if ((uint)bucketIndex < (uint)perCoreBuckets.Length) { PerCoreLockedStacks?b = perCoreBuckets[bucketIndex]; if (b is not null) { buffer = b.TryPop(); if (buffer is not null) { if (log.IsEnabled()) { log.BufferRented(buffer.GetHashCode(), buffer.Length, Id, bucketIndex); } return(buffer); } } // No buffer available. Ensure the length we'll allocate matches that of a bucket // so we can later return it. minimumLength = Utilities.GetMaxSizeForBucket(bucketIndex); } else if (minimumLength == 0) { // We allow requesting zero-length arrays (even though pooling such an array isn't valuable) // as it's a valid length array, and we want the pool to be usable in general instead of using // `new`, even for computed lengths. But, there's no need to log the empty array. Our pool is // effectively infinite for empty arrays and we'll never allocate for rents and never store for returns. return(Array.Empty <T>()); } else if (minimumLength < 0) { throw new ArgumentOutOfRangeException(nameof(minimumLength)); } buffer = GC.AllocateUninitializedArray <T>(minimumLength); if (log.IsEnabled()) { int bufferId = buffer.GetHashCode(); log.BufferRented(bufferId, buffer.Length, Id, ArrayPoolEventSource.NoBucketId); log.BufferAllocated(bufferId, buffer.Length, Id, ArrayPoolEventSource.NoBucketId, bucketIndex >= _buckets.Length ? ArrayPoolEventSource.BufferAllocatedReason.OverMaximumSize : ArrayPoolEventSource.BufferAllocatedReason.PoolExhausted); } return(buffer); }
public override T[] Rent(int minimumLength) { // Arrays can't be smaller than zero. We allow requesting zero-length arrays (even though // pooling such an array isn't valuable) as it's a valid length array, and we want the pool // to be usable in general instead of using `new`, even for computed lengths. if (minimumLength < 0) { throw new ArgumentOutOfRangeException(nameof(minimumLength)); } else if (minimumLength == 0) { // No need for events with the empty array. Our pool is effectively infinite // and we'll never allocate for rents and never store for returns. return(Array.Empty <T>()); } ArrayPoolEventSource log = ArrayPoolEventSource.Log; T[]? buffer; int index = Utilities.SelectBucketIndex(minimumLength); if (index < _buckets.Length) { // Search for an array starting at the 'index' bucket. If the bucket is empty, bump up to the // next higher bucket and try that one, but only try at most a few buckets. const int MaxBucketsToTry = 2; int i = index; do { // Attempt to rent from the bucket. If we get a buffer from it, return it. buffer = _buckets[i].Rent(); if (buffer != null) { if (log.IsEnabled()) { log.BufferRented(buffer.GetHashCode(), buffer.Length, Id, _buckets[i].Id); } return(buffer); } }while (++i < _buckets.Length && i != index + MaxBucketsToTry); // The pool was exhausted for this buffer size. Allocate a new buffer with a size corresponding // to the appropriate bucket. buffer = new T[_buckets[index]._bufferLength]; } else { // The request was for a size too large for the pool. Allocate an array of exactly the requested length. // When it's returned to the pool, we'll simply throw it away. buffer = new T[minimumLength]; } if (log.IsEnabled()) { int bufferId = buffer.GetHashCode(), bucketId = -1; // no bucket for an on-demand allocated buffer log.BufferRented(bufferId, buffer.Length, Id, bucketId); log.BufferAllocated(bufferId, buffer.Length, Id, bucketId, index >= _buckets.Length ? ArrayPoolEventSource.BufferAllocatedReason.OverMaximumSize : ArrayPoolEventSource.BufferAllocatedReason.PoolExhausted); } return(buffer); }
public override void Return(T[] array, bool clearArray = false) { if (array == null) { throw new ArgumentNullException(nameof(array)); } // Determine with what bucket this array length is associated int bucketIndex = Utilities.SelectBucketIndex(array.Length); // If we can tell that the buffer was allocated (or empty), drop it. Otherwise, check if we have space in the pool. bool returned = true; bool haveBucket = bucketIndex < _buckets.Length; if (haveBucket) { // Clear the array if the user requests. if (clearArray) { Array.Clear(array, 0, array.Length); } // Check to see if the buffer is the correct size for this bucket if (array.Length != _bucketArraySizes[bucketIndex]) { throw new ArgumentException(SR.ArgumentException_BufferNotFromPool, nameof(array)); } // Write through the TLS bucket. If there weren't any buckets, create them // and store this array into it. If there were, store this into it, and // if there was a previous one there, push that to the global stack. This // helps to keep LIFO access such that the most recently pushed stack will // be in TLS and the first to be popped next. T[]?[]? tlsBuckets = t_tlsBuckets; if (tlsBuckets == null) { t_tlsBuckets = tlsBuckets = new T[NumBuckets][]; tlsBuckets[bucketIndex] = array; if (s_trimBuffers) { Debug.Assert(s_allTlsBuckets != null, "Should be non-null iff s_trimBuffers is true"); s_allTlsBuckets.Add(tlsBuckets, null); if (Interlocked.Exchange(ref _callbackCreated, 1) != 1) { Gen2GcCallback.Register(Gen2GcCallbackFunc, this); } } } else { T[]? prev = tlsBuckets[bucketIndex]; tlsBuckets[bucketIndex] = array; if (prev != null) { PerCoreLockedStacks stackBucket = _buckets[bucketIndex] ?? CreatePerCoreLockedStacks(bucketIndex); returned = stackBucket.TryPush(prev); } } } // Log that the buffer was returned ArrayPoolEventSource log = ArrayPoolEventSource.Log; if (log.IsEnabled()) { log.BufferReturned(array.GetHashCode(), array.Length, Id); if (!(haveBucket & returned)) { log.BufferDropped(array.GetHashCode(), array.Length, Id, haveBucket ? bucketIndex : ArrayPoolEventSource.NoBucketId, haveBucket ? ArrayPoolEventSource.BufferDroppedReason.Full : ArrayPoolEventSource.BufferDroppedReason.OverMaximumSize); } } }
public override void Return(T[] array, bool clearArray = false) { if (array is null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.array); } // Determine with what bucket this array length is associated int bucketIndex = Utilities.SelectBucketIndex(array.Length); // Make sure our TLS buckets are initialized. Technically we could avoid doing // this if the array being returned is erroneous or too large for the pool, but the // former condition is an error we don't need to optimize for, and the latter is incredibly // rare, given a max size of 1B elements. T[]?[] tlsBuckets = t_tlsBuckets ?? InitializeTlsBucketsAndTrimming(); bool haveBucket = false; bool returned = true; if ((uint)bucketIndex < (uint)tlsBuckets.Length) { haveBucket = true; // Clear the array if the user requested it. if (clearArray) { Array.Clear(array); } // Check to see if the buffer is the correct size for this bucket. if (array.Length != Utilities.GetMaxSizeForBucket(bucketIndex)) { throw new ArgumentException(SR.ArgumentException_BufferNotFromPool, nameof(array)); } // Store the array into the TLS bucket. If there's already an array in it, // push that array down into the per-core stacks, preferring to keep the latest // one in TLS for better locality. T[]? prev = tlsBuckets[bucketIndex]; tlsBuckets[bucketIndex] = array; if (prev is not null) { PerCoreLockedStacks stackBucket = _buckets[bucketIndex] ?? CreatePerCoreLockedStacks(bucketIndex); returned = stackBucket.TryPush(prev); } } // Log that the buffer was returned ArrayPoolEventSource log = ArrayPoolEventSource.Log; if (log.IsEnabled() && array.Length != 0) { log.BufferReturned(array.GetHashCode(), array.Length, Id); if (!(haveBucket & returned)) { log.BufferDropped(array.GetHashCode(), array.Length, Id, haveBucket ? bucketIndex : ArrayPoolEventSource.NoBucketId, haveBucket ? ArrayPoolEventSource.BufferDroppedReason.Full : ArrayPoolEventSource.BufferDroppedReason.OverMaximumSize); } } }