internal RetainedMemory(Memory <T> memory)
{
if (MemoryMarshal.TryGetMemoryManager <T, RetainableMemory <T> >(memory, out var manager))
{
manager.Increment();
_manager = manager;
_start = 0;
_length = memory.Length;
}
else if (MemoryMarshal.TryGetArray <T>(memory, out var segment))
{
_manager = ArrayMemory <T> .Create(segment.Array, segment.Offset, segment.Count, externallyOwned : true);
_manager.Increment();
_start = 0;
_length = _manager.Length;
}
else
{
ThrowNotSupportedMemoryType();
_manager = default;
_start = 0;
_length = 0;
}
#if DETECT_LEAKS
_finalizeChecker = new PanicOnFinalize();
#endif
}
public static unsafe RetainedVec Create <T>(RetainableMemory <T> memorySource, int start, int length, bool externallyOwned = false)
{
if (!memorySource.IsBlittableOffheap)
{
ThrowHelper.ThrowInvalidOperationException("Memory source must have IsBlittableOffheap = true to be used in RetainedVec.");
}
RetainedVec vs;
if (TypeHelper <T> .IsReferenceOrContainsReferences)
{
ThrowHelper.DebugAssert(memorySource.Pointer == default && memorySource._array != default);
// RM's offset goes to _pointerOrOffset
vs = new RetainedVec(externallyOwned ? null : memorySource,
memorySource._array,
(IntPtr)memorySource._offset,
memorySource.Length,
VecTypeHelper <T> .RuntimeTypeId);
}
else
{
ThrowHelper.DebugAssert(memorySource.Pointer != default && memorySource._array == default);
// RM's offset added to _pointerOrOffset
vs = new RetainedVec(externallyOwned ? null : memorySource, array: null, (IntPtr)Unsafe.Add <T>(memorySource.Pointer, memorySource._offset), memorySource.Length,
VecTypeHelper <T> .RuntimeTypeId);
}
return(vs.Clone(start, length, externallyOwned));
}
internal RetainableMemory <T> Rent()
{
RetainableMemory <T>[] buffers = _buffers;
RetainableMemory <T> buffer = null;
// While holding the lock, grab whatever is at the next available index and
// update the index. We do as little work as possible while holding the spin
// lock to minimize contention with other threads. The try/finally is
// necessary to properly handle thread aborts on platforms which have them.
// bool lockTaken = false;
var allocateBuffer = false;
#if !NETCOREAPP
try
#endif
{
var spinner = new SpinWait();
while (0 != Interlocked.CompareExchange(ref _locker, 1, 0))
{
spinner.SpinOnce();
}
if (_index < buffers.Length)
{
buffer = buffers[_index];
buffers[_index++] = null;
allocateBuffer = buffer == null;
}
}
#if !NETCOREAPP
finally
#endif
{
Volatile.Write(ref _locker, 0);
}
// While we were holding the lock, we grabbed whatever was at the next available index, if
// there was one. If we tried and if we got back null, that means we hadn't yet allocated
// for that slot, in which case we should do so now.
if (allocateBuffer)
{
buffer = CreateNew();
var log = RetainableMemoryPoolEventSource.Log;
if (log.IsEnabled())
{
log.BufferAllocated(buffer.GetHashCode(), _bufferLength, _poolId, Id,
RetainableMemoryPoolEventSource.BufferAllocatedReason.Pooled);
}
}
else
{
if (buffer != null && !buffer._isPooled)
{
ThrowNotFromPool <RetainableMemory <T> >();
}
}
return(buffer);
}
internal bool ReturnInternal(RetainableMemory <T> memory, bool clearMemory = true)
{
if (_disposed)
{
return(false);
}
if (memory._poolIdx != PoolIdx)
{
if (memory.IsDisposed)
{
ThrowDisposed <RetainableMemory <T> >();
}
else
{
ThrowNotFromPool <RetainableMemory <T> >();
}
}
if (memory._isPooled)
{
ThrowAlreadyPooled <RetainableMemory <T> >();
}
// Determine with what bucket this array length is associated
int bucket = SelectBucketIndex(memory.LengthPow2);
// If we can tell that the buffer was allocated, drop it. Otherwise, check if we have space in the pool
if (bucket < _buckets.Length)
{
// Clear the array if the user requests regardless of pooling result.
// If not pooled then it should be RM.DisposeFinalize-d and destruction
// is not always GC.
if (clearMemory || IsRentAlwaysClean || _typeHasReferences)
{
if (!memory.SkipCleaning)
{
memory.GetSpan().Clear();
}
}
memory.SkipCleaning = false;
// Return the buffer to its bucket. In the future, we might consider having Return return false
// instead of dropping a bucket, in which case we could try to return to a lower-sized bucket,
// just as how in Rent we allow renting from a higher-sized bucket.
_buckets[bucket].Return(memory);
}
// Log that the buffer was returned
var log = RetainableMemoryPoolEventSource.Log;
if (log.IsEnabled())
{
log.BufferReturned(memory.GetHashCode(), memory.Length, Id);
}
return(memory._isPooled);
}
internal bool ReturnInternal(RetainableMemory <T> memory, bool clearMemory = true)
{
if (_disposed)
{
return(false);
}
if (!memory.IsDisposed)
{
ThrowHelper.ThrowInvalidOperationException("Memory must be disposed before returning to RMP.");
}
if (memory.PoolIndex != PoolIdx)
{
ThrowNotFromPool <RetainableMemory <T> >();
}
// Determine with what bucket this buffer length is associated
var bucketIndex = SelectBucketIndex(memory.LengthPow2);
var pooled = false;
// If we can tell that the buffer was allocated, drop it. Otherwise, check if we have space in the pool
if (bucketIndex < _buckets.Length)
{
var bucket = _buckets[bucketIndex];
if (memory.LengthPow2 != bucket.BufferLength)
{
ThrowNotFromPool <RetainableMemory <T> >();
}
#pragma warning disable 618
// Clear the memory if the user requests regardless of pooling result.
// If not pooled then it should be RM.DisposeFinalize-d and destruction
// is not always GC.
if ((clearMemory || IsRentAlwaysClean || _typeHasReferences) && !memory.SkipCleaning)
{
memory.GetSpan().Clear();
}
memory.SkipCleaning = false;
#pragma warning restore 618
pooled = bucket.Return(memory);
}
// Log that the buffer was returned
var log = RetainableMemoryPoolEventSource.Log;
if (log.IsEnabled())
{
log.BufferReturned(memory.GetHashCode(), memory.Length, Id);
}
return(pooled);
}
internal bool Return(RetainableMemory <T> memory, bool clearArray = false)
{
// These checks for internal code that could Return directly without Dispose on memory
memory.EnsureNotRetainedAndNotDisposed();
if (memory._isPooled)
{
ThrowAlreadyPooled <RetainableMemory <T> >();
}
return(ReturnNoChecks(memory, clearArray));
}
internal void Return(RetainableMemory <T> memory)
{
// Check to see if the buffer is the correct size for this bucket
if (memory.LengthPow2 != _bufferLength)
{
ThrowNotFromPool <RetainableMemory <T> >();
}
// While holding the spin lock, if there's room available in the bucket,
// put the buffer into the next available slot. Otherwise, we just drop it.
// The try/finally is necessary to properly handle thread aborts on platforms
// which have them.
int disposed = 0;
#if !NETCOREAPP
try
#endif
{
var spinner = new SpinWait();
while (0 != Interlocked.CompareExchange(ref _locker, 1, 0))
{
spinner.SpinOnce();
}
var pooled = _index != 0;
if (pooled)
{
if ((disposed = AtomicCounter.TryDispose(ref memory.CounterRef)) == 0)
{
_buffers[--_index] = memory;
Debug.Assert(AtomicCounter.GetIsDisposed(ref memory.CounterRef));
memory._isPooled = true;
}
}
else
{
memory.DisposeFinalize();
}
}
#if !NETCOREAPP
finally
#endif
{
Volatile.Write(ref _locker, 0);
}
// after unlock
if (disposed != 0)
{
AtomicCounter.ThrowNonZeroTryDispose(disposed);
}
}
public static VecStorage Create <T>(RetainableMemory <T> memorySource,
int start,
int length,
bool externallyOwned = false)
{
var ms = memorySource ?? throw new ArgumentNullException(nameof(memorySource));
var vec = ms.GetVec().AsVec().Slice(start, length);
if (!externallyOwned)
{
ms.Increment();
}
var vs = new VecStorage(externallyOwned ? null : ms, vec);
return(vs);
}
public static RetainedVec Create <T>(RetainableMemory <T>?memorySource, int start, int length, bool externallyOwned = false)
{
if (!memorySource.IsBlittableOffheap)
{
ThrowHelper.ThrowInvalidOperationException("Memory source must have IsBlittableOffheap = true to be used in RetainedVec.");
}
var ms = memorySource ?? throw new ArgumentNullException(nameof(memorySource));
var vec = ms.GetVec().AsVec().Slice(start, length);
if (!externallyOwned)
{
ms.Increment();
}
var vs = new RetainedVec(externallyOwned ? null : ms, vec);
return(vs);
}
internal RetainedMemory(RetainableMemory <T> memory, int offset, int length, bool borrow)
#endif
{
Debug.Assert(unchecked ((uint)offset + (uint)length <= memory.Length));
if (borrow)
{
memory.Increment();
}
_manager = memory;
_offset = offset;
_length = length;
// We do not need to Pin arrays, they do not have ref count. Will be pinned when Pointer is accessed.
#if DETECT_LEAKS
_finalizeChecker = borrow ? new PanicOnFinalize() : checker;
#endif
}
public RetainedMemory(Memory <T> memory, bool pin = true) // TODO pin param added later and before it behaved like with true, but better to change to false and review usage
{
if (MemoryMarshal.TryGetMemoryManager <T, RetainableMemory <T> >(memory, out var manager))
{
if (!manager.IsPinned && pin)
{
// TODO review. This uses implementation detail of RetainableMemory:
// if pointer is null then it is an non-pinned array for which we did not create
// a GCHandle (very expensive). Call to Pin() checks if pointer is null and
// creates a GCHandle + pointer. Try to avoid pinning non-pooled ArrayMemory
// because it is very expensive.
manager.Pin();
}
else
{
manager.Increment();
}
_manager = manager;
_offset = 0;
_length = memory.Length;
}
else if (MemoryMarshal.TryGetArray <T>(memory, out var segment))
{
_manager = ArrayMemory <T> .Create(segment.Array, segment.Offset, segment.Count, externallyOwned : true, pin);
_manager.Increment();
_offset = 0;
_length = _manager.Length;
}
else
{
ThrowNotSupportedMemoryType();
_manager = default;
_offset = 0;
_length = 0;
}
#if DETECT_LEAKS
_finalizeChecker = new PanicOnFinalize();
#endif
}
internal RetainedMemory(RetainableMemory <T> memory, int start, int length, bool borrow)
{
if (memory.IsDisposed)
{
BuffersThrowHelper.ThrowDisposed <RetainableMemory <T> >();
}
Debug.Assert(unchecked ((uint)start + (uint)length <= memory.Length));
if (borrow)
{
memory.Increment();
}
_manager = memory;
_start = start;
_length = length;
#if DETECT_LEAKS
_finalizeChecker = new PanicOnFinalize();
#endif
}
public bool Return(RetainableMemory <T> obj)
{
return(Pool.Return(obj));
}
internal RetainedMemory(RetainableMemory <T> memory, int offset, int length, bool borrow, PanicOnFinalize checker = null)
public bool Return(RetainableMemory <T> memory, bool clearMemory = true)
{
return(ReturnInternal(memory, clearMemory));
}