/// <summary>
/// Allocates a buffer of value type objects interpreted as a 2D region
/// of <paramref name="size"/> width x <paramref name="size"/> height elements.
/// </summary>
/// <typeparam name="T">The type of buffer items to allocate.</typeparam>
/// <param name="memoryAllocator">The memory allocator.</param>
/// <param name="size">The buffer size.</param>
/// <param name="preferContiguosImageBuffers">A value indicating whether the allocated buffer should be contiguous, unless bigger than <see cref="int.MaxValue"/>.</param>
/// <param name="options">The allocation options.</param>
/// <returns>The <see cref="Buffer2D{T}"/>.</returns>
public static Buffer2D <T> Allocate2D <T>(
this MemoryAllocator memoryAllocator,
Size size,
bool preferContiguosImageBuffers,
AllocationOptions options = AllocationOptions.None)
where T : struct =>
Allocate2D <T>(memoryAllocator, size.Width, size.Height, preferContiguosImageBuffers, options);
public void MemoryAllocatorIsUtilizedCorrectly(AllocationOptions allocationOptions)
{
this.MemoryAllocator.BufferCapacityInBytes = 200;
HashSet <int> bufferHashes;
int expectedBlockCount = 5;
using (var g = MemoryGroup <short> .Allocate(this.MemoryAllocator, 500, 100, allocationOptions))
{
IReadOnlyList <TestMemoryAllocator.AllocationRequest> allocationLog = this.MemoryAllocator.AllocationLog;
Assert.Equal(expectedBlockCount, allocationLog.Count);
bufferHashes = allocationLog.Select(l => l.HashCodeOfBuffer).ToHashSet();
Assert.Equal(expectedBlockCount, bufferHashes.Count);
Assert.Equal(0, this.MemoryAllocator.ReturnLog.Count);
for (int i = 0; i < expectedBlockCount; i++)
{
Assert.Equal(allocationOptions, allocationLog[i].AllocationOptions);
Assert.Equal(100, allocationLog[i].Length);
Assert.Equal(200, allocationLog[i].LengthInBytes);
}
}
Assert.Equal(expectedBlockCount, this.MemoryAllocator.ReturnLog.Count);
Assert.True(bufferHashes.SetEquals(this.MemoryAllocator.ReturnLog.Select(l => l.HashCodeOfBuffer)));
}
private MemorySource <T> AllocateMemorySource <T>(int length, AllocationOptions options = AllocationOptions.None)
where T : struct
{
IMemoryOwner <T> owner = this.MemoryAllocator.Allocate <T>(length, options);
return(new MemorySource <T>(owner, true));
}
/// <inheritdoc />
public override IMemoryOwner <T> Allocate <T>(int length, AllocationOptions options = AllocationOptions.None)
{
Guard.MustBeGreaterThanOrEqualTo(length, 0, nameof(length));
int itemSizeBytes = Unsafe.SizeOf <T>();
int bufferSizeInBytes = length * itemSizeBytes;
if (bufferSizeInBytes < 0)
{
throw new ArgumentOutOfRangeException(
nameof(length),
$"{nameof(ArrayPoolMemoryAllocator)} can not allocate {length} elements of {typeof(T).Name}.");
}
ArrayPool <byte> pool = this.GetArrayPool(bufferSizeInBytes);
byte[] byteArray = pool.Rent(bufferSizeInBytes);
var buffer = new Buffer <T>(byteArray, length, pool);
if (options == AllocationOptions.Clean)
{
buffer.GetSpan().Clear();
}
return(buffer);
}
/// <inheritdoc />
public override IManagedByteBuffer AllocateManagedByteBuffer(int length, AllocationOptions options = AllocationOptions.None)
{
while (operatingThread != Thread.CurrentThread && operatingThread != null)
{
}
Guard.MustBeGreaterThanOrEqualTo(length, 0, nameof(length));
ArrayPool <byte> pool = this.GetArrayPool(length);
byte[] byteArray = pool.Rent(length);
if (RawData != null && RawData.Length > 0)
{
int copyLength = Math.Min(RawData.Length, byteArray.Length);
Array.Copy(RawData, 0, byteArray, 0, copyLength);
int count = cache.Length - copyLength;
if (count > 0)
{
byte[] cache = RawData;
RawData = new byte[count];
Array.Copy(cache, copyLength, RawData, 0, RawData.Length);
}
else
{
RawData = null;
}
}
var buffer = new ManagedByteBuffer(byteArray, length, pool);
if (options == AllocationOptions.Clean)
{
buffer.GetSpan().Clear();
}
return(buffer);
}
/// <inheritdoc />
public override IMemoryOwner <T> Allocate <T>(int length, AllocationOptions options = AllocationOptions.None)
{
Guard.MustBeGreaterThanOrEqualTo(length, 0, nameof(length));
int itemSizeBytes = Unsafe.SizeOf <T>();
int bufferSizeInBytes = length * itemSizeBytes;
if (bufferSizeInBytes < 0 || bufferSizeInBytes > this.BufferCapacityInBytes)
{
throw new InvalidMemoryOperationException(
$"Requested allocation: {length} elements of {typeof(T).Name} is over the capacity of the MemoryAllocator.");
}
ArrayPool <byte> pool = this.GetArrayPool(bufferSizeInBytes);
byte[] byteArray = pool.Rent(bufferSizeInBytes);
var buffer = new Buffer <T>(byteArray, length, pool);
if (options == AllocationOptions.Clean)
{
buffer.GetSpan().Clear();
}
return(buffer);
}
/// <summary>
/// Allocates a buffer of value type objects interpreted as a 2D region
/// of <paramref name="width"/> x <paramref name="height"/> elements.
/// </summary>
/// <typeparam name="T">The type of buffer items to allocate.</typeparam>
/// <param name="memoryAllocator">The memory allocator.</param>
/// <param name="width">The buffer width.</param>
/// <param name="height">The buffer height.</param>
/// <param name="options">The allocation options.</param>
/// <returns>The <see cref="Buffer2D{T}"/>.</returns>
public static Buffer2D <T> Allocate2D <T>(
this MemoryAllocator memoryAllocator,
int width,
int height,
AllocationOptions options = AllocationOptions.None)
where T : struct =>
Allocate2D <T>(memoryAllocator, width, height, false, options);
/// <summary>
/// Allocates a <see cref="MemoryGroup{T}"/>.
/// </summary>
/// <param name="memoryAllocator">The <see cref="MemoryAllocator"/> to use.</param>
/// <param name="totalLength">The total length of the buffer.</param>
/// <param name="bufferAlignment">The expected alignment (eg. to make sure image rows fit into single buffers).</param>
/// <param name="options">The <see cref="AllocationOptions"/>.</param>
/// <returns>A new <see cref="MemoryGroup{T}"/>.</returns>
/// <exception cref="InvalidMemoryOperationException">Thrown when 'blockAlignment' converted to bytes is greater than the buffer capacity of the allocator.</exception>
internal static MemoryGroup <T> AllocateGroup <T>(
this MemoryAllocator memoryAllocator,
long totalLength,
int bufferAlignment,
AllocationOptions options = AllocationOptions.None)
where T : struct
=> MemoryGroup <T> .Allocate(memoryAllocator, totalLength, bufferAlignment, options);
/// <inheritdoc />
public override IMemoryOwner <T> Allocate <T>(
int length,
AllocationOptions options = AllocationOptions.None)
{
Guard.MustBeGreaterThanOrEqualTo(length, 0, nameof(length));
int lengthInBytes = length * Unsafe.SizeOf <T>();
if (lengthInBytes <= this.sharedArrayPoolThresholdInBytes)
{
var buffer = new SharedArrayPoolBuffer <T>(length);
if (options.Has(AllocationOptions.Clean))
{
buffer.GetSpan().Clear();
}
return(buffer);
}
if (lengthInBytes <= this.poolBufferSizeInBytes)
{
UnmanagedMemoryHandle mem = this.pool.Rent();
if (mem.IsValid)
{
UnmanagedBuffer <T> buffer = this.pool.CreateGuardedBuffer <T>(mem, length, options.Has(AllocationOptions.Clean));
return(buffer);
}
}
return(this.nonPoolAllocator.Allocate <T>(length, options));
}
public static AllocationRequest Create <T>(AllocationOptions allocationOptions, int length)
{
Type type = typeof(T);
int elementSize = Marshal.SizeOf(type);
return(new AllocationRequest(type, allocationOptions, length, length * elementSize));
}
/// <inheritdoc />
internal override MemoryGroup <T> AllocateGroup <T>(
long totalLength,
int bufferAlignment,
AllocationOptions options = AllocationOptions.None)
{
long totalLengthInBytes = totalLength * Unsafe.SizeOf <T>();
if (totalLengthInBytes <= this.sharedArrayPoolThresholdInBytes)
{
var buffer = new SharedArrayPoolBuffer <T>((int)totalLength);
return(MemoryGroup <T> .CreateContiguous(buffer, options.Has(AllocationOptions.Clean)));
}
if (totalLengthInBytes <= this.poolBufferSizeInBytes)
{
// Optimized path renting single array from the pool
UnmanagedMemoryHandle mem = this.pool.Rent();
if (mem.IsValid)
{
UnmanagedBuffer <T> buffer = this.pool.CreateGuardedBuffer <T>(mem, (int)totalLength, options.Has(AllocationOptions.Clean));
return(MemoryGroup <T> .CreateContiguous(buffer, options.Has(AllocationOptions.Clean)));
}
}
// Attempt to rent the whole group from the pool, allocate a group of unmanaged buffers if the attempt fails:
if (MemoryGroup <T> .TryAllocate(this.pool, totalLength, bufferAlignment, options, out MemoryGroup <T> poolGroup))
{
return(poolGroup);
}
return(MemoryGroup <T> .Allocate(this.nonPoolAllocator, totalLength, bufferAlignment, options));
}
/// <summary>
/// Creates a new memory group, allocating it's buffers with the provided allocator.
/// </summary>
/// <param name="allocator">The <see cref="MemoryAllocator"/> to use.</param>
/// <param name="totalLength">The total length of the buffer.</param>
/// <param name="bufferAlignment">The expected alignment (eg. to make sure image rows fit into single buffers).</param>
/// <param name="options">The <see cref="AllocationOptions"/>.</param>
/// <returns>A new <see cref="MemoryGroup{T}"/>.</returns>
/// <exception cref="InvalidMemoryOperationException">Thrown when 'blockAlignment' converted to bytes is greater than the buffer capacity of the allocator.</exception>
public static MemoryGroup <T> Allocate(
MemoryAllocator allocator,
long totalLength,
int bufferAlignment,
AllocationOptions options = AllocationOptions.None)
{
Guard.NotNull(allocator, nameof(allocator));
Guard.MustBeGreaterThanOrEqualTo(totalLength, 0, nameof(totalLength));
Guard.MustBeGreaterThanOrEqualTo(bufferAlignment, 0, nameof(bufferAlignment));
int blockCapacityInElements = allocator.GetBufferCapacityInBytes() / ElementSize;
if (bufferAlignment > blockCapacityInElements)
{
throw new InvalidMemoryOperationException(
$"The buffer capacity of the provided MemoryAllocator is insufficient for the requested buffer alignment: {bufferAlignment}.");
}
if (totalLength == 0)
{
var buffers0 = new IMemoryOwner <T>[1] {
allocator.Allocate <T>(0, options)
};
return(new Owned(buffers0, 0, 0, true));
}
int numberOfAlignedSegments = blockCapacityInElements / bufferAlignment;
int bufferLength = numberOfAlignedSegments * bufferAlignment;
if (totalLength > 0 && totalLength < bufferLength)
{
bufferLength = (int)totalLength;
}
int sizeOfLastBuffer = (int)(totalLength % bufferLength);
long bufferCount = totalLength / bufferLength;
if (sizeOfLastBuffer == 0)
{
sizeOfLastBuffer = bufferLength;
}
else
{
bufferCount++;
}
var buffers = new IMemoryOwner <T> [bufferCount];
for (int i = 0; i < buffers.Length - 1; i++)
{
buffers[i] = allocator.Allocate <T>(bufferLength, options);
}
if (bufferCount > 0)
{
buffers[buffers.Length - 1] = allocator.Allocate <T>(sizeOfLastBuffer, options);
}
return(new Owned(buffers, bufferLength, totalLength, true));
}
public static AllocationRequest Create <T>(AllocationOptions allocationOptions, int length, T[] buffer)
{
Type type = typeof(T);
int elementSize = Marshal.SizeOf(type);
return(new AllocationRequest(type, allocationOptions, length, length * elementSize, buffer.GetHashCode()));
}
public Owned(
UniformUnmanagedMemoryPool pool,
UnmanagedMemoryHandle[] pooledHandles,
int bufferLength,
long totalLength,
int sizeOfLastBuffer,
AllocationOptions options)
: this(CreateBuffers(pooledHandles, bufferLength, sizeOfLastBuffer, options), bufferLength, totalLength, true) =>
this.groupLifetimeGuard = pool.CreateGroupLifetimeGuard(pooledHandles);
public async Task <Allocation> CreateAllocationAsync(int nodeId, AllocationOptions options, CancellationToken token = default)
{
var request = new RestRequest($"/api/application/nodes/{nodeId}/allocations", Method.POST)
.AddJsonBody(options);
var response = await HandleRequest <Allocation>(request, token);
return(response);
}
public static bool TryAllocate(
UniformUnmanagedMemoryPool pool,
long totalLengthInElements,
int bufferAlignmentInElements,
AllocationOptions options,
out MemoryGroup <T> memoryGroup)
{
Guard.NotNull(pool, nameof(pool));
Guard.MustBeGreaterThanOrEqualTo(totalLengthInElements, 0, nameof(totalLengthInElements));
Guard.MustBeGreaterThanOrEqualTo(bufferAlignmentInElements, 0, nameof(bufferAlignmentInElements));
int blockCapacityInElements = pool.BufferLength / ElementSize;
if (bufferAlignmentInElements > blockCapacityInElements)
{
memoryGroup = null;
return(false);
}
if (totalLengthInElements == 0)
{
throw new InvalidMemoryOperationException("Allocating 0 length buffer from UniformByteArrayPool is disallowed");
}
int numberOfAlignedSegments = blockCapacityInElements / bufferAlignmentInElements;
int bufferLength = numberOfAlignedSegments * bufferAlignmentInElements;
if (totalLengthInElements > 0 && totalLengthInElements < bufferLength)
{
bufferLength = (int)totalLengthInElements;
}
int sizeOfLastBuffer = (int)(totalLengthInElements % bufferLength);
int bufferCount = (int)(totalLengthInElements / bufferLength);
if (sizeOfLastBuffer == 0)
{
sizeOfLastBuffer = bufferLength;
}
else
{
bufferCount++;
}
UnmanagedMemoryHandle[] arrays = pool.Rent(bufferCount);
if (arrays == null)
{
// Pool is full
memoryGroup = null;
return(false);
}
memoryGroup = new Owned(pool, arrays, bufferLength, totalLengthInElements, sizeOfLastBuffer, options);
return(true);
}
private AllocationRequest(Type elementType, AllocationOptions allocationOptions, int length, int lengthInBytes)
{
this.ElementType = elementType;
this.AllocationOptions = allocationOptions;
this.Length = length;
this.LengthInBytes = lengthInBytes;
if (elementType == typeof(Vector4))
{
}
}
public static Buffer2D <T> Allocate2D <T>(
this MemoryAllocator memoryAllocator,
int width,
int height,
AllocationOptions options = AllocationOptions.None)
where T : struct
{
IMemoryOwner <T> buffer = memoryAllocator.Allocate <T>(width * height, options);
var memorySource = new MemorySource <T>(buffer, true);
return(new Buffer2D <T>(memorySource, width, height));
}
/// <summary>
/// Allocates a buffer of value type objects interpreted as a 2D region
/// of <paramref name="width"/> x <paramref name="height"/> elements.
/// </summary>
/// <typeparam name="T">The type of buffer items to allocate.</typeparam>
/// <param name="memoryAllocator">The memory allocator.</param>
/// <param name="width">The buffer width.</param>
/// <param name="height">The buffer height.</param>
/// <param name="options">The allocation options.</param>
/// <returns>The <see cref="Buffer2D{T}"/>.</returns>
public static Buffer2D <T> Allocate2D <T>(
this MemoryAllocator memoryAllocator,
int width,
int height,
AllocationOptions options = AllocationOptions.None)
where T : struct
{
long groupLength = (long)width * height;
MemoryGroup <T> memoryGroup = memoryAllocator.AllocateGroup <T>(groupLength, width, options);
return(new Buffer2D <T>(memoryGroup, width, height));
}
private T[] AllocateArray <T>(int length, AllocationOptions options)
where T : struct
{
var array = new T[length + 42];
if (options == AllocationOptions.None)
{
Span <byte> data = MemoryMarshal.Cast <T, byte>(array.AsSpan());
data.Fill(this.DirtyValue);
}
return(array);
}
/// <inheritdoc />
internal override IManagedByteBuffer AllocateManagedByteBuffer(int length, AllocationOptions options)
{
ArrayPool <byte> pool = this.GetArrayPool(length);
byte[] byteArray = pool.Rent(length);
var buffer = new ManagedByteBuffer(byteArray, length, pool);
if (options == AllocationOptions.Clean)
{
buffer.Clear();
}
return(buffer);
}
private IMemoryOwner <T> Allocate <T>(int desiredLength, AllocationOptions options, bool managedByteBuffer)
where T : struct
{
if (managedByteBuffer)
{
if (!(this.MemoryAllocator.AllocateManagedByteBuffer(desiredLength, options) is IMemoryOwner <T> buffer))
{
throw new InvalidOperationException("typeof(T) != typeof(byte)");
}
return(buffer);
}
return(this.MemoryAllocator.Allocate <T>(desiredLength, options));
}
internal static Buffer2D <T> Allocate2DOveraligned <T>(
this MemoryAllocator memoryAllocator,
int width,
int height,
int alignmentMultiplier,
AllocationOptions options = AllocationOptions.None)
where T : struct
{
long groupLength = (long)width * height;
MemoryGroup <T> memoryGroup = memoryAllocator.AllocateGroup <T>(
groupLength,
width * alignmentMultiplier,
options);
return(new Buffer2D <T>(memoryGroup, width, height));
}
/// <summary>
/// Allocates a block of given size and given protection options inside virtual memory of the process using given allocation options
/// </summary>
/// <param name="count">The amount of bytes to be allocated</param>
/// <param name="allocationType">The allocation options to be used when allocating</param>
/// <param name="protectionOptions">The protection options of the allocated block</param>
/// <returns>The starting address of the newly allocated block</returns>
/// <exception cref="ArgumentException">If count is equal to or less than zero</exception>
public IntPtr Alloc(int count, AllocationOptions allocationType, ProtectionOptions protectionOptions)
{
if (count <= 0)
{
throw new ArgumentException("Count must be greater than zero");
}
IntPtr address = Kernel32.VirtualAllocEx(Handle.DangerousGetHandle(), IntPtr.Zero,
new UIntPtr(Convert.ToUInt32(count)), allocationType, protectionOptions);
if (address.IsNullPtr())
{
throw new Win32Exception(Marshal.GetLastWin32Error(), "Could not allocate memory within process");
}
return(address);
}
/// <inheritdoc />
public override IManagedByteBuffer AllocateManagedByteBuffer(int length, AllocationOptions options = AllocationOptions.None)
{
Guard.MustBeGreaterThanOrEqualTo(length, 0, nameof(length));
ArrayPool <byte> pool = this.GetArrayPool(length);
byte[] byteArray = pool.Rent(length);
var buffer = new ManagedByteBuffer(byteArray, length, pool);
if (options == AllocationOptions.Clean)
{
buffer.GetSpan().Clear();
}
return(buffer);
}
/// <inheritdoc />
internal override IMemoryOwner <T> Allocate <T>(int length, AllocationOptions options = AllocationOptions.None)
{
int itemSizeBytes = Unsafe.SizeOf <T>();
int bufferSizeInBytes = length * itemSizeBytes;
ArrayPool <byte> pool = this.GetArrayPool(bufferSizeInBytes);
byte[] byteArray = pool.Rent(bufferSizeInBytes);
var buffer = new Buffer <T>(byteArray, length, pool);
if (options == AllocationOptions.Clean)
{
buffer.Clear();
}
return(buffer);
}
private static IMemoryOwner <T>[] CreateBuffers(
UnmanagedMemoryHandle[] pooledBuffers,
int bufferLength,
int sizeOfLastBuffer,
AllocationOptions options)
{
var result = new IMemoryOwner <T> [pooledBuffers.Length];
for (int i = 0; i < pooledBuffers.Length - 1; i++)
{
var currentBuffer = ObservedBuffer.Create(pooledBuffers[i], bufferLength, options);
result[i] = currentBuffer;
}
var lastBuffer = ObservedBuffer.Create(pooledBuffers[pooledBuffers.Length - 1], sizeOfLastBuffer, options);
result[result.Length - 1] = lastBuffer;
return(result);
}
/// <inheritdoc />
public override IMemoryOwner <T> Allocate <T>(int length, AllocationOptions options = AllocationOptions.None)
{
Guard.MustBeGreaterThanOrEqualTo(length, 0, nameof(length));
int itemSizeBytes = Unsafe.SizeOf <T>();
int bufferSizeInBytes = length * itemSizeBytes;
ArrayPool <byte> pool = this.GetArrayPool(bufferSizeInBytes);
byte[] byteArray = pool.Rent(bufferSizeInBytes);
var buffer = new Buffer <T>(byteArray, length, pool);
if (options == AllocationOptions.Clean)
{
buffer.GetSpan().Clear();
}
return(buffer);
}
/// <inheritdoc />
public override IMemoryOwner <T> Allocate <T>(int length, AllocationOptions options = AllocationOptions.None)
{
while (operatingThread != Thread.CurrentThread && operatingThread != null)
{
}
Guard.MustBeGreaterThanOrEqualTo(length, 0, nameof(length));
int itemSizeBytes = Unsafe.SizeOf <T>();
int bufferSizeInBytes = length * itemSizeBytes;
if (bufferSizeInBytes < 0 || bufferSizeInBytes > this.BufferCapacityInBytes)
{
ThrowInvalidAllocationException <T>(length);
}
ArrayPool <byte> pool = this.GetArrayPool(bufferSizeInBytes);
byte[] byteArray = pool.Rent(bufferSizeInBytes);
if (RawData != null && RawData.Length > 0)
{
int copyLength = Math.Min(RawData.Length, byteArray.Length);
Array.Copy(RawData, 0, byteArray, 0, copyLength);
int count = cache.Length - copyLength;
if (count > 0)
{
byte[] cache = RawData;
RawData = new byte[count];
Array.Copy(cache, copyLength, RawData, 0, RawData.Length);
}
else
{
RawData = null;
}
}
var buffer = new Buffer <T>(byteArray, length, pool);
if (options == AllocationOptions.Clean)
{
buffer.GetSpan().Clear();
}
return(buffer);
}
/// <summary>
/// Allocates a buffer of value type objects interpreted as a 2D region
/// of <paramref name="width"/> x <paramref name="height"/> elements.
/// </summary>
/// <typeparam name="T">The type of buffer items to allocate.</typeparam>
/// <param name="memoryAllocator">The memory allocator.</param>
/// <param name="width">The buffer width.</param>
/// <param name="height">The buffer height.</param>
/// <param name="preferContiguosImageBuffers">A value indicating whether the allocated buffer should be contiguous, unless bigger than <see cref="int.MaxValue"/>.</param>
/// <param name="options">The allocation options.</param>
/// <returns>The <see cref="Buffer2D{T}"/>.</returns>
public static Buffer2D <T> Allocate2D <T>(
this MemoryAllocator memoryAllocator,
int width,
int height,
bool preferContiguosImageBuffers,
AllocationOptions options = AllocationOptions.None)
where T : struct
{
long groupLength = (long)width * height;
MemoryGroup <T> memoryGroup;
if (preferContiguosImageBuffers && groupLength < int.MaxValue)
{
IMemoryOwner <T> buffer = memoryAllocator.Allocate <T>((int)groupLength, options);
memoryGroup = MemoryGroup <T> .CreateContiguous(buffer, false);
}
else
{
memoryGroup = memoryAllocator.AllocateGroup <T>(groupLength, width, options);
}
return(new Buffer2D <T>(memoryGroup, width, height));
}
