public MemoryPoolIterator BeginWrite(int minimumSize = 0)
{
MemoryPoolBlock block = null;
if (_tail != null)
{
int remaining = _tail.Data.Offset + _tail.Data.Count - _tail.End;
if (minimumSize <= remaining && remaining > 0)
{
block = _tail;
}
}
if (block == null)
{
block = _memory.Lease();
}
lock (_sync)
{
if (_head == null)
{
_head = block;
}
else if (block != _tail)
{
Volatile.Write(ref _tail.Next, block);
_tail = block;
}
return(new MemoryPoolIterator(block, block.End));
}
}
public async Task CopyToAsync(Stream stream, MemoryPoolBlock end)
{
if (IsDefault)
{
return;
}
var block = _block;
var index = _index;
while (true)
{
// Determine if we might attempt to copy data from block.Next before
// calculating "following" so we don't risk skipping data that could
// be added after block.End when we decide to copy from block.Next.
// block.End will always be advanced before block.Next is set.
var wasLastBlock = block.Next == null || block == end;
var following = block.End - index;
if (wasLastBlock)
{
await stream.WriteAsync(block.Array, index, following);
break;
}
else
{
await stream.WriteAsync(block.Array, index, following);
block = block.Next;
index = block.Start;
}
}
}
/// <summary>
/// Internal method called when a block is requested and the pool is empty. It allocates one additional slab, creates all of the
/// block tracking objects, and adds them all to the pool.
/// </summary>
private MemoryPoolBlock AllocateSlab()
{
var slab = MemoryPoolSlab.Create(_slabLength);
_slabs.Push(slab);
var basePtr = slab.ArrayPtr;
var firstOffset = (int)((_blockStride - 1) - ((ulong)(basePtr + _blockStride - 1) % _blockStride));
var poolAllocationLength = _slabLength - _blockStride;
var offset = firstOffset;
for (;
offset + _blockLength < poolAllocationLength;
offset += _blockStride)
{
var block = MemoryPoolBlock.Create(
new ArraySegment <byte>(slab.Array, offset, _blockLength),
basePtr,
this,
slab);
Return(block);
}
// return last block rather than adding to pool
var newBlock = MemoryPoolBlock.Create(
new ArraySegment <byte>(slab.Array, offset, _blockLength),
basePtr,
this,
slab);
return(newBlock);
}
/// <summary>
/// Save the data at the current location then move to the next available space.
/// </summary>
/// <param name="data">The byte to be saved.</param>
/// <returns>true if the operation successes. false if can't find available space.</returns>
public bool Put(byte data)
{
if (_block == null)
{
return(false);
}
var block = _block;
var index = _index;
while (true)
{
var wasLastBlock = block.Next == null;
if (index < block.End)
{
_block = block;
_index = index + 1;
block.Array[index] = data;
return(true);
}
else if (wasLastBlock)
{
return(false);
}
else
{
block = block.Next;
index = block.Start;
}
}
}
// Cloning ctor
private MemoryBlockSegment(MemoryPoolBlock block, int start, int end)
{
Block = block;
Start = start;
End = end;
block.AddReference();
ReadOnly = true;
}
/// <summary>
/// Called to return a block to the pool. Once Return has been called the memory no longer belongs to the caller, and
/// Very Bad Things will happen if the memory is read of modified subsequently. If a caller fails to call Return and the
/// block tracking object is garbage collected, the block tracking object's finalizer will automatically re-create and return
/// a new tracking object into the pool. This will only happen if there is a bug in the server, however it is necessary to avoid
/// leaving "dead zones" in the slab due to lost block tracking objects.
/// </summary>
/// <param name="block">The block to return. It must have been acquired by calling Lease on the same memory pool instance.</param>
public void Return(MemoryPoolBlock block)
{
Debug.Assert(block.Pool == this, "Returned block was not leased from this pool");
if (block.Slab != null && block.Slab.IsActive)
{
block.Reset();
_blocks.Enqueue(block);
}
}
public void UpdateEnd(int bytesWritten)
{
Debug.Assert(_block != null);
Debug.Assert(_block.Next == null);
Debug.Assert(_block.End == _index);
var block = _block;
var blockIndex = _index + bytesWritten;
Debug.Assert(blockIndex <= block.Data.Offset + block.Data.Count);
block.End = blockIndex;
_block = block;
_index = blockIndex;
}
public Task EndWriteAsync(MemoryPoolIterator end)
{
lock (_sync)
{
if (!end.IsDefault)
{
_tail = end.Block;
_tail.End = end.Index;
}
Complete();
return(Task.FromResult(0));
}
}
/// <summary>
/// Called to return a block to the pool. Once Return has been called the memory no longer belongs to the caller, and
/// Very Bad Things will happen if the memory is read of modified subsequently. If a caller fails to call Return and the
/// block tracking object is garbage collected, the block tracking object's finalizer will automatically re-create and return
/// a new tracking object into the pool. This will only happen if there is a bug in the server, however it is necessary to avoid
/// leaving "dead zones" in the slab due to lost block tracking objects.
/// </summary>
/// <param name="block">The block to return. It must have been acquired by calling Lease on the same memory pool instance.</param>
public void Return(MemoryPoolBlock block)
{
#if DEBUG
Debug.Assert(block.Pool == this, "Returned block was not leased from this pool");
Debug.Assert(block.IsLeased, $"Block being returned to pool twice: {block.Leaser}{Environment.NewLine}");
block.IsLeased = false;
#endif
if (block.Slab != null && block.Slab.IsActive)
{
_blocks.Enqueue(block);
}
else
{
GC.SuppressFinalize(block);
}
}
public void CopyFrom(byte[] data, int offset, int count)
{
if (IsDefault)
{
return;
}
Debug.Assert(_block != null);
Debug.Assert(_block.Next == null);
Debug.Assert(_block.End == _index);
var pool = _block.Pool;
var block = _block;
var blockIndex = _index;
var bufferIndex = offset;
var remaining = count;
var bytesLeftInBlock = block.Data.Offset + block.Data.Count - blockIndex;
while (remaining > 0)
{
if (bytesLeftInBlock == 0)
{
var nextBlock = pool.Lease();
block.End = blockIndex;
Volatile.Write(ref block.Next, nextBlock);
block = nextBlock;
blockIndex = block.Data.Offset;
bytesLeftInBlock = block.Data.Count;
}
var bytesToCopy = remaining < bytesLeftInBlock ? remaining : bytesLeftInBlock;
Buffer.BlockCopy(data, bufferIndex, block.Array, blockIndex, bytesToCopy);
blockIndex += bytesToCopy;
bufferIndex += bytesToCopy;
remaining -= bytesToCopy;
bytesLeftInBlock -= bytesToCopy;
}
block.End = blockIndex;
_block = block;
_index = blockIndex;
}
public void Skip(int bytesToSkip)
{
if (_block == null)
{
return;
}
var wasLastBlock = _block.Next == null;
var following = _block.End - _index;
if (following >= bytesToSkip)
{
_index += bytesToSkip;
return;
}
var block = _block;
var index = _index;
while (true)
{
if (wasLastBlock)
{
throw new InvalidOperationException("Attempted to skip more bytes than available.");
}
else
{
bytesToSkip -= following;
block = block.Next;
index = block.Start;
}
wasLastBlock = block.Next == null;
following = block.End - index;
if (following >= bytesToSkip)
{
_block = block;
_index = index + bytesToSkip;
return;
}
}
}
public void EndRead(
MemoryPoolIterator consumed,
MemoryPoolIterator examined)
{
MemoryPoolBlock returnStart = null;
MemoryPoolBlock returnEnd = null;
lock (_sync)
{
if (!consumed.IsDefault)
{
returnStart = _head;
returnEnd = consumed.Block;
_head = consumed.Block;
_head.Start = consumed.Index;
}
if (!examined.IsDefault &&
examined.IsEnd &&
Completion.Status == TaskStatus.WaitingForActivation)
{
_manualResetEvent.Reset();
Interlocked.CompareExchange(
ref _awaitableState,
_awaitableIsNotCompleted,
_awaitableIsCompleted);
}
}
while (returnStart != returnEnd)
{
var returnBlock = returnStart;
returnStart = returnStart.Next;
returnBlock.Pool.Return(returnBlock);
}
if (Interlocked.CompareExchange(ref _consumingState, 0, 1) != 1)
{
throw new InvalidOperationException("No ongoing consuming operation to complete.");
}
}
/// <summary>
/// Internal method called when a block is requested and the pool is empty. It allocates one additional slab, creates all of the
/// block tracking objects, and adds them all to the pool.
/// </summary>
private MemoryPoolBlock AllocateSlab()
{
var slab = MemoryPoolSlab.Create(_slabLength);
_slabs.Push(slab);
_slabAllocationCallback?.Invoke(slab);
slab._deallocationCallback = _slabDeallocationCallback;
var basePtr = slab.NativePointer;
var firstOffset = (int)((_blockStride - 1) - ((ulong)(basePtr + _blockStride - 1) % _blockStride));
var poolAllocationLength = _slabLength - _blockStride;
var offset = firstOffset;
for (;
offset + _blockLength < poolAllocationLength;
offset += _blockStride)
{
var block = MemoryPoolBlock.Create(
offset,
_blockLength,
this,
slab);
#if DEBUG
block.IsLeased = true;
#endif
Return(block);
}
// return last block rather than adding to pool
var newBlock = MemoryPoolBlock.Create(
offset,
_blockLength,
this,
slab);
return(newBlock);
}
public int Take()
{
var block = _block;
if (block == null)
{
return(-1);
}
var index = _index;
var wasLastBlock = block.Next == null;
if (index < block.End)
{
_index = index + 1;
return(block.Array[index]);
}
do
{
if (wasLastBlock)
{
return(-1);
}
else
{
block = block.Next;
index = block.Start;
}
wasLastBlock = block.Next == null;
if (index < block.End)
{
_block = block;
_index = index + 1;
return(block.Array[index]);
}
} while (true);
}
public void Dispose()
{
Debug.Assert(_completedWriting, "Not completed writing");
Debug.Assert(_completedReading, "Not completed reading");
lock (_sync)
{
// Return all blocks
var block = _head;
while (block != null)
{
var returnBlock = block;
block = block.Next;
returnBlock.Pool.Return(returnBlock);
}
_head = null;
_tail = null;
Interlocked.Exchange(ref _disposeCallback, null)?.Invoke();
}
}
public void Write(byte data)
{
if (IsDefault)
{
return;
}
Debug.Assert(_block != null);
Debug.Assert(_block.Next == null);
Debug.Assert(_block.End == _index);
var pool = _block.Pool;
var block = _block;
var blockIndex = _index;
var bytesLeftInBlock = block.Data.Offset + block.Data.Count - blockIndex;
if (bytesLeftInBlock == 0)
{
var nextBlock = pool.Lease();
block.End = blockIndex;
Volatile.Write(ref block.Next, nextBlock);
block = nextBlock;
blockIndex = block.Data.Offset;
bytesLeftInBlock = block.Data.Count;
}
block.Array[blockIndex] = data;
blockIndex++;
block.End = blockIndex;
_block = block;
_index = blockIndex;
}
/// <summary>
/// called when the block is returned to the pool. mutable values are re-assigned to their guaranteed initialized state.
/// </summary>
public void Reset()
{
Next = null;
Start = Data.Offset;
End = Data.Offset;
}
public MemoryBlockSpan(MemoryPoolBlock block, int start, int length)
{
_block = block;
_start = start;
_length = length;
}
public MemoryPoolIterator(MemoryPoolBlock block, int index)
{
_block = block;
_index = index;
}
public unsafe int Seek(ref Vector <byte> byte0Vector, ref Vector <byte> byte1Vector, ref Vector <byte> byte2Vector)
{
if (IsDefault)
{
return(-1);
}
var block = _block;
var index = _index;
var wasLastBlock = block.Next == null;
var following = block.End - index;
byte[] array;
int byte0Index = int.MaxValue;
int byte1Index = int.MaxValue;
int byte2Index = int.MaxValue;
var byte0 = byte0Vector[0];
var byte1 = byte1Vector[0];
var byte2 = byte2Vector[0];
while (true)
{
while (following == 0)
{
if (wasLastBlock)
{
_block = block;
_index = index;
return(-1);
}
block = block.Next;
index = block.Start;
wasLastBlock = block.Next == null;
following = block.End - index;
}
array = block.Array;
while (following > 0)
{
// Need unit tests to test Vector path
#if !DEBUG
// Check will be Jitted away https://github.com/dotnet/coreclr/issues/1079
if (Vector.IsHardwareAccelerated)
{
#endif
if (following >= _vectorSpan)
{
var data = new Vector <byte>(array, index);
var byte0Equals = Vector.Equals(data, byte0Vector);
var byte1Equals = Vector.Equals(data, byte1Vector);
var byte2Equals = Vector.Equals(data, byte2Vector);
if (!byte0Equals.Equals(Vector <byte> .Zero))
{
byte0Index = FindFirstEqualByte(ref byte0Equals);
}
if (!byte1Equals.Equals(Vector <byte> .Zero))
{
byte1Index = FindFirstEqualByte(ref byte1Equals);
}
if (!byte2Equals.Equals(Vector <byte> .Zero))
{
byte2Index = FindFirstEqualByte(ref byte2Equals);
}
if (byte0Index == int.MaxValue && byte1Index == int.MaxValue && byte2Index == int.MaxValue)
{
following -= _vectorSpan;
index += _vectorSpan;
continue;
}
_block = block;
int toReturn, toMove;
if (byte0Index < byte1Index)
{
if (byte0Index < byte2Index)
{
toReturn = byte0;
toMove = byte0Index;
}
else
{
toReturn = byte2;
toMove = byte2Index;
}
}
else
{
if (byte1Index < byte2Index)
{
toReturn = byte1;
toMove = byte1Index;
}
else
{
toReturn = byte2;
toMove = byte2Index;
}
}
_index = index + toMove;
return(toReturn);
}
// Need unit tests to test Vector path
#if !DEBUG
}
#endif
var pCurrent = (block.DataFixedPtr + index);
var pEnd = pCurrent + following;
do
{
if (*pCurrent == byte0)
{
_block = block;
_index = index;
return(byte0);
}
if (*pCurrent == byte1)
{
_block = block;
_index = index;
return(byte1);
}
if (*pCurrent == byte2)
{
_block = block;
_index = index;
return(byte2);
}
pCurrent++;
index++;
} while (pCurrent != pEnd);
following = 0;
break;
}
}
}
// Leasing ctor
public MemoryBlockSegment(MemoryPoolBlock block)
{
Block = block;
Start = block.Data.Offset;
End = block.Data.Offset;
}
public unsafe void CopyFromAscii(string data)
{
if (IsDefault)
{
return;
}
Debug.Assert(_block != null);
Debug.Assert(_block.Next == null);
Debug.Assert(_block.End == _index);
var pool = _block.Pool;
var block = _block;
var blockIndex = _index;
var length = data.Length;
var bytesLeftInBlock = block.Data.Offset + block.Data.Count - blockIndex;
var bytesLeftInBlockMinusSpan = bytesLeftInBlock - 3;
fixed(char *pData = data)
{
var input = pData;
var inputEnd = pData + length;
var inputEndMinusSpan = inputEnd - 3;
while (input < inputEnd)
{
if (bytesLeftInBlock == 0)
{
var nextBlock = pool.Lease();
block.End = blockIndex;
Volatile.Write(ref block.Next, nextBlock);
block = nextBlock;
blockIndex = block.Data.Offset;
bytesLeftInBlock = block.Data.Count;
bytesLeftInBlockMinusSpan = bytesLeftInBlock - 3;
}
var output = (block.DataFixedPtr + block.End);
var copied = 0;
for (; input < inputEndMinusSpan && copied < bytesLeftInBlockMinusSpan; copied += 4)
{
*(output) = (byte)*(input);
*(output + 1) = (byte)*(input + 1);
*(output + 2) = (byte)*(input + 2);
*(output + 3) = (byte)*(input + 3);
output += 4;
input += 4;
}
for (; input < inputEnd && copied < bytesLeftInBlock; copied++)
{
*(output++) = (byte)*(input++);
}
blockIndex += copied;
bytesLeftInBlockMinusSpan -= copied;
bytesLeftInBlock -= copied;
}
}
block.End = blockIndex;
_block = block;
_index = blockIndex;
}
public MemoryPoolIterator(MemoryPoolBlock block)
{
_block = block;
_index = _block?.Start ?? 0;
}
// Leasing ctor
public MemoryBlockSegment(MemoryPoolBlock block)
{
Block = block;
Start = 0;
End = 0;
}