/// <summary>
/// Internal method called when a block is requested and the pool is empty. It allocates one additional slab, creates all of the
/// block tracking objects, and adds them all to the pool.
/// </summary>
private MemoryPoolBlock2 AllocateSlab()
{
var slab = MemoryPoolSlab2.Create(_slabLength);
_slabs.Push(slab);
var basePtr = slab.ArrayPtr;
var firstOffset = (int)((_blockStride - 1) - ((ulong)(basePtr + _blockStride - 1) % _blockStride));
var poolAllocationLength = _slabLength - _blockStride;
var offset = firstOffset;
for (;
offset + _blockLength < poolAllocationLength;
offset += _blockStride)
{
var block = MemoryPoolBlock2.Create(
new ArraySegment <byte>(slab.Array, offset, _blockLength),
basePtr,
this,
slab);
Return(block);
}
// return last block rather than adding to pool
var newBlock = MemoryPoolBlock2.Create(
new ArraySegment <byte>(slab.Array, offset, _blockLength),
basePtr,
this,
slab);
return(newBlock);
}
public int Take()
{
if (_block == null)
{
return(-1);
}
else if (_index < _block.End)
{
return(_block.Array[_index++]);
}
var block = _block;
var index = _index;
while (true)
{
if (index < block.End)
{
_block = block;
_index = index + 1;
return(block.Array[index]);
}
else if (block.Next == null)
{
return(-1);
}
else
{
block = block.Next;
index = block.Start;
}
}
}
public void ConsumingComplete(
MemoryPoolIterator2 consumed,
MemoryPoolIterator2 examined)
{
MemoryPoolBlock2 returnStart = null;
MemoryPoolBlock2 returnEnd = null;
lock (_sync)
{
if (!consumed.IsDefault)
{
returnStart = _head;
returnEnd = consumed.Block;
_head = consumed.Block;
_head.Start = consumed.Index;
}
if (!examined.IsDefault &&
examined.IsEnd &&
RemoteIntakeFin == false &&
_awaitableError == null)
{
_manualResetEvent.Reset();
var awaitableState = Interlocked.CompareExchange(
ref _awaitableState,
_awaitableIsNotCompleted,
_awaitableIsCompleted);
}
}
while (returnStart != returnEnd)
{
var returnBlock = returnStart;
returnStart = returnStart.Next;
returnBlock.Pool.Return(returnBlock);
}
}
/// <summary>
/// Save the data at the current location then move to the next available space.
/// </summary>
/// <param name="data">The byte to be saved.</param>
/// <returns>true if the operation successes. false if can't find available space.</returns>
public bool Put(byte data)
{
if (_block == null)
{
return(false);
}
else if (_index < _block.End)
{
_block.Array[_index++] = data;
return(true);
}
var block = _block;
var index = _index;
while (true)
{
if (index < block.End)
{
_block = block;
_index = index + 1;
block.Array[index] = data;
return(true);
}
else if (block.Next == null)
{
return(false);
}
else
{
block = block.Next;
index = block.Start;
}
}
}
private bool _disposedValue = false; // To detect redundant calls
/// <summary>
/// Called to take a block from the pool.
/// </summary>
/// <param name="minimumSize">The block returned must be at least this size. It may be larger than this minimum size, and if so,
/// the caller may write to the block's entire size rather than being limited to the minumumSize requested.</param>
/// <returns>The block that is reserved for the called. It must be passed to Return when it is no longer being used.</returns>
public MemoryPoolBlock2 Lease(int minimumSize)
{
if (minimumSize > _blockLength)
{
// The requested minimumSize is actually larger then the usable memory of a single block.
// Because this is the degenerate case, a one-time-use byte[] array and tracking object are allocated.
// When this block tracking object is returned it is not added to the pool - instead it will be
// allowed to be garbage collected normally.
return(MemoryPoolBlock2.Create(
new ArraySegment <byte>(new byte[minimumSize]),
dataPtr: IntPtr.Zero,
pool: null,
slab: null));
}
while (true)
{
MemoryPoolBlock2 block;
if (_blocks.TryPop(out block))
{
// block successfully taken from the stack - return it
return(block);
}
// no blocks available - grow the pool and try again
AllocateSlab();
}
}
public void Put()
{
var blocks = new MemoryPoolBlock2[4];
for (var i = 0; i < 4; ++i)
{
blocks[i] = _pool.Lease(16);
blocks[i].End += 16;
for (var j = 0; j < blocks.Length; ++j)
{
blocks[i].Array[blocks[i].Start + j] = 0x00;
}
if (i != 0)
{
blocks[i - 1].Next = blocks[i];
}
}
// put FF at first block's head
var head = blocks[0].GetIterator();
Assert.True(head.Put(0xFF));
// data is put at correct position
Assert.Equal(0xFF, blocks[0].Array[blocks[0].Start]);
Assert.Equal(0x00, blocks[0].Array[blocks[0].Start + 1]);
// iterator is moved to next byte after put
Assert.Equal(1, head.Index - blocks[0].Start);
for (var i = 0; i < 14; ++i)
{
// move itr to the end of the block 0
head.Take();
}
// write to the end of block 0
Assert.True(head.Put(0xFE));
Assert.Equal(0xFE, blocks[0].Array[blocks[0].End - 1]);
Assert.Equal(0x00, blocks[1].Array[blocks[1].Start]);
// put data across the block link
Assert.True(head.Put(0xFD));
Assert.Equal(0xFD, blocks[1].Array[blocks[1].Start]);
Assert.Equal(0x00, blocks[1].Array[blocks[1].Start + 1]);
// paint every block
head = blocks[0].GetIterator();
for (var i = 0; i < 64; ++i)
{
Assert.True(head.Put((byte)i), $"Fail to put data at {i}.");
}
// Can't put anything by the end
Assert.False(head.Put(0xFF));
}
public static async Task<MemoryPoolBlock2> CopyToAsync(this Stream source, Stream destination, MemoryPoolBlock2 block)
{
int bytesRead;
while ((bytesRead = await source.ReadAsync(block.Array, block.Data.Offset, block.Data.Count)) != 0)
{
await destination.WriteAsync(block.Array, block.Data.Offset, bytesRead);
}
return block;
}
private void TestAllLengths(MemoryPoolBlock2 block, int lengths)
{
for (var firstIndex = 0; firstIndex <= lengths; ++firstIndex)
{
for (var lastIndex = firstIndex; lastIndex <= lengths; ++lastIndex)
{
var first = block.GetIterator().Add(firstIndex);
var last = block.GetIterator().Add(lastIndex);
Assert.Equal(firstIndex, block.GetIterator().GetLength(first));
Assert.Equal(lastIndex, block.GetIterator().GetLength(last));
Assert.Equal(lastIndex - firstIndex, first.GetLength(last));
}
}
}
public void CopyFrom(byte[] data, int offset, int count)
{
Debug.Assert(_block != null);
Debug.Assert(_block.Pool != null);
Debug.Assert(_block.Next == null);
Debug.Assert(_block.End == _index);
var pool = _block.Pool;
var block = _block;
var blockIndex = _index;
var bufferIndex = offset;
var remaining = count;
var bytesLeftInBlock = block.Data.Offset + block.Data.Count - blockIndex;
while (remaining > 0)
{
if (bytesLeftInBlock == 0)
{
var nextBlock = pool.Lease();
block.End = blockIndex;
block.Next = nextBlock;
block = nextBlock;
blockIndex = block.Data.Offset;
bytesLeftInBlock = block.Data.Count;
}
var bytesToCopy = remaining < bytesLeftInBlock ? remaining : bytesLeftInBlock;
Buffer.BlockCopy(data, bufferIndex, block.Array, blockIndex, bytesToCopy);
blockIndex += bytesToCopy;
bufferIndex += bytesToCopy;
remaining -= bytesToCopy;
bytesLeftInBlock -= bytesToCopy;
}
block.End = blockIndex;
_block = block;
_index = blockIndex;
}
/// <summary>
/// Internal method called when a block is requested and the pool is empty. It allocates one additional slab, creates all of the
/// block tracking objects, and adds them all to the pool.
/// </summary>
private void AllocateSlab()
{
var slab = MemoryPoolSlab2.Create(_slabLength);
_slabs.Push(slab);
var basePtr = slab.ArrayPtr;
var firstOffset = (int)((_blockStride - 1) - ((ulong)(basePtr + _blockStride - 1) % _blockStride));
for (var offset = firstOffset;
offset + _blockLength <= _slabLength;
offset += _blockStride)
{
var block = MemoryPoolBlock2.Create(
new ArraySegment <byte>(slab.Array, offset, _blockLength),
basePtr,
this,
slab);
Return(block);
}
}
public void Skip(int bytesToSkip)
{
if (_block == null)
{
return;
}
var following = _block.End - _index;
if (following >= bytesToSkip)
{
_index += bytesToSkip;
return;
}
var block = _block;
var index = _index;
while (true)
{
if (block.Next == null)
{
return;
}
else
{
bytesToSkip -= following;
block = block.Next;
index = block.Start;
}
following = block.End - index;
if (following >= bytesToSkip)
{
_block = block;
_index = index + bytesToSkip;
return;
}
}
}
public int Take()
{
var block = _block;
if (block == null)
{
return(-1);
}
var index = _index;
if (index < block.End)
{
_index = index + 1;
return(block.Array[index]);
}
do
{
if (block.Next == null)
{
return(-1);
}
else
{
block = block.Next;
index = block.Start;
}
if (index < block.End)
{
_block = block;
_index = index + 1;
return(block.Array[index]);
}
} while (true);
}
public SocketOutput(
KestrelThread thread,
UvStreamHandle socket,
MemoryPool2 memory,
Connection connection,
long connectionId,
IKestrelTrace log,
IThreadPool threadPool,
Queue<UvWriteReq> writeReqPool)
{
_thread = thread;
_socket = socket;
_connection = connection;
_connectionId = connectionId;
_log = log;
_threadPool = threadPool;
_tasksPending = new Queue<TaskCompletionSource<object>>(_initialTaskQueues);
_tasksCompleted = new Queue<TaskCompletionSource<object>>(_initialTaskQueues);
_writeContextPool = new Queue<WriteContext>(_maxPooledWriteContexts);
_writeReqPool = writeReqPool;
_head = memory.Lease();
_tail = _head;
}
public IncomingBuffer IncomingStart(int minimumSize)
{
lock (_sync)
{
if (_tail != null && minimumSize <= _tail.Data.Offset + _tail.Data.Count - _tail.End)
{
_pinned = _tail;
var data = new ArraySegment<byte>(_pinned.Data.Array, _pinned.End, _pinned.Data.Offset + _pinned.Data.Count - _pinned.End);
var dataPtr = _pinned.Pin();
return new IncomingBuffer
{
Data = data,
DataPtr = dataPtr,
};
}
}
_pinned = _memory.Lease(minimumSize);
return new IncomingBuffer
{
Data = _pinned.Data,
DataPtr = _pinned.Pin()
};
}
public MemoryPoolIterator2(MemoryPoolBlock2 block)
{
_block = block;
_index = _block?.Start ?? 0;
}
private static void ReturnBlocks(MemoryPoolBlock2 block)
{
while (block != null)
{
var returningBlock = block;
block = returningBlock.Next;
returningBlock.Pool?.Return(returningBlock);
}
}
public int Seek(int char0, int char1)
{
if (IsDefault)
{
return -1;
}
var byte0 = (byte)char0;
var byte1 = (byte)char1;
var vectorStride = Vector<byte>.Count;
var ch0Vector = new Vector<byte>(byte0);
var ch1Vector = new Vector<byte>(byte1);
var block = _block;
var index = _index;
var array = block.Array;
while (true)
{
while (block.End == index)
{
if (block.Next == null)
{
_block = block;
_index = index;
return -1;
}
block = block.Next;
index = block.Start;
array = block.Array;
}
while (block.End != index)
{
var following = block.End - index;
if (following >= vectorStride)
{
var data = new Vector<byte>(array, index);
var ch0Equals = Vector.Equals(data, ch0Vector);
var ch0Count = Vector.Dot(ch0Equals, _dotCount);
var ch1Equals = Vector.Equals(data, ch1Vector);
var ch1Count = Vector.Dot(ch1Equals, _dotCount);
if (ch0Count == 0 && ch1Count == 0)
{
index += vectorStride;
continue;
}
else if (ch0Count < 2 && ch1Count < 2)
{
var ch0Index = ch0Count == 1 ? Vector.Dot(ch0Equals, _dotIndex) : byte.MaxValue;
var ch1Index = ch1Count == 1 ? Vector.Dot(ch1Equals, _dotIndex) : byte.MaxValue;
if (ch0Index < ch1Index)
{
_block = block;
_index = index + ch0Index;
return char0;
}
else
{
_block = block;
_index = index + ch1Index;
return char1;
}
}
else
{
following = vectorStride;
}
}
for (; following != 0; following--, index++)
{
var byteIndex = block.Array[index];
if (byteIndex == byte0)
{
_block = block;
_index = index;
return char0;
}
else if (byteIndex == byte1)
{
_block = block;
_index = index;
return char1;
}
}
}
}
}
public MemoryPoolIterator2(MemoryPoolBlock2 block, int index)
{
_block = block;
_index = index;
}
public unsafe int Seek(ref Vector<byte> byte0Vector, ref Vector<byte> byte1Vector, ref Vector<byte> byte2Vector)
{
if (IsDefault)
{
return -1;
}
var block = _block;
var index = _index;
var following = block.End - index;
byte[] array;
int byte0Index = int.MaxValue;
int byte1Index = int.MaxValue;
int byte2Index = int.MaxValue;
var byte0 = byte0Vector[0];
var byte1 = byte1Vector[0];
var byte2 = byte2Vector[0];
while (true)
{
while (following == 0)
{
var newBlock = block.Next;
if (newBlock == null)
{
_block = block;
_index = index;
return -1;
}
index = newBlock.Start;
following = newBlock.End - index;
block = newBlock;
}
array = block.Array;
while (following > 0)
{
// Need unit tests to test Vector path
#if !DEBUG
// Check will be Jitted away https://github.com/dotnet/coreclr/issues/1079
if (Vector.IsHardwareAccelerated)
{
#endif
if (following >= _vectorSpan)
{
var data = new Vector<byte>(array, index);
var byte0Equals = Vector.Equals(data, byte0Vector);
var byte1Equals = Vector.Equals(data, byte1Vector);
var byte2Equals = Vector.Equals(data, byte2Vector);
if (!byte0Equals.Equals(Vector<byte>.Zero))
{
byte0Index = FindFirstEqualByte(ref byte0Equals);
}
if (!byte1Equals.Equals(Vector<byte>.Zero))
{
byte1Index = FindFirstEqualByte(ref byte1Equals);
}
if (!byte2Equals.Equals(Vector<byte>.Zero))
{
byte2Index = FindFirstEqualByte(ref byte2Equals);
}
if (byte0Index == int.MaxValue && byte1Index == int.MaxValue && byte2Index == int.MaxValue)
{
following -= _vectorSpan;
index += _vectorSpan;
continue;
}
_block = block;
int toReturn, toMove;
if (byte0Index < byte1Index)
{
if (byte0Index < byte2Index)
{
toReturn = byte0;
toMove = byte0Index;
}
else
{
toReturn = byte2;
toMove = byte2Index;
}
}
else
{
if (byte1Index < byte2Index)
{
toReturn = byte1;
toMove = byte1Index;
}
else
{
toReturn = byte2;
toMove = byte2Index;
}
}
_index = index + toMove;
return toReturn;
}
// Need unit tests to test Vector path
#if !DEBUG
}
#endif
fixed (byte* ptr = &block.Array[index])
{
var pCurrent = ptr;
var pEnd = pCurrent + following;
do
{
if (*pCurrent == byte0)
{
_block = block;
_index = index;
return byte0;
}
if (*pCurrent == byte1)
{
_block = block;
_index = index;
return byte1;
}
if (*pCurrent == byte2)
{
_block = block;
_index = index;
return byte2;
}
pCurrent++;
index++;
} while (pCurrent != pEnd);
}
following = 0;
break;
}
}
}
public int Take()
{
var block = _block;
if (block == null)
{
return -1;
}
var index = _index;
if (index < block.End)
{
_index = index + 1;
return block.Array[index];
}
do
{
if (block.Next == null)
{
return -1;
}
else
{
block = block.Next;
index = block.Start;
}
if (index < block.End)
{
_block = block;
_index = index + 1;
return block.Array[index];
}
} while (true);
}
public MemoryPoolIterator2(MemoryPoolBlock2 block)
{
_block = block;
_index = _block?.Start ?? 0;
}
private static void ReturnWrittenBlocks(MemoryPoolBlock2 block)
{
while (block != null)
{
var returnBlock = block;
block = block.Next;
returnBlock.Unpin();
returnBlock.Pool?.Return(returnBlock);
}
}
/// <summary>
/// Called to return a block to the pool. Once Return has been called the memory no longer belongs to the caller, and
/// Very Bad Things will happen if the memory is read of modified subsequently. If a caller fails to call Return and the
/// block tracking object is garbage collected, the block tracking object's finalizer will automatically re-create and return
/// a new tracking object into the pool. This will only happen if there is a bug in the server, however it is necessary to avoid
/// leaving "dead zones" in the slab due to lost block tracking objects.
/// </summary>
/// <param name="block">The block to return. It must have been acquired by calling Lease on the same memory pool instance.</param>
public void Return(MemoryPoolBlock2 block)
{
block.Reset();
_blocks.Push(block);
}
public int Seek(int char0)
{
if (IsDefault)
{
return(-1);
}
var byte0 = (byte)char0;
var vectorStride = Vector <byte> .Count;
var ch0Vector = new Vector <byte>(byte0);
var block = _block;
var index = _index;
var array = block.Array;
while (true)
{
while (block.End == index)
{
if (block.Next == null)
{
_block = block;
_index = index;
return(-1);
}
block = block.Next;
index = block.Start;
array = block.Array;
}
while (block.End != index)
{
var following = block.End - index;
if (following >= vectorStride)
{
var data = new Vector <byte>(array, index);
var ch0Equals = Vector.Equals(data, ch0Vector);
var ch0Count = Vector.Dot(ch0Equals, _dotCount);
if (ch0Count == 0)
{
index += vectorStride;
continue;
}
else if (ch0Count == 1)
{
_block = block;
_index = index + Vector.Dot(ch0Equals, _dotIndex);
return(char0);
}
else
{
following = vectorStride;
}
}
while (following > 0)
{
if (block.Array[index] == byte0)
{
_block = block;
_index = index;
return(char0);
}
following--;
index++;
}
}
}
}
public int Seek(int char0, int char1)
{
if (IsDefault)
{
return(-1);
}
var byte0 = (byte)char0;
var byte1 = (byte)char1;
var vectorStride = Vector <byte> .Count;
var ch0Vector = new Vector <byte>(byte0);
var ch1Vector = new Vector <byte>(byte1);
var block = _block;
var index = _index;
var array = block.Array;
while (true)
{
while (block.End == index)
{
if (block.Next == null)
{
_block = block;
_index = index;
return(-1);
}
block = block.Next;
index = block.Start;
array = block.Array;
}
while (block.End != index)
{
var following = block.End - index;
if (following >= vectorStride)
{
var data = new Vector <byte>(array, index);
var ch0Equals = Vector.Equals(data, ch0Vector);
var ch0Count = Vector.Dot(ch0Equals, _dotCount);
var ch1Equals = Vector.Equals(data, ch1Vector);
var ch1Count = Vector.Dot(ch1Equals, _dotCount);
if (ch0Count == 0 && ch1Count == 0)
{
index += vectorStride;
continue;
}
else if (ch0Count < 2 && ch1Count < 2)
{
var ch0Index = ch0Count == 1 ? Vector.Dot(ch0Equals, _dotIndex) : byte.MaxValue;
var ch1Index = ch1Count == 1 ? Vector.Dot(ch1Equals, _dotIndex) : byte.MaxValue;
if (ch0Index < ch1Index)
{
_block = block;
_index = index + ch0Index;
return(char0);
}
else
{
_block = block;
_index = index + ch1Index;
return(char1);
}
}
else
{
following = vectorStride;
}
}
for (; following != 0; following--, index++)
{
var byteIndex = block.Array[index];
if (byteIndex == byte0)
{
_block = block;
_index = index;
return(char0);
}
else if (byteIndex == byte1)
{
_block = block;
_index = index;
return(char1);
}
}
}
}
}
public void Skip(int bytesToSkip)
{
if (_block == null)
{
return;
}
var following = _block.End - _index;
if (following >= bytesToSkip)
{
_index += bytesToSkip;
return;
}
var block = _block;
var index = _index;
while (true)
{
if (block.Next == null)
{
return;
}
else
{
bytesToSkip -= following;
block = block.Next;
index = block.Start;
}
following = block.End - index;
if (following >= bytesToSkip)
{
_block = block;
_index = index + bytesToSkip;
return;
}
}
}
public unsafe void CopyFromAscii(string data)
{
Debug.Assert(_block != null);
Debug.Assert(_block.Pool != null);
Debug.Assert(_block.Next == null);
Debug.Assert(_block.End == _index);
var pool = _block.Pool;
var block = _block;
var blockIndex = _index;
var length = data.Length;
var bytesLeftInBlock = block.Data.Offset + block.Data.Count - blockIndex;
var bytesLeftInBlockMinusSpan = bytesLeftInBlock - 3;
fixed (char* pData = data)
{
var input = pData;
var inputEnd = pData + length;
var inputEndMinusSpan = inputEnd - 3;
while (input < inputEnd)
{
if (bytesLeftInBlock == 0)
{
var nextBlock = pool.Lease();
block.End = blockIndex;
block.Next = nextBlock;
block = nextBlock;
blockIndex = block.Data.Offset;
bytesLeftInBlock = block.Data.Count;
bytesLeftInBlockMinusSpan = bytesLeftInBlock - 3;
}
fixed (byte* pOutput = &block.Data.Array[block.End])
{
//this line is needed to allow output be an register var
var output = pOutput;
var copied = 0;
for (; input < inputEndMinusSpan && copied < bytesLeftInBlockMinusSpan; copied += 4)
{
*(output) = (byte)*(input);
*(output + 1) = (byte)*(input + 1);
*(output + 2) = (byte)*(input + 2);
*(output + 3) = (byte)*(input + 3);
output += 4;
input += 4;
}
for (; input < inputEnd && copied < bytesLeftInBlock; copied++)
{
*(output++) = (byte)*(input++);
}
blockIndex += copied;
bytesLeftInBlockMinusSpan -= copied;
bytesLeftInBlock -= copied;
}
}
}
block.End = blockIndex;
_block = block;
_index = blockIndex;
}
public void CopyFrom(byte[] data, int offset, int count)
{
Debug.Assert(_block != null);
Debug.Assert(_block.Pool != null);
Debug.Assert(_block.Next == null);
Debug.Assert(_block.End == _index);
var pool = _block.Pool;
var block = _block;
var blockIndex = _index;
var bufferIndex = offset;
var remaining = count;
var bytesLeftInBlock = block.Data.Offset + block.Data.Count - blockIndex;
while (remaining > 0)
{
if (bytesLeftInBlock == 0)
{
var nextBlock = pool.Lease();
block.End = blockIndex;
block.Next = nextBlock;
block = nextBlock;
blockIndex = block.Data.Offset;
bytesLeftInBlock = block.Data.Count;
}
var bytesToCopy = remaining < bytesLeftInBlock ? remaining : bytesLeftInBlock;
Buffer.BlockCopy(data, bufferIndex, block.Array, blockIndex, bytesToCopy);
blockIndex += bytesToCopy;
bufferIndex += bytesToCopy;
remaining -= bytesToCopy;
bytesLeftInBlock -= bytesToCopy;
}
block.End = blockIndex;
_block = block;
_index = blockIndex;
}
public MemoryPoolIterator2(MemoryPoolBlock2 block, int index)
{
_block = block;
_index = index;
}
private void ProducingCompleteNoPreComplete(MemoryPoolIterator2 end)
{
MemoryPoolBlock2 blockToReturn = null;
lock (_returnLock)
{
Debug.Assert(!_lastStart.IsDefault);
// If the socket has been closed, return the produced blocks
// instead of advancing the now non-existent tail.
if (_tail != null)
{
_tail = end.Block;
_tail.End = end.Index;
}
else
{
blockToReturn = _lastStart.Block;
}
_lastStart = default(MemoryPoolIterator2);
}
if (blockToReturn != null)
{
ThreadPool.QueueUserWorkItem(_returnBlocks, blockToReturn);
}
}
public int Take()
{
if (_block == null)
{
return -1;
}
else if (_index < _block.End)
{
return _block.Array[_index++];
}
var block = _block;
var index = _index;
while (true)
{
if (index < block.End)
{
_block = block;
_index = index + 1;
return block.Array[index];
}
else if (block.Next == null)
{
return -1;
}
else
{
block = block.Next;
index = block.Start;
}
}
}
/// <summary>
/// Save the data at the current location then move to the next available space.
/// </summary>
/// <param name="data">The byte to be saved.</param>
/// <returns>true if the operation successes. false if can't find available space.</returns>
public bool Put(byte data)
{
if (_block == null)
{
return false;
}
else if (_index < _block.End)
{
_block.Array[_index++] = data;
return true;
}
var block = _block;
var index = _index;
while (true)
{
if (index < block.End)
{
_block = block;
_index = index + 1;
block.Array[index] = data;
return true;
}
else if (block.Next == null)
{
return false;
}
else
{
block = block.Next;
index = block.Start;
}
}
}
private void AssertIterator(MemoryPoolIterator2 iter, MemoryPoolBlock2 block, int index)
{
Assert.Same(block, iter.Block);
Assert.Equal(index, iter.Index);
}
public void IncomingComplete(int count, Exception error)
{
Action awaitableState;
lock (_sync)
{
// Unpin may called without an earlier Pin
if (_pinned != null)
{
_pinned.Unpin();
_pinned.End += count;
if (_head == null)
{
_head = _tail = _pinned;
}
else if (_tail == _pinned)
{
// NO-OP: this was a read into unoccupied tail-space
}
else
{
_tail.Next = _pinned;
_tail = _pinned;
}
}
_pinned = null;
if (count == 0)
{
RemoteIntakeFin = true;
}
if (error != null)
{
_awaitableError = error;
}
awaitableState = Interlocked.Exchange(
ref _awaitableState,
_awaitableIsCompleted);
_manualResetEvent.Set();
}
if (awaitableState != _awaitableIsCompleted &&
awaitableState != _awaitableIsNotCompleted)
{
Task.Run(awaitableState);
}
}
public int Seek(int char0)
{
if (IsDefault)
{
return -1;
}
var byte0 = (byte)char0;
var vectorStride = Vector<byte>.Count;
var ch0Vector = new Vector<byte>(byte0);
var block = _block;
var index = _index;
var array = block.Array;
while (true)
{
while (block.End == index)
{
if (block.Next == null)
{
_block = block;
_index = index;
return -1;
}
block = block.Next;
index = block.Start;
array = block.Array;
}
while (block.End != index)
{
var following = block.End - index;
if (following >= vectorStride)
{
var data = new Vector<byte>(array, index);
var ch0Equals = Vector.Equals(data, ch0Vector);
var ch0Count = Vector.Dot(ch0Equals, _dotCount);
if (ch0Count == 0)
{
index += vectorStride;
continue;
}
else if (ch0Count == 1)
{
_block = block;
_index = index + Vector.Dot(ch0Equals, _dotIndex);
return char0;
}
else
{
following = vectorStride;
}
}
while (following > 0)
{
if (block.Array[index] == byte0)
{
_block = block;
_index = index;
return char0;
}
following--;
index++;
}
}
}
}
/// <summary>
/// Called to return a block to the pool. Once Return has been called the memory no longer belongs to the caller, and
/// Very Bad Things will happen if the memory is read of modified subsequently. If a caller fails to call Return and the
/// block tracking object is garbage collected, the block tracking object's finalizer will automatically re-create and return
/// a new tracking object into the pool. This will only happen if there is a bug in the server, however it is necessary to avoid
/// leaving "dead zones" in the slab due to lost block tracking objects.
/// </summary>
/// <param name="block">The block to return. It must have been acquired by calling Lease on the same memory pool instance.</param>
public void Return(MemoryPoolBlock2 block)
{
block.Reset();
_blocks.Push(block);
}
public unsafe int Seek(ref Vector <byte> byte0Vector)
{
if (IsDefault)
{
return(-1);
}
var block = _block;
var index = _index;
var following = block.End - index;
byte[] array;
var byte0 = byte0Vector[0];
while (true)
{
while (following == 0)
{
var newBlock = block.Next;
if (newBlock == null)
{
_block = block;
_index = index;
return(-1);
}
index = newBlock.Start;
following = newBlock.End - index;
block = newBlock;
}
array = block.Array;
while (following > 0)
{
// Need unit tests to test Vector path
#if !DEBUG
// Check will be Jitted away https://github.com/dotnet/coreclr/issues/1079
if (Vector.IsHardwareAccelerated)
{
#endif
if (following >= _vectorSpan)
{
var byte0Equals = Vector.Equals(new Vector <byte>(array, index), byte0Vector);
if (byte0Equals.Equals(Vector <byte> .Zero))
{
following -= _vectorSpan;
index += _vectorSpan;
continue;
}
_block = block;
_index = index + FindFirstEqualByte(ref byte0Equals);
return(byte0);
}
// Need unit tests to test Vector path
#if !DEBUG
}
#endif
fixed(byte *ptr = &block.Array[index])
{
var pCurrent = ptr;
var pEnd = pCurrent + following;
do
{
if (*pCurrent == byte0)
{
_block = block;
_index = index;
return(byte0);
}
pCurrent++;
index++;
} while (pCurrent < pEnd);
}
following = 0;
break;
}
}
}
// This is called on the libuv event loop
private void ReturnAllBlocks()
{
lock (_returnLock)
{
var block = _head;
while (block != _tail)
{
var returnBlock = block;
block = block.Next;
returnBlock.Pool?.Return(returnBlock);
}
// Only return the _tail if we aren't between ProducingStart/Complete calls
if (_lastStart.IsDefault)
{
_tail.Pool?.Return(_tail);
}
_head = null;
_tail = null;
}
}
public unsafe int Seek(ref Vector <byte> byte0Vector, ref Vector <byte> byte1Vector, ref Vector <byte> byte2Vector)
{
if (IsDefault)
{
return(-1);
}
var block = _block;
var index = _index;
var following = block.End - index;
byte[] array;
int byte0Index = int.MaxValue;
int byte1Index = int.MaxValue;
int byte2Index = int.MaxValue;
var byte0 = byte0Vector[0];
var byte1 = byte1Vector[0];
var byte2 = byte2Vector[0];
while (true)
{
while (following == 0)
{
var newBlock = block.Next;
if (newBlock == null)
{
_block = block;
_index = index;
return(-1);
}
index = newBlock.Start;
following = newBlock.End - index;
block = newBlock;
}
array = block.Array;
while (following > 0)
{
// Need unit tests to test Vector path
#if !DEBUG
// Check will be Jitted away https://github.com/dotnet/coreclr/issues/1079
if (Vector.IsHardwareAccelerated)
{
#endif
if (following >= _vectorSpan)
{
var data = new Vector <byte>(array, index);
var byte0Equals = Vector.Equals(data, byte0Vector);
var byte1Equals = Vector.Equals(data, byte1Vector);
var byte2Equals = Vector.Equals(data, byte2Vector);
if (!byte0Equals.Equals(Vector <byte> .Zero))
{
byte0Index = FindFirstEqualByte(ref byte0Equals);
}
if (!byte1Equals.Equals(Vector <byte> .Zero))
{
byte1Index = FindFirstEqualByte(ref byte1Equals);
}
if (!byte2Equals.Equals(Vector <byte> .Zero))
{
byte2Index = FindFirstEqualByte(ref byte2Equals);
}
if (byte0Index == int.MaxValue && byte1Index == int.MaxValue && byte2Index == int.MaxValue)
{
following -= _vectorSpan;
index += _vectorSpan;
continue;
}
_block = block;
int toReturn, toMove;
if (byte0Index < byte1Index)
{
if (byte0Index < byte2Index)
{
toReturn = byte0;
toMove = byte0Index;
}
else
{
toReturn = byte2;
toMove = byte2Index;
}
}
else
{
if (byte1Index < byte2Index)
{
toReturn = byte1;
toMove = byte1Index;
}
else
{
toReturn = byte2;
toMove = byte2Index;
}
}
_index = index + toMove;
return(toReturn);
}
// Need unit tests to test Vector path
#if !DEBUG
}
#endif
fixed(byte *ptr = &block.Array[index])
{
var pCurrent = ptr;
var pEnd = pCurrent + following;
do
{
if (*pCurrent == byte0)
{
_block = block;
_index = index;
return(byte0);
}
if (*pCurrent == byte1)
{
_block = block;
_index = index;
return(byte1);
}
if (*pCurrent == byte2)
{
_block = block;
_index = index;
return(byte2);
}
pCurrent++;
index++;
} while (pCurrent != pEnd);
}
following = 0;
break;
}
}
}
public unsafe int Seek(ref Vector<byte> byte0Vector)
{
if (IsDefault)
{
return -1;
}
var block = _block;
var index = _index;
var following = block.End - index;
byte[] array;
var byte0 = byte0Vector[0];
while (true)
{
while (following == 0)
{
var newBlock = block.Next;
if (newBlock == null)
{
_block = block;
_index = index;
return -1;
}
index = newBlock.Start;
following = newBlock.End - index;
block = newBlock;
}
array = block.Array;
while (following > 0)
{
// Need unit tests to test Vector path
#if !DEBUG
// Check will be Jitted away https://github.com/dotnet/coreclr/issues/1079
if (Vector.IsHardwareAccelerated)
{
#endif
if (following >= _vectorSpan)
{
var byte0Equals = Vector.Equals(new Vector<byte>(array, index), byte0Vector);
if (byte0Equals.Equals(Vector<byte>.Zero))
{
following -= _vectorSpan;
index += _vectorSpan;
continue;
}
_block = block;
_index = index + FindFirstEqualByte(ref byte0Equals);
return byte0;
}
// Need unit tests to test Vector path
#if !DEBUG
}
#endif
fixed (byte* ptr = &block.Array[index])
{
var pCurrent = ptr;
var pEnd = pCurrent + following;
do
{
if (*pCurrent == byte0)
{
_block = block;
_index = index;
return byte0;
}
pCurrent++;
index++;
} while (pCurrent < pEnd);
}
following = 0;
break;
}
}
}
/// <summary>
/// Called to return a block to the pool. Once Return has been called the memory no longer belongs to the caller, and
/// Very Bad Things will happen if the memory is read of modified subsequently. If a caller fails to call Return and the
/// block tracking object is garbage collected, the block tracking object's finalizer will automatically re-create and return
/// a new tracking object into the pool. This will only happen if there is a bug in the server, however it is necessary to avoid
/// leaving "dead zones" in the slab due to lost block tracking objects.
/// </summary>
/// <param name="block">The block to return. It must have been acquired by calling Lease on the same memory pool instance.</param>
public void Return(MemoryPoolBlock2 block)
{
block.Reset();
_blocks.Enqueue(block);
}
/// <summary>
/// Called to return a block to the pool. Once Return has been called the memory no longer belongs to the caller, and
/// Very Bad Things will happen if the memory is read of modified subsequently. If a caller fails to call Return and the
/// block tracking object is garbage collected, the block tracking object's finalizer will automatically re-create and return
/// a new tracking object into the pool. This will only happen if there is a bug in the server, however it is necessary to avoid
/// leaving "dead zones" in the slab due to lost block tracking objects.
/// </summary>
/// <param name="block">The block to return. It must have been acquired by calling Lease on the same memory pool instance.</param>
public void Return(MemoryPoolBlock2 block)
{
block.Reset();
_blocks.Enqueue(block);
}
public unsafe void CopyFromAscii(string data)
{
Debug.Assert(_block != null);
Debug.Assert(_block.Pool != null);
Debug.Assert(_block.Next == null);
Debug.Assert(_block.End == _index);
var pool = _block.Pool;
var block = _block;
var blockIndex = _index;
var length = data.Length;
var bytesLeftInBlock = block.Data.Offset + block.Data.Count - blockIndex;
var bytesLeftInBlockMinusSpan = bytesLeftInBlock - 3;
fixed(char *pData = data)
{
var input = pData;
var inputEnd = pData + length;
var inputEndMinusSpan = inputEnd - 3;
while (input < inputEnd)
{
if (bytesLeftInBlock == 0)
{
var nextBlock = pool.Lease();
block.End = blockIndex;
block.Next = nextBlock;
block = nextBlock;
blockIndex = block.Data.Offset;
bytesLeftInBlock = block.Data.Count;
bytesLeftInBlockMinusSpan = bytesLeftInBlock - 3;
}
fixed(byte *pOutput = &block.Data.Array[block.End])
{
//this line is needed to allow output be an register var
var output = pOutput;
var copied = 0;
for (; input < inputEndMinusSpan && copied < bytesLeftInBlockMinusSpan; copied += 4)
{
*(output) = (byte)*(input);
*(output + 1) = (byte)*(input + 1);
*(output + 2) = (byte)*(input + 2);
*(output + 3) = (byte)*(input + 3);
output += 4;
input += 4;
}
for (; input < inputEnd && copied < bytesLeftInBlock; copied++)
{
*(output++) = (byte)*(input++);
}
blockIndex += copied;
bytesLeftInBlockMinusSpan -= copied;
bytesLeftInBlock -= copied;
}
}
}
block.End = blockIndex;
_block = block;
_index = blockIndex;
}
public MemoryPoolIterator2 ProducingStart()
{
_producingBlock = _memory.Lease();
return new MemoryPoolIterator2(_producingBlock);
}