/// <summary> /// Internal method called when a block is requested and the pool is empty. It allocates one additional slab, creates all of the /// block tracking objects, and adds them all to the pool. /// </summary> private MemoryPoolBlock2 AllocateSlab() { var slab = MemoryPoolSlab2.Create(_slabLength); _slabs.Push(slab); var basePtr = slab.ArrayPtr; var firstOffset = (int)((_blockStride - 1) - ((ulong)(basePtr + _blockStride - 1) % _blockStride)); var poolAllocationLength = _slabLength - _blockStride; var offset = firstOffset; for (; offset + _blockLength < poolAllocationLength; offset += _blockStride) { var block = MemoryPoolBlock2.Create( new ArraySegment <byte>(slab.Array, offset, _blockLength), basePtr, this, slab); Return(block); } // return last block rather than adding to pool var newBlock = MemoryPoolBlock2.Create( new ArraySegment <byte>(slab.Array, offset, _blockLength), basePtr, this, slab); return(newBlock); }
public int Take() { if (_block == null) { return(-1); } else if (_index < _block.End) { return(_block.Array[_index++]); } var block = _block; var index = _index; while (true) { if (index < block.End) { _block = block; _index = index + 1; return(block.Array[index]); } else if (block.Next == null) { return(-1); } else { block = block.Next; index = block.Start; } } }
public void ConsumingComplete( MemoryPoolIterator2 consumed, MemoryPoolIterator2 examined) { MemoryPoolBlock2 returnStart = null; MemoryPoolBlock2 returnEnd = null; lock (_sync) { if (!consumed.IsDefault) { returnStart = _head; returnEnd = consumed.Block; _head = consumed.Block; _head.Start = consumed.Index; } if (!examined.IsDefault && examined.IsEnd && RemoteIntakeFin == false && _awaitableError == null) { _manualResetEvent.Reset(); var awaitableState = Interlocked.CompareExchange( ref _awaitableState, _awaitableIsNotCompleted, _awaitableIsCompleted); } } while (returnStart != returnEnd) { var returnBlock = returnStart; returnStart = returnStart.Next; returnBlock.Pool.Return(returnBlock); } }
/// <summary> /// Save the data at the current location then move to the next available space. /// </summary> /// <param name="data">The byte to be saved.</param> /// <returns>true if the operation successes. false if can't find available space.</returns> public bool Put(byte data) { if (_block == null) { return(false); } else if (_index < _block.End) { _block.Array[_index++] = data; return(true); } var block = _block; var index = _index; while (true) { if (index < block.End) { _block = block; _index = index + 1; block.Array[index] = data; return(true); } else if (block.Next == null) { return(false); } else { block = block.Next; index = block.Start; } } }
private bool _disposedValue = false; // To detect redundant calls /// <summary> /// Called to take a block from the pool. /// </summary> /// <param name="minimumSize">The block returned must be at least this size. It may be larger than this minimum size, and if so, /// the caller may write to the block's entire size rather than being limited to the minumumSize requested.</param> /// <returns>The block that is reserved for the called. It must be passed to Return when it is no longer being used.</returns> public MemoryPoolBlock2 Lease(int minimumSize) { if (minimumSize > _blockLength) { // The requested minimumSize is actually larger then the usable memory of a single block. // Because this is the degenerate case, a one-time-use byte[] array and tracking object are allocated. // When this block tracking object is returned it is not added to the pool - instead it will be // allowed to be garbage collected normally. return(MemoryPoolBlock2.Create( new ArraySegment <byte>(new byte[minimumSize]), dataPtr: IntPtr.Zero, pool: null, slab: null)); } while (true) { MemoryPoolBlock2 block; if (_blocks.TryPop(out block)) { // block successfully taken from the stack - return it return(block); } // no blocks available - grow the pool and try again AllocateSlab(); } }
public void Put() { var blocks = new MemoryPoolBlock2[4]; for (var i = 0; i < 4; ++i) { blocks[i] = _pool.Lease(16); blocks[i].End += 16; for (var j = 0; j < blocks.Length; ++j) { blocks[i].Array[blocks[i].Start + j] = 0x00; } if (i != 0) { blocks[i - 1].Next = blocks[i]; } } // put FF at first block's head var head = blocks[0].GetIterator(); Assert.True(head.Put(0xFF)); // data is put at correct position Assert.Equal(0xFF, blocks[0].Array[blocks[0].Start]); Assert.Equal(0x00, blocks[0].Array[blocks[0].Start + 1]); // iterator is moved to next byte after put Assert.Equal(1, head.Index - blocks[0].Start); for (var i = 0; i < 14; ++i) { // move itr to the end of the block 0 head.Take(); } // write to the end of block 0 Assert.True(head.Put(0xFE)); Assert.Equal(0xFE, blocks[0].Array[blocks[0].End - 1]); Assert.Equal(0x00, blocks[1].Array[blocks[1].Start]); // put data across the block link Assert.True(head.Put(0xFD)); Assert.Equal(0xFD, blocks[1].Array[blocks[1].Start]); Assert.Equal(0x00, blocks[1].Array[blocks[1].Start + 1]); // paint every block head = blocks[0].GetIterator(); for (var i = 0; i < 64; ++i) { Assert.True(head.Put((byte)i), $"Fail to put data at {i}."); } // Can't put anything by the end Assert.False(head.Put(0xFF)); }
public static async Task<MemoryPoolBlock2> CopyToAsync(this Stream source, Stream destination, MemoryPoolBlock2 block) { int bytesRead; while ((bytesRead = await source.ReadAsync(block.Array, block.Data.Offset, block.Data.Count)) != 0) { await destination.WriteAsync(block.Array, block.Data.Offset, bytesRead); } return block; }
private void TestAllLengths(MemoryPoolBlock2 block, int lengths) { for (var firstIndex = 0; firstIndex <= lengths; ++firstIndex) { for (var lastIndex = firstIndex; lastIndex <= lengths; ++lastIndex) { var first = block.GetIterator().Add(firstIndex); var last = block.GetIterator().Add(lastIndex); Assert.Equal(firstIndex, block.GetIterator().GetLength(first)); Assert.Equal(lastIndex, block.GetIterator().GetLength(last)); Assert.Equal(lastIndex - firstIndex, first.GetLength(last)); } } }
public void CopyFrom(byte[] data, int offset, int count) { Debug.Assert(_block != null); Debug.Assert(_block.Pool != null); Debug.Assert(_block.Next == null); Debug.Assert(_block.End == _index); var pool = _block.Pool; var block = _block; var blockIndex = _index; var bufferIndex = offset; var remaining = count; var bytesLeftInBlock = block.Data.Offset + block.Data.Count - blockIndex; while (remaining > 0) { if (bytesLeftInBlock == 0) { var nextBlock = pool.Lease(); block.End = blockIndex; block.Next = nextBlock; block = nextBlock; blockIndex = block.Data.Offset; bytesLeftInBlock = block.Data.Count; } var bytesToCopy = remaining < bytesLeftInBlock ? remaining : bytesLeftInBlock; Buffer.BlockCopy(data, bufferIndex, block.Array, blockIndex, bytesToCopy); blockIndex += bytesToCopy; bufferIndex += bytesToCopy; remaining -= bytesToCopy; bytesLeftInBlock -= bytesToCopy; } block.End = blockIndex; _block = block; _index = blockIndex; }
/// <summary> /// Internal method called when a block is requested and the pool is empty. It allocates one additional slab, creates all of the /// block tracking objects, and adds them all to the pool. /// </summary> private void AllocateSlab() { var slab = MemoryPoolSlab2.Create(_slabLength); _slabs.Push(slab); var basePtr = slab.ArrayPtr; var firstOffset = (int)((_blockStride - 1) - ((ulong)(basePtr + _blockStride - 1) % _blockStride)); for (var offset = firstOffset; offset + _blockLength <= _slabLength; offset += _blockStride) { var block = MemoryPoolBlock2.Create( new ArraySegment <byte>(slab.Array, offset, _blockLength), basePtr, this, slab); Return(block); } }
public void Skip(int bytesToSkip) { if (_block == null) { return; } var following = _block.End - _index; if (following >= bytesToSkip) { _index += bytesToSkip; return; } var block = _block; var index = _index; while (true) { if (block.Next == null) { return; } else { bytesToSkip -= following; block = block.Next; index = block.Start; } following = block.End - index; if (following >= bytesToSkip) { _block = block; _index = index + bytesToSkip; return; } } }
public int Take() { var block = _block; if (block == null) { return(-1); } var index = _index; if (index < block.End) { _index = index + 1; return(block.Array[index]); } do { if (block.Next == null) { return(-1); } else { block = block.Next; index = block.Start; } if (index < block.End) { _block = block; _index = index + 1; return(block.Array[index]); } } while (true); }
public SocketOutput( KestrelThread thread, UvStreamHandle socket, MemoryPool2 memory, Connection connection, long connectionId, IKestrelTrace log, IThreadPool threadPool, Queue<UvWriteReq> writeReqPool) { _thread = thread; _socket = socket; _connection = connection; _connectionId = connectionId; _log = log; _threadPool = threadPool; _tasksPending = new Queue<TaskCompletionSource<object>>(_initialTaskQueues); _tasksCompleted = new Queue<TaskCompletionSource<object>>(_initialTaskQueues); _writeContextPool = new Queue<WriteContext>(_maxPooledWriteContexts); _writeReqPool = writeReqPool; _head = memory.Lease(); _tail = _head; }
public IncomingBuffer IncomingStart(int minimumSize) { lock (_sync) { if (_tail != null && minimumSize <= _tail.Data.Offset + _tail.Data.Count - _tail.End) { _pinned = _tail; var data = new ArraySegment<byte>(_pinned.Data.Array, _pinned.End, _pinned.Data.Offset + _pinned.Data.Count - _pinned.End); var dataPtr = _pinned.Pin(); return new IncomingBuffer { Data = data, DataPtr = dataPtr, }; } } _pinned = _memory.Lease(minimumSize); return new IncomingBuffer { Data = _pinned.Data, DataPtr = _pinned.Pin() }; }
public MemoryPoolIterator2(MemoryPoolBlock2 block) { _block = block; _index = _block?.Start ?? 0; }
private static void ReturnBlocks(MemoryPoolBlock2 block) { while (block != null) { var returningBlock = block; block = returningBlock.Next; returningBlock.Pool?.Return(returningBlock); } }
public int Seek(int char0, int char1) { if (IsDefault) { return -1; } var byte0 = (byte)char0; var byte1 = (byte)char1; var vectorStride = Vector<byte>.Count; var ch0Vector = new Vector<byte>(byte0); var ch1Vector = new Vector<byte>(byte1); var block = _block; var index = _index; var array = block.Array; while (true) { while (block.End == index) { if (block.Next == null) { _block = block; _index = index; return -1; } block = block.Next; index = block.Start; array = block.Array; } while (block.End != index) { var following = block.End - index; if (following >= vectorStride) { var data = new Vector<byte>(array, index); var ch0Equals = Vector.Equals(data, ch0Vector); var ch0Count = Vector.Dot(ch0Equals, _dotCount); var ch1Equals = Vector.Equals(data, ch1Vector); var ch1Count = Vector.Dot(ch1Equals, _dotCount); if (ch0Count == 0 && ch1Count == 0) { index += vectorStride; continue; } else if (ch0Count < 2 && ch1Count < 2) { var ch0Index = ch0Count == 1 ? Vector.Dot(ch0Equals, _dotIndex) : byte.MaxValue; var ch1Index = ch1Count == 1 ? Vector.Dot(ch1Equals, _dotIndex) : byte.MaxValue; if (ch0Index < ch1Index) { _block = block; _index = index + ch0Index; return char0; } else { _block = block; _index = index + ch1Index; return char1; } } else { following = vectorStride; } } for (; following != 0; following--, index++) { var byteIndex = block.Array[index]; if (byteIndex == byte0) { _block = block; _index = index; return char0; } else if (byteIndex == byte1) { _block = block; _index = index; return char1; } } } } }
public MemoryPoolIterator2(MemoryPoolBlock2 block, int index) { _block = block; _index = index; }
public unsafe int Seek(ref Vector<byte> byte0Vector, ref Vector<byte> byte1Vector, ref Vector<byte> byte2Vector) { if (IsDefault) { return -1; } var block = _block; var index = _index; var following = block.End - index; byte[] array; int byte0Index = int.MaxValue; int byte1Index = int.MaxValue; int byte2Index = int.MaxValue; var byte0 = byte0Vector[0]; var byte1 = byte1Vector[0]; var byte2 = byte2Vector[0]; while (true) { while (following == 0) { var newBlock = block.Next; if (newBlock == null) { _block = block; _index = index; return -1; } index = newBlock.Start; following = newBlock.End - index; block = newBlock; } array = block.Array; while (following > 0) { // Need unit tests to test Vector path #if !DEBUG // Check will be Jitted away https://github.com/dotnet/coreclr/issues/1079 if (Vector.IsHardwareAccelerated) { #endif if (following >= _vectorSpan) { var data = new Vector<byte>(array, index); var byte0Equals = Vector.Equals(data, byte0Vector); var byte1Equals = Vector.Equals(data, byte1Vector); var byte2Equals = Vector.Equals(data, byte2Vector); if (!byte0Equals.Equals(Vector<byte>.Zero)) { byte0Index = FindFirstEqualByte(ref byte0Equals); } if (!byte1Equals.Equals(Vector<byte>.Zero)) { byte1Index = FindFirstEqualByte(ref byte1Equals); } if (!byte2Equals.Equals(Vector<byte>.Zero)) { byte2Index = FindFirstEqualByte(ref byte2Equals); } if (byte0Index == int.MaxValue && byte1Index == int.MaxValue && byte2Index == int.MaxValue) { following -= _vectorSpan; index += _vectorSpan; continue; } _block = block; int toReturn, toMove; if (byte0Index < byte1Index) { if (byte0Index < byte2Index) { toReturn = byte0; toMove = byte0Index; } else { toReturn = byte2; toMove = byte2Index; } } else { if (byte1Index < byte2Index) { toReturn = byte1; toMove = byte1Index; } else { toReturn = byte2; toMove = byte2Index; } } _index = index + toMove; return toReturn; } // Need unit tests to test Vector path #if !DEBUG } #endif fixed (byte* ptr = &block.Array[index]) { var pCurrent = ptr; var pEnd = pCurrent + following; do { if (*pCurrent == byte0) { _block = block; _index = index; return byte0; } if (*pCurrent == byte1) { _block = block; _index = index; return byte1; } if (*pCurrent == byte2) { _block = block; _index = index; return byte2; } pCurrent++; index++; } while (pCurrent != pEnd); } following = 0; break; } } }
public int Take() { var block = _block; if (block == null) { return -1; } var index = _index; if (index < block.End) { _index = index + 1; return block.Array[index]; } do { if (block.Next == null) { return -1; } else { block = block.Next; index = block.Start; } if (index < block.End) { _block = block; _index = index + 1; return block.Array[index]; } } while (true); }
private static void ReturnWrittenBlocks(MemoryPoolBlock2 block) { while (block != null) { var returnBlock = block; block = block.Next; returnBlock.Unpin(); returnBlock.Pool?.Return(returnBlock); } }
/// <summary> /// Called to return a block to the pool. Once Return has been called the memory no longer belongs to the caller, and /// Very Bad Things will happen if the memory is read of modified subsequently. If a caller fails to call Return and the /// block tracking object is garbage collected, the block tracking object's finalizer will automatically re-create and return /// a new tracking object into the pool. This will only happen if there is a bug in the server, however it is necessary to avoid /// leaving "dead zones" in the slab due to lost block tracking objects. /// </summary> /// <param name="block">The block to return. It must have been acquired by calling Lease on the same memory pool instance.</param> public void Return(MemoryPoolBlock2 block) { block.Reset(); _blocks.Push(block); }
public int Seek(int char0) { if (IsDefault) { return(-1); } var byte0 = (byte)char0; var vectorStride = Vector <byte> .Count; var ch0Vector = new Vector <byte>(byte0); var block = _block; var index = _index; var array = block.Array; while (true) { while (block.End == index) { if (block.Next == null) { _block = block; _index = index; return(-1); } block = block.Next; index = block.Start; array = block.Array; } while (block.End != index) { var following = block.End - index; if (following >= vectorStride) { var data = new Vector <byte>(array, index); var ch0Equals = Vector.Equals(data, ch0Vector); var ch0Count = Vector.Dot(ch0Equals, _dotCount); if (ch0Count == 0) { index += vectorStride; continue; } else if (ch0Count == 1) { _block = block; _index = index + Vector.Dot(ch0Equals, _dotIndex); return(char0); } else { following = vectorStride; } } while (following > 0) { if (block.Array[index] == byte0) { _block = block; _index = index; return(char0); } following--; index++; } } } }
public int Seek(int char0, int char1) { if (IsDefault) { return(-1); } var byte0 = (byte)char0; var byte1 = (byte)char1; var vectorStride = Vector <byte> .Count; var ch0Vector = new Vector <byte>(byte0); var ch1Vector = new Vector <byte>(byte1); var block = _block; var index = _index; var array = block.Array; while (true) { while (block.End == index) { if (block.Next == null) { _block = block; _index = index; return(-1); } block = block.Next; index = block.Start; array = block.Array; } while (block.End != index) { var following = block.End - index; if (following >= vectorStride) { var data = new Vector <byte>(array, index); var ch0Equals = Vector.Equals(data, ch0Vector); var ch0Count = Vector.Dot(ch0Equals, _dotCount); var ch1Equals = Vector.Equals(data, ch1Vector); var ch1Count = Vector.Dot(ch1Equals, _dotCount); if (ch0Count == 0 && ch1Count == 0) { index += vectorStride; continue; } else if (ch0Count < 2 && ch1Count < 2) { var ch0Index = ch0Count == 1 ? Vector.Dot(ch0Equals, _dotIndex) : byte.MaxValue; var ch1Index = ch1Count == 1 ? Vector.Dot(ch1Equals, _dotIndex) : byte.MaxValue; if (ch0Index < ch1Index) { _block = block; _index = index + ch0Index; return(char0); } else { _block = block; _index = index + ch1Index; return(char1); } } else { following = vectorStride; } } for (; following != 0; following--, index++) { var byteIndex = block.Array[index]; if (byteIndex == byte0) { _block = block; _index = index; return(char0); } else if (byteIndex == byte1) { _block = block; _index = index; return(char1); } } } } }
public unsafe void CopyFromAscii(string data) { Debug.Assert(_block != null); Debug.Assert(_block.Pool != null); Debug.Assert(_block.Next == null); Debug.Assert(_block.End == _index); var pool = _block.Pool; var block = _block; var blockIndex = _index; var length = data.Length; var bytesLeftInBlock = block.Data.Offset + block.Data.Count - blockIndex; var bytesLeftInBlockMinusSpan = bytesLeftInBlock - 3; fixed (char* pData = data) { var input = pData; var inputEnd = pData + length; var inputEndMinusSpan = inputEnd - 3; while (input < inputEnd) { if (bytesLeftInBlock == 0) { var nextBlock = pool.Lease(); block.End = blockIndex; block.Next = nextBlock; block = nextBlock; blockIndex = block.Data.Offset; bytesLeftInBlock = block.Data.Count; bytesLeftInBlockMinusSpan = bytesLeftInBlock - 3; } fixed (byte* pOutput = &block.Data.Array[block.End]) { //this line is needed to allow output be an register var var output = pOutput; var copied = 0; for (; input < inputEndMinusSpan && copied < bytesLeftInBlockMinusSpan; copied += 4) { *(output) = (byte)*(input); *(output + 1) = (byte)*(input + 1); *(output + 2) = (byte)*(input + 2); *(output + 3) = (byte)*(input + 3); output += 4; input += 4; } for (; input < inputEnd && copied < bytesLeftInBlock; copied++) { *(output++) = (byte)*(input++); } blockIndex += copied; bytesLeftInBlockMinusSpan -= copied; bytesLeftInBlock -= copied; } } } block.End = blockIndex; _block = block; _index = blockIndex; }
private void ProducingCompleteNoPreComplete(MemoryPoolIterator2 end) { MemoryPoolBlock2 blockToReturn = null; lock (_returnLock) { Debug.Assert(!_lastStart.IsDefault); // If the socket has been closed, return the produced blocks // instead of advancing the now non-existent tail. if (_tail != null) { _tail = end.Block; _tail.End = end.Index; } else { blockToReturn = _lastStart.Block; } _lastStart = default(MemoryPoolIterator2); } if (blockToReturn != null) { ThreadPool.QueueUserWorkItem(_returnBlocks, blockToReturn); } }
public int Take() { if (_block == null) { return -1; } else if (_index < _block.End) { return _block.Array[_index++]; } var block = _block; var index = _index; while (true) { if (index < block.End) { _block = block; _index = index + 1; return block.Array[index]; } else if (block.Next == null) { return -1; } else { block = block.Next; index = block.Start; } } }
/// <summary> /// Save the data at the current location then move to the next available space. /// </summary> /// <param name="data">The byte to be saved.</param> /// <returns>true if the operation successes. false if can't find available space.</returns> public bool Put(byte data) { if (_block == null) { return false; } else if (_index < _block.End) { _block.Array[_index++] = data; return true; } var block = _block; var index = _index; while (true) { if (index < block.End) { _block = block; _index = index + 1; block.Array[index] = data; return true; } else if (block.Next == null) { return false; } else { block = block.Next; index = block.Start; } } }
private void AssertIterator(MemoryPoolIterator2 iter, MemoryPoolBlock2 block, int index) { Assert.Same(block, iter.Block); Assert.Equal(index, iter.Index); }
public void IncomingComplete(int count, Exception error) { Action awaitableState; lock (_sync) { // Unpin may called without an earlier Pin if (_pinned != null) { _pinned.Unpin(); _pinned.End += count; if (_head == null) { _head = _tail = _pinned; } else if (_tail == _pinned) { // NO-OP: this was a read into unoccupied tail-space } else { _tail.Next = _pinned; _tail = _pinned; } } _pinned = null; if (count == 0) { RemoteIntakeFin = true; } if (error != null) { _awaitableError = error; } awaitableState = Interlocked.Exchange( ref _awaitableState, _awaitableIsCompleted); _manualResetEvent.Set(); } if (awaitableState != _awaitableIsCompleted && awaitableState != _awaitableIsNotCompleted) { Task.Run(awaitableState); } }
public int Seek(int char0) { if (IsDefault) { return -1; } var byte0 = (byte)char0; var vectorStride = Vector<byte>.Count; var ch0Vector = new Vector<byte>(byte0); var block = _block; var index = _index; var array = block.Array; while (true) { while (block.End == index) { if (block.Next == null) { _block = block; _index = index; return -1; } block = block.Next; index = block.Start; array = block.Array; } while (block.End != index) { var following = block.End - index; if (following >= vectorStride) { var data = new Vector<byte>(array, index); var ch0Equals = Vector.Equals(data, ch0Vector); var ch0Count = Vector.Dot(ch0Equals, _dotCount); if (ch0Count == 0) { index += vectorStride; continue; } else if (ch0Count == 1) { _block = block; _index = index + Vector.Dot(ch0Equals, _dotIndex); return char0; } else { following = vectorStride; } } while (following > 0) { if (block.Array[index] == byte0) { _block = block; _index = index; return char0; } following--; index++; } } } }
public unsafe int Seek(ref Vector <byte> byte0Vector) { if (IsDefault) { return(-1); } var block = _block; var index = _index; var following = block.End - index; byte[] array; var byte0 = byte0Vector[0]; while (true) { while (following == 0) { var newBlock = block.Next; if (newBlock == null) { _block = block; _index = index; return(-1); } index = newBlock.Start; following = newBlock.End - index; block = newBlock; } array = block.Array; while (following > 0) { // Need unit tests to test Vector path #if !DEBUG // Check will be Jitted away https://github.com/dotnet/coreclr/issues/1079 if (Vector.IsHardwareAccelerated) { #endif if (following >= _vectorSpan) { var byte0Equals = Vector.Equals(new Vector <byte>(array, index), byte0Vector); if (byte0Equals.Equals(Vector <byte> .Zero)) { following -= _vectorSpan; index += _vectorSpan; continue; } _block = block; _index = index + FindFirstEqualByte(ref byte0Equals); return(byte0); } // Need unit tests to test Vector path #if !DEBUG } #endif fixed(byte *ptr = &block.Array[index]) { var pCurrent = ptr; var pEnd = pCurrent + following; do { if (*pCurrent == byte0) { _block = block; _index = index; return(byte0); } pCurrent++; index++; } while (pCurrent < pEnd); } following = 0; break; } } }
// This is called on the libuv event loop private void ReturnAllBlocks() { lock (_returnLock) { var block = _head; while (block != _tail) { var returnBlock = block; block = block.Next; returnBlock.Pool?.Return(returnBlock); } // Only return the _tail if we aren't between ProducingStart/Complete calls if (_lastStart.IsDefault) { _tail.Pool?.Return(_tail); } _head = null; _tail = null; } }
public unsafe int Seek(ref Vector <byte> byte0Vector, ref Vector <byte> byte1Vector, ref Vector <byte> byte2Vector) { if (IsDefault) { return(-1); } var block = _block; var index = _index; var following = block.End - index; byte[] array; int byte0Index = int.MaxValue; int byte1Index = int.MaxValue; int byte2Index = int.MaxValue; var byte0 = byte0Vector[0]; var byte1 = byte1Vector[0]; var byte2 = byte2Vector[0]; while (true) { while (following == 0) { var newBlock = block.Next; if (newBlock == null) { _block = block; _index = index; return(-1); } index = newBlock.Start; following = newBlock.End - index; block = newBlock; } array = block.Array; while (following > 0) { // Need unit tests to test Vector path #if !DEBUG // Check will be Jitted away https://github.com/dotnet/coreclr/issues/1079 if (Vector.IsHardwareAccelerated) { #endif if (following >= _vectorSpan) { var data = new Vector <byte>(array, index); var byte0Equals = Vector.Equals(data, byte0Vector); var byte1Equals = Vector.Equals(data, byte1Vector); var byte2Equals = Vector.Equals(data, byte2Vector); if (!byte0Equals.Equals(Vector <byte> .Zero)) { byte0Index = FindFirstEqualByte(ref byte0Equals); } if (!byte1Equals.Equals(Vector <byte> .Zero)) { byte1Index = FindFirstEqualByte(ref byte1Equals); } if (!byte2Equals.Equals(Vector <byte> .Zero)) { byte2Index = FindFirstEqualByte(ref byte2Equals); } if (byte0Index == int.MaxValue && byte1Index == int.MaxValue && byte2Index == int.MaxValue) { following -= _vectorSpan; index += _vectorSpan; continue; } _block = block; int toReturn, toMove; if (byte0Index < byte1Index) { if (byte0Index < byte2Index) { toReturn = byte0; toMove = byte0Index; } else { toReturn = byte2; toMove = byte2Index; } } else { if (byte1Index < byte2Index) { toReturn = byte1; toMove = byte1Index; } else { toReturn = byte2; toMove = byte2Index; } } _index = index + toMove; return(toReturn); } // Need unit tests to test Vector path #if !DEBUG } #endif fixed(byte *ptr = &block.Array[index]) { var pCurrent = ptr; var pEnd = pCurrent + following; do { if (*pCurrent == byte0) { _block = block; _index = index; return(byte0); } if (*pCurrent == byte1) { _block = block; _index = index; return(byte1); } if (*pCurrent == byte2) { _block = block; _index = index; return(byte2); } pCurrent++; index++; } while (pCurrent != pEnd); } following = 0; break; } } }
public unsafe int Seek(ref Vector<byte> byte0Vector) { if (IsDefault) { return -1; } var block = _block; var index = _index; var following = block.End - index; byte[] array; var byte0 = byte0Vector[0]; while (true) { while (following == 0) { var newBlock = block.Next; if (newBlock == null) { _block = block; _index = index; return -1; } index = newBlock.Start; following = newBlock.End - index; block = newBlock; } array = block.Array; while (following > 0) { // Need unit tests to test Vector path #if !DEBUG // Check will be Jitted away https://github.com/dotnet/coreclr/issues/1079 if (Vector.IsHardwareAccelerated) { #endif if (following >= _vectorSpan) { var byte0Equals = Vector.Equals(new Vector<byte>(array, index), byte0Vector); if (byte0Equals.Equals(Vector<byte>.Zero)) { following -= _vectorSpan; index += _vectorSpan; continue; } _block = block; _index = index + FindFirstEqualByte(ref byte0Equals); return byte0; } // Need unit tests to test Vector path #if !DEBUG } #endif fixed (byte* ptr = &block.Array[index]) { var pCurrent = ptr; var pEnd = pCurrent + following; do { if (*pCurrent == byte0) { _block = block; _index = index; return byte0; } pCurrent++; index++; } while (pCurrent < pEnd); } following = 0; break; } } }
/// <summary> /// Called to return a block to the pool. Once Return has been called the memory no longer belongs to the caller, and /// Very Bad Things will happen if the memory is read of modified subsequently. If a caller fails to call Return and the /// block tracking object is garbage collected, the block tracking object's finalizer will automatically re-create and return /// a new tracking object into the pool. This will only happen if there is a bug in the server, however it is necessary to avoid /// leaving "dead zones" in the slab due to lost block tracking objects. /// </summary> /// <param name="block">The block to return. It must have been acquired by calling Lease on the same memory pool instance.</param> public void Return(MemoryPoolBlock2 block) { block.Reset(); _blocks.Enqueue(block); }
public unsafe void CopyFromAscii(string data) { Debug.Assert(_block != null); Debug.Assert(_block.Pool != null); Debug.Assert(_block.Next == null); Debug.Assert(_block.End == _index); var pool = _block.Pool; var block = _block; var blockIndex = _index; var length = data.Length; var bytesLeftInBlock = block.Data.Offset + block.Data.Count - blockIndex; var bytesLeftInBlockMinusSpan = bytesLeftInBlock - 3; fixed(char *pData = data) { var input = pData; var inputEnd = pData + length; var inputEndMinusSpan = inputEnd - 3; while (input < inputEnd) { if (bytesLeftInBlock == 0) { var nextBlock = pool.Lease(); block.End = blockIndex; block.Next = nextBlock; block = nextBlock; blockIndex = block.Data.Offset; bytesLeftInBlock = block.Data.Count; bytesLeftInBlockMinusSpan = bytesLeftInBlock - 3; } fixed(byte *pOutput = &block.Data.Array[block.End]) { //this line is needed to allow output be an register var var output = pOutput; var copied = 0; for (; input < inputEndMinusSpan && copied < bytesLeftInBlockMinusSpan; copied += 4) { *(output) = (byte)*(input); *(output + 1) = (byte)*(input + 1); *(output + 2) = (byte)*(input + 2); *(output + 3) = (byte)*(input + 3); output += 4; input += 4; } for (; input < inputEnd && copied < bytesLeftInBlock; copied++) { *(output++) = (byte)*(input++); } blockIndex += copied; bytesLeftInBlockMinusSpan -= copied; bytesLeftInBlock -= copied; } } } block.End = blockIndex; _block = block; _index = blockIndex; }
public MemoryPoolIterator2 ProducingStart() { _producingBlock = _memory.Lease(); return new MemoryPoolIterator2(_producingBlock); }