/// <summary>
        /// Internal method called when a block is requested and the pool is empty. It allocates one additional slab, creates all of the
        /// block tracking objects, and adds them all to the pool.
        /// </summary>
        private MemoryPoolBlock2 AllocateSlab()
        {
            var slab = MemoryPoolSlab2.Create(_slabLength);

            _slabs.Push(slab);

            var basePtr     = slab.ArrayPtr;
            var firstOffset = (int)((_blockStride - 1) - ((ulong)(basePtr + _blockStride - 1) % _blockStride));

            var poolAllocationLength = _slabLength - _blockStride;

            var offset = firstOffset;

            for (;
                 offset + _blockLength < poolAllocationLength;
                 offset += _blockStride)
            {
                var block = MemoryPoolBlock2.Create(
                    new ArraySegment <byte>(slab.Array, offset, _blockLength),
                    basePtr,
                    this,
                    slab);
                Return(block);
            }

            // return last block rather than adding to pool
            var newBlock = MemoryPoolBlock2.Create(
                new ArraySegment <byte>(slab.Array, offset, _blockLength),
                basePtr,
                this,
                slab);

            return(newBlock);
        }
Example #2
0
        public int Take()
        {
            if (_block == null)
            {
                return(-1);
            }
            else if (_index < _block.End)
            {
                return(_block.Array[_index++]);
            }

            var block = _block;
            var index = _index;

            while (true)
            {
                if (index < block.End)
                {
                    _block = block;
                    _index = index + 1;
                    return(block.Array[index]);
                }
                else if (block.Next == null)
                {
                    return(-1);
                }
                else
                {
                    block = block.Next;
                    index = block.Start;
                }
            }
        }
        public void ConsumingComplete(
            MemoryPoolIterator2 consumed,
            MemoryPoolIterator2 examined)
        {
            MemoryPoolBlock2 returnStart = null;
            MemoryPoolBlock2 returnEnd = null;
            lock (_sync)
            {
                if (!consumed.IsDefault)
                {
                    returnStart = _head;
                    returnEnd = consumed.Block;
                    _head = consumed.Block;
                    _head.Start = consumed.Index;
                }
                if (!examined.IsDefault &&
                    examined.IsEnd &&
                    RemoteIntakeFin == false &&
                    _awaitableError == null)
                {
                    _manualResetEvent.Reset();

                    var awaitableState = Interlocked.CompareExchange(
                        ref _awaitableState,
                        _awaitableIsNotCompleted,
                        _awaitableIsCompleted);
                }
            }
            while (returnStart != returnEnd)
            {
                var returnBlock = returnStart;
                returnStart = returnStart.Next;
                returnBlock.Pool.Return(returnBlock);
            }
        }
        /// <summary>
        /// Save the data at the current location then move to the next available space.
        /// </summary>
        /// <param name="data">The byte to be saved.</param>
        /// <returns>true if the operation successes. false if can't find available space.</returns>
        public bool Put(byte data)
        {
            if (_block == null)
            {
                return(false);
            }
            else if (_index < _block.End)
            {
                _block.Array[_index++] = data;
                return(true);
            }

            var block = _block;
            var index = _index;

            while (true)
            {
                if (index < block.End)
                {
                    _block             = block;
                    _index             = index + 1;
                    block.Array[index] = data;
                    return(true);
                }
                else if (block.Next == null)
                {
                    return(false);
                }
                else
                {
                    block = block.Next;
                    index = block.Start;
                }
            }
        }
        private bool _disposedValue = false; // To detect redundant calls

        /// <summary>
        /// Called to take a block from the pool.
        /// </summary>
        /// <param name="minimumSize">The block returned must be at least this size. It may be larger than this minimum size, and if so,
        /// the caller may write to the block's entire size rather than being limited to the minumumSize requested.</param>
        /// <returns>The block that is reserved for the called. It must be passed to Return when it is no longer being used.</returns>
        public MemoryPoolBlock2 Lease(int minimumSize)
        {
            if (minimumSize > _blockLength)
            {
                // The requested minimumSize is actually larger then the usable memory of a single block.
                // Because this is the degenerate case, a one-time-use byte[] array and tracking object are allocated.
                // When this block tracking object is returned it is not added to the pool - instead it will be
                // allowed to be garbage collected normally.
                return(MemoryPoolBlock2.Create(
                           new ArraySegment <byte>(new byte[minimumSize]),
                           dataPtr: IntPtr.Zero,
                           pool: null,
                           slab: null));
            }

            while (true)
            {
                MemoryPoolBlock2 block;
                if (_blocks.TryPop(out block))
                {
                    // block successfully taken from the stack - return it
                    return(block);
                }
                // no blocks available - grow the pool and try again
                AllocateSlab();
            }
        }
        public void Put()
        {
            var blocks = new MemoryPoolBlock2[4];
            for (var i = 0; i < 4; ++i)
            {
                blocks[i] = _pool.Lease(16);
                blocks[i].End += 16;

                for (var j = 0; j < blocks.Length; ++j)
                {
                    blocks[i].Array[blocks[i].Start + j] = 0x00;
                }

                if (i != 0)
                {
                    blocks[i - 1].Next = blocks[i];
                }
            }

            // put FF at first block's head
            var head = blocks[0].GetIterator();
            Assert.True(head.Put(0xFF));

            // data is put at correct position
            Assert.Equal(0xFF, blocks[0].Array[blocks[0].Start]);
            Assert.Equal(0x00, blocks[0].Array[blocks[0].Start + 1]);

            // iterator is moved to next byte after put
            Assert.Equal(1, head.Index - blocks[0].Start);

            for (var i = 0; i < 14; ++i)
            {
                // move itr to the end of the block 0
                head.Take();
            }

            // write to the end of block 0
            Assert.True(head.Put(0xFE));
            Assert.Equal(0xFE, blocks[0].Array[blocks[0].End - 1]);
            Assert.Equal(0x00, blocks[1].Array[blocks[1].Start]);

            // put data across the block link
            Assert.True(head.Put(0xFD));
            Assert.Equal(0xFD, blocks[1].Array[blocks[1].Start]);
            Assert.Equal(0x00, blocks[1].Array[blocks[1].Start + 1]);

            // paint every block
            head = blocks[0].GetIterator();
            for (var i = 0; i < 64; ++i)
            {
                Assert.True(head.Put((byte)i), $"Fail to put data at {i}.");
            }

            // Can't put anything by the end
            Assert.False(head.Put(0xFF));
        }
        public static async Task<MemoryPoolBlock2> CopyToAsync(this Stream source, Stream destination, MemoryPoolBlock2 block)
        {
            int bytesRead;
            while ((bytesRead = await source.ReadAsync(block.Array, block.Data.Offset, block.Data.Count)) != 0)
            {
                await destination.WriteAsync(block.Array, block.Data.Offset, bytesRead);
            }

            return block;
        }
 private void TestAllLengths(MemoryPoolBlock2 block, int lengths)
 {
     for (var firstIndex = 0; firstIndex <= lengths; ++firstIndex)
     {
         for (var lastIndex = firstIndex; lastIndex <= lengths; ++lastIndex)
         {
             var first = block.GetIterator().Add(firstIndex);
             var last = block.GetIterator().Add(lastIndex);
             Assert.Equal(firstIndex, block.GetIterator().GetLength(first));
             Assert.Equal(lastIndex, block.GetIterator().GetLength(last));
             Assert.Equal(lastIndex - firstIndex, first.GetLength(last));
         }
     }
 }
        public void CopyFrom(byte[] data, int offset, int count)
        {
            Debug.Assert(_block != null);
            Debug.Assert(_block.Pool != null);
            Debug.Assert(_block.Next == null);
            Debug.Assert(_block.End == _index);

            var pool       = _block.Pool;
            var block      = _block;
            var blockIndex = _index;

            var bufferIndex      = offset;
            var remaining        = count;
            var bytesLeftInBlock = block.Data.Offset + block.Data.Count - blockIndex;

            while (remaining > 0)
            {
                if (bytesLeftInBlock == 0)
                {
                    var nextBlock = pool.Lease();
                    block.End  = blockIndex;
                    block.Next = nextBlock;
                    block      = nextBlock;

                    blockIndex       = block.Data.Offset;
                    bytesLeftInBlock = block.Data.Count;
                }

                var bytesToCopy = remaining < bytesLeftInBlock ? remaining : bytesLeftInBlock;

                Buffer.BlockCopy(data, bufferIndex, block.Array, blockIndex, bytesToCopy);

                blockIndex       += bytesToCopy;
                bufferIndex      += bytesToCopy;
                remaining        -= bytesToCopy;
                bytesLeftInBlock -= bytesToCopy;
            }

            block.End = blockIndex;
            _block    = block;
            _index    = blockIndex;
        }
Example #10
0
        /// <summary>
        /// Internal method called when a block is requested and the pool is empty. It allocates one additional slab, creates all of the
        /// block tracking objects, and adds them all to the pool.
        /// </summary>
        private void AllocateSlab()
        {
            var slab = MemoryPoolSlab2.Create(_slabLength);

            _slabs.Push(slab);

            var basePtr     = slab.ArrayPtr;
            var firstOffset = (int)((_blockStride - 1) - ((ulong)(basePtr + _blockStride - 1) % _blockStride));

            for (var offset = firstOffset;
                 offset + _blockLength <= _slabLength;
                 offset += _blockStride)
            {
                var block = MemoryPoolBlock2.Create(
                    new ArraySegment <byte>(slab.Array, offset, _blockLength),
                    basePtr,
                    this,
                    slab);
                Return(block);
            }
        }
        public void Skip(int bytesToSkip)
        {
            if (_block == null)
            {
                return;
            }
            var following = _block.End - _index;

            if (following >= bytesToSkip)
            {
                _index += bytesToSkip;
                return;
            }

            var block = _block;
            var index = _index;

            while (true)
            {
                if (block.Next == null)
                {
                    return;
                }
                else
                {
                    bytesToSkip -= following;
                    block        = block.Next;
                    index        = block.Start;
                }
                following = block.End - index;
                if (following >= bytesToSkip)
                {
                    _block = block;
                    _index = index + bytesToSkip;
                    return;
                }
            }
        }
        public int Take()
        {
            var block = _block;

            if (block == null)
            {
                return(-1);
            }

            var index = _index;

            if (index < block.End)
            {
                _index = index + 1;
                return(block.Array[index]);
            }

            do
            {
                if (block.Next == null)
                {
                    return(-1);
                }
                else
                {
                    block = block.Next;
                    index = block.Start;
                }

                if (index < block.End)
                {
                    _block = block;
                    _index = index + 1;
                    return(block.Array[index]);
                }
            } while (true);
        }
        public SocketOutput(
            KestrelThread thread,
            UvStreamHandle socket,
            MemoryPool2 memory,
            Connection connection,
            long connectionId,
            IKestrelTrace log,
            IThreadPool threadPool,
            Queue<UvWriteReq> writeReqPool)
        {
            _thread = thread;
            _socket = socket;
            _connection = connection;
            _connectionId = connectionId;
            _log = log;
            _threadPool = threadPool;
            _tasksPending = new Queue<TaskCompletionSource<object>>(_initialTaskQueues);
            _tasksCompleted = new Queue<TaskCompletionSource<object>>(_initialTaskQueues);
            _writeContextPool = new Queue<WriteContext>(_maxPooledWriteContexts);
            _writeReqPool = writeReqPool;

            _head = memory.Lease();
            _tail = _head;
        }
        public IncomingBuffer IncomingStart(int minimumSize)
        {
            lock (_sync)
            {
                if (_tail != null && minimumSize <= _tail.Data.Offset + _tail.Data.Count - _tail.End)
                {
                    _pinned = _tail;
                    var data = new ArraySegment<byte>(_pinned.Data.Array, _pinned.End, _pinned.Data.Offset + _pinned.Data.Count - _pinned.End);
                    var dataPtr = _pinned.Pin();
                    return new IncomingBuffer
                    {
                        Data = data,
                        DataPtr = dataPtr,
                    };
                }
            }

            _pinned = _memory.Lease(minimumSize);
            return new IncomingBuffer
            {
                Data = _pinned.Data,
                DataPtr = _pinned.Pin()
            };
        }
 public MemoryPoolIterator2(MemoryPoolBlock2 block)
 {
     _block = block;
     _index = _block?.Start ?? 0;
 }
        private static void ReturnBlocks(MemoryPoolBlock2 block)
        {
            while (block != null)
            {
                var returningBlock = block;
                block = returningBlock.Next;

                returningBlock.Pool?.Return(returningBlock);
            }
        }
        public int Seek(int char0, int char1)
        {
            if (IsDefault)
            {
                return -1;
            }

            var byte0 = (byte)char0;
            var byte1 = (byte)char1;
            var vectorStride = Vector<byte>.Count;
            var ch0Vector = new Vector<byte>(byte0);
            var ch1Vector = new Vector<byte>(byte1);

            var block = _block;
            var index = _index;
            var array = block.Array;
            while (true)
            {
                while (block.End == index)
                {
                    if (block.Next == null)
                    {
                        _block = block;
                        _index = index;
                        return -1;
                    }
                    block = block.Next;
                    index = block.Start;
                    array = block.Array;
                }
                while (block.End != index)
                {
                    var following = block.End - index;
                    if (following >= vectorStride)
                    {
                        var data = new Vector<byte>(array, index);
                        var ch0Equals = Vector.Equals(data, ch0Vector);
                        var ch0Count = Vector.Dot(ch0Equals, _dotCount);
                        var ch1Equals = Vector.Equals(data, ch1Vector);
                        var ch1Count = Vector.Dot(ch1Equals, _dotCount);

                        if (ch0Count == 0 && ch1Count == 0)
                        {
                            index += vectorStride;
                            continue;
                        }
                        else if (ch0Count < 2 && ch1Count < 2)
                        {
                            var ch0Index = ch0Count == 1 ? Vector.Dot(ch0Equals, _dotIndex) : byte.MaxValue;
                            var ch1Index = ch1Count == 1 ? Vector.Dot(ch1Equals, _dotIndex) : byte.MaxValue;
                            if (ch0Index < ch1Index)
                            {
                                _block = block;
                                _index = index + ch0Index;
                                return char0;
                            }
                            else
                            {
                                _block = block;
                                _index = index + ch1Index;
                                return char1;
                            }
                        }
                        else
                        {
                            following = vectorStride;
                        }
                    }
                    for (; following != 0; following--, index++)
                    {
                        var byteIndex = block.Array[index];
                        if (byteIndex == byte0)
                        {
                            _block = block;
                            _index = index;
                            return char0;
                        }
                        else if (byteIndex == byte1)
                        {
                            _block = block;
                            _index = index;
                            return char1;
                        }
                    }
                }
            }
        }
 public MemoryPoolIterator2(MemoryPoolBlock2 block, int index)
 {
     _block = block;
     _index = index;
 }
        public unsafe int Seek(ref Vector<byte> byte0Vector, ref Vector<byte> byte1Vector, ref Vector<byte> byte2Vector)
        {
            if (IsDefault)
            {
                return -1;
            }

            var block = _block;
            var index = _index;
            var following = block.End - index;
            byte[] array;
            int byte0Index = int.MaxValue;
            int byte1Index = int.MaxValue;
            int byte2Index = int.MaxValue;
            var byte0 = byte0Vector[0];
            var byte1 = byte1Vector[0];
            var byte2 = byte2Vector[0];

            while (true)
            {
                while (following == 0)
                {
                    var newBlock = block.Next;
                    if (newBlock == null)
                    {
                        _block = block;
                        _index = index;
                        return -1;
                    }
                    index = newBlock.Start;
                    following = newBlock.End - index;
                    block = newBlock;
                }
                array = block.Array;
                while (following > 0)
                {
// Need unit tests to test Vector path
#if !DEBUG 
                    // Check will be Jitted away https://github.com/dotnet/coreclr/issues/1079
                    if (Vector.IsHardwareAccelerated)
                    {
#endif
                        if (following >= _vectorSpan)
                        {
                            var data = new Vector<byte>(array, index);
                            var byte0Equals = Vector.Equals(data, byte0Vector);
                            var byte1Equals = Vector.Equals(data, byte1Vector);
                            var byte2Equals = Vector.Equals(data, byte2Vector);

                            if (!byte0Equals.Equals(Vector<byte>.Zero))
                            {
                                byte0Index = FindFirstEqualByte(ref byte0Equals);
                            }
                            if (!byte1Equals.Equals(Vector<byte>.Zero))
                            {
                                byte1Index = FindFirstEqualByte(ref byte1Equals);
                            }
                            if (!byte2Equals.Equals(Vector<byte>.Zero))
                            {
                                byte2Index = FindFirstEqualByte(ref byte2Equals);
                            }

                            if (byte0Index == int.MaxValue && byte1Index == int.MaxValue && byte2Index == int.MaxValue)
                            {
                                following -= _vectorSpan;
                                index += _vectorSpan;
                                continue;
                            }

                            _block = block;

                            int toReturn, toMove;
                            if (byte0Index < byte1Index)
                            {
                                if (byte0Index < byte2Index)
                                {
                                    toReturn = byte0;
                                    toMove = byte0Index;
                                }
                                else
                                {
                                    toReturn = byte2;
                                    toMove = byte2Index;
                                }
                            }
                            else
                            {
                                if (byte1Index < byte2Index)
                                {
                                    toReturn = byte1;
                                    toMove = byte1Index;
                                }
                                else
                                {
                                    toReturn = byte2;
                                    toMove = byte2Index;
                                }
                            }

                            _index = index + toMove;
                            return toReturn;
                        }
// Need unit tests to test Vector path
#if !DEBUG 
                    }
#endif
                    fixed (byte* ptr = &block.Array[index])
                    {
                        var pCurrent = ptr;
                        var pEnd = pCurrent + following;
                        do
                        {
                            if (*pCurrent == byte0)
                            {
                                _block = block;
                                _index = index;
                                return byte0;
                            }
                            if (*pCurrent == byte1)
                            {
                                _block = block;
                                _index = index;
                                return byte1;
                            }
                            if (*pCurrent == byte2)
                            {
                                _block = block;
                                _index = index;
                                return byte2;
                            }
                            pCurrent++;
                            index++;
                        } while (pCurrent != pEnd);
                    }

                    following = 0;
                    break;
                }
            }
        }
        public int Take()
        {
            var block = _block;
            if (block == null)
            {
                return -1;
            }

            var index = _index;

            if (index < block.End)
            {
                _index = index + 1;
                return block.Array[index];
            }

            do
            {
                if (block.Next == null)
                {
                    return -1;
                }
                else
                {
                    block = block.Next;
                    index = block.Start;
                }

                if (index < block.End)
                {
                    _block = block;
                    _index = index + 1;
                    return block.Array[index];
                }
            } while (true);
        }
 public MemoryPoolIterator2(MemoryPoolBlock2 block)
 {
     _block = block;
     _index = _block?.Start ?? 0;
 }
            private static void ReturnWrittenBlocks(MemoryPoolBlock2 block)
            {
                while (block != null)
                {
                    var returnBlock = block;
                    block = block.Next;

                    returnBlock.Unpin();
                    returnBlock.Pool?.Return(returnBlock);
                }
            }
 /// <summary>
 /// Called to return a block to the pool. Once Return has been called the memory no longer belongs to the caller, and
 /// Very Bad Things will happen if the memory is read of modified subsequently. If a caller fails to call Return and the
 /// block tracking object is garbage collected, the block tracking object's finalizer will automatically re-create and return
 /// a new tracking object into the pool. This will only happen if there is a bug in the server, however it is necessary to avoid
 /// leaving "dead zones" in the slab due to lost block tracking objects.
 /// </summary>
 /// <param name="block">The block to return. It must have been acquired by calling Lease on the same memory pool instance.</param>
 public void Return(MemoryPoolBlock2 block)
 {
     block.Reset();
     _blocks.Push(block);
 }
        public int Seek(int char0)
        {
            if (IsDefault)
            {
                return(-1);
            }

            var byte0        = (byte)char0;
            var vectorStride = Vector <byte> .Count;
            var ch0Vector    = new Vector <byte>(byte0);

            var block = _block;
            var index = _index;
            var array = block.Array;

            while (true)
            {
                while (block.End == index)
                {
                    if (block.Next == null)
                    {
                        _block = block;
                        _index = index;
                        return(-1);
                    }
                    block = block.Next;
                    index = block.Start;
                    array = block.Array;
                }
                while (block.End != index)
                {
                    var following = block.End - index;
                    if (following >= vectorStride)
                    {
                        var data      = new Vector <byte>(array, index);
                        var ch0Equals = Vector.Equals(data, ch0Vector);
                        var ch0Count  = Vector.Dot(ch0Equals, _dotCount);

                        if (ch0Count == 0)
                        {
                            index += vectorStride;
                            continue;
                        }
                        else if (ch0Count == 1)
                        {
                            _block = block;
                            _index = index + Vector.Dot(ch0Equals, _dotIndex);
                            return(char0);
                        }
                        else
                        {
                            following = vectorStride;
                        }
                    }
                    while (following > 0)
                    {
                        if (block.Array[index] == byte0)
                        {
                            _block = block;
                            _index = index;
                            return(char0);
                        }
                        following--;
                        index++;
                    }
                }
            }
        }
Example #25
0
        public int Seek(int char0, int char1)
        {
            if (IsDefault)
            {
                return(-1);
            }

            var byte0        = (byte)char0;
            var byte1        = (byte)char1;
            var vectorStride = Vector <byte> .Count;
            var ch0Vector    = new Vector <byte>(byte0);
            var ch1Vector    = new Vector <byte>(byte1);

            var block = _block;
            var index = _index;
            var array = block.Array;

            while (true)
            {
                while (block.End == index)
                {
                    if (block.Next == null)
                    {
                        _block = block;
                        _index = index;
                        return(-1);
                    }
                    block = block.Next;
                    index = block.Start;
                    array = block.Array;
                }
                while (block.End != index)
                {
                    var following = block.End - index;
                    if (following >= vectorStride)
                    {
                        var data      = new Vector <byte>(array, index);
                        var ch0Equals = Vector.Equals(data, ch0Vector);
                        var ch0Count  = Vector.Dot(ch0Equals, _dotCount);
                        var ch1Equals = Vector.Equals(data, ch1Vector);
                        var ch1Count  = Vector.Dot(ch1Equals, _dotCount);

                        if (ch0Count == 0 && ch1Count == 0)
                        {
                            index += vectorStride;
                            continue;
                        }
                        else if (ch0Count < 2 && ch1Count < 2)
                        {
                            var ch0Index = ch0Count == 1 ? Vector.Dot(ch0Equals, _dotIndex) : byte.MaxValue;
                            var ch1Index = ch1Count == 1 ? Vector.Dot(ch1Equals, _dotIndex) : byte.MaxValue;
                            if (ch0Index < ch1Index)
                            {
                                _block = block;
                                _index = index + ch0Index;
                                return(char0);
                            }
                            else
                            {
                                _block = block;
                                _index = index + ch1Index;
                                return(char1);
                            }
                        }
                        else
                        {
                            following = vectorStride;
                        }
                    }
                    for (; following != 0; following--, index++)
                    {
                        var byteIndex = block.Array[index];
                        if (byteIndex == byte0)
                        {
                            _block = block;
                            _index = index;
                            return(char0);
                        }
                        else if (byteIndex == byte1)
                        {
                            _block = block;
                            _index = index;
                            return(char1);
                        }
                    }
                }
            }
        }
        public void Skip(int bytesToSkip)
        {
            if (_block == null)
            {
                return;
            }
            var following = _block.End - _index;
            if (following >= bytesToSkip)
            {
                _index += bytesToSkip;
                return;
            }

            var block = _block;
            var index = _index;
            while (true)
            {
                if (block.Next == null)
                {
                    return;
                }
                else
                {
                    bytesToSkip -= following;
                    block = block.Next;
                    index = block.Start;
                }
                following = block.End - index;
                if (following >= bytesToSkip)
                {
                    _block = block;
                    _index = index + bytesToSkip;
                    return;
                }
            }
        }
        public unsafe void CopyFromAscii(string data)
        {
            Debug.Assert(_block != null);
            Debug.Assert(_block.Pool != null);
            Debug.Assert(_block.Next == null);
            Debug.Assert(_block.End == _index);

            var pool = _block.Pool;
            var block = _block;
            var blockIndex = _index;
            var length = data.Length;

            var bytesLeftInBlock = block.Data.Offset + block.Data.Count - blockIndex;
            var bytesLeftInBlockMinusSpan = bytesLeftInBlock - 3;

            fixed (char* pData = data)
            {
                var input = pData;
                var inputEnd = pData + length;
                var inputEndMinusSpan = inputEnd - 3;

                while (input < inputEnd)
                {
                    if (bytesLeftInBlock == 0)
                    {
                        var nextBlock = pool.Lease();
                        block.End = blockIndex;
                        block.Next = nextBlock;
                        block = nextBlock;

                        blockIndex = block.Data.Offset;
                        bytesLeftInBlock = block.Data.Count;
                        bytesLeftInBlockMinusSpan = bytesLeftInBlock - 3;
                    }

                    fixed (byte* pOutput = &block.Data.Array[block.End])
                    {
                        //this line is needed to allow output be an register var 
                        var output = pOutput;

                        var copied = 0;
                        for (; input < inputEndMinusSpan && copied < bytesLeftInBlockMinusSpan; copied += 4)
                        {
                            *(output) = (byte)*(input);
                            *(output + 1) = (byte)*(input + 1);
                            *(output + 2) = (byte)*(input + 2);
                            *(output + 3) = (byte)*(input + 3);
                            output += 4;
                            input += 4;
                        }
                        for (; input < inputEnd && copied < bytesLeftInBlock; copied++)
                        {
                            *(output++) = (byte)*(input++);
                        }

                        blockIndex += copied;
                        bytesLeftInBlockMinusSpan -= copied;
                        bytesLeftInBlock -= copied;
                    }
                }
            }

            block.End = blockIndex;
            _block = block;
            _index = blockIndex;
        }
        public void CopyFrom(byte[] data, int offset, int count)
        {
            Debug.Assert(_block != null);
            Debug.Assert(_block.Pool != null);
            Debug.Assert(_block.Next == null);
            Debug.Assert(_block.End == _index);

            var pool = _block.Pool;
            var block = _block;
            var blockIndex = _index;

            var bufferIndex = offset;
            var remaining = count;
            var bytesLeftInBlock = block.Data.Offset + block.Data.Count - blockIndex;

            while (remaining > 0)
            {
                if (bytesLeftInBlock == 0)
                {
                    var nextBlock = pool.Lease();
                    block.End = blockIndex;
                    block.Next = nextBlock;
                    block = nextBlock;

                    blockIndex = block.Data.Offset;
                    bytesLeftInBlock = block.Data.Count;
                }

                var bytesToCopy = remaining < bytesLeftInBlock ? remaining : bytesLeftInBlock;

                Buffer.BlockCopy(data, bufferIndex, block.Array, blockIndex, bytesToCopy);

                blockIndex += bytesToCopy;
                bufferIndex += bytesToCopy;
                remaining -= bytesToCopy;
                bytesLeftInBlock -= bytesToCopy;
            }

            block.End = blockIndex;
            _block = block;
            _index = blockIndex;
        }
 public MemoryPoolIterator2(MemoryPoolBlock2 block, int index)
 {
     _block = block;
     _index = index;
 }
        private void ProducingCompleteNoPreComplete(MemoryPoolIterator2 end)
        {
            MemoryPoolBlock2 blockToReturn = null;

            lock (_returnLock)
            {
                Debug.Assert(!_lastStart.IsDefault);

                // If the socket has been closed, return the produced blocks
                // instead of advancing the now non-existent tail.
                if (_tail != null)
                {
                    _tail = end.Block;
                    _tail.End = end.Index;
                }
                else
                {
                    blockToReturn = _lastStart.Block;
                }

                _lastStart = default(MemoryPoolIterator2);
            }

            if (blockToReturn != null)
            {
                ThreadPool.QueueUserWorkItem(_returnBlocks, blockToReturn);
            }
        }
        public int Take()
        {
            if (_block == null)
            {
                return -1;
            }
            else if (_index < _block.End)
            {
                return _block.Array[_index++];
            }

            var block = _block;
            var index = _index;
            while (true)
            {
                if (index < block.End)
                {
                    _block = block;
                    _index = index + 1;
                    return block.Array[index];
                }
                else if (block.Next == null)
                {
                    return -1;
                }
                else
                {
                    block = block.Next;
                    index = block.Start;
                }
            }
        }
        /// <summary>
        /// Save the data at the current location then move to the next available space.
        /// </summary>
        /// <param name="data">The byte to be saved.</param>
        /// <returns>true if the operation successes. false if can't find available space.</returns>
        public bool Put(byte data)
        {
            if (_block == null)
            {
                return false;
            }
            else if (_index < _block.End)
            {
                _block.Array[_index++] = data;
                return true;
            }

            var block = _block;
            var index = _index;
            while (true)
            {
                if (index < block.End)
                {
                    _block = block;
                    _index = index + 1;
                    block.Array[index] = data;
                    return true;
                }
                else if (block.Next == null)
                {
                    return false;
                }
                else
                {
                    block = block.Next;
                    index = block.Start;
                }
            }
        }
 private void AssertIterator(MemoryPoolIterator2 iter, MemoryPoolBlock2 block, int index)
 {
     Assert.Same(block, iter.Block);
     Assert.Equal(index, iter.Index);
 }
        public void IncomingComplete(int count, Exception error)
        {
            Action awaitableState;

            lock (_sync)
            {
                // Unpin may called without an earlier Pin 
                if (_pinned != null)
                {
                    _pinned.Unpin();

                    _pinned.End += count;
                    if (_head == null)
                    {
                        _head = _tail = _pinned;
                    }
                    else if (_tail == _pinned)
                    {
                        // NO-OP: this was a read into unoccupied tail-space
                    }
                    else
                    {
                        _tail.Next = _pinned;
                        _tail = _pinned;
                    }
                }
                _pinned = null;

                if (count == 0)
                {
                    RemoteIntakeFin = true;
                }
                if (error != null)
                {
                    _awaitableError = error;
                }

                awaitableState = Interlocked.Exchange(
                    ref _awaitableState,
                    _awaitableIsCompleted);

                _manualResetEvent.Set();
            }

            if (awaitableState != _awaitableIsCompleted &&
                awaitableState != _awaitableIsNotCompleted)
            {
                Task.Run(awaitableState);
            }
        }
        public int Seek(int char0)
        {
            if (IsDefault)
            {
                return -1;
            }

            var byte0 = (byte)char0;
            var vectorStride = Vector<byte>.Count;
            var ch0Vector = new Vector<byte>(byte0);

            var block = _block;
            var index = _index;
            var array = block.Array;
            while (true)
            {
                while (block.End == index)
                {
                    if (block.Next == null)
                    {
                        _block = block;
                        _index = index;
                        return -1;
                    }
                    block = block.Next;
                    index = block.Start;
                    array = block.Array;
                }
                while (block.End != index)
                {
                    var following = block.End - index;
                    if (following >= vectorStride)
                    {
                        var data = new Vector<byte>(array, index);
                        var ch0Equals = Vector.Equals(data, ch0Vector);
                        var ch0Count = Vector.Dot(ch0Equals, _dotCount);

                        if (ch0Count == 0)
                        {
                            index += vectorStride;
                            continue;
                        }
                        else if (ch0Count == 1)
                        {
                            _block = block;
                            _index = index + Vector.Dot(ch0Equals, _dotIndex);
                            return char0;
                        }
                        else
                        {
                            following = vectorStride;
                        }
                    }
                    while (following > 0)
                    {
                        if (block.Array[index] == byte0)
                        {
                            _block = block;
                            _index = index;
                            return char0;
                        }
                        following--;
                        index++;
                    }
                }
            }
        }
Example #36
0
 /// <summary>
 /// Called to return a block to the pool. Once Return has been called the memory no longer belongs to the caller, and
 /// Very Bad Things will happen if the memory is read of modified subsequently. If a caller fails to call Return and the
 /// block tracking object is garbage collected, the block tracking object's finalizer will automatically re-create and return
 /// a new tracking object into the pool. This will only happen if there is a bug in the server, however it is necessary to avoid
 /// leaving "dead zones" in the slab due to lost block tracking objects.
 /// </summary>
 /// <param name="block">The block to return. It must have been acquired by calling Lease on the same memory pool instance.</param>
 public void Return(MemoryPoolBlock2 block)
 {
     block.Reset();
     _blocks.Push(block);
 }
        public unsafe int Seek(ref Vector <byte> byte0Vector)
        {
            if (IsDefault)
            {
                return(-1);
            }

            var block     = _block;
            var index     = _index;
            var following = block.End - index;

            byte[] array;
            var    byte0 = byte0Vector[0];

            while (true)
            {
                while (following == 0)
                {
                    var newBlock = block.Next;
                    if (newBlock == null)
                    {
                        _block = block;
                        _index = index;
                        return(-1);
                    }
                    index     = newBlock.Start;
                    following = newBlock.End - index;
                    block     = newBlock;
                }
                array = block.Array;
                while (following > 0)
                {
// Need unit tests to test Vector path
#if !DEBUG
                    // Check will be Jitted away https://github.com/dotnet/coreclr/issues/1079
                    if (Vector.IsHardwareAccelerated)
                    {
#endif
                    if (following >= _vectorSpan)
                    {
                        var byte0Equals = Vector.Equals(new Vector <byte>(array, index), byte0Vector);

                        if (byte0Equals.Equals(Vector <byte> .Zero))
                        {
                            following -= _vectorSpan;
                            index     += _vectorSpan;
                            continue;
                        }

                        _block = block;
                        _index = index + FindFirstEqualByte(ref byte0Equals);
                        return(byte0);
                    }
// Need unit tests to test Vector path
#if !DEBUG
                }
#endif
                    fixed(byte *ptr = &block.Array[index])
                    {
                        var pCurrent = ptr;
                        var pEnd     = pCurrent + following;

                        do
                        {
                            if (*pCurrent == byte0)
                            {
                                _block = block;
                                _index = index;
                                return(byte0);
                            }
                            pCurrent++;
                            index++;
                        } while (pCurrent < pEnd);
                    }

                    following = 0;
                    break;
                }
            }
        }
        // This is called on the libuv event loop
        private void ReturnAllBlocks()
        {
            lock (_returnLock)
            {
                var block = _head;
                while (block != _tail)
                {
                    var returnBlock = block;
                    block = block.Next;

                    returnBlock.Pool?.Return(returnBlock);
                }

                // Only return the _tail if we aren't between ProducingStart/Complete calls
                if (_lastStart.IsDefault)
                {
                    _tail.Pool?.Return(_tail);
                }

                _head = null;
                _tail = null;
            }
        }
        public unsafe int Seek(ref Vector <byte> byte0Vector, ref Vector <byte> byte1Vector, ref Vector <byte> byte2Vector)
        {
            if (IsDefault)
            {
                return(-1);
            }

            var block     = _block;
            var index     = _index;
            var following = block.End - index;

            byte[] array;
            int    byte0Index = int.MaxValue;
            int    byte1Index = int.MaxValue;
            int    byte2Index = int.MaxValue;
            var    byte0      = byte0Vector[0];
            var    byte1      = byte1Vector[0];
            var    byte2      = byte2Vector[0];

            while (true)
            {
                while (following == 0)
                {
                    var newBlock = block.Next;
                    if (newBlock == null)
                    {
                        _block = block;
                        _index = index;
                        return(-1);
                    }
                    index     = newBlock.Start;
                    following = newBlock.End - index;
                    block     = newBlock;
                }
                array = block.Array;
                while (following > 0)
                {
// Need unit tests to test Vector path
#if !DEBUG
                    // Check will be Jitted away https://github.com/dotnet/coreclr/issues/1079
                    if (Vector.IsHardwareAccelerated)
                    {
#endif
                    if (following >= _vectorSpan)
                    {
                        var data        = new Vector <byte>(array, index);
                        var byte0Equals = Vector.Equals(data, byte0Vector);
                        var byte1Equals = Vector.Equals(data, byte1Vector);
                        var byte2Equals = Vector.Equals(data, byte2Vector);

                        if (!byte0Equals.Equals(Vector <byte> .Zero))
                        {
                            byte0Index = FindFirstEqualByte(ref byte0Equals);
                        }
                        if (!byte1Equals.Equals(Vector <byte> .Zero))
                        {
                            byte1Index = FindFirstEqualByte(ref byte1Equals);
                        }
                        if (!byte2Equals.Equals(Vector <byte> .Zero))
                        {
                            byte2Index = FindFirstEqualByte(ref byte2Equals);
                        }

                        if (byte0Index == int.MaxValue && byte1Index == int.MaxValue && byte2Index == int.MaxValue)
                        {
                            following -= _vectorSpan;
                            index     += _vectorSpan;
                            continue;
                        }

                        _block = block;

                        int toReturn, toMove;
                        if (byte0Index < byte1Index)
                        {
                            if (byte0Index < byte2Index)
                            {
                                toReturn = byte0;
                                toMove   = byte0Index;
                            }
                            else
                            {
                                toReturn = byte2;
                                toMove   = byte2Index;
                            }
                        }
                        else
                        {
                            if (byte1Index < byte2Index)
                            {
                                toReturn = byte1;
                                toMove   = byte1Index;
                            }
                            else
                            {
                                toReturn = byte2;
                                toMove   = byte2Index;
                            }
                        }

                        _index = index + toMove;
                        return(toReturn);
                    }
// Need unit tests to test Vector path
#if !DEBUG
                }
#endif
                    fixed(byte *ptr = &block.Array[index])
                    {
                        var pCurrent = ptr;
                        var pEnd     = pCurrent + following;

                        do
                        {
                            if (*pCurrent == byte0)
                            {
                                _block = block;
                                _index = index;
                                return(byte0);
                            }
                            if (*pCurrent == byte1)
                            {
                                _block = block;
                                _index = index;
                                return(byte1);
                            }
                            if (*pCurrent == byte2)
                            {
                                _block = block;
                                _index = index;
                                return(byte2);
                            }
                            pCurrent++;
                            index++;
                        } while (pCurrent != pEnd);
                    }

                    following = 0;
                    break;
                }
            }
        }
        public unsafe int Seek(ref Vector<byte> byte0Vector)
        {
            if (IsDefault)
            {
                return -1;
            }

            var block = _block;
            var index = _index;
            var following = block.End - index;
            byte[] array;
            var byte0 = byte0Vector[0];

            while (true)
            {
                while (following == 0)
                {
                    var newBlock = block.Next;
                    if (newBlock == null)
                    {
                        _block = block;
                        _index = index;
                        return -1;
                    }
                    index = newBlock.Start;
                    following = newBlock.End - index;
                    block = newBlock;
                }
                array = block.Array;
                while (following > 0)
                {
// Need unit tests to test Vector path
#if !DEBUG 
                    // Check will be Jitted away https://github.com/dotnet/coreclr/issues/1079
                    if (Vector.IsHardwareAccelerated)
                    {
#endif
                        if (following >= _vectorSpan)
                        {
                            var byte0Equals = Vector.Equals(new Vector<byte>(array, index), byte0Vector);

                            if (byte0Equals.Equals(Vector<byte>.Zero))
                            {
                                following -= _vectorSpan;
                                index += _vectorSpan;
                                continue;
                            }

                            _block = block;
                            _index = index + FindFirstEqualByte(ref byte0Equals);
                            return byte0;
                        }
// Need unit tests to test Vector path
#if !DEBUG 
                    }
#endif
                    fixed (byte* ptr = &block.Array[index])
                    {
                        var pCurrent = ptr;
                        var pEnd = pCurrent + following;
                        do
                        {
                            if (*pCurrent == byte0)
                            {
                                _block = block;
                                _index = index;
                                return byte0;
                            }
                            pCurrent++;
                            index++;
                        } while (pCurrent < pEnd);
                    }

                    following = 0;
                    break;
                }
            }
        }
 /// <summary>
 /// Called to return a block to the pool. Once Return has been called the memory no longer belongs to the caller, and
 /// Very Bad Things will happen if the memory is read of modified subsequently. If a caller fails to call Return and the
 /// block tracking object is garbage collected, the block tracking object's finalizer will automatically re-create and return
 /// a new tracking object into the pool. This will only happen if there is a bug in the server, however it is necessary to avoid
 /// leaving "dead zones" in the slab due to lost block tracking objects.
 /// </summary>
 /// <param name="block">The block to return. It must have been acquired by calling Lease on the same memory pool instance.</param>
 public void Return(MemoryPoolBlock2 block)
 {
     block.Reset();
     _blocks.Enqueue(block);
 }
 /// <summary>
 /// Called to return a block to the pool. Once Return has been called the memory no longer belongs to the caller, and
 /// Very Bad Things will happen if the memory is read of modified subsequently. If a caller fails to call Return and the
 /// block tracking object is garbage collected, the block tracking object's finalizer will automatically re-create and return
 /// a new tracking object into the pool. This will only happen if there is a bug in the server, however it is necessary to avoid
 /// leaving "dead zones" in the slab due to lost block tracking objects.
 /// </summary>
 /// <param name="block">The block to return. It must have been acquired by calling Lease on the same memory pool instance.</param>
 public void Return(MemoryPoolBlock2 block)
 {
     block.Reset();
     _blocks.Enqueue(block);
 }
        public unsafe void CopyFromAscii(string data)
        {
            Debug.Assert(_block != null);
            Debug.Assert(_block.Pool != null);
            Debug.Assert(_block.Next == null);
            Debug.Assert(_block.End == _index);

            var pool       = _block.Pool;
            var block      = _block;
            var blockIndex = _index;
            var length     = data.Length;

            var bytesLeftInBlock          = block.Data.Offset + block.Data.Count - blockIndex;
            var bytesLeftInBlockMinusSpan = bytesLeftInBlock - 3;

            fixed(char *pData = data)
            {
                var input             = pData;
                var inputEnd          = pData + length;
                var inputEndMinusSpan = inputEnd - 3;

                while (input < inputEnd)
                {
                    if (bytesLeftInBlock == 0)
                    {
                        var nextBlock = pool.Lease();
                        block.End  = blockIndex;
                        block.Next = nextBlock;
                        block      = nextBlock;

                        blockIndex                = block.Data.Offset;
                        bytesLeftInBlock          = block.Data.Count;
                        bytesLeftInBlockMinusSpan = bytesLeftInBlock - 3;
                    }

                    fixed(byte *pOutput = &block.Data.Array[block.End])
                    {
                        //this line is needed to allow output be an register var
                        var output = pOutput;

                        var copied = 0;

                        for (; input < inputEndMinusSpan && copied < bytesLeftInBlockMinusSpan; copied += 4)
                        {
                            *(output)     = (byte)*(input);
                            *(output + 1) = (byte)*(input + 1);
                            *(output + 2) = (byte)*(input + 2);
                            *(output + 3) = (byte)*(input + 3);
                            output       += 4;
                            input        += 4;
                        }
                        for (; input < inputEnd && copied < bytesLeftInBlock; copied++)
                        {
                            *(output++) = (byte)*(input++);
                        }

                        blockIndex += copied;
                        bytesLeftInBlockMinusSpan -= copied;
                        bytesLeftInBlock          -= copied;
                    }
                }
            }

            block.End = blockIndex;
            _block    = block;
            _index    = blockIndex;
        }
 public MemoryPoolIterator2 ProducingStart()
 {
     _producingBlock = _memory.Lease();
     return new MemoryPoolIterator2(_producingBlock);
 }