/// <summary> /// Internal method called when a block is requested and the pool is empty. It allocates one additional slab, creates all of the /// block tracking objects, and adds them all to the pool. /// </summary> private MemoryPoolBlock AllocateSlab() { var slab = MemoryPoolSlab.Create(_slabLength); _slabs.Push(slab); var basePtr = slab.ArrayPtr; var firstOffset = (int)((_blockStride - 1) - ((ulong)(basePtr + _blockStride - 1) % _blockStride)); var poolAllocationLength = _slabLength - _blockStride; var offset = firstOffset; for (; offset + _blockLength < poolAllocationLength; offset += _blockStride) { var block = MemoryPoolBlock.Create( new ArraySegment <byte>(slab.Array, offset, _blockLength), basePtr, this, slab); Return(block); } // return last block rather than adding to pool var newBlock = MemoryPoolBlock.Create( new ArraySegment <byte>(slab.Array, offset, _blockLength), basePtr, this, slab); return(newBlock); }
/// <summary> /// Save the data at the current location then move to the next available space. /// </summary> /// <param name="data">The byte to be saved.</param> /// <returns>true if the operation successes. false if can't find available space.</returns> public bool Put(byte data) { if (_block == null) { return(false); } var block = _block; var index = _index; while (true) { var wasLastBlock = block.Next == null; if (index < block.End) { _block = block; _index = index + 1; block.Array[index] = data; return(true); } else if (wasLastBlock) { return(false); } else { block = block.Next; index = block.Start; } } }
/// <summary> /// Called to return a block to the pool. Once Return has been called the memory no longer belongs to the caller, and /// Very Bad Things will happen if the memory is read of modified subsequently. If a caller fails to call Return and the /// block tracking object is garbage collected, the block tracking object's finalizer will automatically re-create and return /// a new tracking object into the pool. This will only happen if there is a bug in the server, however it is necessary to avoid /// leaving "dead zones" in the slab due to lost block tracking objects. /// </summary> /// <param name="block">The block to return. It must have been acquired by calling Lease on the same memory pool instance.</param> public void Return(MemoryPoolBlock block) { Debug.Assert(block.Pool == this, "Returned block was not leased from this pool"); if (block.Slab != null && block.Slab.IsActive) { block.Reset(); _blocks.Enqueue(block); } }
public void CopyFrom(byte[] data, int offset, int count) { if (IsDefault) { return; } Debug.Assert(_block != null); Debug.Assert(_block.Next == null); Debug.Assert(_block.End == _index); var pool = _block.Pool; var block = _block; var blockIndex = _index; var bufferIndex = offset; var remaining = count; var bytesLeftInBlock = block.Data.Offset + block.Data.Count - blockIndex; while (remaining > 0) { if (bytesLeftInBlock == 0) { var nextBlock = pool.Lease(); block.End = blockIndex; Volatile.Write(ref block.Next, nextBlock); block = nextBlock; blockIndex = block.Data.Offset; bytesLeftInBlock = block.Data.Count; } var bytesToCopy = remaining < bytesLeftInBlock ? remaining : bytesLeftInBlock; Buffer.BlockCopy(data, bufferIndex, block.Array, blockIndex, bytesToCopy); blockIndex += bytesToCopy; bufferIndex += bytesToCopy; remaining -= bytesToCopy; bytesLeftInBlock -= bytesToCopy; } block.End = blockIndex; _block = block; _index = blockIndex; }
/// <summary> /// Called to return a block to the pool. Once Return has been called the memory no longer belongs to the caller, and /// Very Bad Things will happen if the memory is read of modified subsequently. If a caller fails to call Return and the /// block tracking object is garbage collected, the block tracking object's finalizer will automatically re-create and return /// a new tracking object into the pool. This will only happen if there is a bug in the server, however it is necessary to avoid /// leaving "dead zones" in the slab due to lost block tracking objects. /// </summary> /// <param name="block">The block to return. It must have been acquired by calling Lease on the same memory pool instance.</param> public void Return(MemoryPoolBlock block) { #if DEBUG Debug.Assert(block.Pool == this, "Returned block was not leased from this pool"); Debug.Assert(block.IsLeased, $"Block being returned to pool twice: {block.Leaser}{Environment.NewLine}"); block.IsLeased = false; #endif if (block.Slab != null && block.Slab.IsActive) { block.Reset(); _blocks.Enqueue(block); } else { GC.SuppressFinalize(block); } }
public void Skip(int bytesToSkip) { if (_block == null) { return; } var wasLastBlock = _block.Next == null; var following = _block.End - _index; if (following >= bytesToSkip) { _index += bytesToSkip; return; } var block = _block; var index = _index; while (true) { if (wasLastBlock) { throw new InvalidOperationException("Attempted to skip more bytes than available."); } else { bytesToSkip -= following; block = block.Next; index = block.Start; } wasLastBlock = block.Next == null; following = block.End - index; if (following >= bytesToSkip) { _block = block; _index = index + bytesToSkip; return; } } }
public int Take() { var block = _block; if (block == null) { return(-1); } var index = _index; var wasLastBlock = block.Next == null; if (index < block.End) { _index = index + 1; return(block.Array[index]); } do { if (wasLastBlock) { return(-1); } else { block = block.Next; index = block.Start; } wasLastBlock = block.Next == null; if (index < block.End) { _block = block; _index = index + 1; return(block.Array[index]); } } while (true); }
public unsafe int Seek( ref Vector <byte> byte0Vector, ref Vector <byte> byte1Vector, ref Vector <byte> byte2Vector, ref MemoryPoolIterator limit) { if (IsDefault) { return(-1); } var block = _block; var index = _index; var wasLastBlock = block.Next == null; var following = block.End - index; byte[] array; int byte0Index = int.MaxValue; int byte1Index = int.MaxValue; int byte2Index = int.MaxValue; var byte0 = byte0Vector[0]; var byte1 = byte1Vector[0]; var byte2 = byte2Vector[0]; while (true) { while (following == 0) { if ((block == limit.Block && index > limit.Index) || wasLastBlock) { _block = block; // Ensure iterator is left at limit position _index = limit.Index; return(-1); } block = block.Next; index = block.Start; wasLastBlock = block.Next == null; following = block.End - index; } array = block.Array; while (following > 0) { // Need unit tests to test Vector path #if !DEBUG // Check will be Jitted away https://github.com/dotnet/coreclr/issues/1079 if (Vector.IsHardwareAccelerated) { #endif if (following >= _vectorSpan) { var data = new Vector <byte>(array, index); var byte0Equals = Vector.Equals(data, byte0Vector); var byte1Equals = Vector.Equals(data, byte1Vector); var byte2Equals = Vector.Equals(data, byte2Vector); if (!byte0Equals.Equals(Vector <byte> .Zero)) { byte0Index = FindFirstEqualByte(ref byte0Equals); } if (!byte1Equals.Equals(Vector <byte> .Zero)) { byte1Index = FindFirstEqualByte(ref byte1Equals); } if (!byte2Equals.Equals(Vector <byte> .Zero)) { byte2Index = FindFirstEqualByte(ref byte2Equals); } if (byte0Index == int.MaxValue && byte1Index == int.MaxValue && byte2Index == int.MaxValue) { following -= _vectorSpan; index += _vectorSpan; if (block == limit.Block && index > limit.Index) { _block = block; // Ensure iterator is left at limit position _index = limit.Index; return(-1); } continue; } _block = block; int toReturn, toMove; if (byte0Index < byte1Index) { if (byte0Index < byte2Index) { toReturn = byte0; toMove = byte0Index; } else { toReturn = byte2; toMove = byte2Index; } } else { if (byte1Index < byte2Index) { toReturn = byte1; toMove = byte1Index; } else { toReturn = byte2; toMove = byte2Index; } } _index = index + toMove; if (block == limit.Block && _index > limit.Index) { // Ensure iterator is left at limit position _index = limit.Index; return(-1); } return(toReturn); } // Need unit tests to test Vector path #if !DEBUG } #endif var pCurrent = (block.DataFixedPtr + index); var pEnd = block == limit.Block ? block.DataFixedPtr + limit.Index + 1 : pCurrent + following; do { if (*pCurrent == byte0) { _block = block; _index = index; return(byte0); } if (*pCurrent == byte1) { _block = block; _index = index; return(byte1); } if (*pCurrent == byte2) { _block = block; _index = index; return(byte2); } pCurrent++; index++; } while (pCurrent != pEnd); following = 0; break; } } }
public unsafe int Seek( ref Vector <byte> byte0Vector, out int bytesScanned, int limit = int.MaxValue) { bytesScanned = 0; if (IsDefault || limit <= 0) { return(-1); } var block = _block; var index = _index; var wasLastBlock = block.Next == null; var following = block.End - index; byte[] array; var byte0 = byte0Vector[0]; while (true) { while (following == 0) { if (bytesScanned >= limit || wasLastBlock) { _block = block; _index = index; return(-1); } block = block.Next; index = block.Start; wasLastBlock = block.Next == null; following = block.End - index; } array = block.Array; while (following > 0) { // Need unit tests to test Vector path #if !DEBUG // Check will be Jitted away https://github.com/dotnet/coreclr/issues/1079 if (Vector.IsHardwareAccelerated) { #endif if (following >= _vectorSpan) { var byte0Equals = Vector.Equals(new Vector <byte>(array, index), byte0Vector); if (byte0Equals.Equals(Vector <byte> .Zero)) { if (bytesScanned + _vectorSpan >= limit) { _block = block; // Ensure iterator is left at limit position _index = index + (limit - bytesScanned); bytesScanned = limit; return(-1); } bytesScanned += _vectorSpan; following -= _vectorSpan; index += _vectorSpan; continue; } _block = block; var firstEqualByteIndex = FindFirstEqualByte(ref byte0Equals); var vectorBytesScanned = firstEqualByteIndex + 1; if (bytesScanned + vectorBytesScanned > limit) { // Ensure iterator is left at limit position _index = index + (limit - bytesScanned); bytesScanned = limit; return(-1); } _index = index + firstEqualByteIndex; bytesScanned += vectorBytesScanned; return(byte0); } // Need unit tests to test Vector path #if !DEBUG } #endif var pCurrent = (block.DataFixedPtr + index); var pEnd = pCurrent + Math.Min(following, limit - bytesScanned); do { bytesScanned++; if (*pCurrent == byte0) { _block = block; _index = index; return(byte0); } pCurrent++; index++; } while (pCurrent < pEnd); following = 0; break; } } }
public MemoryPoolIterator(MemoryPoolBlock block, int index) { _block = block; _index = index; }
public MemoryPoolIterator(MemoryPoolBlock block) { _block = block; _index = _block?.Start ?? 0; }
public unsafe void CopyFromAscii(string data) { if (IsDefault) { return; } Debug.Assert(_block != null); Debug.Assert(_block.Next == null); Debug.Assert(_block.End == _index); var pool = _block.Pool; var block = _block; var blockIndex = _index; var length = data.Length; var bytesLeftInBlock = block.Data.Offset + block.Data.Count - blockIndex; var bytesLeftInBlockMinusSpan = bytesLeftInBlock - 3; fixed(char *pData = data) { var input = pData; var inputEnd = pData + length; var inputEndMinusSpan = inputEnd - 3; while (input < inputEnd) { if (bytesLeftInBlock == 0) { var nextBlock = pool.Lease(); block.End = blockIndex; Volatile.Write(ref block.Next, nextBlock); block = nextBlock; blockIndex = block.Data.Offset; bytesLeftInBlock = block.Data.Count; bytesLeftInBlockMinusSpan = bytesLeftInBlock - 3; } var output = (block.DataFixedPtr + block.End); var copied = 0; for (; input < inputEndMinusSpan && copied < bytesLeftInBlockMinusSpan; copied += 4) { *(output) = (byte)*(input); *(output + 1) = (byte)*(input + 1); *(output + 2) = (byte)*(input + 2); *(output + 3) = (byte)*(input + 3); output += 4; input += 4; } for (; input < inputEnd && copied < bytesLeftInBlock; copied++) { *(output++) = (byte)*(input++); } blockIndex += copied; bytesLeftInBlockMinusSpan -= copied; bytesLeftInBlock -= copied; } } block.End = blockIndex; _block = block; _index = blockIndex; }
public unsafe int Seek(ref Vector <byte> byte0Vector, ref Vector <byte> byte1Vector) { if (IsDefault) { return(-1); } var block = _block; var index = _index; var wasLastBlock = block.Next == null; var following = block.End - index; byte[] array; int byte0Index = int.MaxValue; int byte1Index = int.MaxValue; var byte0 = byte0Vector[0]; var byte1 = byte1Vector[0]; while (true) { while (following == 0) { if (wasLastBlock) { _block = block; _index = index; return(-1); } block = block.Next; index = block.Start; wasLastBlock = block.Next == null; following = block.End - index; } array = block.Array; while (following > 0) { // Need unit tests to test Vector path #if !DEBUG // Check will be Jitted away https://github.com/dotnet/coreclr/issues/1079 if (Vector.IsHardwareAccelerated) { #endif if (following >= _vectorSpan) { var data = new Vector <byte>(array, index); var byte0Equals = Vector.Equals(data, byte0Vector); var byte1Equals = Vector.Equals(data, byte1Vector); if (!byte0Equals.Equals(Vector <byte> .Zero)) { byte0Index = FindFirstEqualByte(ref byte0Equals); } if (!byte1Equals.Equals(Vector <byte> .Zero)) { byte1Index = FindFirstEqualByte(ref byte1Equals); } if (byte0Index == int.MaxValue && byte1Index == int.MaxValue) { following -= _vectorSpan; index += _vectorSpan; continue; } _block = block; if (byte0Index < byte1Index) { _index = index + byte0Index; return(byte0); } _index = index + byte1Index; return(byte1); } // Need unit tests to test Vector path #if !DEBUG } #endif var pCurrent = (block.DataFixedPtr + index); var pEnd = pCurrent + following; do { if (*pCurrent == byte0) { _block = block; _index = index; return(byte0); } if (*pCurrent == byte1) { _block = block; _index = index; return(byte1); } pCurrent++; index++; } while (pCurrent != pEnd); following = 0; break; } } }
/// <summary> /// called when the block is returned to the pool. mutable values are re-assigned to their guaranteed initialized state. /// </summary> public void Reset() { Next = null; Start = Data.Offset; End = Data.Offset; }