public void ConsumingComplete(
MemoryPoolIterator consumed,
MemoryPoolIterator examined)
{
MemoryPoolBlock returnStart = null;
MemoryPoolBlock returnEnd = null;
lock (_sync)
{
if (!_disposed)
{
if (!consumed.IsDefault)
{
// Compute lengthConsumed before modifying _head or consumed
var lengthConsumed = 0;
if (_bufferSizeControl != null)
{
lengthConsumed = new MemoryPoolIterator(_head).GetLength(consumed);
}
returnStart = _head;
returnEnd = consumed.Block;
_head = consumed.Block;
_head.Start = consumed.Index;
// Must call Subtract() after _head has been advanced, to avoid producer starting too early and growing
// buffer beyond max length.
_bufferSizeControl?.Subtract(lengthConsumed);
}
if (!examined.IsDefault &&
examined.IsEnd &&
RemoteIntakeFin == false &&
_awaitableError == null)
{
_manualResetEvent.Reset();
Interlocked.CompareExchange(
ref _awaitableState,
_awaitableIsNotCompleted,
_awaitableIsCompleted);
}
}
else
{
returnStart = _head;
returnEnd = null;
_head = null;
_tail = null;
}
ReturnBlocks(returnStart, returnEnd);
if (!_consuming)
{
throw new InvalidOperationException("No ongoing consuming operation to complete.");
}
_consuming = false;
}
}
private void ProducingCompleteNoPreComplete(MemoryPoolIterator end)
{
MemoryPoolBlock blockToReturn = null;
lock (_returnLock)
{
// Both ProducingComplete and WriteAsync should not call this method
// if _lastStart was not set.
Debug.Assert(!_lastStart.IsDefault);
// If the socket has been closed, return the produced blocks
// instead of advancing the now non-existent tail.
if (_tail != null)
{
_tail = end.Block;
_tail.End = end.Index;
}
else
{
blockToReturn = _lastStart.Block;
}
_lastStart = default(MemoryPoolIterator);
}
if (blockToReturn != null)
{
_threadPool.UnsafeRun(_returnBlocks, blockToReturn);
}
}
private void LargeAllocationProducesCorrectResults()
{
var byteRange = Enumerable.Range(0, 16384 + 64).Select(x => (byte)x).ToArray();
var expectedByteRange = byteRange.Concat(byteRange).ToArray();
var mem0 = MemoryPoolBlock.Create(new ArraySegment <byte>(byteRange), IntPtr.Zero, null, null);
var mem1 = MemoryPoolBlock.Create(new ArraySegment <byte>(byteRange), IntPtr.Zero, null, null);
mem0.End = byteRange.Length;
mem1.End = byteRange.Length;
mem0.Next = mem1;
var begin = mem0.GetIterator();
var end = GetIterator(begin, expectedByteRange.Length);
var s = begin.GetAsciiString(end);
Assert.Equal(s.Length, expectedByteRange.Length);
for (var i = 0; i < expectedByteRange.Length; i++)
{
var sb = (byte)s[i];
var b = expectedByteRange[i];
Assert.Equal(sb, b);
}
}
private void ProducingCompleteNoPreComplete(MemoryPoolIterator end)
{
MemoryPoolBlock blockToReturn = null;
lock (_returnLock)
{
Debug.Assert(!_lastStart.IsDefault);
// If the socket has been closed, return the produced blocks
// instead of advancing the now non-existent tail.
if (_tail != null)
{
_tail = end.Block;
_tail.End = end.Index;
}
else
{
blockToReturn = _lastStart.Block;
}
_lastStart = default(MemoryPoolIterator);
}
if (blockToReturn != null)
{
ThreadPool.QueueUserWorkItem(_returnBlocks, blockToReturn);
}
}
public void IncomingData(byte[] buffer, int offset, int count)
{
lock (_sync)
{
if (count > 0)
{
if (_tail == null)
{
_tail = _memory.Lease();
}
var iterator = new MemoryPoolIterator(_tail, _tail.End);
iterator.CopyFrom(buffer, offset, count);
if (_head == null)
{
_head = _tail;
}
_tail = iterator.Block;
}
else
{
RemoteIntakeFin = true;
}
Complete();
}
}
public void IncomingData(byte[] buffer, int offset, int count)
{
lock (_sync)
{
// Must call Add() before bytes are available to consumer, to ensure that Length is >= 0
_bufferSizeControl?.Add(count);
if (count > 0)
{
if (_tail == null)
{
_tail = _memory.Lease();
}
var iterator = new MemoryPoolIterator(_tail, _tail.End);
iterator.CopyFrom(buffer, offset, count);
if (_head == null)
{
_head = _tail;
}
_tail = iterator.Block;
}
else
{
RemoteIntakeFin = true;
}
Complete();
}
}
private MemoryPoolIterator BuildSample(MemoryPoolBlock mem, string data)
{
var store = data.Select(c => (byte)c).ToArray();
mem.GetIterator().CopyFrom(new ArraySegment <byte>(store));
return(mem.GetIterator());
}
private MemoryPoolIterator BuildSample(string data)
{
var store = data.Select(c => (byte)c).ToArray();
var mem = MemoryPoolBlock.Create(new ArraySegment <byte>(store), IntPtr.Zero, null, null);
mem.End = store.Length;
return(mem.GetIterator());
}
private void PositiveAssert(MemoryPoolBlock mem, string raw)
{
var begin = BuildSample(mem, raw);
var end = GetIterator(begin, raw.Length);
var result = UrlPathDecoder.Unescape(begin, end);
Assert.NotEqual(raw.Length, begin.GetUtf8String(result).Length);
}
private async Task FilterInputAsync(MemoryPoolBlock block)
{
int bytesRead;
while ((bytesRead = await _filteredStream.ReadAsync(block.Array, block.Data.Offset, block.Data.Count)) != 0)
{
SocketInput.IncomingData(block.Array, block.Data.Offset, bytesRead);
}
}
public Task ReadInputAsync()
{
_block = _memory.Lease();
// Use pooled block for copy
return(FilterInputAsync(_block).ContinueWith((task, state) =>
{
((FilteredStreamAdapter)state).OnStreamClose(task);
}, this));
}
public static async Task CopyToAsync(this Stream source, Stream destination, MemoryPoolBlock block)
{
int bytesRead;
while ((bytesRead = await source.ReadAsync(block.Array, block.Data.Offset, block.Data.Count)) != 0)
{
await destination.WriteAsync(block.Array, block.Data.Offset, bytesRead);
}
}
private static void ReturnWrittenBlocks(MemoryPoolBlock block)
{
while (block != null)
{
var returnBlock = block;
block = block.Next;
returnBlock.Pool.Return(returnBlock);
}
}
private static void ReturnBlocks(MemoryPoolBlock block, MemoryPoolBlock end)
{
while (block != end)
{
var returnBlock = block;
block = block.Next;
returnBlock.Pool.Return(returnBlock);
}
}
internal void Return(MemoryPoolBlock block)
{
#if BLOCK_LEASE_TRACKING
Debug.Assert(block.Pool == this, "Returned block was not leased from this pool");
Debug.Assert(block.IsLeased, $"Block being returned to pool twice: {block.Leaser}{Environment.NewLine}");
block.IsLeased = false;
#endif
if (!_isDisposed)
{
_blocks.Enqueue(block);
}
}
public void IncomingDeferred()
{
Debug.Assert(_pinned != null);
if (_pinned != null)
{
if (_pinned != _tail)
{
_memory.Return(_pinned);
}
_pinned = null;
}
}
private void TestAllLengths(MemoryPoolBlock block, int lengths)
{
for (var firstIndex = 0; firstIndex <= lengths; ++firstIndex)
{
for (var lastIndex = firstIndex; lastIndex <= lengths; ++lastIndex)
{
var first = block.GetIterator().Add(firstIndex);
var last = block.GetIterator().Add(lastIndex);
Assert.Equal(firstIndex, block.GetIterator().GetLength(first));
Assert.Equal(lastIndex, block.GetIterator().GetLength(last));
Assert.Equal(lastIndex - firstIndex, first.GetLength(last));
}
}
}
public void Dispose()
{
lock (_sync)
{
AbortAwaiting();
if (!_consuming)
{
ReturnBlocks(_head, null);
_head = null;
_tail = null;
}
_disposed = true;
}
}
public MemoryPoolBlock IncomingStart()
{
const int minimumSize = 2048;
if (_tail != null && minimumSize <= _tail.Data.Offset + _tail.Data.Count - _tail.End)
{
_pinned = _tail;
}
else
{
_pinned = _memory.Lease();
}
return(_pinned);
}
public void Dispose()
{
AbortAwaiting();
// Return all blocks
var block = _head;
while (block != null)
{
var returnBlock = block;
block = block.Next;
returnBlock.Pool.Return(returnBlock);
}
_head = null;
_tail = null;
}
public void IncomingComplete(int count, Exception error)
{
Action awaitableState;
lock (_sync)
{
// Must call Add() before bytes are available to consumer, to ensure that Length is >= 0
_bufferSizeControl?.Add(count);
if (_pinned != null)
{
_pinned.End += count;
if (_head == null)
{
_head = _tail = _pinned;
}
else if (_tail == _pinned)
{
// NO-OP: this was a read into unoccupied tail-space
}
else
{
Volatile.Write(ref _tail.Next, _pinned);
_tail = _pinned;
}
_pinned = null;
}
if (error != null)
{
SetConnectionError(error);
}
else if (count == 0)
{
FinReceived();
}
awaitableState = Interlocked.Exchange(ref _awaitableState, _awaitableIsCompleted);
}
Complete(awaitableState);
}
public void ConsumingComplete(
MemoryPoolIterator consumed,
MemoryPoolIterator examined)
{
lock (_sync)
{
MemoryPoolBlock returnStart = null;
MemoryPoolBlock returnEnd = null;
if (!consumed.IsDefault)
{
returnStart = _head;
returnEnd = consumed.Block;
_head = consumed.Block;
_head.Start = consumed.Index;
}
if (!examined.IsDefault &&
examined.IsEnd &&
RemoteIntakeFin == false &&
_awaitableError == null)
{
_manualResetEvent.Reset();
Interlocked.CompareExchange(
ref _awaitableState,
_awaitableIsNotCompleted,
_awaitableIsCompleted);
}
while (returnStart != returnEnd)
{
var returnBlock = returnStart;
returnStart = returnStart.Next;
returnBlock.Pool.Return(returnBlock);
}
if (Interlocked.CompareExchange(ref _consumingState, 0, 1) != 1)
{
throw new InvalidOperationException("No ongoing consuming operation to complete.");
}
}
}
public void Dispose()
{
AbortAwaiting();
MemoryPoolBlock block = null;
lock (_sync)
{
if (!_consuming)
{
block = _head;
_head = null;
_tail = null;
}
_disposed = true;
}
ReturnBlocks(block, null);
}
public void IncomingComplete(int count, Exception error)
{
lock (_sync)
{
// Must call Add() before bytes are available to consumer, to ensure that Length is >= 0
_bufferSizeControl?.Add(count);
if (_pinned != null)
{
_pinned.End += count;
if (_head == null)
{
_head = _tail = _pinned;
}
else if (_tail == _pinned)
{
// NO-OP: this was a read into unoccupied tail-space
}
else
{
Volatile.Write(ref _tail.Next, _pinned);
_tail = _pinned;
}
_pinned = null;
}
if (count == 0)
{
RemoteIntakeFin = true;
}
if (error != null)
{
_awaitableError = error;
}
Complete();
}
}
public MemoryPoolIterator ProducingStart()
{
lock (_returnLock)
{
Debug.Assert(_lastStart.IsDefault);
if (_closed)
{
return(default(MemoryPoolIterator));
}
if (_tail == null)
{
_head = _thread.Memory.Lease();
_tail = _head;
}
_lastStart = new MemoryPoolIterator(_tail, _tail.End);
return(_lastStart);
}
}
private void FullByteRangeSupported()
{
var byteRange = Enumerable.Range(0, 256).Select(x => (byte)x).ToArray();
var mem = MemoryPoolBlock.Create(new ArraySegment <byte>(byteRange), IntPtr.Zero, null, null);
mem.End = byteRange.Length;
var begin = mem.GetIterator();
var end = GetIterator(begin, byteRange.Length);
var s = begin.GetAsciiString(end);
Assert.Equal(s.Length, byteRange.Length);
for (var i = 0; i < byteRange.Length; i++)
{
var sb = (byte)s[i];
var b = byteRange[i];
Assert.Equal(sb, b);
}
}
// This is called on the libuv event loop
private void ReturnAllBlocks()
{
lock (_returnLock)
{
var block = _head;
while (block != _tail)
{
var returnBlock = block;
block = block.Next;
returnBlock.Pool.Return(returnBlock);
}
// Only return the _tail if we aren't between ProducingStart/Complete calls
if (_lastStart.IsDefault)
{
_tail.Pool.Return(_tail);
}
_head = null;
_tail = null;
}
}
public SocketOutput(
KestrelThread thread,
UvStreamHandle socket,
MemoryPool memory,
Connection connection,
string connectionId,
IKestrelTrace log,
IThreadPool threadPool,
Queue <UvWriteReq> writeReqPool)
{
_thread = thread;
_socket = socket;
_connection = connection;
_connectionId = connectionId;
_log = log;
_threadPool = threadPool;
_tasksPending = new Queue <WaitingTask>(_initialTaskQueues);
_writeContextPool = new Queue <WriteContext>(_maxPooledWriteContexts);
_writeReqPool = writeReqPool;
_head = memory.Lease();
_tail = _head;
}
public void IncomingComplete(int count, Exception error)
{
lock (_sync)
{
if (_pinned != null)
{
_pinned.End += count;
if (_head == null)
{
_head = _tail = _pinned;
}
else if (_tail == _pinned)
{
// NO-OP: this was a read into unoccupied tail-space
}
else
{
Volatile.Write(ref _tail.Next, _pinned);
_tail = _pinned;
}
_pinned = null;
}
if (count == 0)
{
RemoteIntakeFin = true;
}
if (error != null)
{
_awaitableError = error;
}
Complete();
}
}
public MemoryPoolIterator ProducingStart()
{
_producingBlock = _memory.Lease();
return(new MemoryPoolIterator(_producingBlock));
}
/// <summary>
/// Called to return a block to the pool. Once Return has been called the memory no longer belongs to the caller, and
/// Very Bad Things will happen if the memory is read of modified subsequently. If a caller fails to call Return and the
/// block tracking object is garbage collected, the block tracking object's finalizer will automatically re-create and return
/// a new tracking object into the pool. This will only happen if there is a bug in the server, however it is necessary to avoid
/// leaving "dead zones" in the slab due to lost block tracking objects.
/// </summary>
/// <param name="block">The block to return. It must have been acquired by calling Lease on the same memory pool instance.</param>
public void Return(MemoryPoolBlock block)
{
#if DEBUG
Debug.Assert(block.Pool == this, "Returned block was not leased from this pool");
Debug.Assert(block.IsLeased, $"Block being returned to pool twice: {block.Leaser}{Environment.NewLine}");
block.IsLeased = false;
#endif
if (block.Slab != null && block.Slab.IsActive)
{
_blocks.Enqueue(block);
}
else
{
GC.SuppressFinalize(block);
}
}
