public Action OnCompleted(Action continuation, ref PipeCompletion completion)
{
var awaitableState = _state;
if (ReferenceEquals(awaitableState, _awaitableIsNotCompleted))
{
_state = continuation;
}
if (ReferenceEquals(awaitableState, _awaitableIsCompleted))
{
return(continuation);
}
if (!ReferenceEquals(awaitableState, _awaitableIsNotCompleted))
{
completion.TryComplete(PipelinesThrowHelper.GetInvalidOperationException(ExceptionResource.NoConcurrentOperation));
_state = _awaitableIsCompleted;
Task.Run(continuation);
Task.Run(awaitableState);
}
return(null);
}
/// <summary>
/// Allocates memory from the pipeline to write into.
/// </summary>
/// <param name="minimumSize">The minimum size buffer to allocate</param>
/// <returns>A <see cref="WritableBuffer"/> that can be written to.</returns>
WritableBuffer IPipeWriter.Alloc(int minimumSize)
{
if (_writerCompletion.IsCompleted)
{
PipelinesThrowHelper.ThrowInvalidOperationException(ExceptionResource.NoWritingAllowed, _writerCompletion.Location);
}
if (minimumSize < 0)
{
PipelinesThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.minimumSize);
}
lock (_sync)
{
// CompareExchange not required as its setting to current value if test fails
_writingState.Begin(ExceptionResource.AlreadyWriting);
if (minimumSize > 0)
{
try
{
AllocateWriteHeadUnsynchronized(minimumSize);
}
catch (Exception)
{
// Reset producing state if allocation failed
_writingState.End(ExceptionResource.NoWriteToComplete);
throw;
}
}
_currentWriteLength = 0;
return(new WritableBuffer(this));
}
}
private void EnsureAlloc()
{
if (!_writingState.IsStarted)
{
PipelinesThrowHelper.ThrowInvalidOperationException(ExceptionResource.NotWritingNoAlloc);
}
}
/// <summary>
/// Marks the pipeline as being complete, meaning no more items will be written to it.
/// </summary>
/// <param name="exception">Optional Exception indicating a failure that's causing the pipeline to complete.</param>
internal void Complete(Exception exception)
{
if (_writingState.IsStarted && _currentWriteLength > 0)
{
PipelinesThrowHelper.ThrowInvalidOperationException(ExceptionResource.CompleteWriterActiveWriter, _writingState.Location);
}
Action awaitable;
PipeCompletionCallbacks completionCallbacks;
bool readerCompleted;
lock (_sync)
{
completionCallbacks = _writerCompletion.TryComplete(exception);
awaitable = _readerAwaitable.Complete();
readerCompleted = _readerCompletion.IsCompleted;
}
if (completionCallbacks != null)
{
TrySchedule(_readerScheduler, _invokeCompletionCallbacks, completionCallbacks);
}
TrySchedule(_readerScheduler, awaitable);
if (readerCompleted)
{
CompletePipe();
}
}
/// <summary>
/// Signal to the producer that the consumer is done reading.
/// </summary>
/// <param name="exception">Optional Exception indicating a failure that's causing the pipeline to complete.</param>
internal void CompleteReader(Exception exception)
{
if (_readingState.IsActive)
{
PipelinesThrowHelper.ThrowInvalidOperationException(ExceptionResource.CompleteReaderActiveReader, _readingState.Location);
}
PipeCompletionCallbacks completionCallbacks;
Action awaitable;
bool writerCompleted;
lock (_sync)
{
completionCallbacks = _readerCompletion.TryComplete(exception);
awaitable = _writerAwaitable.Complete();
writerCompleted = _writerCompletion.IsCompleted;
}
if (completionCallbacks != null)
{
TrySchedule(_writerScheduler, _invokeCompletionCallbacks, completionCallbacks);
}
TrySchedule(_writerScheduler, awaitable);
if (writerCompleted)
{
CompletePipe();
}
}
public void Skip(int length)
{
if (length < 0)
{
PipelinesThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.length);
}
_consumedBytes += length;
while (!_end && length > 0)
{
if ((_index + length) < _currentSpan.Length)
{
_index += length;
length = 0;
break;
}
length -= (_currentSpan.Length - _index);
MoveNext();
}
if (length > 0)
{
PipelinesThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.length);
}
}
internal void AdvanceWriter(int bytesWritten)
{
EnsureAlloc();
if (bytesWritten > 0)
{
if (_writingHead == null)
{
PipelinesThrowHelper.ThrowInvalidOperationException(ExceptionResource.AdvancingWithNoBuffer);
}
Debug.Assert(!_writingHead.ReadOnly);
Debug.Assert(_writingHead.Next == null);
var buffer = _writingHead.Buffer;
var bufferIndex = _writingHead.End + bytesWritten;
if (bufferIndex > buffer.Length)
{
PipelinesThrowHelper.ThrowInvalidOperationException(ExceptionResource.AdvancingPastBufferSize);
}
_writingHead.End = bufferIndex;
_currentWriteLength += bytesWritten;
}
else if (bytesWritten < 0)
{
PipelinesThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.bytesWritten);
} // and if zero, just do nothing; don't need to validate tail etc
}
public override void Retain()
{
if (IsDisposed)
{
PipelinesThrowHelper.ThrowObjectDisposedException(nameof(UnownedBuffer));
}
Interlocked.Increment(ref _referenceCount);
}
protected override unsafe bool TryGetPointerInternal(out void *pointer)
{
if (IsDisposed)
{
PipelinesThrowHelper.ThrowObjectDisposedException(nameof(MemoryPoolBlock));
}
pointer = (Slab.NativePointer + _offset).ToPointer();
return(true);
}
/// <summary>
/// Create a <see cref="ReadableBuffer"/> over an array.
/// </summary>
public static ReadableBuffer Create(byte[] data, int offset, int length)
{
if (data == null)
{
PipelinesThrowHelper.ThrowArgumentNullException(ExceptionArgument.data);
}
return(Create((OwnedBuffer <byte>)data, offset, length));
}
protected override bool TryGetArray(out ArraySegment <byte> buffer)
{
if (IsDisposed)
{
PipelinesThrowHelper.ThrowObjectDisposedException(nameof(MemoryPoolBlock));
}
buffer = new ArraySegment <byte>(Slab.Array, _offset, _length);
return(true);
}
protected override bool TryGetArray(out ArraySegment <byte> buffer)
{
if (IsDisposed)
{
PipelinesThrowHelper.ThrowObjectDisposedException(nameof(UnownedBuffer));
}
buffer = _buffer;
return(true);
}
/// <summary>
/// Create a <see cref="ReadableBuffer"/> over an array.
/// </summary>
public static ReadableBuffer Create(byte[] data)
{
if (data == null)
{
PipelinesThrowHelper.ThrowArgumentNullException(ExceptionArgument.data);
}
return(new ReadableBuffer(data, 0, data.Length));
}
// Reading
void IPipeReader.Advance(ReadCursor consumed, ReadCursor examined)
{
BufferSegment returnStart = null;
BufferSegment returnEnd = null;
int consumedBytes = 0;
if (!consumed.IsDefault)
{
returnStart = _readHead;
consumedBytes = ReadCursor.GetLength(returnStart, returnStart.Start, consumed.Segment, consumed.Index);
returnEnd = consumed.Segment;
_readHead = consumed.Segment;
_readHead.Start = consumed.Index;
}
// Reading commit head shared with writer
Action continuation = null;
lock (_sync)
{
var oldLength = _length;
_length -= consumedBytes;
if (oldLength >= _maximumSizeLow &&
_length < _maximumSizeLow)
{
continuation = _writerAwaitable.Complete();
}
// We reset the awaitable to not completed if we've examined everything the producer produced so far
if (examined.Segment == _commitHead &&
examined.Index == _commitHeadIndex &&
!_writerCompletion.IsCompleted)
{
if (!_writerAwaitable.IsCompleted)
{
PipelinesThrowHelper.ThrowInvalidOperationException(ExceptionResource.BackpressureDeadlock);
}
_readerAwaitable.Reset();
}
_readingState.End(ExceptionResource.NoReadToComplete);
}
while (returnStart != null && returnStart != returnEnd)
{
var returnSegment = returnStart;
returnStart = returnStart.Next;
returnSegment.Dispose();
}
TrySchedule(_writerScheduler, continuation);
}
/// <summary>
/// Allocates memory from the pipeline to write into.
/// </summary>
/// <param name="minimumSize">The minimum size buffer to allocate</param>
/// <returns>A <see cref="WritableBuffer"/> that can be written to.</returns>
internal Memory <byte> GetMemory(int minimumSize)
{
if (_writerCompletion.IsCompleted)
{
PipelinesThrowHelper.ThrowInvalidOperationException(ExceptionResource.NoWritingAllowed, _writerCompletion.Location);
}
if (minimumSize < 0)
{
PipelinesThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.minimumSize);
}
var segment = _writingHead;
if (segment == null)
{
lock (_sync)
{
// CompareExchange not required as its setting to current value if test fails
_writingState.BeginTentative(ExceptionResource.AlreadyWriting);
try
{
segment = AllocateWriteHeadUnsynchronized(minimumSize);
}
catch (Exception)
{
// Reset producing state if allocation failed
_writingState.End(ExceptionResource.NoWriteToComplete);
throw;
}
}
}
var bytesLeftInBuffer = segment.WritableBytes;
// If inadequate bytes left or if the segment is readonly
if (bytesLeftInBuffer == 0 || bytesLeftInBuffer < minimumSize || segment.ReadOnly)
{
BufferSegment nextSegment;
lock (_sync)
{
nextSegment = CreateSegmentUnsynchronized();
}
nextSegment.SetMemory(_pool.Rent(Math.Max(_minimumSegmentSize, minimumSize)));
segment.SetNext(nextSegment);
_writingHead = nextSegment;
}
return(Buffer);
}
public override OwnedBuffer <byte> Rent(int size)
{
if (size > _blockLength)
{
PipelinesThrowHelper.ThrowArgumentOutOfRangeException_BufferRequestTooLarge(_blockLength);
}
var block = Lease();
return(block);
}
/// <summary>
/// Create a <see cref="ReadableBuffer"/> over an array.
/// </summary>
public static ReadableBuffer Create(byte[] data)
{
if (data == null)
{
PipelinesThrowHelper.ThrowArgumentNullException(ExceptionArgument.data);
}
OwnedBuffer <byte> buffer = data;
return(CreateInternal(buffer, 0, data.Length));
}
public void End(ExceptionResource exception)
{
if (_state == State.Inactive)
{
PipelinesThrowHelper.ThrowInvalidOperationException(exception, Location);
}
_state = State.Inactive;
#if OPERATION_LOCATION_TRACKING
_operationStartLocation = null;
#endif
}
public override BufferHandle Pin()
{
if (IsDisposed)
{
PipelinesThrowHelper.ThrowObjectDisposedException(nameof(MemoryPoolBlock));
}
Retain();
unsafe
{
return(new BufferHandle(this, (Slab.NativePointer + _offset).ToPointer()));
}
}
public override Span <byte> AsSpan(int index, int length)
{
if (IsDisposed)
{
PipelinesThrowHelper.ThrowObjectDisposedException(nameof(MemoryPoolBlock));
}
if (length > _length - index)
{
throw new ArgumentOutOfRangeException();
}
return(new Span <byte>(Slab.Array, _offset + index, length));
}
public override Span <byte> AsSpan(int index, int length)
{
if (IsDisposed)
{
PipelinesThrowHelper.ThrowObjectDisposedException(nameof(UnownedBuffer));
}
if (length > _buffer.Count - index)
{
throw new ArgumentOutOfRangeException();
}
return(new Span <byte>(_buffer.Array, _buffer.Offset + index, length));
}
internal T Get <T>()
{
switch (Segment)
{
case null:
return(default);
case T segment:
return(segment);
}
PipelinesThrowHelper.ThrowInvalidOperationException(ExceptionResource.UnexpectedSegmentType);
return(default);
/// <summary>
/// Copy the <see cref="ReadableBuffer"/> to the specified <see cref="Span{Byte}"/>.
/// </summary>
/// <param name="destination">The destination <see cref="Span{Byte}"/>.</param>
public void CopyTo(Span <byte> destination)
{
if (Length > destination.Length)
{
PipelinesThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.destination);
}
foreach (var buffer in this)
{
buffer.Span.CopyTo(destination);
destination = destination.Slice(buffer.Length);
}
}
public void BeginTentative(ExceptionResource exception)
{
// Inactive and Tenative are allowed
if (_state == State.Active)
{
PipelinesThrowHelper.ThrowInvalidOperationException(exception, Location);
}
_state = State.Tentative;
#if OPERATION_LOCATION_TRACKING
_operationStartLocation = Environment.StackTrace;
#endif
}
public override bool Release()
{
int newRefCount = Interlocked.Decrement(ref _referenceCount);
if (newRefCount < 0)
{
PipelinesThrowHelper.ThrowInvalidOperationException(ExceptionResource.ReferenceCountZero);
}
if (newRefCount == 0)
{
return(false);
}
return(true);
}
void IAwaiter <FlushResult> .OnCompleted(Action continuation)
{
Action awaitable;
bool doubleCompletion;
lock (_sync)
{
awaitable = _writerAwaitable.OnCompleted(continuation, out doubleCompletion);
}
if (doubleCompletion)
{
Reader.Complete(PipelinesThrowHelper.GetInvalidOperationException(ExceptionResource.NoConcurrentOperation));
}
TrySchedule(_writerScheduler, awaitable);
}
ReadResult IAwaiter <ReadResult> .GetResult()
{
if (!_readerAwaitable.IsCompleted)
{
PipelinesThrowHelper.ThrowInvalidOperationException(ExceptionResource.GetResultNotCompleted);
}
var result = new ReadResult();
lock (_sync)
{
GetResult(ref result);
}
return(result);
}
internal ValueAwaiter <ReadResult> ReadAsync(CancellationToken token)
{
CancellationTokenRegistration cancellationTokenRegistration;
if (_readerCompletion.IsCompleted)
{
PipelinesThrowHelper.ThrowInvalidOperationException(ExceptionResource.NoReadingAllowed, _readerCompletion.Location);
}
lock (_sync)
{
cancellationTokenRegistration = _readerAwaitable.AttachToken(token, _signalReaderAwaitable, this);
}
cancellationTokenRegistration.Dispose();
return(new ValueAwaiter <ReadResult>(this));
}
public override BufferHandle Pin(int index = 0)
{
if (IsDisposed)
{
PipelinesThrowHelper.ThrowObjectDisposedException(nameof(MemoryPoolBlock));
}
if (index > _length)
{
PipelinesThrowHelper.ThrowArgumentOutOfRangeException(_length, index);
}
Retain();
unsafe
{
return(new BufferHandle(this, (Slab.NativePointer + _offset + index).ToPointer()));
}
}
/// <summary>
/// Create a <see cref="ReadableBuffer"/> over an <see cref="OwnedMemory{Byte}"/>.
/// </summary>
public static ReadableBuffer Create(OwnedMemory <byte> data, int offset, int length)
{
if (data == null)
{
PipelinesThrowHelper.ThrowArgumentNullException(ExceptionArgument.data);
}
if (offset < 0)
{
PipelinesThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.offset);
}
if (length < 0 || length > data.Length - offset)
{
PipelinesThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.length);
}
return(new ReadableBuffer(data, offset, length));
}
