private SegmentAsValueType(BufferSegment value) => _value = value;
// Reading void IPipeReader.Advance(ReadCursor consumed, ReadCursor examined) { BufferSegment returnStart = null; BufferSegment returnEnd = null; // Reading commit head shared with writer Action continuation = null; lock (_sync) { var examinedEverything = examined.Segment == _commitHead && examined.Index == _commitHeadIndex; if (!consumed.IsDefault) { if (_readHead == null) { PipelinesThrowHelper.ThrowInvalidOperationException(ExceptionResource.AdvanceToInvalidCursor); return; } var consumedSegment = consumed.GetSegment(); returnStart = _readHead; returnEnd = consumedSegment; // Check if we crossed _maximumSizeLow and complete backpressure var consumedBytes = ReadCursor.GetLength(returnStart, returnStart.Start, consumedSegment, consumed.Index); var oldLength = _length; _length -= consumedBytes; if (oldLength >= _maximumSizeLow && _length < _maximumSizeLow) { continuation = _writerAwaitable.Complete(); } // Check if we consumed entire last segment // if we are going to return commit head // we need to check that there is no writing operation that // might be using tailspace if (consumed.Index == returnEnd.End && !(_commitHead == returnEnd && _writingState.IsActive)) { var nextBlock = returnEnd.Next; if (_commitHead == returnEnd) { _commitHead = nextBlock; _commitHeadIndex = nextBlock?.Start ?? 0; } _readHead = nextBlock; returnEnd = nextBlock; } else { _readHead = consumedSegment; _readHead.Start = consumed.Index; } } // We reset the awaitable to not completed if we've examined everything the producer produced so far // but only if writer is not completed yet if (examinedEverything && !_writerCompletion.IsCompleted) { // Prevent deadlock where reader awaits new data and writer await backpressure if (!_writerAwaitable.IsCompleted) { PipelinesThrowHelper.ThrowInvalidOperationException(ExceptionResource.BackpressureDeadlock); } _readerAwaitable.Reset(); } _readingState.End(ExceptionResource.NoReadToComplete); while (returnStart != null && returnStart != returnEnd) { returnStart.ResetMemory(); ReturnSegmentUnsynchronized(returnStart); returnStart = returnStart.Next; } } TrySchedule(_writerScheduler, continuation); }
private void PushWithResize(BufferSegment item) { Array.Resize(ref _array, 2 * _array.Length); _array[_size] = item; _size++; }
/// <summary> /// Writes a new buffer into the pipeline. The task returned by this operation only completes when the next /// Read has been queued, or the Reader has completed, since the buffer provided here needs to be kept alive /// until the matching Read finishes (because we don't have ownership tracking when working with unowned buffers) /// </summary> /// <param name="buffer"></param> /// <param name="cancellationToken"></param> /// <returns></returns> // Called by the WRITER public async Task WriteAsync(OwnedMemory<byte> buffer, CancellationToken cancellationToken) { // If Writing has stopped, why is the caller writing?? if (Writing.Status != TaskStatus.WaitingForActivation) { throw new OperationCanceledException("Writing has ceased on this pipeline"); } // If Reading has stopped, we cancel. We don't write unless there's a reader ready in this pipeline. if (Reading.Status != TaskStatus.WaitingForActivation) { throw new OperationCanceledException("Reading has ceased on this pipeline"); } // Register for cancellation on this token for the duration of the write using (cancellationToken.Register(state => ((UnownedBufferReader)state).CancelWriter(), this)) { // Wait for reading to start await ReadingStarted; // Cancel this task if this write is cancelled cancellationToken.ThrowIfCancellationRequested(); // Allocate a new segment to hold the buffer being written. using (var segment = new BufferSegment(buffer)) { segment.End = buffer.Memory.Length; if (_head == null || _head.ReadableBytes == 0) { // Update the head to point to the head of the buffer. _head = segment; } else if (_tail != null) { // Add this segment to the end of the chain _tail.Next = segment; } // Always update tail to the buffer's tail _tail = segment; // Trigger the continuation Complete(); // Wait for another read to come (or for the end of Reading, which will also trigger this gate to open) in before returning await _readWaiting; if (_head.ReadableBytes > 0) { // We need to preserve any buffers that haven't been consumed _head = BufferSegment.Clone(new ReadCursor(_head), new ReadCursor(_tail, _tail?.End ?? 0), out _tail); } } // Cancel this task if this write is cancelled cancellationToken.ThrowIfCancellationRequested(); } }
// Reading void IPipeReader.Advance(ReadCursor consumed, ReadCursor examined) { BufferSegment returnStart = null; BufferSegment returnEnd = null; int consumedBytes = 0; if (!consumed.IsDefault) { consumedBytes = ReadCursor.GetLength(_readHead, _readHead.Start, consumed.Segment, consumed.Index); returnStart = _readHead; returnEnd = consumed.Segment; _readHead = consumed.Segment; _readHead.Start = consumed.Index; } bool resumeWriter; // Reading commit head shared with writer lock (_sync) { _length -= consumedBytes; resumeWriter = _length < _maximumSizeLow; // Change the state from observed -> not cancelled. We only want to reset the cancelled state if it was observed Interlocked.CompareExchange(ref _cancelledState, CancelledState.NotCancelled, CancelledState.CancellationObserved); var consumedEverything = examined.Segment == _commitHead && examined.Index == _commitHeadIndex && Reading.Status == TaskStatus.WaitingForActivation; // We reset the awaitable to not completed if // 1. We've consumed everything the producer produced so far // 2. Cancellation wasn't requested if (consumedEverything && _cancelledState != CancelledState.CancellationRequested) { Reset(ref _readerCallback); } } while (returnStart != null && returnStart != returnEnd) { var returnSegment = returnStart; returnStart = returnStart.Next; returnSegment.Dispose(); } #if CONSUMING_LOCATION_TRACKING _consumingLocation = null; #endif // CompareExchange not required as its setting to current value if test fails if (Interlocked.Exchange(ref _consumingState, State.NotActive) != State.Active) { ThrowHelper.ThrowInvalidOperationException(ExceptionResource.NotConsumingToComplete); } if (resumeWriter) { Resume(_writerScheduler, ref _writerCallback); } }
internal ReadCursor(BufferSegment segment) { Segment = segment; Index = segment?.Start ?? 0; }
// Called by the READER void IPipelineReader.Advance(ReadCursor consumed, ReadCursor examined) { BufferSegment returnStart = null; BufferSegment returnEnd = null; if (!consumed.IsDefault) { returnStart = _head; returnEnd = consumed.Segment; _head = consumed.Segment; _head.Start = consumed.Index; } // Again, we don't need an interlock here because Read and Write proceed serially. // REVIEW: examined.IsEnd (PipelineReaderWriter has changed this logic) var consumedEverything = examined.IsEnd && Reading.Status == TaskStatus.WaitingForActivation && _awaitableState == _awaitableIsCompleted; CompareExchange(ref _cancelledState, CancelledState.NotCancelled, CancelledState.CancellationObserved); if (consumedEverything && _cancelledState != CancelledState.CancellationRequested) { _awaitableState = _awaitableIsNotCompleted; } while (returnStart != returnEnd) { var returnSegment = returnStart; returnStart = returnStart.Next; returnSegment.Dispose(); } if (!_consuming) { throw new InvalidOperationException("No ongoing consuming operation to complete."); } _consuming = false; }
internal void Append(ReadableBuffer buffer) { if (buffer.IsEmpty) { return; // nothing to do } EnsureAlloc(); BufferSegment clonedEnd; var clonedBegin = BufferSegment.Clone(buffer.Start, buffer.End, out clonedEnd); if (_writingHead == null) { // No active write if (_commitHead == null) { // No allocated buffers yet, not locking as _readHead will be null _commitHead = clonedBegin; } else { Debug.Assert(_commitHead.Next == null); // Allocated buffer, append as next segment _commitHead.Next = clonedBegin; } } else { Debug.Assert(_writingHead.Next == null); // Active write, append as next segment _writingHead.Next = clonedBegin; } // Move write head to end of buffer _writingHead = clonedEnd; }
internal void Commit() { // CompareExchange not required as its setting to current value if test fails if (Interlocked.Exchange(ref _producingState, State.NotActive) != State.Active) { ThrowHelper.ThrowInvalidOperationException(ExceptionResource.NotProducingToComplete); } if (_writingHead == null) { // Nothing written to commit return; } // Changing commit head shared with Reader lock (_sync) { if (_readHead == null) { // Update the head to point to the head of the buffer. // This happens if we called alloc(0) then write _readHead = _commitHead; } // Always move the commit head to the write head _commitHead = _writingHead; _commitHeadIndex = _writingHead.End; } // Clear the writing state _writingHead = null; }
internal void Ensure(int count = 1) { EnsureAlloc(); var segment = _writingHead; if (segment == null) { segment = AllocateWriteHead(count); } var bytesLeftInBuffer = segment.WritableBytes; // If inadequate bytes left or if the segment is readonly if (bytesLeftInBuffer == 0 || bytesLeftInBuffer < count || segment.ReadOnly) { var nextBuffer = _pool.Lease(count); var nextSegment = new BufferSegment(nextBuffer); segment.Next = nextSegment; _writingHead = nextSegment; } }
private BufferSegment AllocateWriteHead(int count) { BufferSegment segment = null; if (_commitHead != null && !_commitHead.ReadOnly) { // Try to return the tail so the calling code can append to it int remaining = _commitHead.WritableBytes; if (count <= remaining) { // Free tail space of the right amount, use that segment = _commitHead; } } if (segment == null) { // No free tail space, allocate a new segment segment = new BufferSegment(_pool.Lease(count)); } // Changing commit head shared with Reader lock (_sync) { if (_commitHead == null) { // No previous writes have occurred _commitHead = segment; } else if (segment != _commitHead && _commitHead.Next == null) { // Append the segment to the commit head if writes have been committed // and it isn't the same segment (unused tail space) _commitHead.Next = segment; } } // Set write head to assigned segment _writingHead = segment; return segment; }
/// <summary> /// Writes a new buffer into the pipeline. The task returned by this operation only completes when the next /// Read has been queued, or the Reader has completed, since the buffer provided here needs to be kept alive /// until the matching Read finishes (because we don't have ownership tracking when working with unowned buffers) /// </summary> /// <param name="buffer"></param> /// <param name="cancellationToken"></param> /// <returns></returns> // Called by the WRITER public async Task WriteAsync(OwnedBuffer <byte> buffer, CancellationToken cancellationToken) { // If Writing has stopped, why is the caller writing?? if (Writing.Status != TaskStatus.WaitingForActivation) { throw new OperationCanceledException("Writing has ceased on this pipeline"); } // If Reading has stopped, we cancel. We don't write unless there's a reader ready in this pipeline. if (Reading.Status != TaskStatus.WaitingForActivation) { throw new OperationCanceledException("Reading has ceased on this pipeline"); } // Register for cancellation on this token for the duration of the write using (cancellationToken.Register(state => ((UnownedBufferReader)state).CancelWriter(), this)) { // Wait for reading to start await ReadingStarted; // Cancel this task if this write is cancelled cancellationToken.ThrowIfCancellationRequested(); // Allocate a new segment to hold the buffer being written. using (var segment = new BufferSegment(buffer)) { segment.End = buffer.Buffer.Length; if (_head == null || _head.ReadableBytes == 0) { // Update the head to point to the head of the buffer. _head = segment; } else if (_tail != null) { // Add this segment to the end of the chain _tail.SetNext(segment); } // Always update tail to the buffer's tail _tail = segment; // Trigger the continuation Complete(); // Wait for another read to come (or for the end of Reading, which will also trigger this gate to open) in before returning await _readWaiting; if (_head.ReadableBytes > 0) { // We need to preserve any buffers that haven't been consumed _head = BufferSegment.Clone(new ReadCursor(_head), new ReadCursor(_tail, _tail?.End ?? 0), out _tail); } else { // Drop segement references before Dispose gets called on the segment _head = _tail = null; } } // Cancel this task if this write is cancelled cancellationToken.ThrowIfCancellationRequested(); } }
public static BufferSegment Clone(ReadCursor beginBuffer, ReadCursor endBuffer, out BufferSegment lastSegment) { var beginOrig = beginBuffer.Segment; var endOrig = endBuffer.Segment; if (beginOrig == endOrig) { lastSegment = new BufferSegment(); lastSegment.SetMemory(beginOrig._owned, beginBuffer.Index, endBuffer.Index); return(lastSegment); } var beginClone = new BufferSegment(); beginClone.SetMemory(beginOrig._owned, beginBuffer.Index, beginOrig.End); var endClone = beginClone; beginOrig = beginOrig.Next; while (beginOrig != endOrig) { var next = new BufferSegment(); next.SetMemory(beginOrig._owned, beginOrig.Start, beginOrig.End); endClone.SetNext(next); endClone = endClone.Next; beginOrig = beginOrig.Next; } lastSegment = new BufferSegment(); lastSegment.SetMemory(endOrig._owned, endOrig.Start, endBuffer.Index); endClone.SetNext(lastSegment); return(beginClone); }
internal ReadableBuffer(BufferSegment startSegment, int startIndex, BufferSegment endSegment, int endIndex) { BufferStart = new ReadCursor(startSegment, startIndex); BufferEnd = new ReadCursor(endSegment, endIndex); }
public static BufferSegment Clone(ReadCursor beginBuffer, ReadCursor endBuffer, out BufferSegment lastSegment) { var beginOrig = beginBuffer.Segment; var endOrig = endBuffer.Segment; if (beginOrig == endOrig) { lastSegment = new BufferSegment(beginOrig._buffer, beginBuffer.Index, endBuffer.Index); return(lastSegment); } var beginClone = new BufferSegment(beginOrig._buffer, beginBuffer.Index, beginOrig.End); var endClone = beginClone; beginOrig = beginOrig.Next; while (beginOrig != endOrig) { endClone.Next = new BufferSegment(beginOrig._buffer, beginOrig.Start, beginOrig.End); endClone = endClone.Next; beginOrig = beginOrig.Next; } lastSegment = new BufferSegment(endOrig._buffer, endOrig.Start, endBuffer.Index); endClone.Next = lastSegment; return(beginClone); }
public void AdvanceReader(ReadCursor consumed, ReadCursor examined) { BufferSegment returnStart = null; BufferSegment returnEnd = null; if (!consumed.IsDefault) { returnStart = _readHead; returnEnd = consumed.Segment; _readHead = consumed.Segment; _readHead.Start = consumed.Index; } // Reading commit head shared with writer lock (_sync) { // Change the state from observed -> not cancelled. We only want to reset the cancelled state if it was observed Interlocked.CompareExchange(ref _cancelledState, CancelledState.NotCancelled, CancelledState.CancellationObserved); var consumedEverything = examined.Segment == _commitHead && examined.Index == _commitHeadIndex && Reading.Status == TaskStatus.WaitingForActivation; // We reset the awaitable to not completed if // 1. We've consumed everything the producer produced so far // 2. Cancellation wasn't requested if (consumedEverything && _cancelledState != CancelledState.CancellationRequested) { Interlocked.CompareExchange( ref _awaitableState, _awaitableIsNotCompleted, _awaitableIsCompleted); } } while (returnStart != null && returnStart != returnEnd) { var returnSegment = returnStart; returnStart = returnStart.Next; returnSegment.Dispose(); } #if DEBUG _consumingLocation = null; #endif // CompareExchange not required as its setting to current value if test fails if (Interlocked.Exchange(ref _consumingState, State.NotActive) != State.Active) { ThrowHelper.ThrowInvalidOperationException(ExceptionResource.NotConsumingToComplete); } }
private async ValueTask <FlushResult> FlushAsyncInternal(bool writeToStream, ReadOnlyMemory <byte> data, CancellationToken cancellationToken = default) { // Write all completed segments and whatever remains in the current segment // and flush the result. CancellationTokenRegistration reg = default; if (cancellationToken.CanBeCanceled) { reg = cancellationToken.UnsafeRegister(state => ((StreamPipeWriter)state !).Cancel(), this); } if (_tailBytesBuffered > 0) { Debug.Assert(_tail != null); // Update any buffered data _tail.End += _tailBytesBuffered; _tailBytesBuffered = 0; } using (reg) { CancellationToken localToken = InternalTokenSource.Token; try { BufferSegment?segment = _head; while (segment != null) { BufferSegment returnSegment = segment; segment = segment.NextSegment; if (returnSegment.Length > 0 && writeToStream) { await InnerStream.WriteAsync(returnSegment.Memory, localToken).ConfigureAwait(false); } returnSegment.ResetMemory(); ReturnSegmentUnsynchronized(returnSegment); // Update the head segment after we return the current segment _head = segment; } if (writeToStream) { // Write data after the buffered data if (data.Length > 0) { await InnerStream.WriteAsync(data, localToken).ConfigureAwait(false); } if (_bytesBuffered > 0 || data.Length > 0) { await InnerStream.FlushAsync(localToken).ConfigureAwait(false); } } // Mark bytes as written *after* flushing _head = null; _tail = null; _tailMemory = default; _bytesBuffered = 0; return(new FlushResult(isCanceled: false, isCompleted: false)); } catch (OperationCanceledException) { // Remove the cancellation token such that the next time Flush is called // A new CTS is created. lock (_lockObject) { _internalTokenSource = null; } if (localToken.IsCancellationRequested && !cancellationToken.IsCancellationRequested) { // Catch cancellation and translate it into setting isCanceled = true return(new FlushResult(isCanceled: true, isCompleted: false)); } throw; } } }
private void Dispose() { Debug.Assert(Writing.IsCompleted, "Not completed writing"); Debug.Assert(Reading.IsCompleted, "Not completed reading"); // TODO: Review throw if not completed? lock (_sync) { // Return all segments var segment = _readHead; while (segment != null) { var returnSegment = segment; segment = segment.Next; returnSegment.Dispose(); } _readHead = null; _commitHead = null; } }
internal ReadCursor(BufferSegment segment, int index) { Segment = segment; Index = index; }
internal void AdvanceReader(SequencePosition consumed, SequencePosition examined) { BufferSegment returnStart = null; BufferSegment returnEnd = null; Action continuation = null; lock (_sync) { var examinedEverything = false; if (examined.Segment == _commitHead) { examinedEverything = _commitHead != null ? examined.Index == _commitHeadIndex - _commitHead.Start : examined.Index == 0; } if (consumed.Segment != null) { if (_readHead == null) { ThrowHelper.ThrowInvalidOperationException_AdvanceToInvalidCursor(); return; } var consumedSegment = (BufferSegment)consumed.Segment; returnStart = _readHead; returnEnd = consumedSegment; // Check if we crossed _maximumSizeLow and complete backpressure long consumedBytes = new ReadOnlySequence <byte>(returnStart, _readHeadIndex, consumedSegment, consumed.Index).Length; long oldLength = _length; _length -= consumedBytes; if (oldLength >= _maximumSizeLow && _length < _maximumSizeLow) { continuation = _writerAwaitable.Complete(); } // Check if we consumed entire last segment // if we are going to return commit head we need to check that there is no writing operation that // might be using tailspace if (consumed.Index == returnEnd.Length && _writingHead != returnEnd) { var nextBlock = returnEnd.NextSegment; if (_commitHead == returnEnd) { _commitHead = nextBlock; _commitHeadIndex = 0; } _readHead = nextBlock; _readHeadIndex = 0; returnEnd = nextBlock; } else { _readHead = consumedSegment; _readHeadIndex = consumed.Index; } } // We reset the awaitable to not completed if we've examined everything the producer produced so far // but only if writer is not completed yet if (examinedEverything && !_writerCompletion.IsCompleted) { // Prevent deadlock where reader awaits new data and writer await backpressure if (!_writerAwaitable.IsCompleted) { ThrowHelper.ThrowInvalidOperationException_BackpressureDeadlock(); } _readerAwaitable.Reset(); } _readingState.End(); while (returnStart != null && returnStart != returnEnd) { returnStart.ResetMemory(); ReturnSegmentUnsynchronized(returnStart); returnStart = returnStart.NextSegment; } } TrySchedule(_writerScheduler, continuation); }
private void Dispose() { Debug.Assert(Writing.IsCompleted, "Not completed writing"); Debug.Assert(Reading.IsCompleted, "Not completed reading"); // Return all segments var segment = _head; while (segment != null) { var returnSegment = segment; segment = segment.Next; returnSegment.Dispose(); } _head = null; _tail = null; }
private void AdvanceTo(BufferSegment consumedSegment, int consumedIndex, BufferSegment examinedSegment, int examinedIndex) { if (consumedSegment == null || examinedSegment == null) { return; } if (_readHead == null) { ThrowHelper.ThrowInvalidOperationException_AdvanceToInvalidCursor(); } BufferSegment returnStart = _readHead; BufferSegment returnEnd = consumedSegment; long consumedBytes = BufferSegment.GetLength(returnStart, _readIndex, consumedSegment, consumedIndex); _bufferedBytes -= consumedBytes; Debug.Assert(_bufferedBytes >= 0); _examinedEverything = false; if (examinedSegment == _readTail) { // If we examined everything, we force ReadAsync to actually read from the underlying stream // instead of returning a ReadResult from TryRead. _examinedEverything = examinedIndex == _readTail.End; } // Two cases here: // 1. All data is consumed. If so, we empty clear everything so we don't hold onto any // excess memory. // 2. A segment is entirely consumed but there is still more data in nextSegments // We are allowed to remove an extra segment. by setting returnEnd to be the next block. // 3. We are in the middle of a segment. // Move _readHead and _readIndex to consumedSegment and index if (_bufferedBytes == 0) { returnEnd = null; _readHead = null; _readTail = null; _readIndex = 0; } else if (consumedIndex == returnEnd.Length) { BufferSegment nextBlock = returnEnd.NextSegment; _readHead = nextBlock; _readIndex = 0; returnEnd = nextBlock; } else { _readHead = consumedSegment; _readIndex = consumedIndex; } // Remove all blocks that are freed (except the last one) while (returnStart != returnEnd) { BufferSegment next = returnStart.NextSegment; returnStart.ResetMemory(); ReturnSegmentUnsynchronized(returnStart); returnStart = next; } }
public static BufferSegment Clone(ReadCursor beginBuffer, ReadCursor endBuffer, out BufferSegment lastSegment) { var beginOrig = beginBuffer.Segment; var endOrig = endBuffer.Segment; if (beginOrig == endOrig) { lastSegment = new BufferSegment(beginOrig._buffer, beginBuffer.Index, endBuffer.Index); return lastSegment; } var beginClone = new BufferSegment(beginOrig._buffer, beginBuffer.Index, beginOrig.End); var endClone = beginClone; beginOrig = beginOrig.Next; while (beginOrig != endOrig) { endClone.Next = new BufferSegment(beginOrig._buffer, beginOrig.Start, beginOrig.End); endClone = endClone.Next; beginOrig = beginOrig.Next; } lastSegment = new BufferSegment(endOrig._buffer, endOrig.Start, endBuffer.Index); endClone.Next = lastSegment; return beginClone; }
public static BufferSegment Clone(BufferSegment start, int startIndex, BufferSegment end, int endIndex, out BufferSegment lastSegment) { var beginOrig = start; var endOrig = end; if (beginOrig == endOrig) { lastSegment = new BufferSegment(); lastSegment.SetMemory(beginOrig._ownedMemory, startIndex, endIndex); return(lastSegment); } var beginClone = new BufferSegment(); beginClone.SetMemory(beginOrig._ownedMemory, startIndex, beginOrig.End); var endClone = beginClone; beginOrig = beginOrig.Next; while (beginOrig != endOrig) { var next = new BufferSegment(); next.SetMemory(beginOrig._ownedMemory, beginOrig.Start, beginOrig.End); endClone.SetNext(next); endClone = endClone.Next; beginOrig = beginOrig.Next; } lastSegment = new BufferSegment(); lastSegment.SetMemory(endOrig._ownedMemory, endOrig.Start, endIndex); endClone.SetNext(lastSegment); return(beginClone); }