private ValueTask <T> ReadAsyncCore(CancellationToken cancellationToken)
{
if (cancellationToken.IsCancellationRequested)
{
return(new ValueTask <T>(Task.FromCanceled <T>(cancellationToken)));
}
lock (SyncObj)
{
// Now that we hold the lock, try reading again.
T item;
if (TryRead(out item))
{
return(new ValueTask <T>(item));
}
// If no more items will be written, fail the read.
if (_doneWriting != null)
{
return(ChannelUtilities.GetInvalidCompletionValueTask <T>(_doneWriting));
}
Debug.Assert(_blockedReader == null || ((_blockedReader as ReaderInteractor <T>)?.Task.IsCanceled ?? false),
"Incorrect usage; multiple outstanding reads were issued against this single-consumer channel");
// Store the reader to be completed by a writer.
ReaderInteractor <T> reader = ReaderInteractor <T> .Create(_runContinuationsAsynchronously, cancellationToken);
_blockedReader = reader;
return(new ValueTask <T>(reader.Task));
}
}
private ValueTask <T> ReadAsync(CancellationToken cancellationToken = default(CancellationToken))
{
// Fast-path cancellation check
if (cancellationToken.IsCancellationRequested)
{
return(new ValueTask <T>(Task.FromCanceled <T>(cancellationToken)));
}
lock (SyncObj)
{
AssertInvariants();
// If there are any items, hand one back.
if (!_items.IsEmpty)
{
return(new ValueTask <T>(DequeueItemAndPostProcess()));
}
// There weren't any items. If we're done writing so that there
// will never be more items, fail.
if (_doneWriting != null)
{
return(ChannelUtilities.GetErrorValueTask <T>(_doneWriting));
}
// Otherwise, queue the reader.
var reader = ReaderInteractor <T> .Create(_runContinuationsAsynchronously, cancellationToken);
_blockedReaders.EnqueueTail(reader);
return(new ValueTask <T>(reader.Task));
}
}
private bool TryWrite(T item)
{
while (true)
{
ReaderInteractor <T> blockedReader = null;
ReaderInteractor <bool> waitingReaders = null;
lock (SyncObj)
{
// If writing has already been marked as done, fail the write.
AssertInvariants();
if (_doneWriting != null)
{
return(false);
}
// If there aren't any blocked readers, just add the data to the queue,
// and let any waiting readers know that they should try to read it.
// We can only complete such waiters here under the lock if they run
// continuations asynchronously (otherwise the synchronous continuations
// could be invoked under the lock). If we don't complete them here, we
// need to do so outside of the lock.
if (_blockedReaders.IsEmpty)
{
_items.Enqueue(item);
waitingReaders = _waitingReaders;
if (waitingReaders == null)
{
return(true);
}
_waitingReaders = null;
}
else
{
// There were blocked readers. Grab one, and then complete it outside of the lock.
blockedReader = _blockedReaders.DequeueHead();
}
}
if (blockedReader != null)
{
// Complete the reader. It's possible the reader was canceled, in which
// case we loop around to try everything again.
if (blockedReader.Success(item))
{
return(true);
}
}
else
{
// Wake up all of the waiters. Since we've released the lock, it's possible
// we could cause some spurious wake-ups here, if we tell a waiter there's
// something available but all data has already been removed. It's a benign
// race condition, though, as consumers already need to account for such things.
waitingReaders.Success(item: true);
return(true);
}
}
}
/// <summary>Wake up all of the waiters and null out the field.</summary>
/// <param name="waiters">The waiters.</param>
/// <param name="result">The value with which to complete each waiter.</param>
internal static void WakeUpWaiters(ref ReaderInteractor <bool> waiters, bool result)
{
ReaderInteractor <bool> w = waiters;
if (w != null)
{
w.Success(result);
waiters = null;
}
}
/// <summary>Wake up all of the waiters and null out the field.</summary>
/// <param name="waiters">The waiters.</param>
/// <param name="result">The success value with which to complete each waiter if <paramref name="error">error</paramref> is null.</param>
/// <param name="error">The failure with which to cmplete each waiter, if non-null.</param>
internal static void WakeUpWaiters(ref ReaderInteractor <bool> waiters, bool result, Exception error = null)
{
ReaderInteractor <bool> w = waiters;
if (w != null)
{
if (error != null)
{
w.Fail(error);
}
else
{
w.Success(result);
}
waiters = null;
}
}
private ValueTask <T> ReadAsyncCore(CancellationToken cancellationToken)
{
if (cancellationToken.IsCancellationRequested)
{
return(new ValueTask <T>(Task.FromCanceled <T>(cancellationToken)));
}
lock (SyncObj)
{
AssertInvariants();
// If we're already completed, nothing to read.
if (_completion.Task.IsCompleted)
{
return(new ValueTask <T>(
_completion.Task.IsCanceled ? Task.FromCanceled <T>(new CancellationToken(true)) :
Task.FromException <T>(
_completion.Task.IsFaulted ?
ChannelUtilities.CreateInvalidCompletionException(_completion.Task.Exception.InnerException) :
ChannelUtilities.CreateInvalidCompletionException())));
}
// If there are any blocked writers, find one to pair up with
// and get its data. Writers that got canceled will remain in the queue,
// so we need to loop to skip past them.
while (!_blockedWriters.IsEmpty)
{
WriterInteractor <T> w = _blockedWriters.DequeueHead();
if (w.Success(default(VoidResult)))
{
return(new ValueTask <T>(w.Item));
}
}
// No writer found to pair with. Queue the reader.
var r = ReaderInteractor <T> .Create(true, cancellationToken);
_blockedReaders.EnqueueTail(r);
// And let any waiting writers know it's their lucky day.
ChannelUtilities.WakeUpWaiters(ref _waitingWriters, result: true);
return(new ValueTask <T>(r.Task));
}
}
/// <summary>Gets or creates a "waiter" (e.g. WaitForRead/WriteAsync) interactor.</summary>
/// <param name="waiter">The field storing the waiter interactor.</param>
/// <param name="runContinuationsAsynchronously">true to force continuations to run asynchronously; otherwise, false.</param>
/// <param name="cancellationToken">The token to use to cancel the wait.</param>
internal static Task <bool> GetOrCreateWaiter(ref ReaderInteractor <bool> waiter, bool runContinuationsAsynchronously, CancellationToken cancellationToken)
{
// Get the existing waiters interactor.
ReaderInteractor <bool> w = waiter;
// If there isn't one, create one. This explicitly does not include the cancellation token,
// as we reuse it for any number of waiters that overlap.
if (w == null)
{
waiter = w = ReaderInteractor <bool> .Create(runContinuationsAsynchronously);
}
// If the cancellation token can't be canceled, then just return the waiter task.
// If it can, we need to return a task that will complete when the waiter task does but that can also be canceled.
// Easiest way to do that is with a cancelable continuation.
return(cancellationToken.CanBeCanceled ?
w.Task.ContinueWith(t => t.Result, cancellationToken, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default) :
w.Task);
}
private Task WriteAsync(T item, CancellationToken cancellationToken = default(CancellationToken))
{
if (cancellationToken.IsCancellationRequested)
{
return(Task.FromCanceled(cancellationToken));
}
lock (SyncObj)
{
// Fail if we've already completed
if (_completion.Task.IsCompleted)
{
return
(_completion.Task.IsCanceled ? Task.FromCanceled <T>(new CancellationToken(true)) :
Task.FromException <T>(
_completion.Task.IsFaulted ?
ChannelUtilities.CreateInvalidCompletionException(_completion.Task.Exception.InnerException) :
ChannelUtilities.CreateInvalidCompletionException()));
}
// Try to find a reader to pair with. Canceled readers remain in the queue,
// so we need to loop until we find one.
while (!_blockedReaders.IsEmpty)
{
ReaderInteractor <T> r = _blockedReaders.DequeueHead();
if (r.Success(item))
{
return(Task.CompletedTask);
}
}
// No reader was available. Queue the writer.
var w = WriterInteractor <T> .Create(true, cancellationToken, item);
_blockedWriters.EnqueueTail(w);
// And let any waiting readers know it's their lucky day.
ChannelUtilities.WakeUpWaiters(ref _waitingReaders, result: true);
return(w.Task);
}
}
private Task <bool> WaitToReadAsync(CancellationToken cancellationToken = default)
{
// Outside of the lock, check if there are any items waiting to be read. If there are, we're done.
if (!_items.IsEmpty)
{
return(ChannelUtilities.TrueTask);
}
// Now check for cancellation.
if (cancellationToken.IsCancellationRequested)
{
return(Task.FromCanceled <bool>(cancellationToken));
}
ReaderInteractor <bool> oldWaiter = null, newWaiter;
lock (SyncObj)
{
// Again while holding the lock, check to see if there are any items available.
if (!_items.IsEmpty)
{
return(ChannelUtilities.TrueTask);
}
// There aren't any items; if we're done writing, there never will be more items.
if (_doneWriting != null)
{
return(_doneWriting != ChannelUtilities.DoneWritingSentinel ?
Task.FromException <bool>(_doneWriting) :
ChannelUtilities.FalseTask);
}
// Create the new waiter. We're a bit more tolerant of a stray waiting reader
// than we are of a blocked reader, as with usage patterns it's easier to leave one
// behind, so we just cancel any that may have been waiting around.
oldWaiter = _waitingReader;
_waitingReader = newWaiter = ReaderInteractor <bool> .Create(_runContinuationsAsynchronously, cancellationToken);
}
oldWaiter?.TrySetCanceled();
return(newWaiter.Task);
}
private bool TryWrite(T item)
{
lock (SyncObj)
{
AssertInvariants();
// Try to find a reader to pair with
while (!_blockedReaders.IsEmpty)
{
ReaderInteractor <T> r = _blockedReaders.DequeueHead();
if (r.Success(item))
{
return(true);
}
}
}
// None found
return(false);
}
private ValueTask <T> ReadAsyncCore(CancellationToken cancellationToken)
{
if (cancellationToken.IsCancellationRequested)
{
return(new ValueTask <T>(Task.FromCanceled <T>(cancellationToken)));
}
lock (SyncObj)
{
AssertInvariants();
// If there are any items, return one.
T item;
if (_items.TryDequeue(out item))
{
// Dequeue an item
if (_doneWriting != null && _items.IsEmpty)
{
// If we've now emptied the items queue and we're not getting any more, complete.
ChannelUtilities.Complete(_completion, _doneWriting);
}
return(new ValueTask <T>(item));
}
// There are no items, so if we're done writing, fail.
if (_doneWriting != null)
{
return(ChannelUtilities.GetInvalidCompletionValueTask <T>(_doneWriting));
}
// Otherwise, queue the reader.
ReaderInteractor <T> reader = ReaderInteractor <T> .Create(_runContinuationsAsynchronously, cancellationToken);
_blockedReaders.EnqueueTail(reader);
return(new ValueTask <T>(reader.Task));
}
}
private Task WriteAsync(T item, CancellationToken cancellationToken = default(CancellationToken))
{
if (cancellationToken.IsCancellationRequested)
{
return(Task.FromCanceled(cancellationToken));
}
ReaderInteractor <T> blockedReader = null;
ReaderInteractor <bool> waitingReaders = null;
lock (SyncObj)
{
AssertInvariants();
// If we're done writing, trying to write is an error.
if (_doneWriting != null)
{
return(Task.FromException(ChannelUtilities.CreateInvalidCompletionException()));
}
// Get the number of items in the channel currently.
int count = _items.Count;
if (count == 0)
{
// There are no items in the channel, which means we may have blocked/waiting readers.
// If there are any blocked readers, find one that's not canceled
// and store it to complete outside of the lock, in case it has
// continuations that'll run synchronously
while (!_blockedReaders.IsEmpty)
{
ReaderInteractor <T> r = _blockedReaders.DequeueHead();
r.UnregisterCancellation(); // ensure that once we grab it, we own its completion
if (!r.Task.IsCompleted)
{
blockedReader = r;
break;
}
}
if (blockedReader == null)
{
// If there wasn't a blocked reader, then store the item. If no one's waiting
// to be notified about a 0-to-1 transition, we're done.
_items.EnqueueTail(item);
waitingReaders = _waitingReaders;
if (waitingReaders == null)
{
return(ChannelUtilities.TrueTask);
}
_waitingReaders = null;
}
}
else if (count < _bufferedCapacity)
{
// There's room in the channel. Since we're not transitioning from 0-to-1 and
// since there's room, we can simply store the item and exit without having to
// worry about blocked/waiting readers.
_items.EnqueueTail(item);
return(ChannelUtilities.TrueTask);
}
else if (_mode == BoundedChannelFullMode.Wait)
{
// The channel is full and we're in a wait mode.
// Queue the writer.
var writer = WriterInteractor <T> .Create(true, cancellationToken, item);
_blockedWriters.EnqueueTail(writer);
return(writer.Task);
}
else
{
// The channel is full, and we're in a dropping mode.
// Drop either the oldest or the newest and write the new item.
T droppedItem = _mode == BoundedChannelFullMode.DropNewest ?
_items.DequeueTail() :
_items.DequeueHead();
_items.EnqueueTail(item);
return(ChannelUtilities.TrueTask);
}
}
// We either wrote the item already, or we're transfering it to the blocked reader we grabbed.
if (blockedReader != null)
{
// Transfer the written item to the blocked reader.
bool success = blockedReader.Success(item);
Debug.Assert(success, "We should always be able to complete the reader.");
}
else
{
// We stored an item bringing the count up from 0 to 1. Alert
// any waiting readers that there may be something for them to consume.
// Since we're no longer holding the lock, it's possible we'll end up
// waking readers that have since come in.
waitingReaders.Success(item: true);
}
return(ChannelUtilities.TrueTask);
}
private bool TryComplete(Exception error = null)
{
object blockedReader = null;
ReaderInteractor <bool> waitingReader = null;
bool completeTask = false;
lock (SyncObj)
{
// If we're already marked as complete, there's nothing more to do.
if (_doneWriting != null)
{
return(false);
}
// Mark as complete for writing.
_doneWriting = error ?? ChannelUtilities.DoneWritingSentinel;
// If we have no more items remaining, then the channel needs to be marked as completed
// and readers need to be informed they'll never get another item. All of that needs
// to happen outside of the lock to avoid invoking continuations under the lock.
if (_items.IsEmpty)
{
completeTask = true;
if (_blockedReader != null)
{
blockedReader = _blockedReader;
_blockedReader = null;
}
if (_waitingReader != null)
{
waitingReader = _waitingReader;
_waitingReader = null;
}
}
}
// Complete the channel task if necessary
if (completeTask)
{
ChannelUtilities.Complete(_completion, error);
}
Debug.Assert(blockedReader == null || waitingReader == null, "There should only ever be at most one reader.");
// Complete a blocked reader if necessary
if (blockedReader != null)
{
error = ChannelUtilities.CreateInvalidCompletionException(error);
ReaderInteractor <T> interactor = blockedReader as ReaderInteractor <T>;
if (interactor != null)
{
interactor.Fail(error);
}
else
{
((AutoResetAwaiter <T>)blockedReader).SetException(error);
}
}
// Complete a waiting reader if necessary. (We really shouldn't have both a blockedReader
// and a waitingReader, but it's more expensive to prevent it than to just tolerate it.)
if (waitingReader != null)
{
if (error != null)
{
waitingReader.Fail(error);
}
else
{
waitingReader.Success(false);
}
}
// Successfully completed the channel
return(true);
}
private bool TryWrite(T item)
{
while (true) // in case a reader was canceled and we need to try again
{
object blockedReader = null;
ReaderInteractor <bool> waitingReader = null;
lock (SyncObj)
{
// If writing is completed, exit out without writing.
if (_doneWriting != null)
{
return(false);
}
// If there's a blocked reader, store it into a local for completion outside of the lock.
// If there isn't a blocked reader, queue the item being written; then if there's a waiting
// reader, store it for notification outside of the lock.
blockedReader = _blockedReader;
if (blockedReader != null)
{
_blockedReader = null;
}
else
{
_items.Enqueue(item);
waitingReader = _waitingReader;
if (waitingReader == null)
{
return(true);
}
_waitingReader = null;
}
}
// If we get here, we grabbed a blocked or a waiting reader.
Debug.Assert((blockedReader != null) ^ (waitingReader != null), "Expected either a blocked or waiting reader, but not both");
// If we have a waiting reader, notify it that an item was written and exit.
if (waitingReader != null)
{
waitingReader.Success(true);
return(true);
}
// Otherwise we have a blocked reader: complete it with the item being written.
// In the case of a ReadAsync(CancellationToken), it's possible the reader could
// have been completed due to cancellation by the time we get here. In that case,
// we'll loop around to try again so as not to lose the item being written.
Debug.Assert(blockedReader != null);
ReaderInteractor <T> interactor = blockedReader as ReaderInteractor <T>;
if (interactor != null)
{
if (interactor.Success(item))
{
return(true);
}
}
else
{
((AutoResetAwaiter <T>)blockedReader).SetResult(item);
return(true);
}
}
}
