public async ValueTask Error(Exception ex)
{
await ResumeHelper.Await(ref _consumed);
ResumeHelper.Clear(ref _consumed);
_error = ex;
_done = true;
ResumeHelper.Resume(ref _valueReady);
}
public async ValueTask <bool> MoveNextAsync()
{
for (; ;)
{
var d = _sourceDone;
var v = _sourceReady;
if (d)
{
var ex = _sourceError;
if (ex != null)
{
throw ex;
}
if (!v)
{
return(false);
}
}
d = _otherDone;
var o = Volatile.Read(ref _otherValue);
if (d)
{
var ex = _otherError;
if (ex != null)
{
throw ex;
}
if (o == EmptyHelper.EmptyIndicator)
{
return(false);
}
}
if (v)
{
_sourceReady = false;
if (o == EmptyHelper.EmptyIndicator)
{
MoveNextMain();
continue;
}
Current = _func(_sourceValue, (TOther)o);
MoveNextMain();
return(true);
}
await ResumeHelper.Await(ref _resume);
ResumeHelper.Clear(ref _resume);
}
}
public ValueTask DisposeAsync()
{
CancellationHelper.Cancel(ref _cts);
if (Interlocked.Increment(ref _disposeWip) == 1)
{
if (_emitLast)
{
_emitLastItem = default;
}
return(_source.DisposeAsync());
}
return(ResumeHelper.Await(ref _disposeTask));
}
private bool TryDispose()
{
if (Interlocked.Decrement(ref _disposeWip) != 0)
{
if (_emitLast)
{
_emitLastItem = default;
}
ResumeHelper.Complete(ref _disposeTask, _source.DisposeAsync());
return(false);
}
return(true);
}
public async ValueTask <bool> MoveNextAsync()
{
for (; ;)
{
var d = _sourceDone;
var curr = _currentInner;
if (curr == null)
{
if (_inners.TryDequeue(out curr))
{
_currentInner = curr;
}
}
if (d && curr == null)
{
var ex = _error;
if (ex != null)
{
_error = null;
throw ex;
}
return(false);
}
if (curr != null)
{
d = curr.Done;
var success = curr.Queue.TryDequeue(out var v);
if (d && !success)
{
curr.Dispose();
_currentInner = null;
SourceConsumedOne();
continue;
}
if (success)
{
Current = v;
curr.ConsumedOne();
return(true);
}
}
await ResumeHelper.Await(ref _resume);
ResumeHelper.Clear(ref _resume);
}
}
public async ValueTask <bool> MoveNextAsync()
{
for (; ;)
{
var d = _done;
var curr = Volatile.Read(ref _current);
if (d && curr == null)
{
var ex = _error;
if (ex != null)
{
_error = null;
throw ex;
}
return(false);
}
if (curr != null)
{
d = curr.Done;
if (curr.HasValue)
{
Current = curr.Value;
curr.Value = default;
curr.HasValue = false;
if (curr == Volatile.Read(ref _current))
{
curr.MoveNext();
}
return(true);
}
if (d)
{
if (Interlocked.CompareExchange(ref _current, null, curr) == curr)
{
curr.Dispose();
}
continue;
}
}
await ResumeHelper.Await(ref _resume);
ResumeHelper.Clear(ref _resume);
}
}
private void HandleNextOther(Task <bool> t)
{
if (t.IsCanceled)
{
if (!Volatile.Read(ref _suppressCancel))
{
ExceptionHelper.AddException(ref _error, new OperationCanceledException());
_queue.Enqueue(new Entry
{
Done = true
});
Volatile.Write(ref _suppressCancel, true);
_mainCancel.Cancel();
}
}
else if (t.IsFaulted)
{
ExceptionHelper.AddException(ref _error, ExceptionHelper.Extract(t.Exception));
_queue.Enqueue(new Entry
{
Done = true
});
Volatile.Write(ref _suppressCancel, true);
_mainCancel.Cancel();
}
else if (t.Result)
{
_queue.Enqueue(new Entry
{
Boundary = true
});
}
else
{
_queue.Enqueue(new Entry
{
Done = true
});
Volatile.Write(ref _suppressCancel, true);
_mainCancel.Cancel();
}
if (TryDisposeOther())
{
ResumeHelper.Resume(ref _resume);
}
}
private void Next(Task t)
{
if (t.IsCanceled || _cts.IsCancellationRequested)
{
return;
}
var value = Interlocked.Increment(ref _available);
ResumeHelper.Resume(ref _resume);
if (value != _end)
{
// FIXME compensate for drifts
Task.Delay(_period, _cts.Token)
.ContinueWith(NextAction, this, _cts.Token);
}
}
public async ValueTask <bool> MoveNextAsync()
{
await ResumeHelper.Await(ref _valueReady);
ResumeHelper.Clear(ref _valueReady);
if (_done)
{
if (_error != null)
{
throw _error;
}
return(false);
}
Current = _current;
ResumeHelper.Resume(ref _consumed);
return(true);
}
public async ValueTask Next(T value)
{
if (_disposeRequested)
{
return;
}
await ResumeHelper.Await(ref _consumed);
ResumeHelper.Clear(ref _consumed);
if (_disposeRequested)
{
return;
}
Current = value;
_hasValue = true;
ResumeHelper.Resume(ref _valueReady);
}
public async ValueTask <bool> MoveNextAsync()
{
for (; ;)
{
var d = _done;
var success = _queue.TryDequeue(out var v);
if (d && !success)
{
if (_error != null)
{
throw _error;
}
return(false);
}
if (success)
{
Current = v;
var c = _consumed + 1;
if (c == _limit)
{
_consumed = 0;
if (Interlocked.Add(ref _outstanding, c) == c)
{
MoveNext();
}
}
else
{
_consumed = c;
}
return(true);
}
await ResumeHelper.Await(ref _resume);
ResumeHelper.Clear(ref _resume);
}
}
internal void SetTask(Task task)
{
_task = task.ContinueWith(async t =>
{
if (_disposeRequested)
{
return;
}
await ResumeHelper.Await(ref _consumed);
ResumeHelper.Clear(ref _consumed);
if (_disposeRequested)
{
return;
}
_error = ExceptionHelper.Extract(t.Exception);
ResumeHelper.Resume(ref _valueReady);
}, Token);
}
private void InnerNextHandler(Task <bool> t)
{
if (t.IsCanceled)
{
ExceptionHelper.AddException(ref _parent._error, new OperationCanceledException());
Done = true;
if (TryDispose())
{
ResumeHelper.Resume(ref _parent._resume);
}
}
else if (t.IsFaulted)
{
ExceptionHelper.AddException(ref _parent._error, ExceptionHelper.Extract(t.Exception));
Done = true;
if (TryDispose())
{
ResumeHelper.Resume(ref _parent._resume);
}
}
else if (t.Result)
{
Queue.Enqueue(_source.Current);
if (TryDispose())
{
if (Interlocked.Decrement(ref _outstanding) != 0)
{
MoveNext();
}
ResumeHelper.Resume(ref _parent._resume);
}
}
else
{
Done = true;
if (TryDispose())
{
ResumeHelper.Resume(ref _parent._resume);
}
}
}
private void MainHandler(Task <bool> t)
{
if (t.IsCanceled)
{
_sourceError = new OperationCanceledException();
_sourceDone = true;
_cancelOther.Cancel();
if (TryDisposeMain())
{
ResumeHelper.Resume(ref _resume);
}
}
else if (t.IsFaulted)
{
_sourceError = ExceptionHelper.Extract(t.Exception);
_sourceDone = true;
_cancelOther.Cancel();
if (TryDisposeMain())
{
ResumeHelper.Resume(ref _resume);
}
}
else if (t.Result)
{
_sourceValue = _source.Current;
_sourceReady = true;
if (TryDisposeMain())
{
ResumeHelper.Resume(ref _resume);
}
}
else
{
_sourceDone = true;
_cancelOther.Cancel();
if (TryDisposeMain())
{
ResumeHelper.Resume(ref _resume);
}
}
}
private void OtherHandler(Task <bool> t)
{
if (t.IsCanceled)
{
_otherError = new OperationCanceledException();
_otherDone = true;
_cancelMain.Cancel();
if (TryDisposeOther())
{
ResumeHelper.Resume(ref _resume);
}
}
if (t.IsFaulted)
{
_otherError = ExceptionHelper.Extract(t.Exception);
_otherDone = true;
_cancelMain.Cancel();
if (TryDisposeOther())
{
ResumeHelper.Resume(ref _resume);
}
}
else if (t.Result)
{
Interlocked.Exchange(ref _otherValue, _other.Current);
if (TryDisposeOther())
{
ResumeHelper.Resume(ref _resume);
MoveNextOther();
}
}
else
{
_otherDone = true;
if (TryDisposeOther())
{
ResumeHelper.Resume(ref _resume);
}
}
}
public async ValueTask <bool> MoveNextAsync()
{
for (; ;)
{
var ex = Volatile.Read(ref _error);
if (ex != null && ex != ExceptionHelper.Terminated)
{
throw ex;
}
var d = Volatile.Read(ref _done);
var e = Volatile.Read(ref _hasValue);
if (d && !e)
{
return(false);
}
if (e)
{
_hasValue = false;
var next = false;
if (Volatile.Read(ref _gate) != 0)
{
next = true;
Current = _source.Current;
}
MoveNextMain();
if (next)
{
return(true);
}
}
await ResumeHelper.Await(ref _resume);
ResumeHelper.Clear(ref _resume);
}
}
private void SourceHandler(Task <bool> t)
{
var next = false;
if (t.IsFaulted)
{
_error = ExceptionHelper.Extract(t.Exception);
_done = true;
}
else if (t.IsCanceled)
{
_error = new OperationCanceledException();
_done = true;
}
else if (t.Result)
{
_queue.Enqueue(_source.Current);
next = true;
}
else
{
_done = true;
}
// release the MoveNext, just in case
if (Interlocked.Decrement(ref _disposeWip) != 0)
{
ResumeHelper.Complete(ref _disposeTask, _source.DisposeAsync());
}
else
{
if (next && Interlocked.Decrement(ref _outstanding) != 0)
{
MoveNext();
}
Signal();
}
}
public async ValueTask <bool> MoveNextAsync()
{
if (!_once)
{
_once = true;
foreach (var inner in _sources)
{
inner.MoveNext();
}
}
for (; ;)
{
var d = Volatile.Read(ref _done) == 0;
var success = _queue.TryDequeue(out var v);
if (d && !success)
{
var ex = _error;
if (ex != null)
{
_error = null;
throw ex;
}
return(false);
}
if (success)
{
Current = v.Value;
v.Sender.MoveNext();
return(true);
}
await ResumeHelper.Await(ref _resume);
ResumeHelper.Clear(ref _resume);
}
}
private void HandleMain(Task <bool> t)
{
if (t.IsCanceled)
{
_error = new OperationCanceledException();
_done = true;
if (TryDispose())
{
ResumeHelper.Resume(ref _resumeTask);
}
}
else if (t.IsFaulted)
{
_error = ExceptionHelper.Extract(t.Exception);
_done = true;
if (TryDispose())
{
ResumeHelper.Resume(ref _resumeTask);
}
}
else if (t.Result)
{
Interlocked.Exchange(ref _latest, _source.Current);
if (TryDispose())
{
ResumeHelper.Resume(ref _resumeTask);
}
MoveNext();
}
else
{
_done = true;
if (TryDispose())
{
ResumeHelper.Resume(ref _resumeTask);
}
}
}
public async ValueTask <bool> MoveNextAsync()
{
for (; ;)
{
var a = Volatile.Read(ref _available);
var b = _index;
if (a != b)
{
Current = b;
_index = b + 1;
return(true);
}
if (b == _end)
{
return(false);
}
await ResumeHelper.Await(ref _resume);
ResumeHelper.Clear(ref _resume);
}
}
public async ValueTask <bool> MoveNextAsync()
{
ResumeHelper.Resume(ref _consumed);
await ResumeHelper.Await(ref _valueReady);
ResumeHelper.Clear(ref _valueReady);
if (_hasValue)
{
_hasValue = false;
return(true);
}
Current = default;
var ex = _error;
if (ex != null)
{
_error = null;
throw ex;
}
return(false);
}
private void Signal()
{
ResumeHelper.Resume(ref _resume);
}
private void HandleMain(Task <bool> t)
{
if (t.IsCanceled)
{
_error = new OperationCanceledException();
_done = true;
if (TryDispose())
{
ResumeHelper.Resume(ref _resume);
}
}
else if (t.IsFaulted)
{
CancellationHelper.Cancel(ref _cts);
if (_emitLast)
{
var idx = _sourceIndex;
if (idx != 0)
{
SetLatest(_emitLastItem, idx + 1);
_emitLastItem = default;
}
}
_error = ExceptionHelper.Extract(t.Exception);
_done = true;
if (TryDispose())
{
ResumeHelper.Resume(ref _resume);
}
}
else if (t.Result)
{
Volatile.Read(ref _cts)?.Cancel();
var v = _source.Current;
if (TryDispose())
{
if (_emitLast)
{
_emitLastItem = v;
}
var idx = ++_sourceIndex;
var newCts = CancellationTokenSource.CreateLinkedTokenSource(_sourceCTS.Token);
if (CancellationHelper.Replace(ref _cts, newCts))
{
Task.Delay(_delay, newCts.Token)
.ContinueWith(tt => TimerHandler(tt, v, idx), newCts.Token);
MoveNext();
}
}
}
else
{
CancellationHelper.Cancel(ref _cts);
if (_emitLast)
{
var idx = _sourceIndex;
if (idx != 0)
{
SetLatest(_emitLastItem, idx + 1);
_emitLastItem = default;
}
}
_done = true;
if (TryDispose())
{
ResumeHelper.Resume(ref _resume);
}
}
}
public async ValueTask DisposeAsync()
{
await _parent.Remove(this);
ResumeHelper.Resume(ref _consumed);
}
private void DisposeTask()
{
ResumeHelper.Complete(ref _disposeTask, _source.DisposeAsync());
}
public Group(TKey key, GroupByEnumerator parent)
{
Key = key;
_parent = parent;
ResumeHelper.Resume(ref _consumed);
}
private void NextHandler(Task <bool> t)
{
if (t.IsFaulted)
{
ExceptionHelper.AddException(ref _error, ExceptionHelper.Extract(t.Exception));
_sourceDone = true;
if (TryDispose())
{
ResumeHelper.Resume(ref _resume);
}
}
else if (t.Result)
{
IAsyncEnumerator <TResult> src;
try
{
src = _mapper(_source.Current).GetAsyncEnumerator();
}
catch (Exception ex)
{
ExceptionHelper.AddException(ref _error, ex);
_sourceDone = true;
src = null;
if (TryDispose())
{
ResumeHelper.Resume(ref _resume);
return;
}
}
if (src != null)
{
Interlocked.Increment(ref _disposeWip);
var inner = new InnerHandler(src, this);
_inners.Enqueue(inner);
if (_disposeRequested)
{
while (_inners.TryDequeue(out var inner2))
{
inner2.Dispose();
}
}
if (TryDispose())
{
inner.MoveNext();
if (Interlocked.Decrement(ref _sourceOutstanding) != 0)
{
MoveNextSource();
}
ResumeHelper.Resume(ref _resume);
}
}
}
else
{
_sourceDone = true;
if (TryDispose())
{
ResumeHelper.Resume(ref _resume);
}
}
}
public async ValueTask <bool> MoveNextAsync()
{
if (!_once)
{
_once = true;
MoveNextAll();
}
var latest = _latest;
var n = latest.Length;
for (; ;)
{
if (_done == 0)
{
var ex = ExceptionHelper.Terminate(ref _error);
if (ex != null)
{
throw ex;
}
return(false);
}
var success = _queue.TryDequeue(out var entry);
if (success)
{
var inner = _sources[entry.Index];
if (entry.Done)
{
if (inner.HasLatest)
{
_done--;
}
else
{
_done = 0;
}
continue;
}
if (!inner.HasLatest)
{
inner.HasLatest = true;
_latestRemaining--;
}
latest[entry.Index] = entry.Value;
if (_latestRemaining == 0)
{
var copy = new TSource[n];
Array.Copy(latest, 0, copy, 0, n);
Current = _combiner(copy);
inner.MoveNext();
return(true);
}
inner.MoveNext();
continue;
}
await ResumeHelper.Await(ref _resume);
ResumeHelper.Clear(ref _resume);
}
}
public async ValueTask <bool> MoveNextAsync()
{
if (_done)
{
var ex = ExceptionHelper.Terminate(ref _error);
if (ex != null)
{
throw ex;
}
return(false);
}
for (; ;)
{
var success = _queue.TryDequeue(out var entry);
if (success)
{
var b = _buffer;
if (entry.Done)
{
_done = true;
if (b != null)
{
Current = b;
_buffer = default;
return(true);
}
var ex = ExceptionHelper.Terminate(ref _error);
if (ex != null)
{
throw ex;
}
return(false);
}
if (entry.Boundary)
{
if (b == null)
{
Current = _collectionSupplier();
}
else
{
Current = b;
_buffer = default;
}
_size = 0;
MoveNextOther();
return(true);
}
if (b == null)
{
b = _collectionSupplier();
_buffer = b;
}
b.Add(entry.Value);
if (++_size == _maxSize)
{
Current = b;
_buffer = default;
_size = 0;
MoveNextSource();
return(true);
}
MoveNextSource();
continue;
}
await ResumeHelper.Await(ref _resume);
ResumeHelper.Clear(ref _resume);
}
}
public ValueTask DisposeAsync()
{
_disposeRequested = true;
ResumeHelper.Resume(ref _consumed);
return(new ValueTask(_task));
}