private async Task<IList<JsonDocument>> GetNextBatch(IAsyncEnumerator<JsonDocument> documentsStream, CancellationToken token) { var documents = new List<JsonDocument>(); var count = 0; while (await documentsStream.MoveNextAsync().ConfigureAwait(false) && count < BatchSize) { documents.Add(documentsStream.Current); count++; token.ThrowIfCancellationRequested(); } return documents; }
private async Task InitializeDocumentsCursor(CancellationToken cancellation) { using (var store = new DocumentStore()) { store.ParseConnectionString(configuration.ConnectionString); store.Initialize(false); using (var session = store.OpenAsyncSession()) { if (String.IsNullOrEmpty(configuration.Query)) { documentsCursor = await session.Advanced.StreamAsync<RavenJObject>(Etag.Empty, 0, int.MaxValue, null, cancellation); } else { var query = String.IsNullOrEmpty(configuration.Index) ? session.Advanced.AsyncDocumentQuery<RavenJObject>() : session.Advanced.AsyncDocumentQuery<RavenJObject>(configuration.Index); // Streaming query API does not create a dynamic index, so user have to specify one if query is provided: http://issues.hibernatingrhinos.com/issue/RavenDB-1410 documentsCursor = await session.Advanced.StreamAsync<RavenJObject>( query.Where(configuration.Query).NoCaching().NoTracking(), cancellation); } } } }
/// <summary> /// Ensure that both <paramref name="enumerator"/> and <paramref name="otherEnumerator"/> are disposed /// correctly and if an exception is thrown and <paramref name="outerException"/> is not null, throw an <see cref="AggregateException"/> /// that aggregate both the exception thrown by the enumerator/s and the exception containted in <paramref name="outerException"/> /// </summary> /// <param name="enumerator"></param> /// <param name="outerException"></param> /// <param name="otherEnumerator"></param> /// <returns></returns> /// <throw ><see cref="AggregateException"/></throw> public static async Task DisposeAsync(this IAsyncEnumerator enumerator, Exception outerException, IAsyncEnumerator otherEnumerator) { try { await Task.WhenAll(enumerator.DisposeAsync(), otherEnumerator.DisposeAsync()); } catch (Exception ex) { if (outerException != null) throw new AggregateException(outerException, ex); throw; } }
async Task <DirectMethodReportGeneratorMetadata> ProcessSenderTestResults( DirectMethodTestResult dmSenderTestResult, NetworkControllerStatus networkControllerStatus, bool isWithinTolerancePeriod, IAsyncEnumerator <TestOperationResult> senderTestResults) { ulong networkOnSuccess = 0; ulong networkOffSuccess = 0; ulong networkOnToleratedSuccess = 0; ulong networkOffToleratedSuccess = 0; ulong networkOnFailure = 0; ulong networkOffFailure = 0; HttpStatusCode statusCode = dmSenderTestResult.HttpStatusCode; if (!NetworkControllerType.Offline.Equals(this.NetworkControllerType)) { if (HttpStatusCode.OK.Equals(statusCode)) { networkOnSuccess++; } else { networkOnFailure++; } } else if (NetworkControllerStatus.Disabled.Equals(networkControllerStatus)) { if (HttpStatusCode.OK.Equals(statusCode)) { networkOnSuccess++; } else { if (isWithinTolerancePeriod) { networkOnToleratedSuccess++; } else { networkOnFailure++; } } } else if (NetworkControllerStatus.Enabled.Equals(networkControllerStatus)) { if (HttpStatusCode.NotFound.Equals(statusCode)) { networkOffSuccess++; } else if (HttpStatusCode.OK.Equals(statusCode)) { if (isWithinTolerancePeriod) { networkOffToleratedSuccess++; } else { networkOffFailure++; } } else { networkOffFailure++; } } bool hasSenderResult = await senderTestResults.MoveNextAsync(); return(new DirectMethodReportGeneratorMetadata { NetworkOnSuccess = networkOnSuccess, NetworkOffSuccess = networkOffSuccess, NetworkOnToleratedSuccess = networkOnToleratedSuccess, NetworkOffToleratedSuccess = networkOffToleratedSuccess, NetworkOnFailure = networkOnFailure, NetworkOffFailure = networkOffFailure, HasSenderResult = hasSenderResult }); }
public SingleByteAsyncEnumerator(IAsyncEnumerator <byte[]> source) { _source = source; position = 0; }
public async Task <bool> MoveNext(CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); if (_sourceEnumerator == null) { _sourceEnumerator = _groupJoinAsyncEnumerable._source.GetEnumerator(); _hasNext = await _sourceEnumerator.MoveNext(); } if (_hasNext) { var outer = _groupJoinAsyncEnumerable._outerFactory(_sourceEnumerator.Current); var inner = _groupJoinAsyncEnumerable._innerFactory(_sourceEnumerator.Current); var inners = new List <TInner>(); if (inner == null) { Current = _groupJoinAsyncEnumerable._resultSelector( outer, AsyncLinqOperatorProvider.ToAsyncEnumerable(inners)); _hasNext = await _sourceEnumerator.MoveNext(); return(true); } var currentGroupKey = _groupJoinAsyncEnumerable._innerKeySelector(inner); inners.Add(inner); while (true) { _hasNext = await _sourceEnumerator.MoveNext(); if (!_hasNext) { break; } inner = _groupJoinAsyncEnumerable._innerFactory(_sourceEnumerator.Current); if (inner == null) { break; } var innerKey = _groupJoinAsyncEnumerable._innerKeySelector(inner); if (!_comparer.Equals(currentGroupKey, innerKey)) { break; } inners.Add(inner); } Current = _groupJoinAsyncEnumerable._resultSelector( outer, AsyncLinqOperatorProvider.ToAsyncEnumerable(inners)); return(true); } return(false); }
public StreamMergeAsyncEnumerator(IAsyncEnumerator <T> source) { _source = source; skip = true; }
private BufferReaderAsync(IAsyncEnumerable <BufferSlice> source, bool utcDate = false) { _source = source.GetAsyncEnumerator(); _utcDate = utcDate; }
private bool IsStreamed(HubConnectionContext connection, ObjectMethodExecutor methodExecutor, object result, Type resultType, out IAsyncEnumerator <object> enumerator) { if (result == null) { enumerator = null; return(false); } // TODO: We need to support cancelling the stream without a client disconnect as well. var observableInterface = IsIObservable(resultType) ? resultType : resultType.GetInterfaces().FirstOrDefault(IsIObservable); if (observableInterface != null) { enumerator = AsyncEnumeratorAdapters.FromObservable(result, observableInterface, connection.ConnectionAbortedToken); return(true); } else if (IsChannel(resultType, out var payloadType)) { enumerator = AsyncEnumeratorAdapters.FromChannel(result, payloadType, connection.ConnectionAbortedToken); return(true); } else { // Not streamed enumerator = null; return(false); } }
static async IAsyncEnumerable <TSource> Core(IAsyncEnumerable <TSource>[] sources, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken = default) #if USE_FAIR_AND_CHEAPER_MERGE // // This new implementation of Merge differs from the original one in a few ways: // // - It's cheaper because: // - no conversion from ValueTask<bool> to Task<bool> takes place using AsTask, // - we don't instantiate Task.WhenAny tasks for each iteration. // - It's fairer because: // - the MoveNextAsync tasks are awaited concurently, but completions are queued, // instead of awaiting a new WhenAny task where "left" sources have preferential // treatment over "right" sources. // { var count = sources.Length; var enumerators = new IAsyncEnumerator <TSource> [count]; var moveNextTasks = new ValueTask <bool> [count]; try { for (var i = 0; i < count; i++) { IAsyncEnumerator <TSource> enumerator = sources[i].GetAsyncEnumerator(cancellationToken); enumerators[i] = enumerator; // REVIEW: This follows the lead of the original implementation where we kick off MoveNextAsync // operations immediately. An alternative would be to do this in a separate stage, thus // preventing concurrency across MoveNextAsync and GetAsyncEnumerator calls and avoiding // any MoveNextAsync calls before all enumerators are acquired (or an exception has // occurred doing so). moveNextTasks[i] = enumerator.MoveNextAsync(); } var whenAny = TaskExt.WhenAny(moveNextTasks); int active = count; while (active > 0) { int index = await whenAny; IAsyncEnumerator <TSource> enumerator = enumerators[index]; ValueTask <bool> moveNextTask = moveNextTasks[index]; if (!await moveNextTask.ConfigureAwait(false)) { // // Replace the task in our array by a completed task to make finally logic easier. Note that // the WhenAnyValueTask object has a reference to our array (i.e. no copy is made), so this // gets rid of any resources the original task may have held onto. However, we *don't* call // whenAny.Replace to set this value, because it'd attach an awaiter to the already completed // task, causing spurious wake-ups when awaiting whenAny. // moveNextTasks[index] = new ValueTask <bool>(); // REVIEW: The original implementation did not dispose eagerly, which could lead to resource // leaks when merged with other long-running sequences. enumerators[index] = null; // NB: Avoids attempt at double dispose in finally if disposing fails. await enumerator.DisposeAsync().ConfigureAwait(false); active--; } else { TSource item = enumerator.Current; // // Replace the task using whenAny.Replace, which will write it to the moveNextTasks array, and // will start awaiting the task. Note we don't have to write to moveNextTasks ourselves because // the whenAny object has a reference to it (i.e. no copy is made). // whenAny.Replace(index, enumerator.MoveNextAsync()); yield return(item); } } } finally { // REVIEW: The original implementation performs a concurrent dispose, which seems undesirable given the // additional uncontrollable source of concurrency and the sequential resource acquisition. In // this modern implementation, we release resources in opposite order as we acquired them, thus // guaranteeing determinism (and mimicking a series of nested `await using` statements). // REVIEW: If we decide to phase GetAsyncEnumerator and the initial MoveNextAsync calls at the start of // the operator implementation, we should make this symmetric and first await all in flight // MoveNextAsync operations, prior to disposing the enumerators. var errors = default(List <Exception>); for (var i = count - 1; i >= 0; i--) { ValueTask <bool> moveNextTask = moveNextTasks[i]; IAsyncEnumerator <TSource> enumerator = enumerators[i]; try { try { // // Await the task to ensure outstanding work is completed prior to performing a dispose // operation. Note that we don't have to do anything special for tasks belonging to // enumerators that have finished; we swapped in a placeholder completed task. // // REVIEW: This adds an additional continuation to all of the pending tasks (note that // whenAny also has registered one). The whenAny object will be collectible // after all of these complete. Alternatively, we could drain via whenAny, by // awaiting it until the active count drops to 0. This saves on attaching the // additional continuations, but we need to decide on order of dispose. Right // now, we dispose in opposite order of acquiring the enumerators, with the // exception of enumerators that were disposed eagerly upon early completion. // Should we care about the dispose order at all? _ = await moveNextTask.ConfigureAwait(false); } finally { if (enumerator != null) { await enumerator.DisposeAsync().ConfigureAwait(false); } } } catch (Exception ex) { if (errors == null) { errors = new List <Exception>(); } errors.Add(ex); } } // NB: If we had any errors during cleaning (and awaiting pending operations), we throw these exceptions // instead of the original exception that may have led to running the finally block. This is similar // to throwing from any finally block (except that we catch all exceptions to ensure cleanup of all // concurrent sequences being merged). if (errors != null) { throw new AggregateException(errors); } } }
public async Task InitializeAsync(CancellationToken cancellation) { documentsCursor = await Client .QueryDocumentsAsync(Configuration.Collection, Configuration.Query, cancellation); }
public SelectManyIterator(IAsyncEnumerable <TSource> source, Func <TSource, int, IAsyncEnumerable <TCollection> > collectionSelector, Func <TSource, TCollection, TResult> resultSelector) { _enumerator = (source ?? throw new ArgumentNullException(nameof(source))).GetEnumerator(); _collectionSelector = collectionSelector ?? throw new ArgumentNullException(nameof(collectionSelector)); _resultSelector = resultSelector ?? throw new ArgumentNullException(nameof(resultSelector)); }
public ValueTask DisposeAsync() { _enumerator?.DisposeAsync(); _enumerator = null; return(default);
public PaginatedAsyncSequence(IAsyncEnumerable <T> sequence) { enumerator = sequence.GetAsyncEnumerator(); }
protected override async ValueTask <bool> MoveNextCore() { switch (_state) { case AsyncIteratorState.Allocated: _sourceCTS = CancellationTokenSource.CreateLinkedTokenSource(_cancellationToken); _enumerator = _source.GetAsyncEnumerator(_sourceCTS.Token); _state = AsyncIteratorState.Iterating; goto case AsyncIteratorState.Iterating; case AsyncIteratorState.Iterating: var moveNext = _enumerator !.MoveNextAsync(); if (!moveNext.IsCompleted) { using var delayCts = CancellationTokenSource.CreateLinkedTokenSource(_cancellationToken); var delay = Task.Delay(_timeout, delayCts.Token); var next = moveNext.AsTask(); var winner = await Task.WhenAny(next, delay).ConfigureAwait(false); if (winner == delay) { // NB: We still have to wait for the MoveNextAsync operation to complete before we can // dispose _enumerator. The resulting task will be used by DisposeAsync. Also note // that throwing an exception here causes a call to DisposeAsync, where we pick up // the task prepared below. // NB: Any exception reported by a timed out MoveNextAsync operation won't be reported // to the caller, but the task's exception is not marked as observed, so unhandled // exception handlers can still observe the exception. // REVIEW: Should exceptions reported by a timed out MoveNextAsync operation come out // when attempting to call DisposeAsync? _loserTask = next.ContinueWith((_, state) => ((IAsyncDisposable)state !).DisposeAsync().AsTask(), _enumerator); _sourceCTS !.Cancel(); throw new TimeoutException(); } delayCts.Cancel(); } if (await moveNext.ConfigureAwait(false)) { _current = _enumerator.Current; return(true); } break; } await DisposeAsync().ConfigureAwait(false); return(false); }
public async Task Handle_Subscription_DataReceived_And_Completed() { // arrange var connection = new SocketConnectionMock(); var services = new ServiceCollection(); services.AddInMemorySubscriptionProvider(); services.AddStarWarsRepositories(); IQueryExecutor executor = SchemaBuilder.New() .AddServices(services.BuildServiceProvider()) .AddStarWarsTypes() .Create() .MakeExecutable(); DocumentNode query = Utf8GraphQLParser.Parse( "subscription { onReview(episode: NEWHOPE) { stars } }"); var handler = new DataStartMessageHandler(executor, null); var message = new DataStartMessage( "123", new GraphQLRequest(query)); // act await handler.HandleAsync( connection, message, CancellationToken.None); // assert Assert.Empty(connection.SentMessages); Assert.NotEmpty(connection.Subscriptions); IResponseStream stream = (IResponseStream)await executor.ExecuteAsync( "subscription { onReview(episode: NEWHOPE) { stars } }"); await executor.ExecuteAsync(@" mutation { createReview(episode:NEWHOPE review: { commentary: ""foo"" stars: 5 }) { stars } }"); using var cts = new CancellationTokenSource(15000); IAsyncEnumerator <IReadOnlyQueryResult> enumerator = stream.GetAsyncEnumerator(cts.Token); Assert.True(await enumerator.MoveNextAsync()); await Task.Delay(2000); Assert.Collection(connection.SentMessages, t => { Assert.True(t.SequenceEqual( new DataResultMessage(message.Id, enumerator.Current).Serialize())); }); }
private bool TryGetStreamingEnumerator(HubConnectionContext connection, string invocationId, HubMethodDescriptor hubMethodDescriptor, object result, out IAsyncEnumerator <object> enumerator, out CancellationTokenSource streamCts) { if (result != null) { if (hubMethodDescriptor.IsChannel) { streamCts = CreateCancellation(); enumerator = hubMethodDescriptor.FromChannel(result, streamCts.Token); return(true); } } streamCts = null; enumerator = null; return(false); CancellationTokenSource CreateCancellation() { var userCts = new CancellationTokenSource(); connection.ActiveRequestCancellationSources.TryAdd(invocationId, userCts); return(CancellationTokenSource.CreateLinkedTokenSource(connection.ConnectionAborted, userCts.Token)); } }
internal Enumerator(IAsyncEnumerator <T> enumerator, bool continueOnCapturedContext) { _enumerator = enumerator; _continueOnCapturedContext = continueOnCapturedContext; }
public WithCancellationAsyncEnumerator(IAsyncEnumerator <T> source, CancellationToken cancellationToken) { _source = source; _cancellationToken = cancellationToken; }
public YieldStream(AsyncDocumentSession parent, IAsyncDocumentQuery <T> query, FieldsToFetchToken fieldsToFetch, IAsyncEnumerator <BlittableJsonReaderObject> enumerator, CancellationToken token) { _parent = parent; _enumerator = enumerator; _token = token; _query = query; _fieldsToFetch = fieldsToFetch; }
public ZipIterator(IAsyncEnumerable <TFirst> first, IAsyncEnumerable <TSecond> second, Func <TFirst, TSecond, TResult> resultSelector) { _enumeratorOne = (first ?? throw new ArgumentNullException(nameof(first))).GetEnumerator(); _enumeratorTwo = (second ?? throw new ArgumentNullException(nameof(second))).GetEnumerator(); _resultSelector = resultSelector ?? throw new ArgumentNullException(nameof(resultSelector)); }
public ConvertingEnumerator(IAsyncEnumerator <TInput> enumerator, Func <TInput, TResult> converter) { this.enumerator = enumerator; this.converter = converter; }
public async Task StreamingPull() { string projectId = _fixture.ProjectId; string topicId = _fixture.CreateTopicId(); string subscriptionId = _fixture.CreateSubscriptionId(); // Snippet: StreamingPull(*, *) PublisherServiceApiClient publisher = PublisherServiceApiClient.Create(); TopicName topicName = new TopicName(projectId, topicId); publisher.CreateTopic(topicName); SubscriberServiceApiClient subscriber = SubscriberServiceApiClient.Create(); SubscriptionName subscriptionName = new SubscriptionName(projectId, subscriptionId); subscriber.CreateSubscription(subscriptionName, topicName, null, 60); // If we don't see all the messages we expect in 10 seconds, we'll cancel the call. CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(10)); CallSettings callSettings = CallSettings.FromCancellationToken(cancellationTokenSource.Token); SubscriberServiceApiClient.StreamingPullStream stream = subscriber.StreamingPull(callSettings); // The first request must include the subscription name and the stream ack deadline await stream.WriteAsync(new StreamingPullRequest { SubscriptionAsSubscriptionName = subscriptionName, StreamAckDeadlineSeconds = 20 }); Task pullingTask = Task.Run(async() => { int messagesSeen = 0; IAsyncEnumerator <StreamingPullResponse> responseStream = stream.ResponseStream; // Handle responses as we see them. while (await responseStream.MoveNext()) { StreamingPullResponse response = responseStream.Current; Console.WriteLine("Received streaming response"); foreach (ReceivedMessage message in response.ReceivedMessages) { // Messages can contain any data. We'll assume that we know this // topic publishes UTF-8-encoded text. Console.WriteLine($"Message text: {message.Message.Data.ToStringUtf8()}"); } // Acknowledge the messages we've just seen await stream.WriteAsync(new StreamingPullRequest { AckIds = { response.ReceivedMessages.Select(rm => rm.AckId) } }); // If we've seen all the messages we expect, we can complete the streaming call, // and our next MoveNext call will return false. messagesSeen += response.ReceivedMessages.Count; if (messagesSeen == 3) { await stream.WriteCompleteAsync(); } } }); publisher.Publish(topicName, new[] { new PubsubMessage { Data = ByteString.CopyFromUtf8("Message 1") } }); publisher.Publish(topicName, new[] { new PubsubMessage { Data = ByteString.CopyFromUtf8("Message 2") } }); publisher.Publish(topicName, new[] { new PubsubMessage { Data = ByteString.CopyFromUtf8("Message 3") } }); await pullingTask; // End snippet }
public AsyncIncludeCollectionIterator( [NotNull] IAsyncEnumerator <ValueBuffer> relatedValuesEnumerator) { _relatedValuesEnumerator = relatedValuesEnumerator; }
public AsycnEnumerableWrapper(IAsyncEnumerator <T> asyncEnumerator) { this.asyncEnumerator = asyncEnumerator; }
public YieldStream(AsyncDocumentSession parent, IAsyncEnumerator <BlittableJsonReaderObject> enumerator, CancellationToken token) { this.Parent = parent; this.Enumerator = enumerator; }
public async Task <IEnumerable <Domain.CustomerInformation> > GetCustomerInformationAsync(CancellationToken ct) { IList <Domain.CustomerInformation> results = new List <Domain.CustomerInformation>(); IReliableDictionary <string, Domain.CustomerInformation> customers = await this.StateManager.GetOrAddAsync <IReliableDictionary <string, Domain.CustomerInformation> >(CustomerInformationDictionary); ServiceEventSource.Current.Message("Called CustomerInformationDictionary to return CustomerInformation"); using (ITransaction tx = this.StateManager.CreateTransaction()) { ServiceEventSource.Current.Message("Generating item views for {0} items", await customers.GetCountAsync(tx)); IAsyncEnumerator <KeyValuePair <string, Domain.CustomerInformation> > enumerator = (await customers.CreateEnumerableAsync(tx)).GetAsyncEnumerator(); while (await enumerator.MoveNextAsync(ct)) { results.Add(enumerator.Current.Value); } } return(results); }
public OeEntityDbEnumerator(IAsyncEnumerator <Object> asyncEnumerator, OeEntryFactory entryFactory) { _asyncEnumerator = asyncEnumerator; EntryFactory = entryFactory; }
public AllTaskEnumerator(IAsyncEnumerator <TSource> source, Func <TSource, Task <bool> > predicate) { _source = source; _predicate = predicate; }
public OeEntityDbEnumerator(IAsyncEnumerator <Object> asyncEnumerator, OeEntryFactory entryFactory, OeEntityDbEnumerator parentEnumerator) : this(asyncEnumerator, entryFactory) { _parentEnumerator = parentEnumerator; }
/// <summary> /// Allow to use the foreach keyword with an IAsyncEnumerator /// </summary> public static IAsyncEnumerator <T> GetAsyncEnumerator <T>(this IAsyncEnumerator <T> enumerator) => enumerator;
private async Task StreamResultsAsync(string invocationId, HubConnectionContext connection, IAsyncEnumerator <object> enumerator) { try { while (await enumerator.MoveNextAsync()) { // Send the stream item await SendMessageAsync(connection, new StreamItemMessage(invocationId, enumerator.Current)); } await SendMessageAsync(connection, CompletionMessage.Empty(invocationId)); } catch (Exception ex) { await SendMessageAsync(connection, CompletionMessage.WithError(invocationId, ex.Message)); } }
public JoinStream(IAsyncEnumerator<ArraySegment<byte>> factory) { Contract.Requires(factory != null); this._arraySegmentEnumerator = factory; }
private async Task StreamResultsAsync(string invocationId, HubConnectionContext connection, IAsyncEnumerator <object> enumerator, CancellationTokenSource streamCts) { string error = null; try { while (await enumerator.MoveNextAsync()) { // Send the stream item await connection.WriteAsync(new StreamItemMessage(invocationId, enumerator.Current)); } } catch (ChannelClosedException ex) { // If the channel closes from an exception in the streaming method, grab the innerException for the error from the streaming method error = ErrorMessageHelper.BuildErrorMessage("An error occurred on the server while streaming results.", ex.InnerException ?? ex, _enableDetailedErrors); } catch (Exception ex) { // If the streaming method was canceled we don't want to send a HubException message - this is not an error case if (!(ex is OperationCanceledException && connection.ActiveRequestCancellationSources.TryGetValue(invocationId, out var cts) && cts.IsCancellationRequested)) { error = ErrorMessageHelper.BuildErrorMessage("An error occurred on the server while streaming results.", ex, _enableDetailedErrors); } } finally { (enumerator as IDisposable)?.Dispose(); // Dispose the linked CTS for the stream. streamCts.Dispose(); await connection.WriteAsync(CompletionMessage.WithError(invocationId, error)); if (connection.ActiveRequestCancellationSources.TryRemove(invocationId, out var cts)) { cts.Dispose(); } } }