/// <summary>Initializes the debug view.</summary> /// <param name="transformManyBlock">The transform being viewed.</param> public DebugView(TransformManyBlock <TInput, TOutput> transformManyBlock) { Contract.Requires(transformManyBlock != null, "Need a block with which to construct the debug view."); _transformManyBlock = transformManyBlock; _targetDebuggingInformation = transformManyBlock._target.GetDebuggingInformation(); _sourceDebuggingInformation = transformManyBlock._source.GetDebuggingInformation(); }
/// <summary>Initializes the debug view.</summary> /// <param name="actionBlock">The target being debugged.</param> public DebugView(ActionBlock <TInput> actionBlock) { Contract.Requires(actionBlock != null, "Need a block with which to construct the debug view."); _actionBlock = actionBlock; if (_actionBlock._defaultTarget != null) { _defaultDebugInfo = actionBlock._defaultTarget.GetDebuggingInformation(); } else { _spscDebugInfo = actionBlock._spscTarget.GetDebuggingInformation(); } }
/// <summary>Initializes the <see cref="ActionBlock{T}"/> with the specified delegate and options.</summary> /// <param name="action">The action to invoke with each data element received.</param> /// <param name="dataflowBlockOptions">The options with which to configure this <see cref="ActionBlock{T}"/>.</param> /// <exception cref="System.ArgumentNullException">The <paramref name="action"/> is null (Nothing in Visual Basic).</exception> /// <exception cref="System.ArgumentNullException">The <paramref name="dataflowBlockOptions"/> is null (Nothing in Visual Basic).</exception> private ActionBlock(Delegate action, ExecutionDataflowBlockOptions dataflowBlockOptions) { // Validate arguments if (action == null) { throw new ArgumentNullException("action"); } if (dataflowBlockOptions == null) { throw new ArgumentNullException("dataflowBlockOptions"); } Contract.Ensures((_spscTarget != null) ^ (_defaultTarget != null), "One and only one of the two targets must be non-null after construction"); Contract.EndContractBlock(); // Ensure we have options that can't be changed by the caller dataflowBlockOptions = dataflowBlockOptions.DefaultOrClone(); // Based on the mode, initialize the target. If the user specifies SingleProducerConstrained, // we'll try to employ an optimized mode under a limited set of circumstances. var syncAction = action as Action <TInput>; if (syncAction != null && dataflowBlockOptions.SingleProducerConstrained && dataflowBlockOptions.MaxDegreeOfParallelism == 1 && !dataflowBlockOptions.CancellationToken.CanBeCanceled && dataflowBlockOptions.BoundedCapacity == DataflowBlockOptions.Unbounded) { // Initialize the SPSC fast target to handle the bulk of the processing. // The SpscTargetCore is only supported when BoundedCapacity, CancellationToken, // and MaxDOP are all their default values. It's also only supported for sync // delegates and not for async delegates. _spscTarget = new SpscTargetCore <TInput>(this, syncAction, dataflowBlockOptions); } else { // Initialize the TargetCore which handles the bulk of the processing. // The default target core can handle all options and delegate flavors. if (syncAction != null) // sync { _defaultTarget = new TargetCore <TInput>(this, messageWithId => ProcessMessage(syncAction, messageWithId), null, dataflowBlockOptions, TargetCoreOptions.RepresentsBlockCompletion); } else // async { var asyncAction = action as Func <TInput, Task>; Debug.Assert(asyncAction != null, "action is of incorrect delegate type"); _defaultTarget = new TargetCore <TInput>(this, messageWithId => ProcessMessageWithTask(asyncAction, messageWithId), null, dataflowBlockOptions, TargetCoreOptions.RepresentsBlockCompletion | TargetCoreOptions.UsesAsyncCompletion); } // Handle async cancellation requests by declining on the target Common.WireCancellationToComplete( dataflowBlockOptions.CancellationToken, Completion, state => ((TargetCore <TInput>)state).Complete(exception: null, dropPendingMessages: true), _defaultTarget); } #if FEATURE_TRACING DataflowEtwProvider etwLog = DataflowEtwProvider.Log; if (etwLog.IsEnabled()) { etwLog.DataflowBlockCreated(this, dataflowBlockOptions); } #endif }
/// <summary>Stores the next item using the reordering buffer.</summary> /// <param name="id">The ID of the item.</param> /// <param name="item">The completed item.</param> private void StoreOutputItemsReordered(long id, IEnumerable <TOutput> item) { Contract.Requires(_reorderingBuffer != null, "Expected a reordering buffer"); Contract.Requires(id != Common.INVALID_REORDERING_ID, "This ID should never have been handed out."); // Grab info about the transform TargetCore <TInput> target = _target; bool isBounded = target.IsBounded; // Handle invalid items (null enumerables) by delegating to the base if (item == null) { _reorderingBuffer.AddItem(id, null, false); if (isBounded) { target.ChangeBoundingCount(count: -1); } return; } // If we can eagerly get the number of items in the collection, update the bounding count. // This avoids the cost of updating it once per output item (since each update requires synchronization). // Even if we're not bounding, we still want to determine whether the item is trusted so that we // can immediately dump it out once we take the lock if we're the next item. IList <TOutput> itemAsTrustedList = item as TOutput[]; if (itemAsTrustedList == null) { itemAsTrustedList = item as List <TOutput>; } if (itemAsTrustedList != null && isBounded) { UpdateBoundingCountWithOutputCount(count: itemAsTrustedList.Count); } // Determine whether this id is the next item, and if it is and if we have a trusted list, // try to output it immediately on the fast path. If it can be output, we're done. // Otherwise, make forward progress based on whether we're next in line. bool?isNextNullable = _reorderingBuffer.AddItemIfNextAndTrusted(id, itemAsTrustedList, itemAsTrustedList != null); if (!isNextNullable.HasValue) { return; // data was successfully output } bool isNextItem = isNextNullable.Value; // By this point, either we're not the next item, in which case we need to make a copy of the // data and store it, or we are the next item and can store it immediately but we need to enumerate // the items and store them individually because we don't want to enumerate while holding a lock. List <TOutput> itemCopy = null; try { // If this is the next item, we can output it now. if (isNextItem) { StoreOutputItemsNonReorderedWithIteration(item); // here itemCopy remains null, so that base.AddItem will finish our interactions with the reordering buffer } else if (itemAsTrustedList != null) { itemCopy = itemAsTrustedList.ToList(); // we already got the count and updated the bounding count previously } else { // We're not the next item, and we're not trusted, so copy the data into a list. // We need to enumerate outside of the lock in the base class. int itemCount = 0; try { itemCopy = item.ToList(); // itemCopy will remain null in the case of exception itemCount = itemCopy.Count; } finally { // If we're here successfully, then itemCount is the number of output items // we actually received, and we should update the bounding count with it. // If we're here because ToList threw an exception, then itemCount will be 0, // and we still need to update the bounding count with this in order to counteract // the increased bounding count for the corresponding input. if (isBounded) { UpdateBoundingCountWithOutputCount(count: itemCount); } } } // else if the item isn't valid, the finally block will see itemCopy as null and output invalid } finally { // Tell the base reordering buffer that we're done. If we already output // all of the data, itemCopy will be null, and we just pass down the invalid item. // If we haven't, pass down the real thing. We do this even in the case of an exception, // in which case this will be a dummy element. _reorderingBuffer.AddItem(id, itemCopy, itemIsValid: itemCopy != null); } }
/// <summary>Initializes the <see cref="TransformManyBlock{TInput,TOutput}"/> with the specified function and <see cref="ExecutionDataflowBlockOptions"/>.</summary> /// <param name="transformSync">The synchronous function to invoke with each data element received.</param> /// <param name="transformAsync">The asynchronous function to invoke with each data element received.</param> /// <param name="dataflowBlockOptions">The options with which to configure this <see cref="TransformManyBlock{TInput,TOutput}"/>.</param> /// <exception cref="System.ArgumentNullException">The <paramref name="transformSync"/> and <paramref name="transformAsync"/> are both null (Nothing in Visual Basic).</exception> /// <exception cref="System.ArgumentNullException">The <paramref name="dataflowBlockOptions"/> is null (Nothing in Visual Basic).</exception> private TransformManyBlock(Func <TInput, IEnumerable <TOutput> > transformSync, Func <TInput, Task <IEnumerable <TOutput> > > transformAsync, ExecutionDataflowBlockOptions dataflowBlockOptions) { // Validate arguments. It's ok for the filterFunction to be null, but not the other parameters. if (transformSync == null && transformAsync == null) { throw new ArgumentNullException("transform"); } if (dataflowBlockOptions == null) { throw new ArgumentNullException(nameof(dataflowBlockOptions)); } Contract.Requires(transformSync == null ^ transformAsync == null, "Exactly one of transformSync and transformAsync must be null."); Contract.EndContractBlock(); // Ensure we have options that can't be changed by the caller dataflowBlockOptions = dataflowBlockOptions.DefaultOrClone(); // Initialize onItemsRemoved delegate if necessary Action <ISourceBlock <TOutput>, int> onItemsRemoved = null; if (dataflowBlockOptions.BoundedCapacity > 0) { onItemsRemoved = (owningSource, count) => ((TransformManyBlock <TInput, TOutput>)owningSource)._target.ChangeBoundingCount(-count); } // Initialize source component _source = new SourceCore <TOutput>(this, dataflowBlockOptions, owningSource => ((TransformManyBlock <TInput, TOutput>)owningSource)._target.Complete(exception: null, dropPendingMessages: true), onItemsRemoved); // If parallelism is employed, we will need to support reordering messages that complete out-of-order. // However, a developer can override this with EnsureOrdered == false. if (dataflowBlockOptions.SupportsParallelExecution && dataflowBlockOptions.EnsureOrdered) { _reorderingBuffer = new ReorderingBuffer <IEnumerable <TOutput> >( this, (source, messages) => ((TransformManyBlock <TInput, TOutput>)source)._source.AddMessages(messages)); } // Create the underlying target and source if (transformSync != null) // sync { // If an enumerable function was provided, we can use synchronous completion, meaning // that the target will consider a message fully processed as soon as the // delegate returns. _target = new TargetCore <TInput>(this, messageWithId => ProcessMessage(transformSync, messageWithId), _reorderingBuffer, dataflowBlockOptions, TargetCoreOptions.None); } else // async { Debug.Assert(transformAsync != null, "Incorrect delegate type."); // If a task-based function was provided, we need to use asynchronous completion, meaning // that the target won't consider a message completed until the task // returned from that delegate has completed. _target = new TargetCore <TInput>(this, messageWithId => ProcessMessageWithTask(transformAsync, messageWithId), _reorderingBuffer, dataflowBlockOptions, TargetCoreOptions.UsesAsyncCompletion); } // Link up the target half with the source half. In doing so, // ensure exceptions are propagated, and let the source know no more messages will arrive. // As the target has completed, and as the target synchronously pushes work // through the reordering buffer when async processing completes, // we know for certain that no more messages will need to be sent to the source. _target.Completion.ContinueWith((completed, state) => { var sourceCore = (SourceCore <TOutput>)state; if (completed.IsFaulted) { sourceCore.AddAndUnwrapAggregateException(completed.Exception); } sourceCore.Complete(); }, _source, CancellationToken.None, Common.GetContinuationOptions(), TaskScheduler.Default); // It is possible that the source half may fault on its own, e.g. due to a task scheduler exception. // In those cases we need to fault the target half to drop its buffered messages and to release its // reservations. This should not create an infinite loop, because all our implementations are designed // to handle multiple completion requests and to carry over only one. _source.Completion.ContinueWith((completed, state) => { var thisBlock = ((TransformManyBlock <TInput, TOutput>)state) as IDataflowBlock; Debug.Assert(completed.IsFaulted, "The source must be faulted in order to trigger a target completion."); thisBlock.Fault(completed.Exception); }, this, CancellationToken.None, Common.GetContinuationOptions() | TaskContinuationOptions.OnlyOnFaulted, TaskScheduler.Default); // Handle async cancellation requests by declining on the target Common.WireCancellationToComplete( dataflowBlockOptions.CancellationToken, Completion, state => ((TargetCore <TInput>)state).Complete(exception: null, dropPendingMessages: true), _target); #if FEATURE_TRACING DataflowEtwProvider etwLog = DataflowEtwProvider.Log; if (etwLog.IsEnabled()) { etwLog.DataflowBlockCreated(this, dataflowBlockOptions); } #endif }
/// <summary>Stores the next item using the reordering buffer.</summary> /// <param name="id">The ID of the item.</param> /// <param name="item">The async enumerable.</param> private async Task StoreOutputItemsReorderedAsync(long id, IAsyncEnumerable <TOutput>?item) { Debug.Assert(_reorderingBuffer is not null, "Expected a reordering buffer"); Debug.Assert(id != Common.INVALID_REORDERING_ID, "This ID should never have been handed out."); // Grab info about the transform TargetCore <TInput> target = _target; bool isBounded = target.IsBounded; // Handle invalid items (null enumerables) by delegating to the base if (item is null) { _reorderingBuffer.AddItem(id, null, false); if (isBounded) { target.ChangeBoundingCount(count: -1); } return; } // By this point, either we're not the next item, in which case we need to make a copy of the // data and store it, or we are the next item and can store it immediately but we need to enumerate // the items and store them individually because we don't want to enumerate while holding a lock. List <TOutput>?itemCopy = null; try { // If this is the next item, we can output it now. if (_reorderingBuffer.IsNext(id)) { await StoreOutputItemsNonReorderedWithIterationAsync(item).ConfigureAwait(false); // here itemCopy remains null, so that base.AddItem will finish our interactions with the reordering buffer } else { // We're not the next item, and we're not trusted, so copy the data into a list. // We need to enumerate outside of the lock in the base class. int itemCount = 0; try { itemCopy = new List <TOutput>(); await foreach (TOutput element in item.ConfigureAwait(true)) { itemCopy.Add(element); } itemCount = itemCopy.Count; } finally { // If we're here successfully, then itemCount is the number of output items // we actually received, and we should update the bounding count with it. // If we're here because ToList threw an exception, then itemCount will be 0, // and we still need to update the bounding count with this in order to counteract // the increased bounding count for the corresponding input. if (isBounded) { UpdateBoundingCountWithOutputCount(count: itemCount); } } } // else if the item isn't valid, the finally block will see itemCopy as null and output invalid } finally { // Tell the base reordering buffer that we're done. If we already output // all of the data, itemCopy will be null, and we just pass down the invalid item. // If we haven't, pass down the real thing. We do this even in the case of an exception, // in which case this will be a dummy element. _reorderingBuffer.AddItem(id, itemCopy, itemIsValid: itemCopy is not null); } }
private void Initialize( Action <KeyValuePair <TInput, long> > processMessageAction, ExecutionDataflowBlockOptions dataflowBlockOptions, [NotNull] ref SourceCore <TOutput>?source, [NotNull] ref TargetCore <TInput>?target, ref ReorderingBuffer <IEnumerable <TOutput> >?reorderingBuffer, TargetCoreOptions targetCoreOptions) { if (dataflowBlockOptions == null) { throw new ArgumentNullException(nameof(dataflowBlockOptions)); } // Ensure we have options that can't be changed by the caller dataflowBlockOptions = dataflowBlockOptions.DefaultOrClone(); // Initialize onItemsRemoved delegate if necessary Action <ISourceBlock <TOutput>, int>?onItemsRemoved = null; if (dataflowBlockOptions.BoundedCapacity > 0) { onItemsRemoved = (owningSource, count) => ((TransformManyBlock <TInput, TOutput>)owningSource)._target.ChangeBoundingCount(-count); } // Initialize source component source = new SourceCore <TOutput>(this, dataflowBlockOptions, owningSource => ((TransformManyBlock <TInput, TOutput>)owningSource)._target.Complete(exception: null, dropPendingMessages: true), onItemsRemoved); // If parallelism is employed, we will need to support reordering messages that complete out-of-order. // However, a developer can override this with EnsureOrdered == false. if (dataflowBlockOptions.SupportsParallelExecution && dataflowBlockOptions.EnsureOrdered) { reorderingBuffer = new ReorderingBuffer <IEnumerable <TOutput> >( this, (source, messages) => ((TransformManyBlock <TInput, TOutput>)source)._source.AddMessages(messages)); } // Create the underlying target and source target = new TargetCore <TInput>(this, processMessageAction, _reorderingBuffer, dataflowBlockOptions, targetCoreOptions); // Link up the target half with the source half. In doing so, // ensure exceptions are propagated, and let the source know no more messages will arrive. // As the target has completed, and as the target synchronously pushes work // through the reordering buffer when async processing completes, // we know for certain that no more messages will need to be sent to the source. target.Completion.ContinueWith((completed, state) => { var sourceCore = (SourceCore <TOutput>)state !; if (completed.IsFaulted) { sourceCore.AddAndUnwrapAggregateException(completed.Exception !); } sourceCore.Complete(); }, source, CancellationToken.None, Common.GetContinuationOptions(), TaskScheduler.Default); // It is possible that the source half may fault on its own, e.g. due to a task scheduler exception. // In those cases we need to fault the target half to drop its buffered messages and to release its // reservations. This should not create an infinite loop, because all our implementations are designed // to handle multiple completion requests and to carry over only one. source.Completion.ContinueWith((completed, state) => { var thisBlock = ((TransformManyBlock <TInput, TOutput>)state !) as IDataflowBlock; Debug.Assert(completed.IsFaulted, "The source must be faulted in order to trigger a target completion."); thisBlock.Fault(completed.Exception !); }, this, CancellationToken.None, Common.GetContinuationOptions() | TaskContinuationOptions.OnlyOnFaulted, TaskScheduler.Default); // Handle async cancellation requests by declining on the target Common.WireCancellationToComplete( dataflowBlockOptions.CancellationToken, Completion, state => ((TargetCore <TInput>)state !).Complete(exception: null, dropPendingMessages: true), target); DataflowEtwProvider etwLog = DataflowEtwProvider.Log; if (etwLog.IsEnabled()) { etwLog.DataflowBlockCreated(this, dataflowBlockOptions); } }