/// <summary>Initializes the debug view.</summary>
/// <param name="transformManyBlock">The transform being viewed.</param>
public DebugView(TransformManyBlock <TInput, TOutput> transformManyBlock)
{
Contract.Requires(transformManyBlock != null, "Need a block with which to construct the debug view.");
_transformManyBlock = transformManyBlock;
_targetDebuggingInformation = transformManyBlock._target.GetDebuggingInformation();
_sourceDebuggingInformation = transformManyBlock._source.GetDebuggingInformation();
}
/// <summary>Initializes the debug view.</summary>
/// <param name="actionBlock">The target being debugged.</param>
public DebugView(ActionBlock <TInput> actionBlock)
{
Contract.Requires(actionBlock != null, "Need a block with which to construct the debug view.");
_actionBlock = actionBlock;
if (_actionBlock._defaultTarget != null)
{
_defaultDebugInfo = actionBlock._defaultTarget.GetDebuggingInformation();
}
else
{
_spscDebugInfo = actionBlock._spscTarget.GetDebuggingInformation();
}
}
/// <summary>Initializes the <see cref="ActionBlock{T}"/> with the specified delegate and options.</summary>
/// <param name="action">The action to invoke with each data element received.</param>
/// <param name="dataflowBlockOptions">The options with which to configure this <see cref="ActionBlock{T}"/>.</param>
/// <exception cref="System.ArgumentNullException">The <paramref name="action"/> is null (Nothing in Visual Basic).</exception>
/// <exception cref="System.ArgumentNullException">The <paramref name="dataflowBlockOptions"/> is null (Nothing in Visual Basic).</exception>
private ActionBlock(Delegate action, ExecutionDataflowBlockOptions dataflowBlockOptions)
{
// Validate arguments
if (action == null)
{
throw new ArgumentNullException("action");
}
if (dataflowBlockOptions == null)
{
throw new ArgumentNullException("dataflowBlockOptions");
}
Contract.Ensures((_spscTarget != null) ^ (_defaultTarget != null), "One and only one of the two targets must be non-null after construction");
Contract.EndContractBlock();
// Ensure we have options that can't be changed by the caller
dataflowBlockOptions = dataflowBlockOptions.DefaultOrClone();
// Based on the mode, initialize the target. If the user specifies SingleProducerConstrained,
// we'll try to employ an optimized mode under a limited set of circumstances.
var syncAction = action as Action <TInput>;
if (syncAction != null &&
dataflowBlockOptions.SingleProducerConstrained &&
dataflowBlockOptions.MaxDegreeOfParallelism == 1 &&
!dataflowBlockOptions.CancellationToken.CanBeCanceled &&
dataflowBlockOptions.BoundedCapacity == DataflowBlockOptions.Unbounded)
{
// Initialize the SPSC fast target to handle the bulk of the processing.
// The SpscTargetCore is only supported when BoundedCapacity, CancellationToken,
// and MaxDOP are all their default values. It's also only supported for sync
// delegates and not for async delegates.
_spscTarget = new SpscTargetCore <TInput>(this, syncAction, dataflowBlockOptions);
}
else
{
// Initialize the TargetCore which handles the bulk of the processing.
// The default target core can handle all options and delegate flavors.
if (syncAction != null) // sync
{
_defaultTarget = new TargetCore <TInput>(this,
messageWithId => ProcessMessage(syncAction, messageWithId),
null, dataflowBlockOptions, TargetCoreOptions.RepresentsBlockCompletion);
}
else // async
{
var asyncAction = action as Func <TInput, Task>;
Debug.Assert(asyncAction != null, "action is of incorrect delegate type");
_defaultTarget = new TargetCore <TInput>(this,
messageWithId => ProcessMessageWithTask(asyncAction, messageWithId),
null, dataflowBlockOptions, TargetCoreOptions.RepresentsBlockCompletion | TargetCoreOptions.UsesAsyncCompletion);
}
// Handle async cancellation requests by declining on the target
Common.WireCancellationToComplete(
dataflowBlockOptions.CancellationToken, Completion, state => ((TargetCore <TInput>)state).Complete(exception: null, dropPendingMessages: true), _defaultTarget);
}
#if FEATURE_TRACING
DataflowEtwProvider etwLog = DataflowEtwProvider.Log;
if (etwLog.IsEnabled())
{
etwLog.DataflowBlockCreated(this, dataflowBlockOptions);
}
#endif
}
/// <summary>Stores the next item using the reordering buffer.</summary>
/// <param name="id">The ID of the item.</param>
/// <param name="item">The completed item.</param>
private void StoreOutputItemsReordered(long id, IEnumerable <TOutput> item)
{
Contract.Requires(_reorderingBuffer != null, "Expected a reordering buffer");
Contract.Requires(id != Common.INVALID_REORDERING_ID, "This ID should never have been handed out.");
// Grab info about the transform
TargetCore <TInput> target = _target;
bool isBounded = target.IsBounded;
// Handle invalid items (null enumerables) by delegating to the base
if (item == null)
{
_reorderingBuffer.AddItem(id, null, false);
if (isBounded)
{
target.ChangeBoundingCount(count: -1);
}
return;
}
// If we can eagerly get the number of items in the collection, update the bounding count.
// This avoids the cost of updating it once per output item (since each update requires synchronization).
// Even if we're not bounding, we still want to determine whether the item is trusted so that we
// can immediately dump it out once we take the lock if we're the next item.
IList <TOutput> itemAsTrustedList = item as TOutput[];
if (itemAsTrustedList == null)
{
itemAsTrustedList = item as List <TOutput>;
}
if (itemAsTrustedList != null && isBounded)
{
UpdateBoundingCountWithOutputCount(count: itemAsTrustedList.Count);
}
// Determine whether this id is the next item, and if it is and if we have a trusted list,
// try to output it immediately on the fast path. If it can be output, we're done.
// Otherwise, make forward progress based on whether we're next in line.
bool?isNextNullable = _reorderingBuffer.AddItemIfNextAndTrusted(id, itemAsTrustedList, itemAsTrustedList != null);
if (!isNextNullable.HasValue)
{
return; // data was successfully output
}
bool isNextItem = isNextNullable.Value;
// By this point, either we're not the next item, in which case we need to make a copy of the
// data and store it, or we are the next item and can store it immediately but we need to enumerate
// the items and store them individually because we don't want to enumerate while holding a lock.
List <TOutput> itemCopy = null;
try
{
// If this is the next item, we can output it now.
if (isNextItem)
{
StoreOutputItemsNonReorderedWithIteration(item);
// here itemCopy remains null, so that base.AddItem will finish our interactions with the reordering buffer
}
else if (itemAsTrustedList != null)
{
itemCopy = itemAsTrustedList.ToList();
// we already got the count and updated the bounding count previously
}
else
{
// We're not the next item, and we're not trusted, so copy the data into a list.
// We need to enumerate outside of the lock in the base class.
int itemCount = 0;
try
{
itemCopy = item.ToList(); // itemCopy will remain null in the case of exception
itemCount = itemCopy.Count;
}
finally
{
// If we're here successfully, then itemCount is the number of output items
// we actually received, and we should update the bounding count with it.
// If we're here because ToList threw an exception, then itemCount will be 0,
// and we still need to update the bounding count with this in order to counteract
// the increased bounding count for the corresponding input.
if (isBounded)
{
UpdateBoundingCountWithOutputCount(count: itemCount);
}
}
}
// else if the item isn't valid, the finally block will see itemCopy as null and output invalid
}
finally
{
// Tell the base reordering buffer that we're done. If we already output
// all of the data, itemCopy will be null, and we just pass down the invalid item.
// If we haven't, pass down the real thing. We do this even in the case of an exception,
// in which case this will be a dummy element.
_reorderingBuffer.AddItem(id, itemCopy, itemIsValid: itemCopy != null);
}
}
/// <summary>Initializes the <see cref="TransformManyBlock{TInput,TOutput}"/> with the specified function and <see cref="ExecutionDataflowBlockOptions"/>.</summary>
/// <param name="transformSync">The synchronous function to invoke with each data element received.</param>
/// <param name="transformAsync">The asynchronous function to invoke with each data element received.</param>
/// <param name="dataflowBlockOptions">The options with which to configure this <see cref="TransformManyBlock{TInput,TOutput}"/>.</param>
/// <exception cref="System.ArgumentNullException">The <paramref name="transformSync"/> and <paramref name="transformAsync"/> are both null (Nothing in Visual Basic).</exception>
/// <exception cref="System.ArgumentNullException">The <paramref name="dataflowBlockOptions"/> is null (Nothing in Visual Basic).</exception>
private TransformManyBlock(Func <TInput, IEnumerable <TOutput> > transformSync, Func <TInput, Task <IEnumerable <TOutput> > > transformAsync, ExecutionDataflowBlockOptions dataflowBlockOptions)
{
// Validate arguments. It's ok for the filterFunction to be null, but not the other parameters.
if (transformSync == null && transformAsync == null)
{
throw new ArgumentNullException("transform");
}
if (dataflowBlockOptions == null)
{
throw new ArgumentNullException(nameof(dataflowBlockOptions));
}
Contract.Requires(transformSync == null ^ transformAsync == null, "Exactly one of transformSync and transformAsync must be null.");
Contract.EndContractBlock();
// Ensure we have options that can't be changed by the caller
dataflowBlockOptions = dataflowBlockOptions.DefaultOrClone();
// Initialize onItemsRemoved delegate if necessary
Action <ISourceBlock <TOutput>, int> onItemsRemoved = null;
if (dataflowBlockOptions.BoundedCapacity > 0)
{
onItemsRemoved = (owningSource, count) => ((TransformManyBlock <TInput, TOutput>)owningSource)._target.ChangeBoundingCount(-count);
}
// Initialize source component
_source = new SourceCore <TOutput>(this, dataflowBlockOptions,
owningSource => ((TransformManyBlock <TInput, TOutput>)owningSource)._target.Complete(exception: null, dropPendingMessages: true),
onItemsRemoved);
// If parallelism is employed, we will need to support reordering messages that complete out-of-order.
// However, a developer can override this with EnsureOrdered == false.
if (dataflowBlockOptions.SupportsParallelExecution && dataflowBlockOptions.EnsureOrdered)
{
_reorderingBuffer = new ReorderingBuffer <IEnumerable <TOutput> >(
this, (source, messages) => ((TransformManyBlock <TInput, TOutput>)source)._source.AddMessages(messages));
}
// Create the underlying target and source
if (transformSync != null) // sync
{
// If an enumerable function was provided, we can use synchronous completion, meaning
// that the target will consider a message fully processed as soon as the
// delegate returns.
_target = new TargetCore <TInput>(this,
messageWithId => ProcessMessage(transformSync, messageWithId),
_reorderingBuffer, dataflowBlockOptions, TargetCoreOptions.None);
}
else // async
{
Debug.Assert(transformAsync != null, "Incorrect delegate type.");
// If a task-based function was provided, we need to use asynchronous completion, meaning
// that the target won't consider a message completed until the task
// returned from that delegate has completed.
_target = new TargetCore <TInput>(this,
messageWithId => ProcessMessageWithTask(transformAsync, messageWithId),
_reorderingBuffer, dataflowBlockOptions, TargetCoreOptions.UsesAsyncCompletion);
}
// Link up the target half with the source half. In doing so,
// ensure exceptions are propagated, and let the source know no more messages will arrive.
// As the target has completed, and as the target synchronously pushes work
// through the reordering buffer when async processing completes,
// we know for certain that no more messages will need to be sent to the source.
_target.Completion.ContinueWith((completed, state) =>
{
var sourceCore = (SourceCore <TOutput>)state;
if (completed.IsFaulted)
{
sourceCore.AddAndUnwrapAggregateException(completed.Exception);
}
sourceCore.Complete();
}, _source, CancellationToken.None, Common.GetContinuationOptions(), TaskScheduler.Default);
// It is possible that the source half may fault on its own, e.g. due to a task scheduler exception.
// In those cases we need to fault the target half to drop its buffered messages and to release its
// reservations. This should not create an infinite loop, because all our implementations are designed
// to handle multiple completion requests and to carry over only one.
_source.Completion.ContinueWith((completed, state) =>
{
var thisBlock = ((TransformManyBlock <TInput, TOutput>)state) as IDataflowBlock;
Debug.Assert(completed.IsFaulted, "The source must be faulted in order to trigger a target completion.");
thisBlock.Fault(completed.Exception);
}, this, CancellationToken.None, Common.GetContinuationOptions() | TaskContinuationOptions.OnlyOnFaulted, TaskScheduler.Default);
// Handle async cancellation requests by declining on the target
Common.WireCancellationToComplete(
dataflowBlockOptions.CancellationToken, Completion, state => ((TargetCore <TInput>)state).Complete(exception: null, dropPendingMessages: true), _target);
#if FEATURE_TRACING
DataflowEtwProvider etwLog = DataflowEtwProvider.Log;
if (etwLog.IsEnabled())
{
etwLog.DataflowBlockCreated(this, dataflowBlockOptions);
}
#endif
}
/// <summary>Stores the next item using the reordering buffer.</summary>
/// <param name="id">The ID of the item.</param>
/// <param name="item">The async enumerable.</param>
private async Task StoreOutputItemsReorderedAsync(long id, IAsyncEnumerable <TOutput>?item)
{
Debug.Assert(_reorderingBuffer is not null, "Expected a reordering buffer");
Debug.Assert(id != Common.INVALID_REORDERING_ID, "This ID should never have been handed out.");
// Grab info about the transform
TargetCore <TInput> target = _target;
bool isBounded = target.IsBounded;
// Handle invalid items (null enumerables) by delegating to the base
if (item is null)
{
_reorderingBuffer.AddItem(id, null, false);
if (isBounded)
{
target.ChangeBoundingCount(count: -1);
}
return;
}
// By this point, either we're not the next item, in which case we need to make a copy of the
// data and store it, or we are the next item and can store it immediately but we need to enumerate
// the items and store them individually because we don't want to enumerate while holding a lock.
List <TOutput>?itemCopy = null;
try
{
// If this is the next item, we can output it now.
if (_reorderingBuffer.IsNext(id))
{
await StoreOutputItemsNonReorderedWithIterationAsync(item).ConfigureAwait(false);
// here itemCopy remains null, so that base.AddItem will finish our interactions with the reordering buffer
}
else
{
// We're not the next item, and we're not trusted, so copy the data into a list.
// We need to enumerate outside of the lock in the base class.
int itemCount = 0;
try
{
itemCopy = new List <TOutput>();
await foreach (TOutput element in item.ConfigureAwait(true))
{
itemCopy.Add(element);
}
itemCount = itemCopy.Count;
}
finally
{
// If we're here successfully, then itemCount is the number of output items
// we actually received, and we should update the bounding count with it.
// If we're here because ToList threw an exception, then itemCount will be 0,
// and we still need to update the bounding count with this in order to counteract
// the increased bounding count for the corresponding input.
if (isBounded)
{
UpdateBoundingCountWithOutputCount(count: itemCount);
}
}
}
// else if the item isn't valid, the finally block will see itemCopy as null and output invalid
}
finally
{
// Tell the base reordering buffer that we're done. If we already output
// all of the data, itemCopy will be null, and we just pass down the invalid item.
// If we haven't, pass down the real thing. We do this even in the case of an exception,
// in which case this will be a dummy element.
_reorderingBuffer.AddItem(id, itemCopy, itemIsValid: itemCopy is not null);
}
}
private void Initialize(
Action <KeyValuePair <TInput, long> > processMessageAction,
ExecutionDataflowBlockOptions dataflowBlockOptions,
[NotNull] ref SourceCore <TOutput>?source,
[NotNull] ref TargetCore <TInput>?target,
ref ReorderingBuffer <IEnumerable <TOutput> >?reorderingBuffer,
TargetCoreOptions targetCoreOptions)
{
if (dataflowBlockOptions == null)
{
throw new ArgumentNullException(nameof(dataflowBlockOptions));
}
// Ensure we have options that can't be changed by the caller
dataflowBlockOptions = dataflowBlockOptions.DefaultOrClone();
// Initialize onItemsRemoved delegate if necessary
Action <ISourceBlock <TOutput>, int>?onItemsRemoved = null;
if (dataflowBlockOptions.BoundedCapacity > 0)
{
onItemsRemoved = (owningSource, count) => ((TransformManyBlock <TInput, TOutput>)owningSource)._target.ChangeBoundingCount(-count);
}
// Initialize source component
source = new SourceCore <TOutput>(this, dataflowBlockOptions,
owningSource => ((TransformManyBlock <TInput, TOutput>)owningSource)._target.Complete(exception: null, dropPendingMessages: true),
onItemsRemoved);
// If parallelism is employed, we will need to support reordering messages that complete out-of-order.
// However, a developer can override this with EnsureOrdered == false.
if (dataflowBlockOptions.SupportsParallelExecution && dataflowBlockOptions.EnsureOrdered)
{
reorderingBuffer = new ReorderingBuffer <IEnumerable <TOutput> >(
this, (source, messages) => ((TransformManyBlock <TInput, TOutput>)source)._source.AddMessages(messages));
}
// Create the underlying target and source
target = new TargetCore <TInput>(this, processMessageAction, _reorderingBuffer, dataflowBlockOptions, targetCoreOptions);
// Link up the target half with the source half. In doing so,
// ensure exceptions are propagated, and let the source know no more messages will arrive.
// As the target has completed, and as the target synchronously pushes work
// through the reordering buffer when async processing completes,
// we know for certain that no more messages will need to be sent to the source.
target.Completion.ContinueWith((completed, state) =>
{
var sourceCore = (SourceCore <TOutput>)state !;
if (completed.IsFaulted)
{
sourceCore.AddAndUnwrapAggregateException(completed.Exception !);
}
sourceCore.Complete();
}, source, CancellationToken.None, Common.GetContinuationOptions(), TaskScheduler.Default);
// It is possible that the source half may fault on its own, e.g. due to a task scheduler exception.
// In those cases we need to fault the target half to drop its buffered messages and to release its
// reservations. This should not create an infinite loop, because all our implementations are designed
// to handle multiple completion requests and to carry over only one.
source.Completion.ContinueWith((completed, state) =>
{
var thisBlock = ((TransformManyBlock <TInput, TOutput>)state !) as IDataflowBlock;
Debug.Assert(completed.IsFaulted, "The source must be faulted in order to trigger a target completion.");
thisBlock.Fault(completed.Exception !);
}, this, CancellationToken.None, Common.GetContinuationOptions() | TaskContinuationOptions.OnlyOnFaulted, TaskScheduler.Default);
// Handle async cancellation requests by declining on the target
Common.WireCancellationToComplete(
dataflowBlockOptions.CancellationToken, Completion, state => ((TargetCore <TInput>)state !).Complete(exception: null, dropPendingMessages: true), target);
DataflowEtwProvider etwLog = DataflowEtwProvider.Log;
if (etwLog.IsEnabled())
{
etwLog.DataflowBlockCreated(this, dataflowBlockOptions);
}
}
