internal static MergeExecutor <TInputOutput> Execute <TKey>(PartitionedStream <TInputOutput, TKey> partitions, bool ignoreOutput, ParallelMergeOptions options, TaskScheduler taskScheduler, bool isOrdered, CancellationState cancellationState, int queryId)
{
MergeExecutor <TInputOutput> executor = new MergeExecutor <TInputOutput>();
if (isOrdered && !ignoreOutput)
{
if ((options != ParallelMergeOptions.FullyBuffered) && !partitions.OrdinalIndexState.IsWorseThan(OrdinalIndexState.Increasing))
{
bool autoBuffered = options == ParallelMergeOptions.AutoBuffered;
if (partitions.PartitionCount > 1)
{
executor.m_mergeHelper = new OrderPreservingPipeliningMergeHelper <TInputOutput>((PartitionedStream <TInputOutput, int>)partitions, taskScheduler, cancellationState, autoBuffered, queryId);
}
else
{
executor.m_mergeHelper = new DefaultMergeHelper <TInputOutput, TKey>(partitions, false, options, taskScheduler, cancellationState, queryId);
}
}
else
{
executor.m_mergeHelper = new OrderPreservingMergeHelper <TInputOutput, TKey>(partitions, taskScheduler, cancellationState, queryId);
}
}
else
{
executor.m_mergeHelper = new DefaultMergeHelper <TInputOutput, TKey>(partitions, ignoreOutput, options, taskScheduler, cancellationState, queryId);
}
executor.Execute();
return(executor);
}
internal DefaultMergeHelper(PartitionedStream <TInputOutput, TIgnoreKey> partitions, bool ignoreOutput, ParallelMergeOptions options, TaskScheduler taskScheduler, CancellationState cancellationState, int queryId)
{
this.m_taskGroupState = new QueryTaskGroupState(cancellationState, queryId);
this.m_partitions = partitions;
this.m_taskScheduler = taskScheduler;
this.m_ignoreOutput = ignoreOutput;
if (!ignoreOutput)
{
if (options != ParallelMergeOptions.FullyBuffered)
{
if (partitions.PartitionCount > 1)
{
this.m_asyncChannels = MergeExecutor <TInputOutput> .MakeAsynchronousChannels(partitions.PartitionCount, options, cancellationState.MergedCancellationToken);
this.m_channelEnumerator = new AsynchronousChannelMergeEnumerator <TInputOutput>(this.m_taskGroupState, this.m_asyncChannels);
}
else
{
this.m_channelEnumerator = ExceptionAggregator.WrapQueryEnumerator <TInputOutput, TIgnoreKey>(partitions[0], this.m_taskGroupState.CancellationState).GetEnumerator();
}
}
else
{
this.m_syncChannels = MergeExecutor <TInputOutput> .MakeSynchronousChannels(partitions.PartitionCount);
this.m_channelEnumerator = new SynchronousChannelMergeEnumerator <TInputOutput>(this.m_taskGroupState, this.m_syncChannels);
}
}
}
public void Receive <TKey>(PartitionedStream <TOutput, TKey> partitionedStream)
{
#if DEBUG
m_received = true;
#endif
m_mergeExecutor = MergeExecutor <TOutput> .Execute <TKey>(
partitionedStream, m_forEffectMerge, m_mergeOptions, m_taskScheduler, m_isOrdered, m_cancellationState, m_queryId);
TraceHelpers.TraceInfo("[timing]: {0}: finished opening - QueryOperator<>::GetEnumerator", DateTime.Now.Ticks);
}
private readonly bool _ignoreOutput; // Whether we're enumerating "for effect".
//-----------------------------------------------------------------------------------
// Instantiates a new merge helper.
//
// Arguments:
// partitions - the source partitions from which to consume data.
// ignoreOutput - whether we're enumerating "for effect" or for output.
// pipeline - whether to use a pipelined merge.
//
internal DefaultMergeHelper(PartitionedStream <TInputOutput, TIgnoreKey> partitions, bool ignoreOutput, ParallelMergeOptions options,
TaskScheduler taskScheduler, CancellationState cancellationState, int queryId)
{
Debug.Assert(partitions != null);
_taskGroupState = new QueryTaskGroupState(cancellationState, queryId);
_partitions = partitions;
_taskScheduler = taskScheduler;
_ignoreOutput = ignoreOutput;
IntValueEvent consumerEvent = new IntValueEvent();
TraceHelpers.TraceInfo("DefaultMergeHelper::.ctor(..): creating a default merge helper");
// If output won't be ignored, we need to manufacture a set of channels for the consumer.
// Otherwise, when the merge is executed, we'll just invoke the activities themselves.
if (!ignoreOutput)
{
// Create the asynchronous or synchronous channels, based on whether we're pipelining.
if (options != ParallelMergeOptions.FullyBuffered)
{
if (partitions.PartitionCount > 1)
{
Debug.Assert(!ParallelEnumerable.SinglePartitionMode);
_asyncChannels =
MergeExecutor <TInputOutput> .MakeAsynchronousChannels(partitions.PartitionCount, options, consumerEvent, cancellationState.MergedCancellationToken);
_channelEnumerator = new AsynchronousChannelMergeEnumerator <TInputOutput>(_taskGroupState, _asyncChannels, consumerEvent);
}
else
{
// If there is only one partition, we don't need to create channels. The only producer enumerator
// will be used as the result enumerator.
_channelEnumerator = ExceptionAggregator.WrapQueryEnumerator(partitions[0], _taskGroupState.CancellationState).GetEnumerator();
}
}
else
{
_syncChannels =
MergeExecutor <TInputOutput> .MakeSynchronousChannels(partitions.PartitionCount);
_channelEnumerator = new SynchronousChannelMergeEnumerator <TInputOutput>(_taskGroupState, _syncChannels);
}
Debug.Assert(_asyncChannels == null || _asyncChannels.Length == partitions.PartitionCount);
Debug.Assert(_syncChannels == null || _syncChannels.Length == partitions.PartitionCount);
Debug.Assert(_channelEnumerator != null, "enumerator can't be null if we're not ignoring output");
}
}
//-----------------------------------------------------------------------------------
// Creates and executes a new merge executor object.
//
// Arguments:
// partitions - the partitions whose data will be merged into one stream
// ignoreOutput - if true, we are enumerating "for effect", and we won't actually
// generate data in the output stream
// pipeline - whether to use a pipelined merge or not.
// isOrdered - whether to perform an ordering merge.
//
internal static MergeExecutor <TInputOutput> Execute <TKey>(
PartitionedStream <TInputOutput, TKey> partitions, bool ignoreOutput, ParallelMergeOptions options, TaskScheduler taskScheduler, bool isOrdered,
CancellationState cancellationState, int queryId)
{
Debug.Assert(partitions != null);
Debug.Assert(partitions.PartitionCount > 0);
Debug.Assert(!ignoreOutput || options == ParallelMergeOptions.FullyBuffered, "Pipelining with no output is not supported.");
MergeExecutor <TInputOutput> mergeExecutor = new MergeExecutor <TInputOutput>();
if (isOrdered && !ignoreOutput)
{
if (options != ParallelMergeOptions.FullyBuffered && !partitions.OrdinalIndexState.IsWorseThan(OrdinalIndexState.Increasing))
{
Debug.Assert(options == ParallelMergeOptions.NotBuffered || options == ParallelMergeOptions.AutoBuffered);
bool autoBuffered = (options == ParallelMergeOptions.AutoBuffered);
if (partitions.PartitionCount > 1)
{
Debug.Assert(!ParallelEnumerable.SinglePartitionMode);
// We use a pipelining ordered merge
mergeExecutor._mergeHelper = new OrderPreservingPipeliningMergeHelper <TInputOutput, TKey>(
partitions, taskScheduler, cancellationState, autoBuffered, queryId, partitions.KeyComparer);
}
else
{
// When DOP=1, the default merge simply returns the single producer enumerator to the consumer. This way, ordering
// does not add any extra overhead, and no producer task needs to be scheduled.
mergeExecutor._mergeHelper = new DefaultMergeHelper <TInputOutput, TKey>(
partitions, false, options, taskScheduler, cancellationState, queryId);
}
}
else
{
// We use a stop-and-go ordered merge helper
mergeExecutor._mergeHelper = new OrderPreservingMergeHelper <TInputOutput, TKey>(partitions, taskScheduler, cancellationState, queryId);
}
}
else
{
// We use a default - unordered - merge helper.
mergeExecutor._mergeHelper = new DefaultMergeHelper <TInputOutput, TKey>(partitions, ignoreOutput, options, taskScheduler, cancellationState, queryId);
}
mergeExecutor.Execute();
return(mergeExecutor);
}
//---------------------------------------------------------------------------------------
// A helper method that executes the query rooted at the openedChild operator, and returns
// the results as ListQueryResults<TSource>.
//
internal static ListQueryResults <TOutput> ExecuteAndCollectResults <TKey>(
PartitionedStream <TOutput, TKey> openedChild,
int partitionCount,
bool outputOrdered,
bool useStriping,
QuerySettings settings)
{
TaskScheduler taskScheduler = settings.TaskScheduler;
MergeExecutor <TOutput> executor = MergeExecutor <TOutput> .Execute <TKey>(
openedChild, false, ParallelMergeOptions.FullyBuffered, taskScheduler, outputOrdered,
settings.CancellationState, settings.QueryId);
return(new ListQueryResults <TOutput>(executor.GetResultsAsArray(), partitionCount, useStriping));
}
public void Receive <TKey>(PartitionedStream <TOutput, TKey> partitionedStream)
{
this.m_mergeExecutor = MergeExecutor <TOutput> .Execute <TKey>(partitionedStream, this.m_forEffectMerge, this.m_mergeOptions, this.m_taskScheduler, this.m_isOrdered, this.m_cancellationState, this.m_queryId);
}