private readonly TaskScheduler _taskScheduler; // The task manager to execute the query.
//-----------------------------------------------------------------------------------
// Instantiates a new merge helper.
//
// Arguments:
// partitions - the source partitions from which to consume data.
// ignoreOutput - whether we're enumerating "for effect" or for output.
//
internal OrderPreservingMergeHelper(PartitionedStream <TInputOutput, TKey> partitions, TaskScheduler taskScheduler,
CancellationState cancellationState, int queryId)
{
Debug.Assert(partitions != null);
TraceHelpers.TraceInfo("KeyOrderPreservingMergeHelper::.ctor(..): creating an order preserving merge helper");
_taskGroupState = new QueryTaskGroupState(cancellationState, queryId);
_partitions = partitions;
_results = new Shared <TInputOutput[]>(null);
_taskScheduler = taskScheduler;
}
// This method is called only once on the 'head operator' which is the last specified operator in the query
// This method then recursively uses Open() to prepare itself and the other enumerators.
private QueryResults <TOutput> GetQueryResults(QuerySettings querySettings)
{
TraceHelpers.TraceInfo("[timing]: {0}: starting execution - QueryOperator<>::GetQueryResults", DateTime.Now.Ticks);
// All mandatory query settings must be specified
Debug.Assert(querySettings.TaskScheduler != null);
Debug.Assert(querySettings.DegreeOfParallelism.HasValue);
Debug.Assert(querySettings.ExecutionMode.HasValue);
// Now just open the query tree's root operator, supplying a specific DOP
return(Open(querySettings, false));
}
//-----------------------------------------------------------------------------------
// Creates and begins execution of a new spooling task. If pipelineMerges is specified,
// we will execute the task asynchronously; otherwise, this is done synchronously,
// and by the time this API has returned all of the results have been produced.
//
// Arguments:
// source - the producer enumerator
// destination - the destination channel into which to spool elements
// ordinalIndexState - state of the index of the input to the merge
//
// Assumptions:
// Source cannot be null, although the other arguments may be.
//
internal static void Spool(
QueryTaskGroupState groupState, PartitionedStream <TInputOutput, TKey> partitions,
Shared <TInputOutput[]> results, TaskScheduler taskScheduler)
{
Contract.Assert(groupState != null);
Contract.Assert(partitions != null);
Contract.Assert(results != null);
Contract.Assert(results.Value == null);
// Determine how many async tasks to create.
int maxToRunInParallel = partitions.PartitionCount - 1;
// Generate a set of sort helpers.
SortHelper <TInputOutput, TKey>[] sortHelpers =
SortHelper <TInputOutput, TKey> .GenerateSortHelpers(partitions, groupState);
// Ensure all tasks in this query are parented under a common root.
Task rootTask = new Task(
() =>
{
// Create tasks that will enumerate the partitions in parallel. We'll use the current
// thread for one task and then block before returning to the caller, until all results
// have been accumulated. Pipelining is not supported by sort merges.
for (int i = 0; i < maxToRunInParallel; i++)
{
TraceHelpers.TraceInfo("OrderPreservingSpoolingTask::Spool: Running partition[{0}] asynchronously", i);
QueryTask asyncTask = new OrderPreservingSpoolingTask <TInputOutput, TKey>(
i, groupState, results, sortHelpers[i]);
asyncTask.RunAsynchronously(taskScheduler);
}
// Run one task synchronously on the current thread.
TraceHelpers.TraceInfo("OrderPreservingSpoolingTask::Spool: Running partition[{0}] synchronously", maxToRunInParallel);
QueryTask syncTask = new OrderPreservingSpoolingTask <TInputOutput, TKey>(
maxToRunInParallel, groupState, results, sortHelpers[maxToRunInParallel]);
syncTask.RunSynchronously(taskScheduler);
});
// Begin the query on the calling thread.
groupState.QueryBegin(rootTask);
// We don't want to return until the task is finished. Run it on the calling thread.
rootTask.RunSynchronously(taskScheduler);
// Destroy the state associated with our sort helpers.
for (int i = 0; i < sortHelpers.Length; i++)
{
sortHelpers[i].Dispose();
}
// End the query, which has the effect of propagating any unhandled exceptions.
groupState.QueryEnd(false);
}
//-----------------------------------------------------------------------------------
// This internal helper method is used to generate a set of synchronous channels.
// The channel data structure used has been optimized for sequential execution and
// does not support pipelining.
//
// Arguments:
// partitionsCount - the number of partitions for which to create new channels.
//
// Return Value:
// An array of synchronous channels, one for each partition.
//
internal static SynchronousChannel<TInputOutput>[] MakeSynchronousChannels(int partitionCount)
{
SynchronousChannel<TInputOutput>[] channels = new SynchronousChannel<TInputOutput>[partitionCount];
TraceHelpers.TraceInfo("MergeExecutor::MakeChannels: setting up {0} channels in prep for stop-and-go", partitionCount);
// We just build up the results in memory using simple, dynamically growable FIFO queues.
for (int i = 0; i < channels.Length; i++)
{
channels[i] = new SynchronousChannel<TInputOutput>();
}
return channels;
}
private readonly bool _ignoreOutput; // Whether we're enumerating "for effect".
//-----------------------------------------------------------------------------------
// Instantiates a new merge helper.
//
// Arguments:
// partitions - the source partitions from which to consume data.
// ignoreOutput - whether we're enumerating "for effect" or for output.
// pipeline - whether to use a pipelined merge.
//
internal DefaultMergeHelper(PartitionedStream <TInputOutput, TIgnoreKey> partitions, bool ignoreOutput, ParallelMergeOptions options,
TaskScheduler taskScheduler, CancellationState cancellationState, int queryId)
{
Debug.Assert(partitions != null);
_taskGroupState = new QueryTaskGroupState(cancellationState, queryId);
_partitions = partitions;
_taskScheduler = taskScheduler;
_ignoreOutput = ignoreOutput;
IntValueEvent consumerEvent = new IntValueEvent();
TraceHelpers.TraceInfo("DefaultMergeHelper::.ctor(..): creating a default merge helper");
// If output won't be ignored, we need to manufacture a set of channels for the consumer.
// Otherwise, when the merge is executed, we'll just invoke the activities themselves.
if (!ignoreOutput)
{
// Create the asynchronous or synchronous channels, based on whether we're pipelining.
if (options != ParallelMergeOptions.FullyBuffered)
{
if (partitions.PartitionCount > 1)
{
Debug.Assert(!ParallelEnumerable.SinglePartitionMode);
_asyncChannels =
MergeExecutor <TInputOutput> .MakeAsynchronousChannels(partitions.PartitionCount, options, consumerEvent, cancellationState.MergedCancellationToken);
_channelEnumerator = new AsynchronousChannelMergeEnumerator <TInputOutput>(_taskGroupState, _asyncChannels, consumerEvent);
}
else
{
// If there is only one partition, we don't need to create channels. The only producer enumerator
// will be used as the result enumerator.
_channelEnumerator = ExceptionAggregator.WrapQueryEnumerator(partitions[0], _taskGroupState.CancellationState).GetEnumerator();
}
}
else
{
_syncChannels =
MergeExecutor <TInputOutput> .MakeSynchronousChannels(partitions.PartitionCount);
_channelEnumerator = new SynchronousChannelMergeEnumerator <TInputOutput>(_taskGroupState, _syncChannels);
}
Debug.Assert(_asyncChannels == null || _asyncChannels.Length == partitions.PartitionCount);
Debug.Assert(_syncChannels == null || _syncChannels.Length == partitions.PartitionCount);
Debug.Assert(_channelEnumerator != null, "enumerator can't be null if we're not ignoring output");
}
}
protected void BuildBaseHashLookup <TBaseBuilder, TBaseElement, TBaseOrderKey>(
QueryOperatorEnumerator <Pair <TBaseElement, THashKey>, TBaseOrderKey> dataSource,
TBaseBuilder baseHashBuilder,
CancellationToken cancellationToken) where TBaseBuilder : IBaseHashBuilder <TBaseElement, TBaseOrderKey>
{
Debug.Assert(dataSource != null);
#if DEBUG
int hashLookupCount = 0;
int hashKeyCollisions = 0;
#endif
Pair <TBaseElement, THashKey> currentPair = default(Pair <TBaseElement, THashKey>);
TBaseOrderKey orderKey = default(TBaseOrderKey);
int i = 0;
while (dataSource.MoveNext(ref currentPair, ref orderKey))
{
if ((i++ & CancellationState.POLL_INTERVAL) == 0)
{
CancellationState.ThrowIfCanceled(cancellationToken);
}
TBaseElement element = currentPair.First;
THashKey hashKey = currentPair.Second;
// We ignore null keys.
if (hashKey != null)
{
#if DEBUG
hashLookupCount++;
#endif
if (baseHashBuilder.Add(hashKey, element, orderKey))
{
#if DEBUG
hashKeyCollisions++;
#endif
}
}
}
#if DEBUG
TraceHelpers.TraceInfo("HashLookupBuilder::BuildBaseHashLookup - built hash table [count = {0}, collisions = {1}]",
hashLookupCount, hashKeyCollisions);
#endif
}
//-----------------------------------------------------------------------------------
// Common function called regardless of sync or async execution. Just wraps some
// amount of tracing around the call to the real work API.
//
private void BaseWork(object unused)
{
Contract.Assert(unused == null);
TraceHelpers.TraceInfo("[timing]: {0}: Start work {1}", DateTime.Now.Ticks, _taskIndex);
PlinqEtwProvider.Log.ParallelQueryFork(_groupState.QueryId);
try
{
Work();
}
finally
{
PlinqEtwProvider.Log.ParallelQueryJoin(_groupState.QueryId);
}
TraceHelpers.TraceInfo("[timing]: {0}: End work {1}", DateTime.Now.Ticks, _taskIndex);
}
//-----------------------------------------------------------------------------------
// Just waits until the queue is non-full.
//
private void WaitUntilNonFull()
{
Debug.Assert(_producerEvent != null);
// We must loop; sometimes the producer event will have been set
// prematurely due to the way waiting flags are managed. By looping,
// we will only return from this method when space is truly available.
do
{
// If the queue is full, we have to wait for a consumer to make room.
// Reset the event to unsignaled state before waiting.
_producerEvent.Reset();
// We have to handle the case where a producer and consumer are racing to
// wait simultaneously. For instance, a producer might see a full queue (by
// reading IsFull just above), but meanwhile a consumer might drain the queue
// very quickly, suddenly seeing an empty queue. This would lead to deadlock
// if we aren't careful. Therefore we check the empty/full state AGAIN after
// setting our flag to see if a real wait is warranted.
Interlocked.Exchange(ref _producerIsWaiting, 1);
// (We have to prevent the reads that go into determining whether the buffer
// is full from moving before the write to the producer-wait flag. Hence the CAS.)
// Because we might be racing with a consumer that is transitioning the
// buffer from full to non-full, we must check that the queue is full once
// more. Otherwise, we might decide to wait and never be woken up (since
// we just reset the event).
if (IsFull)
{
// Assuming a consumer didn't make room for us, we can wait on the event.
TraceHelpers.TraceInfo("AsynchronousChannel::EnqueueChunk - producer waiting, buffer full");
_producerEvent.Wait(_cancellationToken);
}
else
{
// Reset the flags, we don't actually have to wait after all.
_producerIsWaiting = 0;
}
}
while (IsFull);
}
//-----------------------------------------------------------------------------------
// Creates and begins execution of a new spooling task. Executes synchronously,
// and by the time this API has returned all of the results have been produced.
//
// Arguments:
// groupState - values for inter-task communication
// partitions - the producer enumerators
// channels - the producer-consumer channels
// taskScheduler - the task manager on which to execute
//
internal static void SpoolStopAndGo <TInputOutput, TIgnoreKey>(
QueryTaskGroupState groupState, PartitionedStream <TInputOutput, TIgnoreKey> partitions,
SynchronousChannel <TInputOutput>[] channels, TaskScheduler taskScheduler)
{
Contract.Requires(partitions.PartitionCount == channels.Length);
Contract.Requires(groupState != null);
// Ensure all tasks in this query are parented under a common root.
Task rootTask = new Task(
() =>
{
int maxToRunInParallel = partitions.PartitionCount - 1;
// A stop-and-go merge uses the current thread for one task and then blocks before
// returning to the caller, until all results have been accumulated. We do this by
// running the last partition on the calling thread.
for (int i = 0; i < maxToRunInParallel; i++)
{
TraceHelpers.TraceInfo("SpoolingTask::Spool: Running partition[{0}] asynchronously", i);
QueryTask asyncTask = new StopAndGoSpoolingTask <TInputOutput, TIgnoreKey>(i, groupState, partitions[i], channels[i]);
asyncTask.RunAsynchronously(taskScheduler);
}
TraceHelpers.TraceInfo("SpoolingTask::Spool: Running partition[{0}] synchronously", maxToRunInParallel);
// Run one task synchronously on the current thread.
QueryTask syncTask = new StopAndGoSpoolingTask <TInputOutput, TIgnoreKey>(
maxToRunInParallel, groupState, partitions[maxToRunInParallel], channels[maxToRunInParallel]);
syncTask.RunSynchronously(taskScheduler);
});
// Begin the query on the calling thread.
groupState.QueryBegin(rootTask);
// We don't want to return until the task is finished. Run it on the calling thread.
rootTask.RunSynchronously(taskScheduler);
// Wait for the query to complete, propagate exceptions, and so on.
// For pipelined queries, this step happens in the async enumerator.
groupState.QueryEnd(false);
}
//-----------------------------------------------------------------------------------
// Common function called regardless of sync or async execution. Just wraps some
// amount of tracing around the call to the real work API.
//
private void BaseWork(object unused)
{
Contract.Assert(unused == null);
TraceHelpers.TraceInfo("[timing]: {0}: Start work {1}", DateTime.Now.Ticks, m_taskIndex);
#if !FEATURE_PAL // PAL doesn't support eventing
PlinqEtwProvider.Log.ParallelQueryFork(m_groupState.QueryId);
#endif
try
{
Work();
}
finally
{
#if !FEATURE_PAL // PAL doesn't support eventing
PlinqEtwProvider.Log.ParallelQueryJoin(m_groupState.QueryId);
#endif
}
TraceHelpers.TraceInfo("[timing]: {0}: End work {1}", DateTime.Now.Ticks, m_taskIndex);
}
//-----------------------------------------------------------------------------------
// Creates and begins execution of a new spooling task. This is a for-all style
// execution, meaning that the query will be run fully (for effect) before returning
// and that there are no channels into which data will be queued.
//
// Arguments:
// groupState - values for inter-task communication
// partitions - the producer enumerators
// taskScheduler - the task manager on which to execute
//
internal static void SpoolForAll <TInputOutput, TIgnoreKey>(
QueryTaskGroupState groupState, PartitionedStream <TInputOutput, TIgnoreKey> partitions, TaskScheduler taskScheduler)
{
Contract.Requires(groupState != null);
// Ensure all tasks in this query are parented under a common root.
Task rootTask = new Task(
() =>
{
int maxToRunInParallel = partitions.PartitionCount - 1;
// Create tasks that will enumerate the partitions in parallel "for effect"; in other words,
// no data will be placed into any kind of producer-consumer channel.
for (int i = 0; i < maxToRunInParallel; i++)
{
TraceHelpers.TraceInfo("SpoolingTask::Spool: Running partition[{0}] asynchronously", i);
QueryTask asyncTask = new ForAllSpoolingTask <TInputOutput, TIgnoreKey>(i, groupState, partitions[i]);
asyncTask.RunAsynchronously(taskScheduler);
}
TraceHelpers.TraceInfo("SpoolingTask::Spool: Running partition[{0}] synchronously", maxToRunInParallel);
// Run one task synchronously on the current thread.
QueryTask syncTask = new ForAllSpoolingTask <TInputOutput, TIgnoreKey>(maxToRunInParallel, groupState, partitions[maxToRunInParallel]);
syncTask.RunSynchronously(taskScheduler);
});
// Begin the query on the calling thread.
groupState.QueryBegin(rootTask);
// We don't want to return until the task is finished. Run it on the calling thread.
rootTask.RunSynchronously(taskScheduler);
// Wait for the query to complete, propagate exceptions, and so on.
// For pipelined queries, this step happens in the async enumerator.
groupState.QueryEnd(false);
}
//-----------------------------------------------------------------------------------
// Used by a producer to signal that it is done producing new elements. This will
// also wake up any consumers that have gone to sleep.
//
internal void SetDone()
{
TraceHelpers.TraceInfo("tid {0}: AsynchronousChannel<T>::SetDone() called",
Thread.CurrentThread.ManagedThreadId);
// This is set with a volatile write to ensure that, after the consumer
// sees done, they can re-read the enqueued chunks and see the last one we
// enqueued just above.
m_done = true;
// Because we can race with threads trying to Dispose of the event, we must
// acquire a lock around our setting, and double-check that the event isn't null.
lock (this)
{
if (m_consumerEvent != null)
{
// We set the event to ensure consumers that may have waited or are
// considering waiting will notice that the producer is done. This is done
// after setting the done flag to facilitate a Dekker-style check/recheck.
m_consumerEvent.Set();
}
}
}
//-----------------------------------------------------------------------------------
// Positions the enumerator over the next element. This includes merging as we
// enumerate, by just incrementing indexes, etc.
//
// Return Value:
// True if there's a current element, false if we've reached the end.
//
public override bool MoveNext()
{
Debug.Assert(_channels != null);
// If we're at the start, initialize the index.
if (_channelIndex == -1)
{
_channelIndex = 0;
}
// If the index has reached the end, we bail.
while (_channelIndex != _channels.Length)
{
SynchronousChannel <T> current = _channels[_channelIndex];
Debug.Assert(current != null);
if (current.Count == 0)
{
// We're done with this channel, move on to the next one. We don't
// have to check that it's "done" since this is a synchronous consumer.
_channelIndex++;
}
else
{
// Remember the "current" element and return.
_currentElement = current.Dequeue();
return(true);
}
}
TraceHelpers.TraceInfo("[timing]: {0}: Completed the merge", DateTime.Now.Ticks);
// If we got this far, it means we've exhausted our channels.
Debug.Assert(_channelIndex == _channels.Length);
return(false);
}
//-----------------------------------------------------------------------------------
// Internal helper to queue a real chunk, not just an element.
//
// Arguments:
// chunk - the chunk to make visible to consumers
// timeoutMilliseconds - an optional timeout; we return false if it expires
//
// Notes:
// This API will block if the buffer is full. A chunk must contain only valid
// elements; if the chunk wasn't filled, it should be trimmed to size before
// enqueueing it for consumers to observe.
//
private void EnqueueChunk(T[] chunk)
{
Debug.Assert(chunk != null);
Debug.Assert(!_done, "can't continue producing after the production is over");
if (IsFull)
{
WaitUntilNonFull();
}
Debug.Assert(!IsFull, "expected a non-full buffer");
// We can safely store into the current producer index because we know no consumers
// will be reading from it concurrently.
int bufferIndex = _producerBufferIndex;
_buffer[bufferIndex] = chunk;
// Increment the producer index, taking into count wrapping back to 0. This is a shared
// write; the CLR 2.0 memory model ensures the write won't move before the write to the
// corresponding element, so a consumer won't see the new index but the corresponding
// element in the array as empty.
#pragma warning disable 0420
Interlocked.Exchange(ref _producerBufferIndex, (bufferIndex + 1) % _buffer.Length);
#pragma warning restore 0420
// (If there is a consumer waiting, we have to ensure to signal the event. Unfortunately,
// this requires that we issue a memory barrier: We need to guarantee that the write to
// our producer index doesn't pass the read of the consumer waiting flags; the CLR memory
// model unfortunately permits this reordering. That is handled by using a CAS above.)
if (_consumerIsWaiting == 1 && !IsChunkBufferEmpty)
{
TraceHelpers.TraceInfo("AsynchronousChannel::EnqueueChunk - producer waking consumer");
_consumerIsWaiting = 0;
_consumerEvent.Set(_index);
}
}
//-----------------------------------------------------------------------------------
// Instantiates a new merge helper.
//
// Arguments:
// partitions - the source partitions from which to consume data.
// ignoreOutput - whether we're enumerating "for effect" or for output.
//
internal OrderPreservingPipeliningMergeHelper(
PartitionedStream <TOutput, int> partitions,
TaskScheduler taskScheduler,
CancellationState cancellationState,
bool autoBuffered,
int queryId)
{
Contract.Assert(partitions != null);
TraceHelpers.TraceInfo("KeyOrderPreservingMergeHelper::.ctor(..): creating an order preserving merge helper");
m_taskGroupState = new QueryTaskGroupState(cancellationState, queryId);
m_partitions = partitions;
m_taskScheduler = taskScheduler;
m_autoBuffered = autoBuffered;
int partitionCount = m_partitions.PartitionCount;
m_buffers = new Queue <Pair <int, TOutput> > [partitionCount];
m_producerDone = new bool[partitionCount];
m_consumerWaiting = new bool[partitionCount];
m_producerWaiting = new bool[partitionCount];
m_bufferLocks = new object[partitionCount];
}
//-----------------------------------------------------------------------------------
// Gets the recommended "chunk size" for a particular CLR type.
//
// Notes:
// We try to recommend some reasonable "chunk size" for the data, but this is
// clearly a tradeoff, and requires a bit of experimentation. A larger chunk size
// can help locality, but if it's too big we may end up either stalling another
// partition (if enumerators are calculating data on demand) or skewing the
// distribution of data among the partitions.
//
internal static int GetDefaultChunkSize <T>()
{
int chunkSize;
if (typeof(T).IsValueType)
{
// Marshal.SizeOf fails for value types that don't have explicit layouts. We
// just fall back to some arbitrary constant in that case. Is there a better way?
{
// We choose '128' because this ensures, no matter the actual size of the value type,
// the total bytes used will be a multiple of 128. This ensures it's cache aligned.
chunkSize = 128;
}
}
else
{
Debug.Assert((DEFAULT_BYTES_PER_CHUNK % IntPtr.Size) == 0, "bytes per chunk should be a multiple of pointer size");
chunkSize = (DEFAULT_BYTES_PER_CHUNK / IntPtr.Size);
}
TraceHelpers.TraceInfo("Scheduling::GetDefaultChunkSize({0}) -- returning {1}", typeof(T), chunkSize);
return(chunkSize);
}
//---------------------------------------------------------------------------------------
// This method just creates the individual partitions given a data source.
//
// Notes:
// We check whether the data source is an IList<T> and, if so, we can partition
// "in place" by calculating a set of indexes. Otherwise, we return an enumerator that
// performs partitioning lazily. Depending on which case it is, the enumerator may
// contain synchronization (i.e. the latter case), meaning callers may occ----ionally
// block when enumerating it.
//
private void InitializePartitions(IEnumerable <T> source, int partitionCount, bool useStriping)
{
Contract.Assert(source != null);
Contract.Assert(partitionCount > 0);
// If this is a wrapper, grab the internal wrapped data source so we can uncover its real type.
ParallelEnumerableWrapper <T> wrapper = source as ParallelEnumerableWrapper <T>;
if (wrapper != null)
{
source = wrapper.WrappedEnumerable;
Contract.Assert(source != null);
}
// Check whether we have an indexable data source.
IList <T> sourceAsList = source as IList <T>;
if (sourceAsList != null)
{
QueryOperatorEnumerator <T, int>[] partitions = new QueryOperatorEnumerator <T, int> [partitionCount];
int listCount = sourceAsList.Count;
// We use this below to specialize enumerators when possible.
T[] sourceAsArray = source as T[];
// If range partitioning is used, chunk size will be unlimited, i.e. -1.
int maxChunkSize = -1;
if (useStriping)
{
maxChunkSize = Scheduling.GetDefaultChunkSize <T>();
// The minimum chunk size is 1.
if (maxChunkSize < 1)
{
maxChunkSize = 1;
}
}
// Calculate indexes and construct enumerators that walk a subset of the input.
for (int i = 0; i < partitionCount; i++)
{
if (sourceAsArray != null)
{
// If the source is an array, we can use a fast path below to index using
// 'ldelem' instructions rather than making interface method calls.
if (useStriping)
{
partitions[i] = new ArrayIndexRangeEnumerator(sourceAsArray, partitionCount, i, maxChunkSize);
}
else
{
partitions[i] = new ArrayContiguousIndexRangeEnumerator(sourceAsArray, partitionCount, i);
}
TraceHelpers.TraceInfo("ContigousRangePartitionExchangeStream::MakePartitions - (array) #{0} {1}", i, maxChunkSize);
}
else
{
// Create a general purpose list enumerator object.
if (useStriping)
{
partitions[i] = new ListIndexRangeEnumerator(sourceAsList, partitionCount, i, maxChunkSize);
}
else
{
partitions[i] = new ListContiguousIndexRangeEnumerator(sourceAsList, partitionCount, i);
}
TraceHelpers.TraceInfo("ContigousRangePartitionExchangeStream::MakePartitions - (list) #{0} {1})", i, maxChunkSize);
}
}
Contract.Assert(partitions.Length == partitionCount);
m_partitions = partitions;
}
else
{
// We couldn't use an in-place partition. Shucks. Defer to the other overload which
// accepts an enumerator as input instead.
m_partitions = MakePartitions(source.GetEnumerator(), partitionCount);
}
}
//-----------------------------------------------------------------------------------
// The slow path used when a quick loop through the channels didn't come up
// with anything. We may need to block and/or mark channels as done.
//
private bool MoveNextSlowPath()
{
int doneChannels = 0;
// Remember the first channel we are looking at. If we pass through all of the
// channels without finding an element, we will go to sleep.
int firstChannelIndex = m_channelIndex;
int currChannelIndex;
while ((currChannelIndex = m_channelIndex) != m_channels.Length)
{
AsynchronousChannel <T> current = m_channels[currChannelIndex];
bool isDone = m_done[currChannelIndex];
if (!isDone && current.TryDequeue(ref m_currentElement))
{
// The channel has an item to be processed. We already remembered the current
// element (Dequeue stores it as an out-parameter), so we just return true
// after advancing to the next channel.
m_channelIndex = (currChannelIndex + 1) % m_channels.Length;
return(true);
}
else
{
// There isn't an element in the current channel. Check whether the channel
// is done before possibly waiting for an element to arrive.
if (!isDone && current.IsDone)
{
// We must check to ensure an item didn't get enqueued after originally
// trying to dequeue above and reading the IsDone flag. If there are still
// elements, the producer may have marked the channel as done but of course
// we still need to continue processing them.
if (!current.IsChunkBufferEmpty)
{
bool dequeueResult = current.TryDequeue(ref m_currentElement);
Contract.Assert(dequeueResult, "channel isn't empty, yet the dequeue failed, hmm");
return(true);
}
// Mark this channel as being truly done. We won't consider it any longer.
m_done[currChannelIndex] = true;
if (m_channelEvents != null)
{
m_channelEvents[currChannelIndex] = null; //we definitely never want to wait on this (soon to be disposed) event.
}
isDone = true;
current.Dispose();
}
if (isDone)
{
Contract.Assert(m_channels[currChannelIndex].IsDone, "thought this channel was done");
Contract.Assert(m_channels[currChannelIndex].IsChunkBufferEmpty, "thought this channel was empty");
// Increment the count of done channels that we've seen. If this reaches the
// total number of channels, we know we're finally done.
if (++doneChannels == m_channels.Length)
{
// Remember that we are done by setting the index past the end.
m_channelIndex = currChannelIndex = m_channels.Length;
break;
}
}
// Still no element. Advance to the next channel and continue searching.
m_channelIndex = currChannelIndex = (currChannelIndex + 1) % m_channels.Length;
// If the channels aren't done, and we've inspected all of the queues and still
// haven't found anything, we will go ahead and wait on all the queues.
if (currChannelIndex == firstChannelIndex)
{
// On our first pass through the queues, we didn't have any side-effects
// that would let a producer know we are waiting. Now we go through and
// accumulate a set of events to wait on.
try
{
// If this is the first time we must block, lazily allocate and cache
// a list of events to be reused for next time.
if (m_channelEvents == null)
{
m_channelEvents = new ManualResetEventSlim[m_channels.Length];
}
// Reset our done channels counter; we need to tally them again during the
// second pass through.
doneChannels = 0;
for (int i = 0; i < m_channels.Length; i++)
{
if (!m_done[i] && m_channels[i].TryDequeue(ref m_currentElement, ref m_channelEvents[i]))
{
Contract.Assert(m_channelEvents[i] == null);
// The channel has received an item since the last time we checked.
// Just return and let the consumer process the element returned.
return(true);
}
else if (m_channelEvents[i] == null)
{
// The channel in question is done and empty.
Contract.Assert(m_channels[i].IsDone, "expected channel to be done");
Contract.Assert(m_channels[i].IsChunkBufferEmpty, "expected channel to be empty");
if (!m_done[i])
{
m_done[i] = true;
m_channels[i].Dispose();
}
if (++doneChannels == m_channels.Length)
{
// No need to wait. All channels are done. Remember this by setting
// the index past the end of the channel list.
m_channelIndex = currChannelIndex = m_channels.Length;
break;
}
}
}
// If all channels are done, we can break out of the loop entirely.
if (currChannelIndex == m_channels.Length)
{
break;
}
// Finally, we have accumulated a set of events. Perform a wait-any on them.
Contract.Assert(m_channelEvents.Length <= 63,
"WaitForMultipleObjects only supports 63 threads if running on an STA thread (64 otherwise).");
//This WaitAny() does not require cancellation support as it will wake up when all the producers into the
//channel have finished. Hence, if all the producers wake up on cancellation, so will this.
m_channelIndex = currChannelIndex = WaitAny(m_channelEvents);
Contract.Assert(0 <= m_channelIndex && m_channelIndex < m_channelEvents.Length);
Contract.Assert(0 <= currChannelIndex && currChannelIndex < m_channelEvents.Length);
Contract.Assert(m_channelEvents[currChannelIndex] != null);
//
// We have woken up, and the channel that caused this is contained in the
// returned index. This could be due to one of two reasons. Either the channel's
// producer has notified that it is done, in which case we just have to take it
// out of our current wait-list and redo the wait, or a channel actually has an
// item which we will go ahead and process.
//
// We just go back 'round the loop to accomplish this logic. Reset the channel
// index and # of done channels. Go back to the beginning, starting with the channel
// that caused us to wake up.
//
firstChannelIndex = currChannelIndex;
doneChannels = 0;
}
finally
{
// We have to guarantee that any waits we said we would perform are undone.
for (int i = 0; i < m_channelEvents.Length; i++)
{
// If we retrieved an event from a channel, we need to reset the wait.
if (m_channelEvents[i] != null)
{
m_channels[i].DoneWithDequeueWait();
}
}
}
}
}
}
TraceHelpers.TraceInfo("[timing]: {0}: Completed the merge", DateTime.Now.Ticks);
// If we got this far, it means we've exhausted our channels.
Contract.Assert(currChannelIndex == m_channels.Length);
// If any tasks failed, propagate the failure now. We must do it here, because the merge
// executor returns control back to the caller before the query has completed; contrast
// this with synchronous enumeration where we can wait before returning.
m_taskGroupState.QueryEnd(false);
return(false);
}
//---------------------------------------------------------------------------------------
// MoveNext implements all the hash-join logic noted earlier. When it is called first, it
// will execute the entire inner query tree, and build a hash-table lookup. This is the
// Building phase. Then for the first call and all subsequent calls to MoveNext, we will
// incrementally perform the Probing phase. We'll keep getting elements from the outer
// data source, looking into the hash-table we built, and enumerating the full results.
//
// This routine supports both inner and outer (group) joins. An outer join will yield a
// (possibly empty) list of matching elements from the inner instead of one-at-a-time,
// as we do for inner joins.
//
internal override bool MoveNext(ref TOutput currentElement, ref TLeftKey currentKey)
{
Contract.Assert(_singleResultSelector != null || _groupResultSelector != null, "expected a compiled result selector");
Contract.Assert(_leftSource != null);
Contract.Assert(_rightSource != null);
// BUILD phase: If we haven't built the hash-table yet, create that first.
Mutables mutables = _mutables;
if (mutables == null)
{
mutables = _mutables = new Mutables();
#if DEBUG
int hashLookupCount = 0;
int hashKeyCollisions = 0;
#endif
mutables._rightHashLookup = new HashLookup <THashKey, Pair>(_keyComparer);
Pair rightPair = new Pair(default(TRightInput), default(THashKey));
int rightKeyUnused = default(int);
int i = 0;
while (_rightSource.MoveNext(ref rightPair, ref rightKeyUnused))
{
if ((i++ & CancellationState.POLL_INTERVAL) == 0)
{
CancellationState.ThrowIfCanceled(_cancellationToken);
}
TRightInput rightElement = (TRightInput)rightPair.First;
THashKey rightHashKey = (THashKey)rightPair.Second;
// We ignore null keys.
if (rightHashKey != null)
{
#if DEBUG
hashLookupCount++;
#endif
// See if we've already stored an element under the current key. If not, we
// lazily allocate a pair to hold the elements mapping to the same key.
const int INITIAL_CHUNK_SIZE = 2;
Pair currentValue = new Pair(default(TRightInput), default(ListChunk <TRightInput>));
if (!mutables._rightHashLookup.TryGetValue(rightHashKey, ref currentValue))
{
currentValue = new Pair(rightElement, null);
if (_groupResultSelector != null)
{
// For group joins, we also add the element to the list. This makes
// it easier later to yield the list as-is.
currentValue.Second = new ListChunk <TRightInput>(INITIAL_CHUNK_SIZE);
((ListChunk <TRightInput>)currentValue.Second).Add((TRightInput)rightElement);
}
mutables._rightHashLookup.Add(rightHashKey, currentValue);
}
else
{
if (currentValue.Second == null)
{
// Lazily allocate a list to hold all but the 1st value. We need to
// re-store this element because the pair is a value type.
currentValue.Second = new ListChunk <TRightInput>(INITIAL_CHUNK_SIZE);
mutables._rightHashLookup[rightHashKey] = currentValue;
}
((ListChunk <TRightInput>)currentValue.Second).Add((TRightInput)rightElement);
#if DEBUG
hashKeyCollisions++;
#endif
}
}
}
#if DEBUG
TraceHelpers.TraceInfo("ParallelJoinQueryOperator::MoveNext - built hash table [count = {0}, collisions = {1}]",
hashLookupCount, hashKeyCollisions);
#endif
}
// PROBE phase: So long as the source has a next element, return the match.
ListChunk <TRightInput> currentRightChunk = mutables._currentRightMatches;
if (currentRightChunk != null && mutables._currentRightMatchesIndex == currentRightChunk.Count)
{
currentRightChunk = mutables._currentRightMatches = currentRightChunk.Next;
mutables._currentRightMatchesIndex = 0;
}
if (mutables._currentRightMatches == null)
{
// We have to look up the next list of matches in the hash-table.
Pair leftPair = new Pair(default(TLeftInput), default(THashKey));
TLeftKey leftKey = default(TLeftKey);
while (_leftSource.MoveNext(ref leftPair, ref leftKey))
{
if ((mutables._outputLoopCount++ & CancellationState.POLL_INTERVAL) == 0)
{
CancellationState.ThrowIfCanceled(_cancellationToken);
}
// Find the match in the hash table.
Pair matchValue = new Pair(default(TRightInput), default(ListChunk <TRightInput>));
TLeftInput leftElement = (TLeftInput)leftPair.First;
THashKey leftHashKey = (THashKey)leftPair.Second;
// Ignore null keys.
if (leftHashKey != null)
{
if (mutables._rightHashLookup.TryGetValue(leftHashKey, ref matchValue))
{
// We found a new match. For inner joins, we remember the list in case
// there are multiple value under this same key -- the next iteration will pick
// them up. For outer joins, we will use the list momentarily.
if (_singleResultSelector != null)
{
mutables._currentRightMatches = (ListChunk <TRightInput>)matchValue.Second;
Contract.Assert(mutables._currentRightMatches == null || mutables._currentRightMatches.Count > 0,
"we were expecting that the list would be either null or empty");
mutables._currentRightMatchesIndex = 0;
// Yield the value.
currentElement = _singleResultSelector(leftElement, (TRightInput)matchValue.First);
currentKey = leftKey;
// If there is a list of matches, remember the left values for next time.
if (matchValue.Second != null)
{
mutables._currentLeft = leftElement;
mutables._currentLeftKey = leftKey;
}
return(true);
}
}
}
// For outer joins, we always yield a result.
if (_groupResultSelector != null)
{
// Grab the matches, or create an empty list if there are none.
IEnumerable <TRightInput> matches = (ListChunk <TRightInput>)matchValue.Second;
if (matches == null)
{
matches = ParallelEnumerable.Empty <TRightInput>();
}
// Generate the current value.
currentElement = _groupResultSelector(leftElement, matches);
currentKey = leftKey;
return(true);
}
}
// If we've reached the end of the data source, we're done.
return(false);
}
// Produce the next element and increment our index within the matches.
Contract.Assert(_singleResultSelector != null);
Contract.Assert(mutables._currentRightMatches != null);
Contract.Assert(0 <= mutables._currentRightMatchesIndex && mutables._currentRightMatchesIndex < mutables._currentRightMatches.Count);
currentElement = _singleResultSelector(
mutables._currentLeft, mutables._currentRightMatches._chunk[mutables._currentRightMatchesIndex]);
currentKey = mutables._currentLeftKey;
mutables._currentRightMatchesIndex++;
return(true);
}
//-----------------------------------------------------------------------------------
// Internal helper method to dequeue a whole chunk. This version of the API is used
// when the caller will wait for a new chunk to be enqueued.
//
// Arguments:
// chunk - a byref for the dequeued chunk
// waitEvent - a byref for the event used to signal blocked consumers
//
// Return Value:
// True if a chunk was found, false otherwise.
//
// Notes:
// If the return value is false, it doesn't always mean waitEvent will be non-
// null. If the producer is done enqueueing, the return will be false and the
// event will remain null. A caller must check for this condition.
//
// If the return value is false and an event is returned, there have been
// side-effects on the channel. Namely, the flag telling producers a consumer
// might be waiting will have been set. DequeueEndAfterWait _must_ be called
// eventually regardless of whether the caller actually waits or not.
//
private bool TryDequeueChunk(ref T[] chunk, ref bool isDone)
{
isDone = false;
// We will register our interest in waiting, and then return an event
// that the caller can use to wait.
while (IsChunkBufferEmpty)
{
// If the producer is done and we've drained the queue, we can bail right away.
if (IsDone)
{
// We have to see if the buffer is empty AFTER we've seen that it's done.
// Otherwise, we would possibly miss the elements enqueued before the
// producer signaled that it's done. This is done with a volatile load so
// that the read of empty doesn't move before the read of done.
if (IsChunkBufferEmpty)
{
// Return isDone=true so callers know not to wait
isDone = true;
return(false);
}
}
// We have to handle the case where a producer and consumer are racing to
// wait simultaneously. For instance, a consumer might see an empty queue (by
// reading IsChunkBufferEmpty just above), but meanwhile a producer might fill the queue
// very quickly, suddenly seeing a full queue. This would lead to deadlock
// if we aren't careful. Therefore we check the empty/full state AGAIN after
// setting our flag to see if a real wait is warranted.
#pragma warning disable 0420
Interlocked.Exchange(ref _consumerIsWaiting, 1);
#pragma warning restore 0420
// (We have to prevent the reads that go into determining whether the buffer
// is full from moving before the write to the producer-wait flag. Hence the CAS.)
// Because we might be racing with a producer that is transitioning the
// buffer from empty to non-full, we must check that the queue is empty once
// more. Similarly, if the queue has been marked as done, we must not wait
// because we just reset the event, possibly losing as signal. In both cases,
// we would otherwise decide to wait and never be woken up (i.e. deadlock).
if (IsChunkBufferEmpty && !IsDone)
{
// Note that the caller must eventually call DequeueEndAfterWait to set the
// flags back to a state where no consumer is waiting, whether they choose
// to wait or not.
TraceHelpers.TraceInfo("AsynchronousChannel::DequeueChunk - consumer possibly waiting");
return(false);
}
else
{
// Reset the wait flags, we don't need to wait after all. We loop back around
// and recheck that the queue isn't empty, done, etc.
_consumerIsWaiting = 0;
}
}
Debug.Assert(!IsChunkBufferEmpty, "single-consumer should never witness an empty queue here");
chunk = InternalDequeueChunk();
return(true);
}
