//---------------------------------------------------------------------------------------
// MoveNext implements all the hash-join logic noted earlier. When it is called first, it
// will execute the entire inner query tree, and build a hash-table lookup. This is the
// Building phase. Then for the first call and all subsequent calls to MoveNext, we will
// incrementally perform the Probing phase. We'll keep getting elements from the outer
// data source, looking into the hash-table we built, and enumerating the full results.
//
// This routine supports both inner and outer (group) joins. An outer join will yield a
// (possibly empty) list of matching elements from the inner instead of one-at-a-time,
// as we do for inner joins.
//
internal override bool MoveNext(ref TOutput currentElement, ref TLeftKey currentKey)
{
Contract.Assert(m_singleResultSelector != null || m_groupResultSelector != null, "expected a compiled result selector");
Contract.Assert(m_leftSource != null);
Contract.Assert(m_rightSource != null);
// BUILD phase: If we haven't built the hash-table yet, create that first.
Mutables mutables = m_mutables;
if (mutables == null)
{
mutables = m_mutables = new Mutables();
#if DEBUG
int hashLookupCount = 0;
int hashKeyCollisions = 0;
#endif
mutables.m_rightHashLookup = new HashLookup <THashKey, Pair <TRightInput, ListChunk <TRightInput> > >(m_keyComparer);
Pair <TRightInput, THashKey> rightPair = default(Pair <TRightInput, THashKey>);
int rightKeyUnused = default(int);
int i = 0;
while (m_rightSource.MoveNext(ref rightPair, ref rightKeyUnused))
{
if ((i++ & CancellationState.POLL_INTERVAL) == 0)
{
CancellationState.ThrowIfCanceled(m_cancellationToken);
}
TRightInput rightElement = rightPair.First;
THashKey rightHashKey = rightPair.Second;
// We ignore null keys.
if (rightHashKey != null)
{
#if DEBUG
hashLookupCount++;
#endif
// See if we've already stored an element under the current key. If not, we
// lazily allocate a pair to hold the elements mapping to the same key.
const int INITIAL_CHUNK_SIZE = 2;
Pair <TRightInput, ListChunk <TRightInput> > currentValue = default(Pair <TRightInput, ListChunk <TRightInput> >);
if (!mutables.m_rightHashLookup.TryGetValue(rightHashKey, ref currentValue))
{
currentValue = new Pair <TRightInput, ListChunk <TRightInput> >(rightElement, null);
if (m_groupResultSelector != null)
{
// For group joins, we also add the element to the list. This makes
// it easier later to yield the list as-is.
currentValue.Second = new ListChunk <TRightInput>(INITIAL_CHUNK_SIZE);
currentValue.Second.Add(rightElement);
}
mutables.m_rightHashLookup.Add(rightHashKey, currentValue);
}
else
{
if (currentValue.Second == null)
{
// Lazily allocate a list to hold all but the 1st value. We need to
// re-store this element because the pair is a value type.
currentValue.Second = new ListChunk <TRightInput>(INITIAL_CHUNK_SIZE);
mutables.m_rightHashLookup[rightHashKey] = currentValue;
}
currentValue.Second.Add(rightElement);
#if DEBUG
hashKeyCollisions++;
#endif
}
}
}
#if DEBUG
TraceHelpers.TraceInfo("ParallelJoinQueryOperator::MoveNext - built hash table [count = {0}, collisions = {1}]",
hashLookupCount, hashKeyCollisions);
#endif
}
// PROBE phase: So long as the source has a next element, return the match.
ListChunk <TRightInput> currentRightChunk = mutables.m_currentRightMatches;
if (currentRightChunk != null && mutables.m_currentRightMatchesIndex == currentRightChunk.Count)
{
currentRightChunk = mutables.m_currentRightMatches = currentRightChunk.Next;
mutables.m_currentRightMatchesIndex = 0;
}
if (mutables.m_currentRightMatches == null)
{
// We have to look up the next list of matches in the hash-table.
Pair <TLeftInput, THashKey> leftPair = default(Pair <TLeftInput, THashKey>);
TLeftKey leftKey = default(TLeftKey);
while (m_leftSource.MoveNext(ref leftPair, ref leftKey))
{
if ((mutables.m_outputLoopCount++ & CancellationState.POLL_INTERVAL) == 0)
{
CancellationState.ThrowIfCanceled(m_cancellationToken);
}
// Find the match in the hash table.
Pair <TRightInput, ListChunk <TRightInput> > matchValue = default(Pair <TRightInput, ListChunk <TRightInput> >);
TLeftInput leftElement = leftPair.First;
THashKey leftHashKey = leftPair.Second;
// Ignore null keys.
if (leftHashKey != null)
{
if (mutables.m_rightHashLookup.TryGetValue(leftHashKey, ref matchValue))
{
// We found a new match. For inner joins, we remember the list in case
// there are multiple value under this same key -- the next iteration will pick
// them up. For outer joins, we will use the list momentarily.
if (m_singleResultSelector != null)
{
mutables.m_currentRightMatches = matchValue.Second;
Contract.Assert(mutables.m_currentRightMatches == null || mutables.m_currentRightMatches.Count > 0,
"we were expecting that the list would be either null or empty");
mutables.m_currentRightMatchesIndex = 0;
// Yield the value.
currentElement = m_singleResultSelector(leftElement, matchValue.First);
currentKey = leftKey;
// If there is a list of matches, remember the left values for next time.
if (matchValue.Second != null)
{
mutables.m_currentLeft = leftElement;
mutables.m_currentLeftKey = leftKey;
}
return(true);
}
}
}
// For outer joins, we always yield a result.
if (m_groupResultSelector != null)
{
// Grab the matches, or create an empty list if there are none.
IEnumerable <TRightInput> matches = matchValue.Second;
if (matches == null)
{
matches = ParallelEnumerable.Empty <TRightInput>();
}
// Generate the current value.
currentElement = m_groupResultSelector(leftElement, matches);
currentKey = leftKey;
return(true);
}
}
// If we've reached the end of the data source, we're done.
return(false);
}
// Produce the next element and increment our index within the matches.
Contract.Assert(m_singleResultSelector != null);
Contract.Assert(mutables.m_currentRightMatches != null);
Contract.Assert(0 <= mutables.m_currentRightMatchesIndex && mutables.m_currentRightMatchesIndex < mutables.m_currentRightMatches.Count);
currentElement = m_singleResultSelector(
mutables.m_currentLeft, mutables.m_currentRightMatches.m_chunk[mutables.m_currentRightMatchesIndex]);
currentKey = mutables.m_currentLeftKey;
mutables.m_currentRightMatchesIndex++;
return(true);
}
/// <summary>
/// Wait until a producer's buffer is non-empty, or until that producer is done.
/// </summary>
/// <returns>false if there is no element to yield because the producer is done, true otherwise</returns>
private bool TryWaitForElement(int producer, ref Pair <TKey, TOutput> element)
{
Queue <Pair <TKey, TOutput> > buffer = _mergeHelper._buffers[producer];
object bufferLock = _mergeHelper._bufferLocks[producer];
lock (bufferLock)
{
// If the buffer is empty, we need to wait on the producer
if (buffer.Count == 0)
{
// If the producer is already done, return false
if (_mergeHelper._producerDone[producer])
{
element = default(Pair <TKey, TOutput>);
return(false);
}
if (ParallelEnumerable.SinglePartitionMode)
{
return(false);
}
_mergeHelper._consumerWaiting[producer] = true;
Monitor.Wait(bufferLock);
// If the buffer is still empty, the producer is done
if (buffer.Count == 0)
{
Debug.Assert(_mergeHelper._producerDone[producer]);
element = default(Pair <TKey, TOutput>);
return(false);
}
}
Debug.Assert(buffer.Count > 0, "Producer's buffer should not be empty here.");
// If the producer is waiting, wake it up
if (_mergeHelper._producerWaiting[producer])
{
Debug.Assert(!ParallelEnumerable.SinglePartitionMode);
Monitor.Pulse(bufferLock);
_mergeHelper._producerWaiting[producer] = false;
}
if (buffer.Count < STEAL_BUFFER_SIZE)
{
element = buffer.Dequeue();
return(true);
}
else
{
// Privatize the entire buffer
_privateBuffer[producer] = _mergeHelper._buffers[producer];
// Give an empty buffer to the producer
_mergeHelper._buffers[producer] = new Queue <Pair <TKey, TOutput> >(INITIAL_BUFFER_SIZE);
// No return statement.
// This is the only branch that continues below of the lock region.
}
}
// Get an element out of the private buffer.
bool gotElement = TryGetPrivateElement(producer, ref element);
Debug.Assert(gotElement);
return(true);
}
//---------------------------------------------------------------------------------------
// Walks the two data sources, left and then right, to produce the intersection.
//
internal override bool MoveNext(ref TInputOutput currentElement, ref TLeftKey currentKey)
{
Contract.Assert(m_leftSource != null);
Contract.Assert(m_rightSource != null);
// Build the set out of the left data source, if we haven't already.
int i = 0;
if (m_hashLookup == null)
{
m_hashLookup = new Dictionary <Wrapper <TInputOutput>, Pair <TInputOutput, TLeftKey> >(m_comparer);
Pair <TInputOutput, NoKeyMemoizationRequired> leftElement = default(Pair <TInputOutput, NoKeyMemoizationRequired>);
TLeftKey leftKey = default(TLeftKey);
while (m_leftSource.MoveNext(ref leftElement, ref leftKey))
{
if ((i++ & CancellationState.POLL_INTERVAL) == 0)
{
CancellationState.ThrowIfCanceled(m_cancellationToken);
}
// For each element, we track the smallest order key for that element that we saw so far
Pair <TInputOutput, TLeftKey> oldEntry;
Wrapper <TInputOutput> wrappedLeftElem = new Wrapper <TInputOutput>(leftElement.First);
// If this is the first occurence of this element, or the order key is lower than all keys we saw previously,
// update the order key for this element.
if (!m_hashLookup.TryGetValue(wrappedLeftElem, out oldEntry) || m_leftKeyComparer.Compare(leftKey, oldEntry.Second) < 0)
{
// For each "elem" value, we store the smallest key, and the element value that had that key.
// Note that even though two element values are "equal" according to the EqualityComparer,
// we still cannot choose arbitrarily which of the two to yield.
m_hashLookup[wrappedLeftElem] = new Pair <TInputOutput, TLeftKey>(leftElement.First, leftKey);
}
}
}
// Now iterate over the right data source, looking for matches.
Pair <TInputOutput, NoKeyMemoizationRequired> rightElement = default(Pair <TInputOutput, NoKeyMemoizationRequired>);
int rightKeyUnused = default(int);
while (m_rightSource.MoveNext(ref rightElement, ref rightKeyUnused))
{
if ((i++ & CancellationState.POLL_INTERVAL) == 0)
{
CancellationState.ThrowIfCanceled(m_cancellationToken);
}
// If we found the element in our set, and if we haven't returned it yet,
// we can yield it to the caller. We also mark it so we know we've returned
// it once already and never will again.
Pair <TInputOutput, TLeftKey> entry;
Wrapper <TInputOutput> wrappedRightElem = new Wrapper <TInputOutput>(rightElement.First);
if (m_hashLookup.TryGetValue(wrappedRightElem, out entry))
{
currentElement = entry.First;
currentKey = entry.Second;
m_hashLookup.Remove(new Wrapper <TInputOutput>(entry.First));
return(true);
}
}
return(false);
}
//---------------------------------------------------------------------------------------
// MoveNext implements all the hash-join logic noted earlier. When it is called first, it
// will execute the entire inner query tree, and build a hash-table lookup. This is the
// Building phase. Then for the first call and all subsequent calls to MoveNext, we will
// incrementally perform the Probing phase. We'll keep getting elements from the outer
// data source, looking into the hash-table we built, and enumerating the full results.
//
// This routine supports both inner and outer (group) joins. An outer join will yield a
// (possibly empty) list of matching elements from the inner instead of one-at-a-time,
// as we do for inner joins.
//
internal override bool MoveNext([MaybeNullWhen(false), AllowNull] ref TOutput currentElement, ref TOutputKey currentKey)
{
Debug.Assert(_resultSelector != null, "expected a compiled result selector");
Debug.Assert(_leftSource != null);
Debug.Assert(_rightLookupBuilder != null);
// BUILD phase: If we haven't built the hash-table yet, create that first.
Mutables?mutables = _mutables;
if (mutables == null)
{
mutables = _mutables = new Mutables();
mutables._rightHashLookup = _rightLookupBuilder.BuildHashLookup(_cancellationToken);
}
// PROBE phase: So long as the source has a next element, return the match.
ListChunk <Pair <TRightInput, TRightKey> >?currentRightChunk = mutables._currentRightMatches;
if (currentRightChunk != null && mutables._currentRightMatchesIndex == currentRightChunk.Count)
{
mutables._currentRightMatches = currentRightChunk.Next;
mutables._currentRightMatchesIndex = 0;
}
if (mutables._currentRightMatches == null)
{
// We have to look up the next list of matches in the hash-table.
Pair <TLeftInput, THashKey> leftPair = default(Pair <TLeftInput, THashKey>);
TLeftKey leftKey = default(TLeftKey) !;
while (_leftSource.MoveNext(ref leftPair, ref leftKey))
{
if ((mutables._outputLoopCount++ & CancellationState.POLL_INTERVAL) == 0)
{
CancellationState.ThrowIfCanceled(_cancellationToken);
}
// Find the match in the hash table.
HashLookupValueList <TRightInput, TRightKey> matchValue = default(HashLookupValueList <TRightInput, TRightKey>);
TLeftInput leftElement = leftPair.First;
THashKey leftHashKey = leftPair.Second;
// Ignore null keys.
if (leftHashKey != null)
{
Debug.Assert(mutables._rightHashLookup != null);
if (mutables._rightHashLookup.TryGetValue(leftHashKey, ref matchValue))
{
// We found a new match. We remember the list in case there are multiple
// values under this same key -- the next iteration will pick them up.
mutables._currentRightMatches = matchValue.Tail;
Debug.Assert(mutables._currentRightMatches == null || mutables._currentRightMatches.Count > 0,
"we were expecting that the list would be either null or empty");
mutables._currentRightMatchesIndex = 0;
// Yield the value.
currentElement = _resultSelector(leftElement, matchValue.Head.First);
currentKey = _outputKeyBuilder.Combine(leftKey, matchValue.Head.Second);
// If there is a list of matches, remember the left values for next time.
if (matchValue.Tail != null)
{
mutables._currentLeft = leftElement;
mutables._currentLeftKey = leftKey;
}
return(true);
}
}
}
// If we've reached the end of the data source, we're done.
return(false);
}
// Produce the next element.
Debug.Assert(mutables._currentRightMatches != null);
Debug.Assert(0 <= mutables._currentRightMatchesIndex && mutables._currentRightMatchesIndex < mutables._currentRightMatches.Count);
Pair <TRightInput, TRightKey> rightMatch = mutables._currentRightMatches._chunk[mutables._currentRightMatchesIndex];
currentElement = _resultSelector(mutables._currentLeft, rightMatch.First);
currentKey = _outputKeyBuilder.Combine(mutables._currentLeftKey, rightMatch.Second);
mutables._currentRightMatchesIndex++;
return(true);
}
/// <summary>
/// Moves the enumerator to the next result, or returns false if there are no more results to yield.
/// </summary>
public override bool MoveNext()
{
if (!_initialized)
{
//
// Initialization: wait until each producer has produced at least one element. Since the order indices
// are increasing, we cannot start yielding until we have at least one element from each producer.
//
_initialized = true;
for (int producer = 0; producer < _mergeHelper._partitions.PartitionCount; producer++)
{
Pair <TKey, TOutput> element = default(Pair <TKey, TOutput>);
// Get the first element from this producer
if (TryWaitForElement(producer, ref element))
{
// Update the producer heap and its helper array with the received element
_producerHeap.Insert(new Producer <TKey>(element.First, producer));
_producerNextElement[producer] = element.Second;
}
else
{
// If this producer didn't produce any results because it encountered an exception,
// cancellation would have been initiated by now. If cancellation has started, we will
// propagate the exception now.
ThrowIfInTearDown();
}
}
}
else
{
// If the producer heap is empty, we are done. In fact, we know that a previous MoveNext() call
// already returned false.
if (_producerHeap.Count == 0)
{
return(false);
}
//
// Get the next element from the producer that yielded a value last. Update the producer heap.
// The next producer to yield will be in the front of the producer heap.
//
// The last producer whose result the merge yielded
int lastProducer = _producerHeap.MaxValue.ProducerIndex;
// Get the next element from the same producer
Pair <TKey, TOutput> element = default(Pair <TKey, TOutput>);
if (TryGetPrivateElement(lastProducer, ref element) ||
TryWaitForElement(lastProducer, ref element))
{
// Update the producer heap and its helper array with the received element
_producerHeap.ReplaceMax(new Producer <TKey>(element.First, lastProducer));
_producerNextElement[lastProducer] = element.Second;
}
else
{
// If this producer is done because it encountered an exception, cancellation
// would have been initiated by now. If cancellation has started, we will propagate
// the exception now.
ThrowIfInTearDown();
// This producer is done. Remove it from the producer heap.
_producerHeap.RemoveMax();
}
}
return(_producerHeap.Count > 0);
}
//---------------------------------------------------------------------------------------
// Retrieves the next element from this partition. All repartitioning operators across
// all partitions cooperate in a barrier-style algorithm. The first time an element is
// requested, the repartitioning operator will enter the 1st phase: during this phase, it
// scans its entire input and compute the destination partition for each element. During
// the 2nd phase, each partition scans the elements found by all other partitions for
// it, and yield this to callers. The only synchronization required is the barrier itself
// -- all other parts of this algorithm are synchronization-free.
//
// Notes: One rather large penalty that this algorithm incurs is higher memory usage and a
// larger time-to-first-element latency, at least compared with our old implementation; this
// happens because all input elements must be fetched before we can produce a single output
// element. In many cases this isn't too terrible: e.g. a GroupBy requires this to occur
// anyway, so having the repartitioning operator do so isn't complicating matters much at all.
//
internal override bool MoveNext(ref Pair <TInputOutput, THashKey> currentElement, ref int currentKey)
{
if (m_partitionCount == 1)
{
// If there's only one partition, no need to do any sort of exchanges.
TIgnoreKey keyUnused = default(TIgnoreKey);
TInputOutput current = default(TInputOutput);
#if DEBUG
currentKey = unchecked ((int)0xdeadbeef);
#endif
if (m_source.MoveNext(ref current, ref keyUnused))
{
currentElement = new Pair <TInputOutput, THashKey>(
current, m_keySelector == null ? default(THashKey) : m_keySelector(current));
return(true);
}
return(false);
}
Mutables mutables = m_mutables;
if (mutables == null)
{
mutables = m_mutables = new Mutables();
}
// If we haven't enumerated the source yet, do that now. This is the first phase
// of a two-phase barrier style operation.
if (mutables.m_currentBufferIndex == ENUMERATION_NOT_STARTED)
{
EnumerateAndRedistributeElements();
Contract.Assert(mutables.m_currentBufferIndex != ENUMERATION_NOT_STARTED);
}
// Once we've enumerated our contents, we can then go back and walk the buffers that belong
// to the current partition. This is phase two. Note that we slyly move on to the first step
// of phase two before actually waiting for other partitions. That's because we can enumerate
// the buffer we wrote to above, as already noted.
while (mutables.m_currentBufferIndex < m_partitionCount)
{
// If the queue is non-null and still has elements, yield them.
if (mutables.m_currentBuffer != null)
{
if (++mutables.m_currentIndex < mutables.m_currentBuffer.Count)
{
// Return the current element.
currentElement = mutables.m_currentBuffer.m_chunk[mutables.m_currentIndex];
return(true);
}
else
{
// If the chunk is empty, advance to the next one (if any).
mutables.m_currentIndex = ENUMERATION_NOT_STARTED;
mutables.m_currentBuffer = mutables.m_currentBuffer.Next;
Contract.Assert(mutables.m_currentBuffer == null || mutables.m_currentBuffer.Count > 0);
continue; // Go back around and invoke this same logic.
}
}
// We're done with the current partition. Slightly different logic depending on whether
// we're on our own buffer or one that somebody else found for us.
if (mutables.m_currentBufferIndex == m_partitionIndex)
{
// We now need to wait at the barrier, in case some other threads aren't done.
// Once we wake up, we reset our index and will increment it immediately after.
m_barrier.Wait(m_cancellationToken);
mutables.m_currentBufferIndex = ENUMERATION_NOT_STARTED;
}
// Advance to the next buffer.
mutables.m_currentBufferIndex++;
mutables.m_currentIndex = ENUMERATION_NOT_STARTED;
if (mutables.m_currentBufferIndex == m_partitionIndex)
{
// Skip our current buffer (since we already enumerated it).
mutables.m_currentBufferIndex++;
}
// Assuming we're within bounds, retrieve the next buffer object.
if (mutables.m_currentBufferIndex < m_partitionCount)
{
mutables.m_currentBuffer = m_valueExchangeMatrix[mutables.m_currentBufferIndex, m_partitionIndex];
}
}
// We're done. No more buffers to enumerate.
return(false);
}
// constructor used to build a new list.
internal HashLookupValueList(TElement firstValue, TOrderKey firstOrderKey)
{
_head = CreatePair(firstValue, firstOrderKey);
_tail = null;
}
internal override bool MoveNext(ref TResult currentElement, ref int currentKey)
{
if ((this.m_buffer == null) && (this.m_count > 0))
{
List <Pair <TResult, int> > list = new List <Pair <TResult, int> >();
TResult local = default(TResult);
int num = 0;
int num2 = 0;
while ((list.Count < this.m_count) && this.m_source.MoveNext(ref local, ref num))
{
if ((num2++ & 0x3f) == 0)
{
CancellationState.ThrowIfCanceled(this.m_cancellationToken);
}
list.Add(new Pair <TResult, int>(local, num));
lock (this.m_sharedIndices)
{
if (!this.m_sharedIndices.Insert(num))
{
break;
}
continue;
}
}
this.m_sharedBarrier.Signal();
this.m_sharedBarrier.Wait(this.m_cancellationToken);
this.m_buffer = list;
this.m_bufferIndex = new Shared <int>(-1);
}
if (this.m_take)
{
if ((this.m_count == 0) || (this.m_bufferIndex.Value >= (this.m_buffer.Count - 1)))
{
return(false);
}
this.m_bufferIndex.Value += 1;
Pair <TResult, int> pair = this.m_buffer[this.m_bufferIndex.Value];
currentElement = pair.First;
Pair <TResult, int> pair2 = this.m_buffer[this.m_bufferIndex.Value];
currentKey = pair2.Second;
int maxValue = this.m_sharedIndices.MaxValue;
if (maxValue != -1)
{
Pair <TResult, int> pair3 = this.m_buffer[this.m_bufferIndex.Value];
return(pair3.Second <= maxValue);
}
return(true);
}
int num4 = -1;
if (this.m_count > 0)
{
if (this.m_sharedIndices.Count < this.m_count)
{
return(false);
}
num4 = this.m_sharedIndices.MaxValue;
if (this.m_bufferIndex.Value < (this.m_buffer.Count - 1))
{
this.m_bufferIndex.Value += 1;
while (this.m_bufferIndex.Value < this.m_buffer.Count)
{
Pair <TResult, int> pair4 = this.m_buffer[this.m_bufferIndex.Value];
if (pair4.Second > num4)
{
Pair <TResult, int> pair5 = this.m_buffer[this.m_bufferIndex.Value];
currentElement = pair5.First;
Pair <TResult, int> pair6 = this.m_buffer[this.m_bufferIndex.Value];
currentKey = pair6.Second;
return(true);
}
this.m_bufferIndex.Value += 1;
}
}
}
return(this.m_source.MoveNext(ref currentElement, ref currentKey));
}
//---------------------------------------------------------------------------------------
// Retrieves the next element from this partition. All repartitioning operators across
// all partitions cooperate in a barrier-style algorithm. The first time an element is
// requested, the repartitioning operator will enter the 1st phase: during this phase, it
// scans its entire input and compute the destination partition for each element. During
// the 2nd phase, each partition scans the elements found by all other partitions for
// it, and yield this to callers. The only synchronization required is the barrier itself
// -- all other parts of this algorithm are synchronization-free.
//
// Notes: One rather large penalty that this algorithm incurs is higher memory usage and a
// larger time-to-first-element latency, at least compared with our old implementation; this
// happens because all input elements must be fetched before we can produce a single output
// element. In many cases this isn't too terrible: e.g. a GroupBy requires this to occur
// anyway, so having the repartitioning operator do so isn't complicating matters much at all.
//
internal override bool MoveNext(ref Pair <TInputOutput, THashKey> currentElement, [AllowNull] ref TOrderKey currentKey)
{
if (_partitionCount == 1)
{
TInputOutput current = default(TInputOutput) !;
// If there's only one partition, no need to do any sort of exchanges.
if (_source.MoveNext(ref current !, ref currentKey))
{
currentElement = new Pair <TInputOutput, THashKey>(
current, _keySelector == null ? default ! : _keySelector(current));
return(true);
}
return(false);
}
Debug.Assert(!ParallelEnumerable.SinglePartitionMode);
Mutables mutables = _mutables ??= new Mutables();
// If we haven't enumerated the source yet, do that now. This is the first phase
// of a two-phase barrier style operation.
if (mutables._currentBufferIndex == ENUMERATION_NOT_STARTED)
{
EnumerateAndRedistributeElements();
Debug.Assert(mutables._currentBufferIndex != ENUMERATION_NOT_STARTED);
}
// Once we've enumerated our contents, we can then go back and walk the buffers that belong
// to the current partition. This is phase two. Note that we slyly move on to the first step
// of phase two before actually waiting for other partitions. That's because we can enumerate
// the buffer we wrote to above, as already noted.
while (mutables._currentBufferIndex < _partitionCount)
{
// If the queue is non-null and still has elements, yield them.
if (mutables._currentBuffer != null)
{
Debug.Assert(mutables._currentKeyBuffer != null);
if (++mutables._currentIndex < mutables._currentBuffer.Count)
{
// Return the current element.
currentElement = mutables._currentBuffer._chunk[mutables._currentIndex];
Debug.Assert(mutables._currentKeyBuffer != null, "expected same # of buffers/key-buffers");
currentKey = mutables._currentKeyBuffer._chunk[mutables._currentIndex];
return(true);
}
else
{
// If the chunk is empty, advance to the next one (if any).
mutables._currentIndex = ENUMERATION_NOT_STARTED;
mutables._currentBuffer = mutables._currentBuffer.Next;
mutables._currentKeyBuffer = mutables._currentKeyBuffer.Next;
Debug.Assert(mutables._currentBuffer == null || mutables._currentBuffer.Count > 0);
Debug.Assert((mutables._currentBuffer == null) == (mutables._currentKeyBuffer == null));
Debug.Assert(mutables._currentBuffer == null || mutables._currentBuffer.Count == mutables._currentKeyBuffer !.Count);
continue; // Go back around and invoke this same logic.
}
}
// We're done with the current partition. Slightly different logic depending on whether
// we're on our own buffer or one that somebody else found for us.
if (mutables._currentBufferIndex == _partitionIndex)
{
// We now need to wait at the barrier, in case some other threads aren't done.
// Once we wake up, we reset our index and will increment it immediately after.
_barrier.Wait(_cancellationToken);
mutables._currentBufferIndex = ENUMERATION_NOT_STARTED;
}
// Advance to the next buffer.
mutables._currentBufferIndex++;
mutables._currentIndex = ENUMERATION_NOT_STARTED;
if (mutables._currentBufferIndex == _partitionIndex)
{
// Skip our current buffer (since we already enumerated it).
mutables._currentBufferIndex++;
}
// Assuming we're within bounds, retrieve the next buffer object.
if (mutables._currentBufferIndex < _partitionCount)
{
mutables._currentBuffer = _valueExchangeMatrix[mutables._currentBufferIndex][_partitionIndex];
mutables._currentKeyBuffer = _keyExchangeMatrix[mutables._currentBufferIndex][_partitionIndex];
}
}
// We're done. No more buffers to enumerate.
return(false);
}
//---------------------------------------------------------------------------------------
// Walks the two data sources, left and then right, to produce the union.
//
internal override bool MoveNext(ref TInputOutput currentElement, ref ConcatKey <TLeftKey, TRightKey> currentKey)
{
Debug.Assert(_leftSource != null);
Debug.Assert(_rightSource != null);
if (_outputEnumerator == null)
{
IEqualityComparer <Wrapper <TInputOutput> > wrapperComparer = new WrapperEqualityComparer <TInputOutput>(_comparer);
Dictionary <Wrapper <TInputOutput>, Pair <TInputOutput, ConcatKey <TLeftKey, TRightKey> > > union =
new Dictionary <Wrapper <TInputOutput>, Pair <TInputOutput, ConcatKey <TLeftKey, TRightKey> > >(wrapperComparer);
Pair <TInputOutput, NoKeyMemoizationRequired> elem = default(Pair <TInputOutput, NoKeyMemoizationRequired>);
TLeftKey leftKey = default(TLeftKey);
int i = 0;
while (_leftSource.MoveNext(ref elem, ref leftKey))
{
if ((i++ & CancellationState.POLL_INTERVAL) == 0)
{
CancellationState.ThrowIfCanceled(_cancellationToken);
}
ConcatKey <TLeftKey, TRightKey> key =
ConcatKey <TLeftKey, TRightKey> .MakeLeft(_leftOrdered?leftKey : default(TLeftKey));
Pair <TInputOutput, ConcatKey <TLeftKey, TRightKey> > oldEntry;
Wrapper <TInputOutput> wrappedElem = new Wrapper <TInputOutput>(elem.First);
if (!union.TryGetValue(wrappedElem, out oldEntry) || _keyComparer.Compare(key, oldEntry.Second) < 0)
{
union[wrappedElem] = new Pair <TInputOutput, ConcatKey <TLeftKey, TRightKey> >(elem.First, key);
}
}
TRightKey rightKey = default(TRightKey);
while (_rightSource.MoveNext(ref elem, ref rightKey))
{
if ((i++ & CancellationState.POLL_INTERVAL) == 0)
{
CancellationState.ThrowIfCanceled(_cancellationToken);
}
ConcatKey <TLeftKey, TRightKey> key =
ConcatKey <TLeftKey, TRightKey> .MakeRight(_rightOrdered?rightKey : default(TRightKey));
Pair <TInputOutput, ConcatKey <TLeftKey, TRightKey> > oldEntry;
Wrapper <TInputOutput> wrappedElem = new Wrapper <TInputOutput>(elem.First);
if (!union.TryGetValue(wrappedElem, out oldEntry) || _keyComparer.Compare(key, oldEntry.Second) < 0)
{
union[wrappedElem] = new Pair <TInputOutput, ConcatKey <TLeftKey, TRightKey> >(elem.First, key);
}
}
_outputEnumerator = union.GetEnumerator();
}
if (_outputEnumerator.MoveNext())
{
Pair <TInputOutput, ConcatKey <TLeftKey, TRightKey> > current = _outputEnumerator.Current.Value;
currentElement = current.First;
currentKey = current.Second;
return(true);
}
return(false);
}
//---------------------------------------------------------------------------------------
// Walks the two data sources, left and then right, to produce the union.
//
internal override bool MoveNext(ref TInputOutput currentElement, ref int currentKey)
{
if (_hashLookup == null)
{
_hashLookup = new Set <TInputOutput>(_comparer);
_outputLoopCount = new Shared <int>(0);
}
Debug.Assert(_hashLookup != null);
// Enumerate the left and then right data source. When each is done, we set the
// field to null so we will skip it upon subsequent calls to MoveNext.
if (_leftSource != null)
{
// Iterate over this set's elements until we find a unique element.
TLeftKey keyUnused = default(TLeftKey);
Pair <TInputOutput, NoKeyMemoizationRequired> currentLeftElement = default(Pair <TInputOutput, NoKeyMemoizationRequired>);
int i = 0;
while (_leftSource.MoveNext(ref currentLeftElement, ref keyUnused))
{
if ((i++ & CancellationState.POLL_INTERVAL) == 0)
{
CancellationState.ThrowIfCanceled(_cancellationToken);
}
// We ensure we never return duplicates by tracking them in our set.
if (_hashLookup.Add(currentLeftElement.First))
{
#if DEBUG
currentKey = unchecked ((int)0xdeadbeef);
#endif
currentElement = currentLeftElement.First;
return(true);
}
}
_leftSource.Dispose();
_leftSource = null;
}
if (_rightSource != null)
{
// Iterate over this set's elements until we find a unique element.
TRightKey keyUnused = default(TRightKey);
Pair <TInputOutput, NoKeyMemoizationRequired> currentRightElement = default(Pair <TInputOutput, NoKeyMemoizationRequired>);
while (_rightSource.MoveNext(ref currentRightElement, ref keyUnused))
{
if ((_outputLoopCount.Value++ & CancellationState.POLL_INTERVAL) == 0)
{
CancellationState.ThrowIfCanceled(_cancellationToken);
}
// We ensure we never return duplicates by tracking them in our set.
if (_hashLookup.Add(currentRightElement.First))
{
#if DEBUG
currentKey = unchecked ((int)0xdeadbeef);
#endif
currentElement = currentRightElement.First;
return(true);
}
}
_rightSource.Dispose();
_rightSource = null;
}
return(false);
}
//---------------------------------------------------------------------------------------
// Straightforward IEnumerator<T> methods.
//
internal override bool MoveNext([MaybeNullWhen(false), AllowNull] ref TOutput currentElement, ref Pair <TLeftKey, int> currentKey)
{
while (true)
{
if (_currentRightSource == null)
{
_mutables = new Mutables();
// Check cancellation every few lhs-enumerations in case none of them are producing
// any outputs. Otherwise, we rely on the consumer of this operator to be performing the checks.
if ((_mutables._lhsCount++ & CancellationState.POLL_INTERVAL) == 0)
{
CancellationState.ThrowIfCanceled(_cancellationToken);
}
// We don't have a "current" right enumerator to use. We have to fetch the next
// one. If the left has run out of elements, however, we're done and just return
// false right away.
if (!_leftSource.MoveNext(ref _mutables._currentLeftElement !, ref _mutables._currentLeftKey))
{
return(false);
}
Debug.Assert(_selectManyOperator._rightChildSelector != null);
// Use the source selection routine to create a right child.
IEnumerable <TRightInput> rightChild = _selectManyOperator._rightChildSelector(_mutables._currentLeftElement);
Debug.Assert(rightChild != null);
_currentRightSource = rightChild.GetEnumerator();
Debug.Assert(_currentRightSource != null);
// If we have no result selector, we will need to access the Current element of the right
// data source as though it is a TOutput. Unfortunately, we know that TRightInput must
// equal TOutput (we check it during operator construction), but the type system doesn't.
// Thus we would have to cast the result of invoking Current from type TRightInput to
// TOutput. This is no good, since the results could be value types. Instead, we save the
// enumerator object as an IEnumerator<TOutput> and access that later on.
if (_selectManyOperator._resultSelector == null)
{
_currentRightSourceAsOutput = (IEnumerator <TOutput>)_currentRightSource;
Debug.Assert(_currentRightSourceAsOutput == _currentRightSource,
"these must be equal, otherwise the surrounding logic will be broken");
}
}
if (_currentRightSource.MoveNext())
{
Debug.Assert(_mutables != null);
_mutables._currentRightSourceIndex++;
// If the inner data source has an element, we can yield it.
if (_selectManyOperator._resultSelector != null)
{
// In the case of a selection function, use that to yield the next element.
currentElement = _selectManyOperator._resultSelector(_mutables._currentLeftElement, _currentRightSource.Current);
}
else
{
// Otherwise, the right input and output types must be the same. We use the
// casted copy of the current right source and just return its current element.
Debug.Assert(_currentRightSourceAsOutput != null);
currentElement = _currentRightSourceAsOutput.Current;
}
currentKey = new Pair <TLeftKey, int>(_mutables._currentLeftKey, _mutables._currentRightSourceIndex);
return(true);
}
else
{
// Otherwise, we have exhausted the right data source. Loop back around and try
// to get the next left element, then its right, and so on.
_currentRightSource.Dispose();
_currentRightSource = null;
_currentRightSourceAsOutput = null;
}
}
}
