//---------------------------------------------------------------------------------------
// Instantiates a new union operator.
//
internal OrderedUnionQueryOperatorEnumerator(
QueryOperatorEnumerator <Pair, TLeftKey> leftSource,
QueryOperatorEnumerator <Pair, TRightKey> rightSource,
bool leftOrdered, bool rightOrdered, IEqualityComparer <TInputOutput> comparer, IComparer <ConcatKey> keyComparer,
CancellationToken cancellationToken)
{
Debug.Assert(leftSource != null);
Debug.Assert(rightSource != null);
_leftSource = leftSource;
_rightSource = rightSource;
_keyComparer = keyComparer;
_leftOrdered = leftOrdered;
_rightOrdered = rightOrdered;
_comparer = comparer;
if (_comparer == null)
{
_comparer = EqualityComparer <TInputOutput> .Default;
}
_cancellationToken = cancellationToken;
}
//---------------------------------------------------------------------------------------
// This method just creates the individual partitions given a data source. See the base
// class for more details on this method's contracts. This version takes an enumerator,
// and so it can't actually do an in-place partition. We'll instead create enumerators
// that coordinate with one another to lazily (on demand) grab chunks from the enumerator.
// This clearly is much less efficient than the fast path above since it requires
// synchronization. We try to amortize that cost by retrieving many elements at once
// instead of just one-at-a-time.
//
private static QueryOperatorEnumerator <T, int>[] MakePartitions(IEnumerator <T> source, int partitionCount)
{
Debug.Assert(source != null);
Debug.Assert(partitionCount > 0);
// At this point we were unable to efficiently partition the data source. Instead, we
// will return enumerators that lazily partition the data source.
QueryOperatorEnumerator <T, int>[] partitions = new QueryOperatorEnumerator <T, int> [partitionCount];
// The following is used for synchronization between threads.
object sharedSyncLock = new object();
Shared <int> sharedCurrentIndex = new Shared <int>(0);
Shared <int> sharedPartitionCount = new Shared <int>(partitionCount);
Shared <bool> sharedExceptionTracker = new Shared <bool>(false);
// Create a new lazy chunking enumerator per partition, sharing the same lock.
for (int i = 0; i < partitionCount; i++)
{
partitions[i] = new ContiguousChunkLazyEnumerator(
source, sharedExceptionTracker, sharedSyncLock, sharedCurrentIndex, sharedPartitionCount);
}
return(partitions);
}
//---------------------------------------------------------------------------------------
// Creates a new repartitioning enumerator.
//
// Arguments:
// source - the data stream from which to pull elements
// useOrdinalOrderPreservation - whether order preservation is required
// partitionCount - total number of partitions
// partitionIndex - this operator's unique partition index
// repartitionStream - the stream object to use for partition selection
// barrier - a latch used to signal task completion
// buffers - a set of buffers for inter-task communication
//
internal OrderedHashRepartitionEnumerator(
QueryOperatorEnumerator <TInputOutput, TOrderKey> source, int partitionCount, int partitionIndex,
Func <TInputOutput, THashKey>?keySelector, OrderedHashRepartitionStream <TInputOutput, THashKey, TOrderKey> repartitionStream, CountdownEvent barrier,
ListChunk <Pair <TInputOutput, THashKey> >[][] valueExchangeMatrix, ListChunk <TOrderKey>[][] keyExchangeMatrix, CancellationToken cancellationToken)
{
Debug.Assert(source != null);
Debug.Assert(keySelector != null || typeof(THashKey) == typeof(NoKeyMemoizationRequired));
Debug.Assert(repartitionStream != null);
Debug.Assert(barrier != null);
Debug.Assert(valueExchangeMatrix != null);
Debug.Assert(valueExchangeMatrix.GetLength(0) == partitionCount, "expected square matrix of buffers (NxN)");
Debug.Assert(partitionCount > 0 && valueExchangeMatrix[0].Length == partitionCount, "expected square matrix of buffers (NxN)");
Debug.Assert(0 <= partitionIndex && partitionIndex < partitionCount);
_source = source;
_partitionCount = partitionCount;
_partitionIndex = partitionIndex;
_keySelector = keySelector;
_repartitionStream = repartitionStream;
_barrier = barrier;
_valueExchangeMatrix = valueExchangeMatrix;
_keyExchangeMatrix = keyExchangeMatrix;
_cancellationToken = cancellationToken;
}
protected override bool MoveNextCore(ref Pair <double, long> currentElement)
{
double first = 0.0;
long second = 0L;
QueryOperatorEnumerator <double?, TKey> source = this.m_source;
double?nullable = null;
TKey currentKey = default(TKey);
int num3 = 0;
while (source.MoveNext(ref nullable, ref currentKey))
{
if (nullable.HasValue)
{
if ((num3++ & 0x3f) == 0)
{
CancellationState.ThrowIfCanceled(base.m_cancellationToken);
}
first += nullable.GetValueOrDefault();
second += 1L;
}
}
currentElement = new Pair <double, long>(first, second);
return(second > 0L);
}
//---------------------------------------------------------------------------------------
// Instantiates a new union operator.
//
internal OrderedUnionQueryOperatorEnumerator(
QueryOperatorEnumerator <Pair <TInputOutput, NoKeyMemoizationRequired>, TLeftKey> leftSource,
QueryOperatorEnumerator <Pair <TInputOutput, NoKeyMemoizationRequired>, TRightKey> rightSource,
bool leftOrdered, bool rightOrdered, IEqualityComparer <TInputOutput> comparer, IComparer <ConcatKey <TLeftKey, TRightKey> > keyComparer,
CancellationToken cancellationToken)
{
Contract.Assert(leftSource != null);
Contract.Assert(rightSource != null);
m_leftSource = leftSource;
m_rightSource = rightSource;
m_keyComparer = keyComparer;
m_leftOrdered = leftOrdered;
m_rightOrdered = rightOrdered;
m_comparer = comparer;
if (m_comparer == null)
{
m_comparer = EqualityComparer <TInputOutput> .Default;
}
m_cancellationToken = cancellationToken;
}
protected override bool MoveNextCore(ref double currentElement)
{
double num = 0.0;
TKey currentKey = default(TKey);
QueryOperatorEnumerator <double, TKey> source = this.m_source;
if (!source.MoveNext(ref num, ref currentKey))
{
return(false);
}
double num2 = 0.0;
int num3 = 0;
do
{
if ((num3++ & 0x3f) == 0)
{
CancellationState.ThrowIfCanceled(base.m_cancellationToken);
}
num2 += num;
}while (source.MoveNext(ref num, ref currentKey));
currentElement = num2;
return(true);
}
protected override bool MoveNextCore(ref long?currentElement)
{
long?nullable = null;
TKey currentKey = default(TKey);
QueryOperatorEnumerator <long?, TKey> source = this.m_source;
if (!source.MoveNext(ref nullable, ref currentKey))
{
return(false);
}
long num = 0L;
int num2 = 0;
do
{
if ((num2++ & 0x3f) == 0)
{
CancellationState.ThrowIfCanceled(base.m_cancellationToken);
}
num += nullable.GetValueOrDefault();
}while (source.MoveNext(ref nullable, ref currentKey));
currentElement = new long?(num);
return(true);
}
protected override bool MoveNextCore(ref int currentElement)
{
TSource local = default(TSource);
TKey currentKey = default(TKey);
QueryOperatorEnumerator <TSource, TKey> source = this.m_source;
if (!source.MoveNext(ref local, ref currentKey))
{
return(false);
}
int num = 0;
int num2 = 0;
do
{
if ((num2++ & 0x3f) == 0)
{
CancellationState.ThrowIfCanceled(base.m_cancellationToken);
}
num++;
}while (source.MoveNext(ref local, ref currentKey));
currentElement = num;
return(true);
}
//---------------------------------------------------------------------------------------
// Creates an enumerator that is used internally for the final aggregation step.
//
protected override QueryOperatorEnumerator <int?, int> CreateEnumerator <TKey>(
int index, int count, QueryOperatorEnumerator <int?, TKey> source, object sharedData, CancellationToken cancellationToken)
{
return(new NullableIntMinMaxAggregationOperatorEnumerator <TKey>(source, index, _sign, cancellationToken));
}
internal SelectQueryOperatorEnumerator(QueryOperatorEnumerator <TInput, TKey> source, Func <TInput, TOutput> selector)
{
this.m_source = source;
this.m_selector = selector;
}
internal FloatMinMaxAggregationOperatorEnumerator(QueryOperatorEnumerator <float, TKey> source, int partitionIndex, int sign, CancellationToken cancellationToken) : base(partitionIndex, cancellationToken)
{
this.m_source = source;
this.m_sign = sign;
}
protected override QueryOperatorEnumerator <float, int> CreateEnumerator <TKey>(int index, int count, QueryOperatorEnumerator <float, TKey> source, object sharedData, CancellationToken cancellationToken)
{
return(new FloatMinMaxAggregationOperatorEnumerator <TKey>(source, index, this.m_sign, cancellationToken));
}
internal ExceptQueryOperatorEnumerator(QueryOperatorEnumerator <Pair <TInputOutput, NoKeyMemoizationRequired>, TLeftKey> leftSource, QueryOperatorEnumerator <Pair <TInputOutput, NoKeyMemoizationRequired>, int> rightSource, IEqualityComparer <TInputOutput> comparer, CancellationToken cancellationToken)
{
this.m_leftSource = leftSource;
this.m_rightSource = rightSource;
this.m_comparer = comparer;
this.m_cancellationToken = cancellationToken;
}
/// <summary>
/// A variant of WrapEnumerable that accepts a QueryOperatorEnumerator{,} instead of an IEnumerable{}.
/// The code duplication is necessary to avoid extra virtual method calls that would otherwise be needed to
/// convert the QueryOperatorEnumerator{,} to an IEnumerator{}.
/// </summary>
internal static IEnumerable <TElement> WrapQueryEnumerator <TElement, TIgnoreKey>(QueryOperatorEnumerator <TElement, TIgnoreKey> source,
CancellationState cancellationState)
{
TElement elem = default(TElement);
TIgnoreKey ignoreKey = default(TIgnoreKey);
try
{
while (true)
{
try
{
if (!source.MoveNext(ref elem, ref ignoreKey))
{
yield break;
}
}
catch (ThreadAbortException)
{
// Do not wrap ThreadAbortExceptions
throw;
}
catch (Exception ex)
{
ThrowOCEorAggregateException(ex, cancellationState);
}
yield return(elem);
}
}
finally
{
source.Dispose();
}
}
internal UnionQueryOperatorEnumerator(QueryOperatorEnumerator <Pair <TInputOutput, NoKeyMemoizationRequired>, TLeftKey> leftSource, QueryOperatorEnumerator <Pair <TInputOutput, NoKeyMemoizationRequired>, TRightKey> rightSource, int partitionIndex, IEqualityComparer <TInputOutput> comparer, CancellationToken cancellationToken)
{
this.m_leftSource = leftSource;
this.m_rightSource = rightSource;
this.m_partitionIndex = partitionIndex;
this.m_comparer = comparer;
this.m_cancellationToken = cancellationToken;
}
//---------------------------------------------------------------------------------------
// Creates an enumerator that is used internally for the final aggregation step.
//
protected override QueryOperatorEnumerator<Pair<long, long>, int> CreateEnumerator<TKey>(
int index, int count, QueryOperatorEnumerator<long, TKey> source, object? sharedData, CancellationToken cancellationToken)
{
return new LongAverageAggregationOperatorEnumerator<TKey>(source, index, cancellationToken);
}
internal StopAndGoSpoolingTask(int taskIndex, QueryTaskGroupState groupState, QueryOperatorEnumerator <TInputOutput, TIgnoreKey> source, SynchronousChannel <TInputOutput> destination) : base(taskIndex, groupState)
{
this.m_source = source;
this.m_destination = destination;
}
internal static IEnumerable <TElement> WrapQueryEnumerator <TElement, TIgnoreKey>(QueryOperatorEnumerator <TElement, TIgnoreKey> source, CancellationState cancellationState)
{
TElement currentElement = default(TElement);
TIgnoreKey currentKey = default(TIgnoreKey);
while (true)
{
try
{
if (!source.MoveNext(ref currentElement, ref currentKey))
{
break;
}
}
catch (ThreadAbortException)
{
throw;
}
catch (Exception exception)
{
ThrowOCEorAggregateException(exception, cancellationState);
}
yield return(currentElement);
}
}
//---------------------------------------------------------------------------------------
// Walks the two data sources, left and then right, to produce the union.
//
internal override bool MoveNext([MaybeNullWhen(false), AllowNull] ref TInputOutput currentElement, ref int currentKey)
{
if (_hashLookup == null)
{
_hashLookup = new Set <TInputOutput>(_comparer);
_outputLoopCount = new Shared <int>(0);
}
Debug.Assert(_hashLookup != null);
// Enumerate the left and then right data source. When each is done, we set the
// field to null so we will skip it upon subsequent calls to MoveNext.
if (_leftSource != null)
{
// Iterate over this set's elements until we find a unique element.
TLeftKey keyUnused = default(TLeftKey) !;
Pair <TInputOutput, NoKeyMemoizationRequired> currentLeftElement = default(Pair <TInputOutput, NoKeyMemoizationRequired>);
int i = 0;
while (_leftSource.MoveNext(ref currentLeftElement, ref keyUnused))
{
if ((i++ & CancellationState.POLL_INTERVAL) == 0)
{
CancellationState.ThrowIfCanceled(_cancellationToken);
}
// We ensure we never return duplicates by tracking them in our set.
if (_hashLookup.Add(currentLeftElement.First))
{
#if DEBUG
currentKey = unchecked ((int)0xdeadbeef);
#endif
currentElement = currentLeftElement.First;
return(true);
}
}
_leftSource.Dispose();
_leftSource = null;
}
if (_rightSource != null)
{
// Iterate over this set's elements until we find a unique element.
TRightKey keyUnused = default(TRightKey) !;
Pair <TInputOutput, NoKeyMemoizationRequired> currentRightElement = default(Pair <TInputOutput, NoKeyMemoizationRequired>);
while (_rightSource.MoveNext(ref currentRightElement, ref keyUnused))
{
Debug.Assert(_outputLoopCount != null);
if ((_outputLoopCount.Value++ & CancellationState.POLL_INTERVAL) == 0)
{
CancellationState.ThrowIfCanceled(_cancellationToken);
}
// We ensure we never return duplicates by tracking them in our set.
if (_hashLookup.Add(currentRightElement.First))
{
#if DEBUG
currentKey = unchecked ((int)0xdeadbeef);
#endif
currentElement = currentRightElement.First;
return(true);
}
}
_rightSource.Dispose();
_rightSource = null;
}
return(false);
}