//---------------------------------------------------------------------------------------
            // Instantiates a new union operator.
            //

            internal OrderedUnionQueryOperatorEnumerator(
                QueryOperatorEnumerator <Pair, TLeftKey> leftSource,
                QueryOperatorEnumerator <Pair, TRightKey> rightSource,
                bool leftOrdered, bool rightOrdered, IEqualityComparer <TInputOutput> comparer, IComparer <ConcatKey> keyComparer,
                CancellationToken cancellationToken)
            {
                Debug.Assert(leftSource != null);
                Debug.Assert(rightSource != null);

                _leftSource  = leftSource;
                _rightSource = rightSource;
                _keyComparer = keyComparer;

                _leftOrdered  = leftOrdered;
                _rightOrdered = rightOrdered;
                _comparer     = comparer;

                if (_comparer == null)
                {
                    _comparer = EqualityComparer <TInputOutput> .Default;
                }

                _cancellationToken = cancellationToken;
            }
Exemple #2
0
        //---------------------------------------------------------------------------------------
        // This method just creates the individual partitions given a data source. See the base
        // class for more details on this method's contracts. This version takes an enumerator,
        // and so it can't actually do an in-place partition. We'll instead create enumerators
        // that coordinate with one another to lazily (on demand) grab chunks from the enumerator.
        // This clearly is much less efficient than the fast path above since it requires
        // synchronization. We try to amortize that cost by retrieving many elements at once
        // instead of just one-at-a-time.
        //

        private static QueryOperatorEnumerator <T, int>[] MakePartitions(IEnumerator <T> source, int partitionCount)
        {
            Debug.Assert(source != null);
            Debug.Assert(partitionCount > 0);

            // At this point we were unable to efficiently partition the data source. Instead, we
            // will return enumerators that lazily partition the data source.
            QueryOperatorEnumerator <T, int>[] partitions = new QueryOperatorEnumerator <T, int> [partitionCount];

            // The following is used for synchronization between threads.
            object        sharedSyncLock         = new object();
            Shared <int>  sharedCurrentIndex     = new Shared <int>(0);
            Shared <int>  sharedPartitionCount   = new Shared <int>(partitionCount);
            Shared <bool> sharedExceptionTracker = new Shared <bool>(false);

            // Create a new lazy chunking enumerator per partition, sharing the same lock.
            for (int i = 0; i < partitionCount; i++)
            {
                partitions[i] = new ContiguousChunkLazyEnumerator(
                    source, sharedExceptionTracker, sharedSyncLock, sharedCurrentIndex, sharedPartitionCount);
            }

            return(partitions);
        }
        //---------------------------------------------------------------------------------------
        // Creates a new repartitioning enumerator.
        //
        // Arguments:
        //     source            - the data stream from which to pull elements
        //     useOrdinalOrderPreservation - whether order preservation is required
        //     partitionCount    - total number of partitions
        //     partitionIndex    - this operator's unique partition index
        //     repartitionStream - the stream object to use for partition selection
        //     barrier           - a latch used to signal task completion
        //     buffers           - a set of buffers for inter-task communication
        //

        internal OrderedHashRepartitionEnumerator(
            QueryOperatorEnumerator <TInputOutput, TOrderKey> source, int partitionCount, int partitionIndex,
            Func <TInputOutput, THashKey>?keySelector, OrderedHashRepartitionStream <TInputOutput, THashKey, TOrderKey> repartitionStream, CountdownEvent barrier,
            ListChunk <Pair <TInputOutput, THashKey> >[][] valueExchangeMatrix, ListChunk <TOrderKey>[][] keyExchangeMatrix, CancellationToken cancellationToken)
        {
            Debug.Assert(source != null);
            Debug.Assert(keySelector != null || typeof(THashKey) == typeof(NoKeyMemoizationRequired));
            Debug.Assert(repartitionStream != null);
            Debug.Assert(barrier != null);
            Debug.Assert(valueExchangeMatrix != null);
            Debug.Assert(valueExchangeMatrix.GetLength(0) == partitionCount, "expected square matrix of buffers (NxN)");
            Debug.Assert(partitionCount > 0 && valueExchangeMatrix[0].Length == partitionCount, "expected square matrix of buffers (NxN)");
            Debug.Assert(0 <= partitionIndex && partitionIndex < partitionCount);

            _source              = source;
            _partitionCount      = partitionCount;
            _partitionIndex      = partitionIndex;
            _keySelector         = keySelector;
            _repartitionStream   = repartitionStream;
            _barrier             = barrier;
            _valueExchangeMatrix = valueExchangeMatrix;
            _keyExchangeMatrix   = keyExchangeMatrix;
            _cancellationToken   = cancellationToken;
        }
Exemple #4
0
            protected override bool MoveNextCore(ref Pair <double, long> currentElement)
            {
                double first  = 0.0;
                long   second = 0L;
                QueryOperatorEnumerator <double?, TKey> source = this.m_source;
                double?nullable   = null;
                TKey   currentKey = default(TKey);
                int    num3       = 0;

                while (source.MoveNext(ref nullable, ref currentKey))
                {
                    if (nullable.HasValue)
                    {
                        if ((num3++ & 0x3f) == 0)
                        {
                            CancellationState.ThrowIfCanceled(base.m_cancellationToken);
                        }
                        first  += nullable.GetValueOrDefault();
                        second += 1L;
                    }
                }
                currentElement = new Pair <double, long>(first, second);
                return(second > 0L);
            }
            //---------------------------------------------------------------------------------------
            // Instantiates a new union operator.
            //

            internal OrderedUnionQueryOperatorEnumerator(
                QueryOperatorEnumerator <Pair <TInputOutput, NoKeyMemoizationRequired>, TLeftKey> leftSource,
                QueryOperatorEnumerator <Pair <TInputOutput, NoKeyMemoizationRequired>, TRightKey> rightSource,
                bool leftOrdered, bool rightOrdered, IEqualityComparer <TInputOutput> comparer, IComparer <ConcatKey <TLeftKey, TRightKey> > keyComparer,
                CancellationToken cancellationToken)
            {
                Contract.Assert(leftSource != null);
                Contract.Assert(rightSource != null);

                m_leftSource  = leftSource;
                m_rightSource = rightSource;
                m_keyComparer = keyComparer;

                m_leftOrdered  = leftOrdered;
                m_rightOrdered = rightOrdered;
                m_comparer     = comparer;

                if (m_comparer == null)
                {
                    m_comparer = EqualityComparer <TInputOutput> .Default;
                }

                m_cancellationToken = cancellationToken;
            }
Exemple #6
0
            protected override bool MoveNextCore(ref double currentElement)
            {
                double num        = 0.0;
                TKey   currentKey = default(TKey);
                QueryOperatorEnumerator <double, TKey> source = this.m_source;

                if (!source.MoveNext(ref num, ref currentKey))
                {
                    return(false);
                }
                double num2 = 0.0;
                int    num3 = 0;

                do
                {
                    if ((num3++ & 0x3f) == 0)
                    {
                        CancellationState.ThrowIfCanceled(base.m_cancellationToken);
                    }
                    num2 += num;
                }while (source.MoveNext(ref num, ref currentKey));
                currentElement = num2;
                return(true);
            }
            protected override bool MoveNextCore(ref long?currentElement)
            {
                long?nullable   = null;
                TKey currentKey = default(TKey);
                QueryOperatorEnumerator <long?, TKey> source = this.m_source;

                if (!source.MoveNext(ref nullable, ref currentKey))
                {
                    return(false);
                }
                long num  = 0L;
                int  num2 = 0;

                do
                {
                    if ((num2++ & 0x3f) == 0)
                    {
                        CancellationState.ThrowIfCanceled(base.m_cancellationToken);
                    }
                    num += nullable.GetValueOrDefault();
                }while (source.MoveNext(ref nullable, ref currentKey));
                currentElement = new long?(num);
                return(true);
            }
Exemple #8
0
            protected override bool MoveNextCore(ref int currentElement)
            {
                TSource local      = default(TSource);
                TKey    currentKey = default(TKey);
                QueryOperatorEnumerator <TSource, TKey> source = this.m_source;

                if (!source.MoveNext(ref local, ref currentKey))
                {
                    return(false);
                }
                int num  = 0;
                int num2 = 0;

                do
                {
                    if ((num2++ & 0x3f) == 0)
                    {
                        CancellationState.ThrowIfCanceled(base.m_cancellationToken);
                    }
                    num++;
                }while (source.MoveNext(ref local, ref currentKey));
                currentElement = num;
                return(true);
            }
        //---------------------------------------------------------------------------------------
        // Creates an enumerator that is used internally for the final aggregation step.
        //

        protected override QueryOperatorEnumerator <int?, int> CreateEnumerator <TKey>(
            int index, int count, QueryOperatorEnumerator <int?, TKey> source, object sharedData, CancellationToken cancellationToken)
        {
            return(new NullableIntMinMaxAggregationOperatorEnumerator <TKey>(source, index, _sign, cancellationToken));
        }
Exemple #10
0
 internal SelectQueryOperatorEnumerator(QueryOperatorEnumerator <TInput, TKey> source, Func <TInput, TOutput> selector)
 {
     this.m_source   = source;
     this.m_selector = selector;
 }
 internal FloatMinMaxAggregationOperatorEnumerator(QueryOperatorEnumerator <float, TKey> source, int partitionIndex, int sign, CancellationToken cancellationToken) : base(partitionIndex, cancellationToken)
 {
     this.m_source = source;
     this.m_sign   = sign;
 }
 protected override QueryOperatorEnumerator <float, int> CreateEnumerator <TKey>(int index, int count, QueryOperatorEnumerator <float, TKey> source, object sharedData, CancellationToken cancellationToken)
 {
     return(new FloatMinMaxAggregationOperatorEnumerator <TKey>(source, index, this.m_sign, cancellationToken));
 }
 internal ExceptQueryOperatorEnumerator(QueryOperatorEnumerator <Pair <TInputOutput, NoKeyMemoizationRequired>, TLeftKey> leftSource, QueryOperatorEnumerator <Pair <TInputOutput, NoKeyMemoizationRequired>, int> rightSource, IEqualityComparer <TInputOutput> comparer, CancellationToken cancellationToken)
 {
     this.m_leftSource        = leftSource;
     this.m_rightSource       = rightSource;
     this.m_comparer          = comparer;
     this.m_cancellationToken = cancellationToken;
 }
        /// <summary>
        /// A variant of WrapEnumerable that accepts a QueryOperatorEnumerator{,} instead of an IEnumerable{}.
        /// The code duplication is necessary to avoid extra virtual method calls that would otherwise be needed to
        /// convert the QueryOperatorEnumerator{,} to an IEnumerator{}.
        /// </summary>
        internal static IEnumerable <TElement> WrapQueryEnumerator <TElement, TIgnoreKey>(QueryOperatorEnumerator <TElement, TIgnoreKey> source,
                                                                                          CancellationState cancellationState)
        {
            TElement   elem      = default(TElement);
            TIgnoreKey ignoreKey = default(TIgnoreKey);

            try
            {
                while (true)
                {
                    try
                    {
                        if (!source.MoveNext(ref elem, ref ignoreKey))
                        {
                            yield break;
                        }
                    }
                    catch (ThreadAbortException)
                    {
                        // Do not wrap ThreadAbortExceptions
                        throw;
                    }
                    catch (Exception ex)
                    {
                        ThrowOCEorAggregateException(ex, cancellationState);
                    }

                    yield return(elem);
                }
            }
            finally
            {
                source.Dispose();
            }
        }
 internal UnionQueryOperatorEnumerator(QueryOperatorEnumerator <Pair <TInputOutput, NoKeyMemoizationRequired>, TLeftKey> leftSource, QueryOperatorEnumerator <Pair <TInputOutput, NoKeyMemoizationRequired>, TRightKey> rightSource, int partitionIndex, IEqualityComparer <TInputOutput> comparer, CancellationToken cancellationToken)
 {
     this.m_leftSource        = leftSource;
     this.m_rightSource       = rightSource;
     this.m_partitionIndex    = partitionIndex;
     this.m_comparer          = comparer;
     this.m_cancellationToken = cancellationToken;
 }
        //---------------------------------------------------------------------------------------
        // Creates an enumerator that is used internally for the final aggregation step.
        //

        protected override QueryOperatorEnumerator<Pair<long, long>, int> CreateEnumerator<TKey>(
            int index, int count, QueryOperatorEnumerator<long, TKey> source, object? sharedData, CancellationToken cancellationToken)
        {
            return new LongAverageAggregationOperatorEnumerator<TKey>(source, index, cancellationToken);
        }
 internal StopAndGoSpoolingTask(int taskIndex, QueryTaskGroupState groupState, QueryOperatorEnumerator <TInputOutput, TIgnoreKey> source, SynchronousChannel <TInputOutput> destination) : base(taskIndex, groupState)
 {
     this.m_source      = source;
     this.m_destination = destination;
 }
Exemple #18
0
        internal static IEnumerable <TElement> WrapQueryEnumerator <TElement, TIgnoreKey>(QueryOperatorEnumerator <TElement, TIgnoreKey> source, CancellationState cancellationState)
        {
            TElement   currentElement = default(TElement);
            TIgnoreKey currentKey     = default(TIgnoreKey);

            while (true)
            {
                try
                {
                    if (!source.MoveNext(ref currentElement, ref currentKey))
                    {
                        break;
                    }
                }
                catch (ThreadAbortException)
                {
                    throw;
                }
                catch (Exception exception)
                {
                    ThrowOCEorAggregateException(exception, cancellationState);
                }
                yield return(currentElement);
            }
        }
Exemple #19
0
            //---------------------------------------------------------------------------------------
            // Walks the two data sources, left and then right, to produce the union.
            //

            internal override bool MoveNext([MaybeNullWhen(false), AllowNull] ref TInputOutput currentElement, ref int currentKey)
            {
                if (_hashLookup == null)
                {
                    _hashLookup      = new Set <TInputOutput>(_comparer);
                    _outputLoopCount = new Shared <int>(0);
                }

                Debug.Assert(_hashLookup != null);

                // Enumerate the left and then right data source. When each is done, we set the
                // field to null so we will skip it upon subsequent calls to MoveNext.
                if (_leftSource != null)
                {
                    // Iterate over this set's elements until we find a unique element.
                    TLeftKey keyUnused = default(TLeftKey) !;
                    Pair <TInputOutput, NoKeyMemoizationRequired> currentLeftElement = default(Pair <TInputOutput, NoKeyMemoizationRequired>);

                    int i = 0;
                    while (_leftSource.MoveNext(ref currentLeftElement, ref keyUnused))
                    {
                        if ((i++ & CancellationState.POLL_INTERVAL) == 0)
                        {
                            CancellationState.ThrowIfCanceled(_cancellationToken);
                        }

                        // We ensure we never return duplicates by tracking them in our set.
                        if (_hashLookup.Add(currentLeftElement.First))
                        {
#if DEBUG
                            currentKey = unchecked ((int)0xdeadbeef);
#endif
                            currentElement = currentLeftElement.First;
                            return(true);
                        }
                    }

                    _leftSource.Dispose();
                    _leftSource = null;
                }


                if (_rightSource != null)
                {
                    // Iterate over this set's elements until we find a unique element.
                    TRightKey keyUnused = default(TRightKey) !;
                    Pair <TInputOutput, NoKeyMemoizationRequired> currentRightElement = default(Pair <TInputOutput, NoKeyMemoizationRequired>);

                    while (_rightSource.MoveNext(ref currentRightElement, ref keyUnused))
                    {
                        Debug.Assert(_outputLoopCount != null);
                        if ((_outputLoopCount.Value++ & CancellationState.POLL_INTERVAL) == 0)
                        {
                            CancellationState.ThrowIfCanceled(_cancellationToken);
                        }

                        // We ensure we never return duplicates by tracking them in our set.
                        if (_hashLookup.Add(currentRightElement.First))
                        {
#if DEBUG
                            currentKey = unchecked ((int)0xdeadbeef);
#endif
                            currentElement = currentRightElement.First;
                            return(true);
                        }
                    }

                    _rightSource.Dispose();
                    _rightSource = null;
                }

                return(false);
            }