internal override void WrapPartitionedStream <TKey>( PartitionedStream <TInput, TKey> inputStream, IPartitionedStreamRecipient <bool> recipient, bool preferStriping, QuerySettings settings) { int partitionCount = inputStream.PartitionCount; PartitionedStream <bool, int> outputStream = new PartitionedStream <bool, int>(partitionCount, Util.GetDefaultComparer <int>(), OrdinalIndexState.Correct); // Create a shared cancellation variable Shared <bool> resultFoundFlag = new Shared <bool>(false); for (int i = 0; i < partitionCount; i++) { outputStream[i] = new ContainsSearchOperatorEnumerator <TKey>(inputStream[i], _searchValue, _comparer, i, resultFoundFlag, settings.CancellationState.MergedCancellationToken); } recipient.Receive(outputStream); }
private void WrapHelper <TLeftKey, TRightKey>( PartitionedStream <TSource, TLeftKey> leftStreamInc, PartitionedStream <TSource, TRightKey> rightStream, IPartitionedStreamRecipient <TSource> outputRecipient, QuerySettings settings, bool preferStriping) { // Prematurely merge the right results, if necessary if (_prematureMergeRight) { ListQueryResults <TSource> rightStreamResults = ExecuteAndCollectResults(rightStream, leftStreamInc.PartitionCount, LeftChild.OutputOrdered, preferStriping, settings); PartitionedStream <TSource, int> rightStreamInc = rightStreamResults.GetPartitionedStream(); WrapHelper2 <TLeftKey, int>(leftStreamInc, rightStreamInc, outputRecipient); } else { Debug.Assert(!ExchangeUtilities.IsWorseThan(rightStream.OrdinalIndexState, OrdinalIndexState.Increasing)); WrapHelper2 <TLeftKey, TRightKey>(leftStreamInc, rightStream, outputRecipient); } }
internal override void GivePartitionedStream(IPartitionedStreamRecipient <TOutput> recipient) { if ((((ParallelExecutionMode)this.m_settings.ExecutionMode.Value) == ParallelExecutionMode.Default) && this.m_op.LimitsParallelism) { IEnumerable <TOutput> source = this.m_op.AsSequentialQuery(this.m_settings.CancellationState.ExternalCancellationToken); PartitionedStream <TOutput, int> partitionedStream = ExchangeUtilities.PartitionDataSource <TOutput>(source, this.m_settings.DegreeOfParallelism.Value, this.m_preferStriping); recipient.Receive <int>(partitionedStream); } else if (this.IsIndexible) { PartitionedStream <TOutput, int> stream2 = ExchangeUtilities.PartitionDataSource <TOutput>(this, this.m_settings.DegreeOfParallelism.Value, this.m_preferStriping); recipient.Receive <int>(stream2); } else { this.m_childQueryResults.GivePartitionedStream(new ChildResultsRecipient <TInput, TOutput>(recipient, this.m_op, this.m_preferStriping, this.m_settings)); } }
internal override void WrapPartitionedStream <TKey>( PartitionedStream <TInput, TKey> inputStream, IPartitionedStreamRecipient <bool> recipient, bool preferStriping, QuerySettings settings) { // Create a shared cancellation variable and then return a possibly wrapped new enumerator. Shared <bool> resultFoundFlag = new Shared <bool>(false); int partitionCount = inputStream.PartitionCount; PartitionedStream <bool, int> outputStream = new PartitionedStream <bool, int>( partitionCount, Util.GetDefaultComparer <int>(), OrdinalIndexState.Correct); for (int i = 0; i < partitionCount; i++) { outputStream[i] = new AnyAllSearchOperatorEnumerator <TKey>(inputStream[i], _qualification, _predicate, i, resultFoundFlag, settings.CancellationState.MergedCancellationToken); } recipient.Receive(outputStream); }
internal override void WrapPartitionedStream <TKey>( PartitionedStream <TResult, TKey> inputStream, IPartitionedStreamRecipient <TResult> recipient, bool preferStriping, QuerySettings settings) { Debug.Assert(Child.OrdinalIndexState != OrdinalIndexState.Indexable, "Don't take this code path if the child is indexable."); // If the index is not at least increasing, we need to reindex. if (_prematureMerge) { ListQueryResults <TResult> results = ExecuteAndCollectResults( inputStream, inputStream.PartitionCount, Child.OutputOrdered, preferStriping, settings); PartitionedStream <TResult, int> inputIntStream = results.GetPartitionedStream(); WrapHelper <int>(inputIntStream, recipient, settings); } else { WrapHelper <TKey>(inputStream, recipient, settings); } }
public override void WrapPartitionedStream <TLeftKey, TRightKey>( PartitionedStream <TSource, TLeftKey> leftStream, PartitionedStream <TSource, TRightKey> rightStream, IPartitionedStreamRecipient <TSource> outputRecipient, bool preferStriping, QuerySettings settings) { // Prematurely merge the left results, if necessary if (_prematureMergeLeft) { ListQueryResults <TSource> leftStreamResults = ExecuteAndCollectResults(leftStream, leftStream.PartitionCount, LeftChild.OutputOrdered, preferStriping, settings); PartitionedStream <TSource, int> leftStreamInc = leftStreamResults.GetPartitionedStream(); WrapHelper <int, TRightKey>(leftStreamInc, rightStream, outputRecipient, settings, preferStriping); } else { Contract.Assert(!ExchangeUtilities.IsWorseThan(leftStream.OrdinalIndexState, OrdinalIndexState.Increasing)); WrapHelper <TLeftKey, TRightKey>(leftStream, rightStream, outputRecipient, settings, preferStriping); } }
private void WrapHelper2 <TLeftKey, TRightKey>( PartitionedStream <TSource, TLeftKey> leftStreamInc, PartitionedStream <TSource, TRightKey> rightStreamInc, IPartitionedStreamRecipient <TSource> outputRecipient) { int partitionCount = leftStreamInc.PartitionCount; // Generate the shared data. IComparer <ConcatKey> comparer = ConcatKey.MakeComparer( leftStreamInc.KeyComparer, rightStreamInc.KeyComparer); var outputStream = new PartitionedStream <TSource, ConcatKey>(partitionCount, comparer, OrdinalIndexState); for (int i = 0; i < partitionCount; i++) { outputStream[i] = new ConcatQueryOperatorEnumerator <TLeftKey, TRightKey>(leftStreamInc[i], rightStreamInc[i]); } outputRecipient.Receive(outputStream); }
//--------------------------------------------------------------------------------------- // This is a helper method. WrapPartitionedStream decides what type TLeftKey is going // to be, and then call this method with that key as a generic parameter. // private void WrapPartitionedStreamHelper <TLeftKey, TRightKey>( PartitionedStream <Pair <TLeftInput, TKey>, TLeftKey> leftHashStream, PartitionedStream <TRightInput, TRightKey> rightPartitionedStream, IPartitionedStreamRecipient <TOutput> outputRecipient, int partitionCount, CancellationToken cancellationToken) { PartitionedStream <Pair <TRightInput, TKey>, int> rightHashStream = ExchangeUtilities.HashRepartition( rightPartitionedStream, m_rightKeySelector, m_keyComparer, null, cancellationToken); PartitionedStream <TOutput, TLeftKey> outputStream = new PartitionedStream <TOutput, TLeftKey>( partitionCount, leftHashStream.KeyComparer, OrdinalIndexState); for (int i = 0; i < partitionCount; i++) { outputStream[i] = new HashJoinQueryOperatorEnumerator <TLeftInput, TLeftKey, TRightInput, TKey, TOutput>( leftHashStream[i], rightHashStream[i], null, m_resultSelector, m_keyComparer, cancellationToken); } outputRecipient.Receive(outputStream); }
internal override void WrapPartitionedStream <TKey>( PartitionedStream <TInputOutput, TKey> inputStream, IPartitionedStreamRecipient <TInputOutput> recipient, bool preferStriping, QuerySettings settings) { // Hash-repartition the source stream if (OutputOrdered) { WrapPartitionedStreamHelper <TKey>( ExchangeUtilities.HashRepartitionOrdered <TInputOutput, NoKeyMemoizationRequired, TKey>( inputStream, null, null, _comparer, settings.CancellationState.MergedCancellationToken), recipient, settings.CancellationState.MergedCancellationToken); } else { WrapPartitionedStreamHelper <int>( ExchangeUtilities.HashRepartition <TInputOutput, NoKeyMemoizationRequired, TKey>( inputStream, null, null, _comparer, settings.CancellationState.MergedCancellationToken), recipient, settings.CancellationState.MergedCancellationToken); } }
public override void WrapPartitionedStream <TLeftKey, TRightKey>( PartitionedStream <TLeftInput, TLeftKey> leftStream, PartitionedStream <TRightInput, TRightKey> rightStream, IPartitionedStreamRecipient <TOutput> outputRecipient, bool preferStriping, QuerySettings settings) { Debug.Assert(rightStream.PartitionCount == leftStream.PartitionCount); if (LeftChild.OutputOrdered) { WrapPartitionedStreamHelper <TLeftKey, TRightKey>( ExchangeUtilities.HashRepartitionOrdered(leftStream, _leftKeySelector, _keyComparer, null, settings.CancellationState.MergedCancellationToken), rightStream, outputRecipient, settings.CancellationState.MergedCancellationToken); } else { WrapPartitionedStreamHelper <int, TRightKey>( ExchangeUtilities.HashRepartition(leftStream, _leftKeySelector, _keyComparer, null, settings.CancellationState.MergedCancellationToken), rightStream, outputRecipient, settings.CancellationState.MergedCancellationToken); } }
internal override void WrapPartitionedStream <TKey>( PartitionedStream <TSource, TKey> inputStream, IPartitionedStreamRecipient <TSource> recipient, bool preferStriping, QuerySettings settings) { int partitionCount = inputStream.PartitionCount; // Generate the shared data. Shared <int> sharedEmptyCount = new Shared <int>(0); CountdownEvent sharedLatch = new CountdownEvent(partitionCount - 1); PartitionedStream <TSource, TKey> outputStream = new PartitionedStream <TSource, TKey>(partitionCount, inputStream.KeyComparer, OrdinalIndexState); for (int i = 0; i < partitionCount; i++) { outputStream[i] = new DefaultIfEmptyQueryOperatorEnumerator <TKey>( inputStream[i], _defaultValue, i, partitionCount, sharedEmptyCount, sharedLatch, settings.CancellationState.MergedCancellationToken); } recipient.Receive(outputStream); }
private void WrapPartitionedStreamHelper <TLeftKey, TRightKey, TOutputKey>( PartitionedStream <Pair <TLeftInput, TKey>, TLeftKey> leftHashStream, PartitionedStream <Pair <TRightInput, TKey>, TRightKey> rightHashStream, HashJoinOutputKeyBuilder <TLeftKey, TRightKey, TOutputKey> outputKeyBuilder, IComparer <TOutputKey> outputKeyComparer, IPartitionedStreamRecipient <TOutput> outputRecipient, CancellationToken cancellationToken) { int partitionCount = leftHashStream.PartitionCount; PartitionedStream <TOutput, TOutputKey> outputStream = new PartitionedStream <TOutput, TOutputKey>(partitionCount, outputKeyComparer, OrdinalIndexState); for (int i = 0; i < partitionCount; i++) { JoinHashLookupBuilder <TRightInput, TRightKey, TKey> rightLookupBuilder = new JoinHashLookupBuilder <TRightInput, TRightKey, TKey>(rightHashStream[i], _keyComparer); outputStream[i] = new HashJoinQueryOperatorEnumerator <TLeftInput, TLeftKey, TRightInput, TRightKey, TKey, TOutput, TOutputKey>( leftHashStream[i], rightLookupBuilder, _resultSelector, outputKeyBuilder, cancellationToken); } outputRecipient.Receive(outputStream); }
public override void WrapPartitionedStream <TLeftKey, TRightKey>( PartitionedStream <TInputOutput, TLeftKey> leftPartitionedStream, PartitionedStream <TInputOutput, TRightKey> rightPartitionedStream, IPartitionedStreamRecipient <TInputOutput> outputRecipient, bool preferStriping, QuerySettings settings) { Debug.Assert(leftPartitionedStream.PartitionCount == rightPartitionedStream.PartitionCount); if (OutputOrdered) { WrapPartitionedStreamHelper <TLeftKey, TRightKey>( ExchangeUtilities.HashRepartitionOrdered <TInputOutput, NoKeyMemoizationRequired, TLeftKey>( leftPartitionedStream, null, null, _comparer, settings.CancellationState.MergedCancellationToken), rightPartitionedStream, outputRecipient, settings.CancellationState.MergedCancellationToken); } else { WrapPartitionedStreamHelper <int, TRightKey>( ExchangeUtilities.HashRepartition <TInputOutput, NoKeyMemoizationRequired, TLeftKey>( leftPartitionedStream, null, null, _comparer, settings.CancellationState.MergedCancellationToken), rightPartitionedStream, outputRecipient, settings.CancellationState.MergedCancellationToken); } }
private void WrapHelper <TKey>( PartitionedStream <TSource, TKey> inputStream, IPartitionedStreamRecipient <TSource> recipient, QuerySettings settings) { int partitionCount = inputStream.PartitionCount; // Generate the shared data. FirstQueryOperatorState <TKey> operatorState = new FirstQueryOperatorState <TKey>(); CountdownEvent sharedBarrier = new CountdownEvent(partitionCount); PartitionedStream <TSource, int> outputStream = new PartitionedStream <TSource, int>( partitionCount, Util.GetDefaultComparer <int>(), OrdinalIndexState.Shuffled); for (int i = 0; i < partitionCount; i++) { outputStream[i] = new FirstQueryOperatorEnumerator <TKey>( inputStream[i], _predicate, operatorState, sharedBarrier, settings.CancellationState.MergedCancellationToken, inputStream.KeyComparer, i); } recipient.Receive(outputStream); }
internal override void WrapPartitionedStream <TLeftKey>( PartitionedStream <TLeftInput, TLeftKey> inputStream, IPartitionedStreamRecipient <TOutput> recipient, bool preferStriping, QuerySettings settings) { int partitionCount = inputStream.PartitionCount; if (_indexedRightChildSelector != null) { PartitionedStream <TLeftInput, int> inputStreamInt; // If the index is not correct, we need to reindex. if (_prematureMerge) { ListQueryResults <TLeftInput> listResults = QueryOperator <TLeftInput> .ExecuteAndCollectResults(inputStream, partitionCount, OutputOrdered, preferStriping, settings); inputStreamInt = listResults.GetPartitionedStream(); } else { inputStreamInt = (PartitionedStream <TLeftInput, int>)(object) inputStream; } WrapPartitionedStreamIndexed(inputStreamInt, recipient, settings); return; } // // if (_prematureMerge) { PartitionedStream <TLeftInput, int> inputStreamInt = QueryOperator <TLeftInput> .ExecuteAndCollectResults(inputStream, partitionCount, OutputOrdered, preferStriping, settings) .GetPartitionedStream(); WrapPartitionedStreamNotIndexed(inputStreamInt, recipient, settings); } else { WrapPartitionedStreamNotIndexed(inputStream, recipient, settings); } }
private void WrapPartitionedStreamHelper <TLeftKey, TRightKey>( PartitionedStream <Pair <TLeftInput, TKey>, TLeftKey> leftHashStream, HashLookupBuilder <IEnumerable <TRightInput>, TRightKey, TKey>[] rightLookupBuilders, IComparer <TRightKey>?rightKeyComparer, IPartitionedStreamRecipient <TOutput> outputRecipient, int partitionCount, CancellationToken cancellationToken) { if (RightChild.OutputOrdered && LeftChild.OutputOrdered) { PairOutputKeyBuilder <TLeftKey, TRightKey> outputKeyBuilder = new PairOutputKeyBuilder <TLeftKey, TRightKey>(); IComparer <Pair <TLeftKey, TRightKey> > outputKeyComparer = new PairComparer <TLeftKey, TRightKey>(leftHashStream.KeyComparer, rightKeyComparer); WrapPartitionedStreamHelper <TLeftKey, TRightKey, Pair <TLeftKey, TRightKey> >(leftHashStream, rightLookupBuilders, outputKeyBuilder, outputKeyComparer, outputRecipient, partitionCount, cancellationToken); } else { LeftKeyOutputKeyBuilder <TLeftKey, TRightKey> outputKeyBuilder = new LeftKeyOutputKeyBuilder <TLeftKey, TRightKey>(); WrapPartitionedStreamHelper <TLeftKey, TRightKey, TLeftKey>(leftHashStream, rightLookupBuilders, outputKeyBuilder, leftHashStream.KeyComparer, outputRecipient, partitionCount, cancellationToken); } }
//--------------------------------------------------------------------------------------- // This is a helper method. WrapPartitionedStream decides what type TLeftKey is going // to be, and then call this method with that key as a generic parameter. // private void WrapPartitionedStreamHelper <TLeftKey, TRightKey>( PartitionedStream <Pair <TLeftInput, TKey>, TLeftKey> leftHashStream, PartitionedStream <TRightInput, TRightKey> rightPartitionedStream, IPartitionedStreamRecipient <TOutput> outputRecipient, CancellationToken cancellationToken) { if (RightChild.OutputOrdered && LeftChild.OutputOrdered) { PairOutputKeyBuilder <TLeftKey, TRightKey> outputKeyBuilder = new PairOutputKeyBuilder <TLeftKey, TRightKey>(); IComparer <Pair <TLeftKey, TRightKey> > outputKeyComparer = new PairComparer <TLeftKey, TRightKey>(leftHashStream.KeyComparer, rightPartitionedStream.KeyComparer); WrapPartitionedStreamHelper <TLeftKey, TRightKey, Pair <TLeftKey, TRightKey> >(leftHashStream, ExchangeUtilities.HashRepartitionOrdered(rightPartitionedStream, _rightKeySelector, _keyComparer, null, cancellationToken), outputKeyBuilder, outputKeyComparer, outputRecipient, cancellationToken); } else { LeftKeyOutputKeyBuilder <TLeftKey, int> outputKeyBuilder = new LeftKeyOutputKeyBuilder <TLeftKey, int>(); WrapPartitionedStreamHelper <TLeftKey, int, TLeftKey>(leftHashStream, ExchangeUtilities.HashRepartition(rightPartitionedStream, _rightKeySelector, _keyComparer, null, cancellationToken), outputKeyBuilder, leftHashStream.KeyComparer, outputRecipient, cancellationToken); } }
//--------------------------------------------------------------------------------------- // A helper method that allows WrapPartitionedStream to fix the TLeftKey type parameter. // private void WrapPartitionedStreamFixedLeftType <TLeftKey, TRightKey>( PartitionedStream <Pair <TInputOutput, NoKeyMemoizationRequired>, TLeftKey> leftHashStream, PartitionedStream <TInputOutput, TRightKey> rightStream, IPartitionedStreamRecipient <TInputOutput> outputRecipient, int partitionCount, CancellationToken cancellationToken) { if (RightChild.OutputOrdered) { PartitionedStream <Pair <TInputOutput, NoKeyMemoizationRequired>, TRightKey> rightHashStream = ExchangeUtilities.HashRepartitionOrdered <TInputOutput, NoKeyMemoizationRequired, TRightKey>( rightStream, null, null, _comparer, cancellationToken); WrapPartitionedStreamFixedBothTypes <TLeftKey, TRightKey>( leftHashStream, rightHashStream, outputRecipient, partitionCount, cancellationToken); } else { PartitionedStream <Pair <TInputOutput, NoKeyMemoizationRequired>, int> rightHashStream = ExchangeUtilities.HashRepartition <TInputOutput, NoKeyMemoizationRequired, TRightKey>( rightStream, null, null, _comparer, cancellationToken); WrapPartitionedStreamFixedBothTypes <TLeftKey, int>( leftHashStream, rightHashStream, outputRecipient, partitionCount, cancellationToken); } }
//--------------------------------------------------------------------------------------- // A helper method that allows WrapPartitionedStreamHelper to fix the TRightKey type parameter. // private void WrapPartitionedStreamFixedBothTypes <TLeftKey, TRightKey>( PartitionedStream <Pair <TInputOutput, NoKeyMemoizationRequired>, TLeftKey> leftHashStream, PartitionedStream <Pair <TInputOutput, NoKeyMemoizationRequired>, TRightKey> rightHashStream, IPartitionedStreamRecipient <TInputOutput> outputRecipient, int partitionCount, CancellationToken cancellationToken) { if (LeftChild.OutputOrdered || RightChild.OutputOrdered) { IComparer <ConcatKey <TLeftKey, TRightKey> > compoundKeyComparer = ConcatKey <TLeftKey, TRightKey> .MakeComparer(leftHashStream.KeyComparer, rightHashStream.KeyComparer); PartitionedStream <TInputOutput, ConcatKey <TLeftKey, TRightKey> > outputStream = new PartitionedStream <TInputOutput, ConcatKey <TLeftKey, TRightKey> >(partitionCount, compoundKeyComparer, OrdinalIndexState.Shuffled); for (int i = 0; i < partitionCount; i++) { outputStream[i] = new OrderedUnionQueryOperatorEnumerator <TLeftKey, TRightKey>( leftHashStream[i], rightHashStream[i], LeftChild.OutputOrdered, RightChild.OutputOrdered, _comparer, compoundKeyComparer, cancellationToken); } outputRecipient.Receive(outputStream); } else { PartitionedStream <TInputOutput, int> outputStream = new PartitionedStream <TInputOutput, int>(partitionCount, Util.GetDefaultComparer <int>(), OrdinalIndexState.Shuffled); for (int i = 0; i < partitionCount; i++) { outputStream[i] = new UnionQueryOperatorEnumerator <TLeftKey, TRightKey>( leftHashStream[i], rightHashStream[i], _comparer, cancellationToken); } outputRecipient.Receive(outputStream); } }
//--------------------------------------------------------------------------------------- // This is a helper method. WrapPartitionedStream decides what type TLeftKey is going // to be, and then call this method with that key as a generic parameter. // private void WrapPartitionedStreamHelper <TLeftKey, TRightKey>( PartitionedStream <Pair <TLeftInput, TKey>, TLeftKey> leftHashStream, PartitionedStream <TRightInput, TRightKey> rightPartitionedStream, IPartitionedStreamRecipient <TOutput> outputRecipient, int partitionCount, CancellationToken cancellationToken) { if (RightChild.OutputOrdered) { PartitionedStream <Pair <TRightInput, TKey>, TRightKey> rePartitionedRightStream = ExchangeUtilities.HashRepartitionOrdered( rightPartitionedStream, _rightKeySelector, _keyComparer, null, cancellationToken); HashLookupBuilder <IEnumerable <TRightInput>, Pair <bool, TRightKey>, TKey>[] rightLookupBuilders = new HashLookupBuilder <IEnumerable <TRightInput>, Pair <bool, TRightKey>, TKey> [partitionCount]; for (int i = 0; i < partitionCount; i++) { rightLookupBuilders[i] = new OrderedGroupJoinHashLookupBuilder <TRightInput, TRightKey, TKey>( rePartitionedRightStream[i], _keyComparer, rePartitionedRightStream.KeyComparer); } WrapPartitionedStreamHelper <TLeftKey, Pair <bool, TRightKey> >(leftHashStream, rightLookupBuilders, CreateComparer(rightPartitionedStream.KeyComparer), outputRecipient, partitionCount, cancellationToken); } else { PartitionedStream <Pair <TRightInput, TKey>, int> rePartitionedRightStream = ExchangeUtilities.HashRepartition( rightPartitionedStream, _rightKeySelector, _keyComparer, null, cancellationToken); HashLookupBuilder <IEnumerable <TRightInput>, int, TKey>[] rightLookupBuilders = new HashLookupBuilder <IEnumerable <TRightInput>, int, TKey> [partitionCount]; for (int i = 0; i < partitionCount; i++) { rightLookupBuilders[i] = new GroupJoinHashLookupBuilder <TRightInput, int, TKey>( rePartitionedRightStream[i], _keyComparer); } WrapPartitionedStreamHelper <TLeftKey, int>(leftHashStream, rightLookupBuilders, null, outputRecipient, partitionCount, cancellationToken); } }
internal override void GivePartitionedStream(IPartitionedStreamRecipient <TOutput> recipient) { Contract.Assert(IsIndexible == (m_op.OrdinalIndexState == OrdinalIndexState.Indexible)); if (m_settings.ExecutionMode.Value == ParallelExecutionMode.Default && m_op.LimitsParallelism) { // We need to run the query sequentially up to and including this operator IEnumerable <TOutput> opSequential = m_op.AsSequentialQuery(m_settings.CancellationState.ExternalCancellationToken); PartitionedStream <TOutput, int> result = ExchangeUtilities.PartitionDataSource( opSequential, m_settings.DegreeOfParallelism.Value, m_preferStriping); recipient.Receive <int>(result); } else if (IsIndexible) { // The output of this operator is indexible. Pass the partitioned output into the IPartitionedStreamRecipient. PartitionedStream <TOutput, int> result = ExchangeUtilities.PartitionDataSource(this, m_settings.DegreeOfParallelism.Value, m_preferStriping); recipient.Receive <int>(result); } else { // The common case: get partitions from the child and wrap each partition. m_leftChildQueryResults.GivePartitionedStream(new LeftChildResultsRecipient(recipient, this, m_preferStriping, m_settings)); } }
internal override void WrapPartitionedStream <TKey>( PartitionedStream <TSource, TKey> inputStream, IPartitionedStreamRecipient <IGrouping <TGroupKey, TElement> > recipient, bool preferStriping, QuerySettings settings) { // Hash-repartition the source stream if (Child.OutputOrdered) { WrapPartitionedStreamHelperOrdered <TKey>( ExchangeUtilities.HashRepartitionOrdered <TSource, TGroupKey, TKey>( inputStream, _keySelector, _keyComparer, null, settings.CancellationState.MergedCancellationToken), recipient, settings.CancellationState.MergedCancellationToken ); } else { WrapPartitionedStreamHelper <TKey, int>( ExchangeUtilities.HashRepartition <TSource, TGroupKey, TKey>( inputStream, _keySelector, _keyComparer, null, settings.CancellationState.MergedCancellationToken), recipient, settings.CancellationState.MergedCancellationToken ); } }
//--------------------------------------------------------------------------------------- // This is a helper method. WrapPartitionedStream decides what type TKey is going // to be, and then call this method with that key as a generic parameter. // private void WrapPartitionedStreamHelper <TKey>( PartitionedStream <Pair <TInputOutput, NoKeyMemoizationRequired>, TKey> hashStream, IPartitionedStreamRecipient <TInputOutput> recipient, CancellationToken cancellationToken) { int partitionCount = hashStream.PartitionCount; PartitionedStream <TInputOutput, TKey> outputStream = new PartitionedStream <TInputOutput, TKey>(partitionCount, hashStream.KeyComparer, OrdinalIndexState.Shuffled); for (int i = 0; i < partitionCount; i++) { if (OutputOrdered) { outputStream[i] = new OrderedDistinctQueryOperatorEnumerator <TKey>(hashStream[i], _comparer, hashStream.KeyComparer, cancellationToken); } else { outputStream[i] = (QueryOperatorEnumerator <TInputOutput, TKey>)(object) new DistinctQueryOperatorEnumerator <TKey>(hashStream[i], _comparer, cancellationToken); } } recipient.Receive(outputStream); }
private void WrapHelper <TKey>(PartitionedStream <TResult, TKey> inputStream, IPartitionedStreamRecipient <TResult> recipient, QuerySettings settings) { int partitionCount = inputStream.PartitionCount; // Create shared data. OperatorState <TKey> operatorState = new OperatorState <TKey>(); CountdownEvent sharedBarrier = new CountdownEvent(partitionCount); Debug.Assert(_indexedPredicate == null || typeof(TKey) == typeof(int)); Func <TResult, TKey, bool>?convertedIndexedPredicate = (Func <TResult, TKey, bool>?)(object?) _indexedPredicate; PartitionedStream <TResult, TKey> partitionedStream = new PartitionedStream <TResult, TKey>(partitionCount, inputStream.KeyComparer, OrdinalIndexState); for (int i = 0; i < partitionCount; i++) { partitionedStream[i] = new TakeOrSkipWhileQueryOperatorEnumerator <TKey>( inputStream[i], _predicate, convertedIndexedPredicate, _take, operatorState, sharedBarrier, settings.CancellationState.MergedCancellationToken, inputStream.KeyComparer); } recipient.Receive(partitionedStream); }
public override void WrapPartitionedStream <TLeftKey, TRightKey>(PartitionedStream <TInputOutput, TLeftKey> leftStream, PartitionedStream <TInputOutput, TRightKey> rightStream, IPartitionedStreamRecipient <TInputOutput> outputRecipient, bool preferStriping, QuerySettings settings) { int partitionCount = leftStream.PartitionCount; if (base.LeftChild.OutputOrdered) { PartitionedStream <Pair <TInputOutput, NoKeyMemoizationRequired>, TLeftKey> leftHashStream = ExchangeUtilities.HashRepartitionOrdered <TInputOutput, NoKeyMemoizationRequired, TLeftKey>(leftStream, null, null, this.m_comparer, settings.CancellationState.MergedCancellationToken); this.WrapPartitionedStreamFixedLeftType <TLeftKey, TRightKey>(leftHashStream, rightStream, outputRecipient, partitionCount, settings.CancellationState.MergedCancellationToken); } else { PartitionedStream <Pair <TInputOutput, NoKeyMemoizationRequired>, int> stream2 = ExchangeUtilities.HashRepartition <TInputOutput, NoKeyMemoizationRequired, TLeftKey>(leftStream, null, null, this.m_comparer, settings.CancellationState.MergedCancellationToken); this.WrapPartitionedStreamFixedLeftType <int, TRightKey>(stream2, rightStream, outputRecipient, partitionCount, settings.CancellationState.MergedCancellationToken); } }
internal override void GivePartitionedStream(IPartitionedStreamRecipient <TElement> recipient) { Contract.Assert(m_settings.DegreeOfParallelism.HasValue); int partitionCount = m_settings.DegreeOfParallelism.Value; OrderablePartitioner <TElement> orderablePartitioner = m_partitioner as OrderablePartitioner <TElement>; // If the partitioner is not orderable, it will yield zeros as order keys. The order index state // is irrelevant. OrdinalIndexState indexState = (orderablePartitioner != null) ? GetOrdinalIndexState(orderablePartitioner) : OrdinalIndexState.Shuffled; PartitionedStream <TElement, int> partitions = new PartitionedStream <TElement, int>( partitionCount, Util.GetDefaultComparer <int>(), indexState); if (orderablePartitioner != null) { IList <IEnumerator <KeyValuePair <long, TElement> > > partitionerPartitions = orderablePartitioner.GetOrderablePartitions(partitionCount); if (partitionerPartitions == null) { throw new InvalidOperationException(SR.GetString(SR.PartitionerQueryOperator_NullPartitionList)); } if (partitionerPartitions.Count != partitionCount) { throw new InvalidOperationException(SR.GetString(SR.PartitionerQueryOperator_WrongNumberOfPartitions)); } for (int i = 0; i < partitionCount; i++) { IEnumerator <KeyValuePair <long, TElement> > partition = partitionerPartitions[i]; if (partition == null) { throw new InvalidOperationException(SR.GetString(SR.PartitionerQueryOperator_NullPartition)); } partitions[i] = new OrderablePartitionerEnumerator(partition); } } else { IList <IEnumerator <TElement> > partitionerPartitions = m_partitioner.GetPartitions(partitionCount); if (partitionerPartitions == null) { throw new InvalidOperationException(SR.GetString(SR.PartitionerQueryOperator_NullPartitionList)); } if (partitionerPartitions.Count != partitionCount) { throw new InvalidOperationException(SR.GetString(SR.PartitionerQueryOperator_WrongNumberOfPartitions)); } for (int i = 0; i < partitionCount; i++) { IEnumerator <TElement> partition = partitionerPartitions[i]; if (partition == null) { throw new InvalidOperationException(SR.GetString(SR.PartitionerQueryOperator_NullPartition)); } partitions[i] = new PartitionerEnumerator(partition); } } recipient.Receive <int>(partitions); }
internal override void GivePartitionedStream(IPartitionedStreamRecipient <T> recipient) { PartitionedStream <T, int> partitionedStream = GetPartitionedStream(); recipient.Receive <int>(partitionedStream); }
internal override void GivePartitionedStream(IPartitionedStreamRecipient <TOutput> recipient) { PartitionedStream <TOutput, int> partitionedStream = ExchangeUtilities.PartitionDataSource(this, _partitionCount, _preferStriping); recipient.Receive(partitionedStream); }
//--------------------------------------------------------------------------------------- // This method wraps accepts two child partitioned streams, and constructs an output // partitioned stream. However, instead of returning the transformed partitioned // stream, we pass it to a recipient object by calling recipient.Give<TNewKey>(..). That // way, we can "return" a partitioned stream that uses an order key selected by the operator. // public abstract void WrapPartitionedStream <TLeftKey, TRightKey>( PartitionedStream <TLeftInput, TLeftKey> leftPartitionedStream, PartitionedStream <TRightInput, TRightKey> rightPartitionedStream, IPartitionedStreamRecipient <TOutput> outputRecipient, bool preferStriping, QuerySettings settings);
public override void WrapPartitionedStream <TLeftKey, TRightKey>(PartitionedStream <TLeftInput, TLeftKey> leftStream, PartitionedStream <TRightInput, TRightKey> rightStream, IPartitionedStreamRecipient <TOutput> outputRecipient, bool preferStriping, QuerySettings settings) { if (base.LeftChild.OutputOrdered) { this.WrapPartitionedStreamHelper <TLeftKey, TRightKey>(ExchangeUtilities.HashRepartitionOrdered <TLeftInput, TKey, TLeftKey>(leftStream, this.m_leftKeySelector, this.m_keyComparer, null, settings.CancellationState.MergedCancellationToken), rightStream, outputRecipient, settings.CancellationState.MergedCancellationToken); } else { this.WrapPartitionedStreamHelper <int, TRightKey>(ExchangeUtilities.HashRepartition <TLeftInput, TKey, TLeftKey>(leftStream, this.m_leftKeySelector, this.m_keyComparer, null, settings.CancellationState.MergedCancellationToken), rightStream, outputRecipient, settings.CancellationState.MergedCancellationToken); } }