internal override void WrapPartitionedStream <TKey>( PartitionedStream <TInput, TKey> inputStream, IPartitionedStreamRecipient <TOutput> recipient, bool preferStriping, QuerySettings settings) { int partitionCount = inputStream.PartitionCount; // If the index is not correct, we need to reindex. PartitionedStream <TInput, int> inputStreamInt; if (_prematureMerge) { ListQueryResults <TInput> listResults = QueryOperator <TInput> .ExecuteAndCollectResults( inputStream, partitionCount, Child.OutputOrdered, preferStriping, settings); inputStreamInt = listResults.GetPartitionedStream(); } else { Debug.Assert(typeof(TKey) == typeof(int)); inputStreamInt = (PartitionedStream <TInput, int>)(object) inputStream; } // Since the index is correct, the type of the index must be int PartitionedStream <TOutput, int> outputStream = new PartitionedStream <TOutput, int>(partitionCount, Util.GetDefaultComparer <int>(), OrdinalIndexState); for (int i = 0; i < partitionCount; i++) { outputStream[i] = new IndexedSelectQueryOperatorEnumerator(inputStreamInt[i], _selector); } recipient.Receive(outputStream); }
internal override void WrapPartitionedStream <TKey>( PartitionedStream <TSource, TKey> inputStream, IPartitionedStreamRecipient <TSource> recipient, bool preferStriping, QuerySettings settings) { OrdinalIndexState inputIndexState = inputStream.OrdinalIndexState; PartitionedStream <TSource, int> intKeyStream; int partitionCount = inputStream.PartitionCount; // If the index is not at least increasing, we need to reindex. if (m_prematureMergeNeeded) { ListQueryResults <TSource> listResults = ExecuteAndCollectResults(inputStream, partitionCount, Child.OutputOrdered, preferStriping, settings); intKeyStream = listResults.GetPartitionedStream(); } else { Contract.Assert(typeof(TKey) == typeof(int)); intKeyStream = (PartitionedStream <TSource, int>)(object) inputStream; } // Generate the shared data. Shared <int> sharedFirstCandidate = new Shared <int>(-1); CountdownEvent sharedBarrier = new CountdownEvent(partitionCount); PartitionedStream <TSource, int> outputStream = new PartitionedStream <TSource, int>( partitionCount, Util.GetDefaultComparer <int>(), OrdinalIndexState.Shuffled); for (int i = 0; i < partitionCount; i++) { outputStream[i] = new FirstQueryOperatorEnumerator( intKeyStream[i], m_predicate, sharedFirstCandidate, sharedBarrier, settings.CancellationState.MergedCancellationToken); } recipient.Receive(outputStream); }
//--------------------------------------------------------------------------------------- // Just opens the current operator, including opening the children and wrapping them with // partitions as needed. // internal override QueryResults <TOutput> Open(QuerySettings settings, bool preferStriping) { // We just open our child operators, left and then right. QueryResults <TLeftInput> leftChildResults = _leftChild.Open(settings, preferStriping); QueryResults <TRightInput> rightChildResults = _rightChild.Open(settings, preferStriping); int partitionCount = settings.DegreeOfParallelism.Value; if (_prematureMergeLeft) { PartitionedStreamMerger <TLeftInput> merger = new PartitionedStreamMerger <TLeftInput>( false, ParallelMergeOptions.FullyBuffered, settings.TaskScheduler, _leftChild.OutputOrdered, settings.CancellationState, settings.QueryId); leftChildResults.GivePartitionedStream(merger); leftChildResults = new ListQueryResults <TLeftInput>( merger.MergeExecutor.GetResultsAsArray(), partitionCount, preferStriping); } if (_prematureMergeRight) { PartitionedStreamMerger <TRightInput> merger = new PartitionedStreamMerger <TRightInput>( false, ParallelMergeOptions.FullyBuffered, settings.TaskScheduler, _rightChild.OutputOrdered, settings.CancellationState, settings.QueryId); rightChildResults.GivePartitionedStream(merger); rightChildResults = new ListQueryResults <TRightInput>( merger.MergeExecutor.GetResultsAsArray(), partitionCount, preferStriping); } return(new ZipQueryOperatorResults(leftChildResults, rightChildResults, _resultSelector, partitionCount, preferStriping)); }
internal override void WrapPartitionedStream <TKey>( PartitionedStream <TResult, TKey> inputStream, IPartitionedStreamRecipient <TResult> recipient, bool preferStriping, QuerySettings settings) { int partitionCount = inputStream.PartitionCount; PartitionedStream <TResult, int> listInputStream; if (m_prematureMerge) { ListQueryResults <TResult> results = ExecuteAndCollectResults(inputStream, partitionCount, Child.OutputOrdered, preferStriping, settings); listInputStream = results.GetPartitionedStream(); } else { Contract.Assert(typeof(int) == typeof(TKey)); listInputStream = (PartitionedStream <TResult, int>)(object) inputStream; } // Create shared data. One is an index that represents the lowest false value found, // while the other is a latch used as a barrier. Shared <int> sharedLowFalse = new Shared <int>(-1); // Note that -1 is a sentinel to mean "not set yet". CountdownEvent sharedBarrier = new CountdownEvent(partitionCount); PartitionedStream <TResult, int> partitionedStream = new PartitionedStream <TResult, int>(partitionCount, Util.GetDefaultComparer <int>(), OrdinalIndexState); for (int i = 0; i < partitionCount; i++) { partitionedStream[i] = new TakeOrSkipWhileQueryOperatorEnumerator( listInputStream[i], m_predicate, m_indexedPredicate, m_take, sharedLowFalse, sharedBarrier, settings.CancellationState.MergedCancellationToken); } recipient.Receive(partitionedStream); }
internal override void WrapPartitionedStream <TKey>( PartitionedStream <TResult, TKey> inputStream, IPartitionedStreamRecipient <TResult> recipient, bool preferStriping, QuerySettings settings) { Contract.Assert(Child.OrdinalIndexState != OrdinalIndexState.Indexible, "Don't take this code path if the child is indexible."); PartitionedStream <TResult, int> inputIntStream; // If the index is not at least increasing, we need to reindex. if (m_prematureMerge) { ListQueryResults <TResult> results = ExecuteAndCollectResults( inputStream, inputStream.PartitionCount, Child.OutputOrdered, preferStriping, settings); inputIntStream = results.GetPartitionedStream(); } else { Contract.Assert(typeof(TKey) == typeof(int)); inputIntStream = (PartitionedStream <TResult, int>)((object)inputStream); } int partitionCount = inputStream.PartitionCount; FixedMaxHeap <int> sharedIndices = new FixedMaxHeap <int>(m_count); // an array used to track the sequence of indices leading up to the Nth index CountdownEvent sharredBarrier = new CountdownEvent(partitionCount); // a barrier to synchronize before yielding PartitionedStream <TResult, int> outputStream = new PartitionedStream <TResult, int>(partitionCount, Util.GetDefaultComparer <int>(), OrdinalIndexState); for (int i = 0; i < partitionCount; i++) { outputStream[i] = new TakeOrSkipQueryOperatorEnumerator( inputIntStream[i], m_count, m_take, sharedIndices, sharredBarrier, settings.CancellationState.MergedCancellationToken); } recipient.Receive(outputStream); }
internal override void WrapPartitionedStream <TKey>( PartitionedStream <TResult, TKey> inputStream, IPartitionedStreamRecipient <TResult> recipient, bool preferStriping, QuerySettings settings) { if (_prematureMerge) { ListQueryResults <TResult> results = ExecuteAndCollectResults(inputStream, inputStream.PartitionCount, Child.OutputOrdered, preferStriping, settings); PartitionedStream <TResult, int> listInputStream = results.GetPartitionedStream(); WrapHelper <int>(listInputStream, recipient, settings); } else { WrapHelper <TKey>(inputStream, recipient, settings); } }
internal override void WrapPartitionedStream <TKey>( PartitionedStream <TSource, TKey> inputStream, IPartitionedStreamRecipient <TSource> recipient, bool preferStriping, QuerySettings settings) { // If the index is not at least increasing, we need to reindex. if (_prematureMergeNeeded) { ListQueryResults <TSource> listResults = ExecuteAndCollectResults(inputStream, inputStream.PartitionCount, Child.OutputOrdered, preferStriping, settings); WrapHelper <int>(listResults.GetPartitionedStream(), recipient, settings); } else { WrapHelper <TKey>(inputStream, recipient, settings); } }
public override void WrapPartitionedStream <TLeftKey, TRightKey>( PartitionedStream <TSource, TLeftKey> leftStream, PartitionedStream <TSource, TRightKey> rightStream, IPartitionedStreamRecipient <TSource> outputRecipient, bool preferStriping, QuerySettings settings) { OrdinalIndexState leftChildIndexState = leftStream.OrdinalIndexState; int partitionCount = leftStream.PartitionCount; PartitionedStream <TSource, int> leftStreamInc; PartitionedStream <TSource, int> rightStreamInc; // Prematurely merge the left results, if necessary if (m_prematureMergeLeft) { ListQueryResults <TSource> leftStreamResults = ExecuteAndCollectResults(leftStream, partitionCount, LeftChild.OutputOrdered, preferStriping, settings); leftStreamInc = leftStreamResults.GetPartitionedStream(); } else { Contract.Assert(!ExchangeUtilities.IsWorseThan(leftStream.OrdinalIndexState, OrdinalIndexState.Increasing)); leftStreamInc = (PartitionedStream <TSource, int>)(object) leftStream; } // Prematurely merge the right results, if necessary if (m_prematureMergeRight) { ListQueryResults <TSource> rightStreamResults = ExecuteAndCollectResults(rightStream, partitionCount, LeftChild.OutputOrdered, preferStriping, settings); rightStreamInc = rightStreamResults.GetPartitionedStream(); } else { Contract.Assert(!ExchangeUtilities.IsWorseThan(rightStream.OrdinalIndexState, OrdinalIndexState.Increasing)); rightStreamInc = (PartitionedStream <TSource, int>)(object) rightStream; } // Generate the shared data. IComparer <ConcatKey <int, int> > comparer = ConcatKey <int, int> .MakeComparer( leftStreamInc.KeyComparer, rightStreamInc.KeyComparer); var outputStream = new PartitionedStream <TSource, ConcatKey <int, int> >(partitionCount, comparer, OrdinalIndexState); for (int i = 0; i < partitionCount; i++) { outputStream[i] = new ConcatQueryOperatorEnumerator <int, int>(leftStreamInc[i], rightStreamInc[i]); } outputRecipient.Receive(outputStream); }
private void WrapHelper <TLeftKey, TRightKey>( PartitionedStream <TSource, TLeftKey> leftStreamInc, PartitionedStream <TSource, TRightKey> rightStream, IPartitionedStreamRecipient <TSource> outputRecipient, QuerySettings settings, bool preferStriping) { // Prematurely merge the right results, if necessary if (_prematureMergeRight) { ListQueryResults <TSource> rightStreamResults = ExecuteAndCollectResults(rightStream, leftStreamInc.PartitionCount, LeftChild.OutputOrdered, preferStriping, settings); PartitionedStream <TSource, int> rightStreamInc = rightStreamResults.GetPartitionedStream(); WrapHelper2 <TLeftKey, int>(leftStreamInc, rightStreamInc, outputRecipient); } else { Debug.Assert(!ExchangeUtilities.IsWorseThan(rightStream.OrdinalIndexState, OrdinalIndexState.Increasing)); WrapHelper2 <TLeftKey, TRightKey>(leftStreamInc, rightStream, outputRecipient); } }
internal override void WrapPartitionedStream <TKey>( PartitionedStream <TResult, TKey> inputStream, IPartitionedStreamRecipient <TResult> recipient, bool preferStriping, QuerySettings settings) { Debug.Assert(Child.OrdinalIndexState != OrdinalIndexState.Indexable, "Don't take this code path if the child is indexable."); // If the index is not at least increasing, we need to reindex. if (_prematureMerge) { ListQueryResults <TResult> results = ExecuteAndCollectResults( inputStream, inputStream.PartitionCount, Child.OutputOrdered, preferStriping, settings); PartitionedStream <TResult, int> inputIntStream = results.GetPartitionedStream(); WrapHelper <int>(inputIntStream, recipient, settings); } else { WrapHelper <TKey>(inputStream, recipient, settings); } }
public override void WrapPartitionedStream <TLeftKey, TRightKey>( PartitionedStream <TSource, TLeftKey> leftStream, PartitionedStream <TSource, TRightKey> rightStream, IPartitionedStreamRecipient <TSource> outputRecipient, bool preferStriping, QuerySettings settings) { // Prematurely merge the left results, if necessary if (_prematureMergeLeft) { ListQueryResults <TSource> leftStreamResults = ExecuteAndCollectResults(leftStream, leftStream.PartitionCount, LeftChild.OutputOrdered, preferStriping, settings); PartitionedStream <TSource, int> leftStreamInc = leftStreamResults.GetPartitionedStream(); WrapHelper <int, TRightKey>(leftStreamInc, rightStream, outputRecipient, settings, preferStriping); } else { Contract.Assert(!ExchangeUtilities.IsWorseThan(leftStream.OrdinalIndexState, OrdinalIndexState.Increasing)); WrapHelper <TLeftKey, TRightKey>(leftStream, rightStream, outputRecipient, settings, preferStriping); } }
internal override QueryResults <TOutput> Open(QuerySettings settings, bool preferStriping) { QueryResults <TLeftInput> leftChildResults = this.m_leftChild.Open(settings, preferStriping); QueryResults <TRightInput> rightChildResults = this.m_rightChild.Open(settings, preferStriping); int partitionCount = settings.DegreeOfParallelism.Value; if (this.m_prematureMergeLeft) { PartitionedStreamMerger <TLeftInput> recipient = new PartitionedStreamMerger <TLeftInput>(false, ParallelMergeOptions.FullyBuffered, settings.TaskScheduler, this.m_leftChild.OutputOrdered, settings.CancellationState, settings.QueryId); leftChildResults.GivePartitionedStream(recipient); leftChildResults = new ListQueryResults <TLeftInput>(recipient.MergeExecutor.GetResultsAsArray(), partitionCount, preferStriping); } if (this.m_prematureMergeRight) { PartitionedStreamMerger <TRightInput> merger2 = new PartitionedStreamMerger <TRightInput>(false, ParallelMergeOptions.FullyBuffered, settings.TaskScheduler, this.m_rightChild.OutputOrdered, settings.CancellationState, settings.QueryId); rightChildResults.GivePartitionedStream(merger2); rightChildResults = new ListQueryResults <TRightInput>(merger2.MergeExecutor.GetResultsAsArray(), partitionCount, preferStriping); } return(new ZipQueryOperatorResults <TLeftInput, TRightInput, TOutput>(leftChildResults, rightChildResults, this.m_resultSelector, partitionCount, preferStriping)); }
internal override void WrapPartitionedStream <TLeftKey>( PartitionedStream <TLeftInput, TLeftKey> inputStream, IPartitionedStreamRecipient <TOutput> recipient, bool preferStriping, QuerySettings settings) { int partitionCount = inputStream.PartitionCount; if (_indexedRightChildSelector != null) { PartitionedStream <TLeftInput, int> inputStreamInt; // If the index is not correct, we need to reindex. if (_prematureMerge) { ListQueryResults <TLeftInput> listResults = QueryOperator <TLeftInput> .ExecuteAndCollectResults(inputStream, partitionCount, OutputOrdered, preferStriping, settings); inputStreamInt = listResults.GetPartitionedStream(); } else { inputStreamInt = (PartitionedStream <TLeftInput, int>)(object) inputStream; } WrapPartitionedStreamIndexed(inputStreamInt, recipient, settings); return; } // // if (_prematureMerge) { PartitionedStream <TLeftInput, int> inputStreamInt = QueryOperator <TLeftInput> .ExecuteAndCollectResults(inputStream, partitionCount, OutputOrdered, preferStriping, settings) .GetPartitionedStream(); WrapPartitionedStreamNotIndexed(inputStreamInt, recipient, settings); } else { WrapPartitionedStreamNotIndexed(inputStream, recipient, settings); } }