/// <summary> /// Decides whether or not to split a task further or compute it /// directly. If computing directly, calls {@code doLeaf} and pass /// the result to {@code setRawResult}. Otherwise splits off /// subtasks, forking one and continuing as the other. /// /// <para> The method is structured to conserve resources across a /// range of uses. The loop continues with one of the child tasks /// when split, to avoid deep recursion. To cope with spliterators /// that may be systematically biased toward left-heavy or /// right-heavy splits, we alternate which child is forked versus /// continued in the loop. /// </para> /// </summary> public override void Compute() { Spliterator <P_IN> rs = Spliterator, ls; // right, left spliterators long sizeEstimate = rs.EstimateSize(); long sizeThreshold = GetTargetSize(sizeEstimate); bool forkRight = false; //JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: //ORIGINAL LINE: @SuppressWarnings("unchecked") K task = (K) this; K task = (K)this; while (sizeEstimate > sizeThreshold && (ls = rs.TrySplit()) != null) { K leftChild, rightChild, taskToFork; task.leftChild = leftChild = task.makeChild(ls); task.rightChild = rightChild = task.makeChild(rs); task.PendingCount = 1; if (forkRight) { forkRight = false; rs = ls; task = leftChild; taskToFork = rightChild; } else { forkRight = true; task = rightChild; taskToFork = leftChild; } taskToFork.fork(); sizeEstimate = rs.EstimateSize(); } task.LocalResult = task.doLeaf(); task.tryComplete(); }
internal ForEachTask(PipelineHelper <T> helper, Spliterator <S> spliterator, Sink <S> sink) : base(null) { this.Sink = sink; this.Helper = helper; this.Spliterator = spliterator; this.TargetSize = 0L; }
internal override Node <T> opEvaluateParallel <P_IN>(PipelineHelper <T> helper, Spliterator <P_IN> spliterator, IntFunction <T[]> generator) { long size = helper.ExactOutputSizeIfKnown(spliterator); if (size > 0 && spliterator.hasCharacteristics(java.util.Spliterator_Fields.SUBSIZED)) { // Because the pipeline is SIZED the slice spliterator // can be created from the source, this requires matching // to shape of the source, and is potentially more efficient // than creating the slice spliterator from the pipeline // wrapping spliterator Spliterator <P_IN> s = SliceSpliterator(helper.SourceShape, spliterator, Skip, Limit); return(Nodes.Collect(helper, s, Stream_Fields.True, generator)); } else if (!StreamOpFlag.ORDERED.isKnown(helper.StreamAndOpFlags)) { Spliterator <T> s = unorderedSkipLimitSpliterator(helper.WrapSpliterator(spliterator), Skip, Limit, size); // Collect using this pipeline, which is empty and therefore // can be used with the pipeline wrapping spliterator // Note that we cannot create a slice spliterator from // the source spliterator if the pipeline is not SIZED return(Nodes.Collect(this, s, Stream_Fields.True, generator)); } else { return((new SliceTask <>(this, helper, spliterator, generator, Skip, Limit)).invoke()); } }
public override void onCompletion <T1>(CountedCompleter <T1> caller) { if (Node != null) { // Dump buffered elements from this leaf into the sink Node.ForEach(Action); Node = null; } else if (Spliterator != null) { // Dump elements output from this leaf's pipeline into the sink Helper.WrapAndCopyInto(Action, Spliterator); Spliterator = null; } // The completion of this task *and* the dumping of elements // "happens-before" completion of the associated left-most leaf task // of right subtree (if any, which can be this task's right sibling) // ForEachOrderedTask <S, T> leftDescendant = CompletionMap.Remove(this); if (leftDescendant != null) { leftDescendant.TryComplete(); } }
internal ForEachTask(ForEachTask <S, T> parent, Spliterator <S> spliterator) : base(parent) { this.Spliterator = spliterator; this.Sink = parent.Sink; this.TargetSize = parent.TargetSize; this.Helper = parent.Helper; }
//------------------------------------------------------------------------- /// <summary> /// Streams the set of dates included in the range. /// <para> /// This returns a stream consisting of each date in the range. /// The stream is ordered. /// /// </para> /// </summary> /// <param name="startInclusive"> the start date </param> /// <param name="endExclusive"> the end date </param> /// <returns> the stream of dates from the start to the end </returns> internal static Stream <LocalDate> stream(LocalDate startInclusive, LocalDate endExclusive) { IEnumerator <LocalDate> it = new IteratorAnonymousInnerClass(startInclusive, endExclusive); long count = endExclusive.toEpochDay() - startInclusive.toEpochDay() + 1; Spliterator <LocalDate> spliterator = Spliterators.spliterator(it, count, Spliterator.IMMUTABLE | Spliterator.NONNULL | Spliterator.DISTINCT | Spliterator.ORDERED | Spliterator.SORTED | Spliterator.SIZED | Spliterator.SUBSIZED); return(StreamSupport.stream(spliterator, false)); }
internal virtual Node <T> reduce <P_IN>(PipelineHelper <T> helper, Spliterator <P_IN> spliterator) { // If the stream is SORTED then it should also be ORDERED so the following will also // preserve the sort order TerminalOp <T, LinkedHashSet <T> > reduceOp = ReduceOps.MakeRef <T, LinkedHashSet <T> >(LinkedHashSet::new, LinkedHashSet::add, LinkedHashSet::addAll); return(Nodes.Node(reduceOp.evaluateParallel(helper, spliterator))); }
internal ForEachOrderedTask(ForEachOrderedTask <S, T> parent, Spliterator <S> spliterator, ForEachOrderedTask <S, T> leftPredecessor) : base(parent) { this.Helper = parent.Helper; this.Spliterator = spliterator; this.TargetSize = parent.TargetSize; this.CompletionMap = parent.CompletionMap; this.Action = parent.Action; this.LeftPredecessor = leftPredecessor; }
public override Boolean evaluateParallel<S>(PipelineHelper<T> helper, Spliterator<S> spliterator) { // Approach for parallel implementation: // - Decompose as per usual // - run match on leaf chunks, call result "b" // - if b == matchKind.shortCircuitOn, complete early and return b // - else if we complete normally, return !shortCircuitOn return (new MatchTask<>(this, helper, spliterator)).invoke(); }
protected internal ForEachOrderedTask(PipelineHelper <T> helper, Spliterator <S> spliterator, Sink <T> action) : base(null) { this.Helper = helper; this.Spliterator = spliterator; this.TargetSize = AbstractTask.SuggestTargetSize(spliterator.EstimateSize()); // Size map to avoid concurrent re-sizes this.CompletionMap = new ConcurrentDictionary <>(System.Math.Max(16, AbstractTask.LEAF_TARGET << 1)); this.Action = action; this.LeftPredecessor = null; }
internal virtual Spliterator <T> UnorderedSkipLimitSpliterator(Spliterator <T> s, long skip, long limit, long sizeIfKnown) { if (skip <= sizeIfKnown) { // Use just the limit if the number of elements // to skip is <= the known pipeline size limit = limit >= 0 ? System.Math.Min(limit, sizeIfKnown - skip) : sizeIfKnown - skip; skip = 0; } return(new StreamSpliterators.UnorderedSliceSpliterator.OfRef <>(s, skip, limit)); }
public virtual Stream <AvailableAnalyzer> ListAvailableAnalyzers() { Stream <AnalyzerProvider> stream = Accessor.listAvailableAnalyzers(); return(stream.flatMap(provider => { string description = provider.description(); Spliterator <string> spliterator = provider.Keys.spliterator(); return StreamSupport.stream(spliterator, false).map(name => new AvailableAnalyzer(name, description)); })); }
/// <summary> /// Create a stream from the given iterator with given characteristics. /// <para> /// <b>Note:</b> returned stream needs to be closed via <seealso cref="Stream.close()"/> if the given iterator implements /// <seealso cref="Resource"/>. /// /// </para> /// </summary> /// <param name="iterator"> the iterator to convert to stream </param> /// <param name="characteristics"> the logical OR of characteristics for the underlying <seealso cref="Spliterator"/> </param> /// @param <T> the type of elements in the given iterator </param> /// <returns> stream over the iterator elements </returns> /// <exception cref="NullPointerException"> when the given iterator is {@code null} </exception> public static Stream <T> Stream <T>(IEnumerator <T> iterator, int characteristics) { Objects.requireNonNull(iterator); Spliterator <T> spliterator = Spliterators.spliteratorUnknownSize(iterator, characteristics); Stream <T> stream = StreamSupport.stream(spliterator, false); if (iterator is Resource) { return(stream.onClose((( Resource )iterator).close)); } return(stream); }
public override Void evaluateParallel <S>(PipelineHelper <T> helper, Spliterator <S> spliterator) { if (Ordered) { (new ForEachOrderedTask <>(helper, spliterator, this)).Invoke(); } else { (new ForEachTask <>(helper, spliterator, helper.WrapSink(this))).Invoke(); } return(null); }
/// <summary> /// Overrides AbstractTask version to include checks for early /// exits while splitting or computing. /// </summary> public override void Compute() { Spliterator <P_IN> rs = spliterator, ls; long sizeEstimate = rs.EstimateSize(); long sizeThreshold = getTargetSize(sizeEstimate); bool forkRight = false; //JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: //ORIGINAL LINE: @SuppressWarnings("unchecked") K task = (K) this; K task = (K)this; AtomicReference <R> sr = SharedResult; R result; while ((result = sr.Get()) == null) { if (task.taskCanceled()) { result = task.EmptyResult; break; } if (sizeEstimate <= sizeThreshold || (ls = rs.TrySplit()) == null) { result = task.doLeaf(); break; } K leftChild, rightChild, taskToFork; task.leftChild = leftChild = task.makeChild(ls); task.rightChild = rightChild = task.makeChild(rs); task.PendingCount = 1; if (forkRight) { forkRight = false; rs = ls; task = leftChild; taskToFork = rightChild; } else { forkRight = true; task = rightChild; taskToFork = leftChild; } taskToFork.fork(); sizeEstimate = rs.EstimateSize(); } task.LocalResult = result; task.tryComplete(); }
// Similar to AbstractTask but doesn't need to track child tasks public void Compute() { Spliterator <S> rightSplit = Spliterator, leftSplit; long sizeEstimate = rightSplit.EstimateSize(), sizeThreshold; if ((sizeThreshold = TargetSize) == 0L) { TargetSize = sizeThreshold = AbstractTask.SuggestTargetSize(sizeEstimate); } bool isShortCircuit = StreamOpFlag.SHORT_CIRCUIT.isKnown(Helper.StreamAndOpFlags); bool forkRight = false; Sink <S> taskSink = Sink; ForEachTask <S, T> task = this; while (!isShortCircuit || !taskSink.cancellationRequested()) { if (sizeEstimate <= sizeThreshold || (leftSplit = rightSplit.TrySplit()) == null) { task.Helper.CopyInto(taskSink, rightSplit); break; } ForEachTask <S, T> leftTask = new ForEachTask <S, T>(task, leftSplit); task.AddToPendingCount(1); ForEachTask <S, T> taskToFork; if (forkRight) { forkRight = false; rightSplit = leftSplit; taskToFork = task; task = leftTask; } else { forkRight = true; taskToFork = leftTask; } taskToFork.Fork(); sizeEstimate = rightSplit.EstimateSize(); } task.Spliterator = null; task.PropagateCompletion(); }
/// <summary> /// Creates a slice spliterator given a stream shape governing the /// spliterator type. Requires that the underlying Spliterator /// be SUBSIZED. /// </summary> //JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: //ORIGINAL LINE: @SuppressWarnings("unchecked") private static <P_IN> java.util.Spliterator<P_IN> sliceSpliterator(StreamShape shape, java.util.Spliterator<P_IN> s, long skip, long limit) private static Spliterator <P_IN> sliceSpliterator <P_IN>(StreamShape shape, Spliterator <P_IN> s, long skip, long limit) { Debug.Assert(s.hasCharacteristics(java.util.Spliterator_Fields.SUBSIZED)); long sliceFence = CalcSliceFence(skip, limit); switch (shape) { case java.util.stream.StreamShape.REFERENCE: return(new StreamSpliterators.SliceSpliterator.OfRef <>(s, skip, sliceFence)); case java.util.stream.StreamShape.INT_VALUE: return((Spliterator <P_IN>) new StreamSpliterators.SliceSpliterator.OfInt((java.util.Spliterator_OfInt)s, skip, sliceFence)); case java.util.stream.StreamShape.LONG_VALUE: return((Spliterator <P_IN>) new StreamSpliterators.SliceSpliterator.OfLong((java.util.Spliterator_OfLong)s, skip, sliceFence)); case java.util.stream.StreamShape.DOUBLE_VALUE: return((Spliterator <P_IN>) new StreamSpliterators.SliceSpliterator.OfDouble((java.util.Spliterator_OfDouble)s, skip, sliceFence)); default: throw new IllegalStateException("Unknown shape " + shape); } }
internal override Spliterator <T> opEvaluateParallelLazy <P_IN>(PipelineHelper <T> helper, Spliterator <P_IN> spliterator) { long size = helper.ExactOutputSizeIfKnown(spliterator); if (size > 0 && spliterator.hasCharacteristics(java.util.Spliterator_Fields.SUBSIZED)) { return(new StreamSpliterators.SliceSpliterator.OfRef <>(helper.WrapSpliterator(spliterator), Skip, CalcSliceFence(Skip, Limit))); } else if (!StreamOpFlag.ORDERED.isKnown(helper.StreamAndOpFlags)) { return(unorderedSkipLimitSpliterator(helper.WrapSpliterator(spliterator), Skip, Limit, size)); } else { // @@@ OOMEs will occur for LongStream.longs().filter(i -> true).limit(n) // regardless of the value of n // Need to adjust the target size of splitting for the // SliceTask from say (size / k) to say min(size / k, 1 << 14) // This will limit the size of the buffers created at the leaf nodes // cancellation will be more aggressive cancelling later tasks // if the target slice size has been reached from a given task, // cancellation should also clear local results if any return((new SliceTask <>(this, helper, spliterator, CastingArray(), Skip, Limit)).invoke().spliterator()); } }
public override O evaluateParallel <P_IN>(PipelineHelper <T> helper, Spliterator <P_IN> spliterator) { return((new FindTask <>(this, helper, spliterator)).invoke()); }
/// <summary> /// Constructor for root tasks. /// </summary> /// <param name="helper"> the {@code PipelineHelper} describing the stream pipeline /// up to this operation </param> /// <param name="spliterator"> the {@code Spliterator} describing the source for this /// pipeline </param> protected internal AbstractShortCircuitTask(PipelineHelper <P_OUT> helper, Spliterator <P_IN> spliterator) : base(helper, spliterator) { SharedResult = new AtomicReference <>(null); }
/// <summary> /// Constructor for non-root nodes. /// </summary> /// <param name="parent"> parent task in the computation tree </param> /// <param name="spliterator"> the {@code Spliterator} for the portion of the /// computation tree described by this task </param> protected internal AbstractShortCircuitTask(K parent, Spliterator <P_IN> spliterator) : base(parent, spliterator) { SharedResult = parent.sharedResult; }
/// <summary> /// Constructor for non-root node /// </summary> internal MatchTask(MatchTask<P_IN, P_OUT> parent, Spliterator<P_IN> spliterator) : base(parent, spliterator) { this.Op = parent.Op; }
protected internal override MatchTask<P_IN, P_OUT> MakeChild(Spliterator<P_IN> spliterator) { return new MatchTask<>(this, spliterator); }
/// <summary> /// Constructor for root node /// </summary> internal MatchTask(MatchOp<P_OUT> op, PipelineHelper<P_OUT> helper, Spliterator<P_IN> spliterator) : base(helper, spliterator) { this.Op = op; }
internal FindTask(FindOp <P_OUT, O> op, PipelineHelper <P_OUT> helper, Spliterator <P_IN> spliterator) : base(helper, spliterator) { this.Op = op; }
public override Boolean evaluateSequential<S>(PipelineHelper<T> helper, Spliterator<S> spliterator) { return helper.WrapAndCopyInto(SinkSupplier.Get(), spliterator).AndClearState; }
internal FindTask(FindTask <P_IN, P_OUT, O> parent, Spliterator <P_IN> spliterator) : base(parent, spliterator) { this.Op = parent.Op; }
protected internal override FindTask <P_IN, P_OUT, O> MakeChild(Spliterator <P_IN> spliterator) { return(new FindTask <>(this, spliterator)); }
internal override Node <T> opEvaluateParallel <P_IN>(PipelineHelper <T> helper, Spliterator <P_IN> spliterator, IntFunction <T[]> generator) { if (StreamOpFlag.DISTINCT.isKnown(helper.StreamAndOpFlags)) { // No-op return(helper.Evaluate(spliterator, false, generator)); } else if (StreamOpFlag.ORDERED.isKnown(helper.StreamAndOpFlags)) { return(reduce(helper, spliterator)); } else { // Holder of null state since ConcurrentHashMap does not support null values AtomicBoolean seenNull = new AtomicBoolean(false); ConcurrentDictionary <T, Boolean> map = new ConcurrentDictionary <T, Boolean>(); //JAVA TO C# CONVERTER TODO TASK: Java lambdas satisfy functional interfaces, while .NET lambdas satisfy delegates - change the appropriate interface to a delegate: TerminalOp <T, Void> forEachOp = ForEachOps.makeRef(Stream_Fields.t => { if (Stream_Fields.t == null) { seenNull.Set(Stream_Fields.True); } else { map.GetOrAdd(Stream_Fields.t, true); } }, false);
/// <summary> /// Returns a stream that wraps this iterator. /// <para> /// The stream will process any remaining rows in the CSV file. /// As such, it is recommended that callers should use this method or the iterator methods and not both. /// /// </para> /// </summary> /// <returns> the stream wrapping this iterator </returns> public Stream <CsvRow> asStream() { Spliterator <CsvRow> spliterator = Spliterators.spliteratorUnknownSize(this, Spliterator.ORDERED | Spliterator.NONNULL); return(StreamSupport.stream(spliterator, false)); }