/// <summary>
/// Decides whether or not to split a task further or compute it
/// directly. If computing directly, calls {@code doLeaf} and pass
/// the result to {@code setRawResult}. Otherwise splits off
/// subtasks, forking one and continuing as the other.
///
/// <para> The method is structured to conserve resources across a
/// range of uses. The loop continues with one of the child tasks
/// when split, to avoid deep recursion. To cope with spliterators
/// that may be systematically biased toward left-heavy or
/// right-heavy splits, we alternate which child is forked versus
/// continued in the loop.
/// </para>
/// </summary>
public override void Compute()
{
Spliterator <P_IN> rs = Spliterator, ls; // right, left spliterators
long sizeEstimate = rs.EstimateSize();
long sizeThreshold = GetTargetSize(sizeEstimate);
bool forkRight = false;
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @SuppressWarnings("unchecked") K task = (K) this;
K task = (K)this;
while (sizeEstimate > sizeThreshold && (ls = rs.TrySplit()) != null)
{
K leftChild, rightChild, taskToFork;
task.leftChild = leftChild = task.makeChild(ls);
task.rightChild = rightChild = task.makeChild(rs);
task.PendingCount = 1;
if (forkRight)
{
forkRight = false;
rs = ls;
task = leftChild;
taskToFork = rightChild;
}
else
{
forkRight = true;
task = rightChild;
taskToFork = leftChild;
}
taskToFork.fork();
sizeEstimate = rs.EstimateSize();
}
task.LocalResult = task.doLeaf();
task.tryComplete();
}
internal ForEachTask(PipelineHelper <T> helper, Spliterator <S> spliterator, Sink <S> sink) : base(null)
{
this.Sink = sink;
this.Helper = helper;
this.Spliterator = spliterator;
this.TargetSize = 0L;
}
internal override Node <T> opEvaluateParallel <P_IN>(PipelineHelper <T> helper, Spliterator <P_IN> spliterator, IntFunction <T[]> generator)
{
long size = helper.ExactOutputSizeIfKnown(spliterator);
if (size > 0 && spliterator.hasCharacteristics(java.util.Spliterator_Fields.SUBSIZED))
{
// Because the pipeline is SIZED the slice spliterator
// can be created from the source, this requires matching
// to shape of the source, and is potentially more efficient
// than creating the slice spliterator from the pipeline
// wrapping spliterator
Spliterator <P_IN> s = SliceSpliterator(helper.SourceShape, spliterator, Skip, Limit);
return(Nodes.Collect(helper, s, Stream_Fields.True, generator));
}
else if (!StreamOpFlag.ORDERED.isKnown(helper.StreamAndOpFlags))
{
Spliterator <T> s = unorderedSkipLimitSpliterator(helper.WrapSpliterator(spliterator), Skip, Limit, size);
// Collect using this pipeline, which is empty and therefore
// can be used with the pipeline wrapping spliterator
// Note that we cannot create a slice spliterator from
// the source spliterator if the pipeline is not SIZED
return(Nodes.Collect(this, s, Stream_Fields.True, generator));
}
else
{
return((new SliceTask <>(this, helper, spliterator, generator, Skip, Limit)).invoke());
}
}
public override void onCompletion <T1>(CountedCompleter <T1> caller)
{
if (Node != null)
{
// Dump buffered elements from this leaf into the sink
Node.ForEach(Action);
Node = null;
}
else if (Spliterator != null)
{
// Dump elements output from this leaf's pipeline into the sink
Helper.WrapAndCopyInto(Action, Spliterator);
Spliterator = null;
}
// The completion of this task *and* the dumping of elements
// "happens-before" completion of the associated left-most leaf task
// of right subtree (if any, which can be this task's right sibling)
//
ForEachOrderedTask <S, T> leftDescendant = CompletionMap.Remove(this);
if (leftDescendant != null)
{
leftDescendant.TryComplete();
}
}
internal ForEachTask(ForEachTask <S, T> parent, Spliterator <S> spliterator) : base(parent)
{
this.Spliterator = spliterator;
this.Sink = parent.Sink;
this.TargetSize = parent.TargetSize;
this.Helper = parent.Helper;
}
//-------------------------------------------------------------------------
/// <summary>
/// Streams the set of dates included in the range.
/// <para>
/// This returns a stream consisting of each date in the range.
/// The stream is ordered.
///
/// </para>
/// </summary>
/// <param name="startInclusive"> the start date </param>
/// <param name="endExclusive"> the end date </param>
/// <returns> the stream of dates from the start to the end </returns>
internal static Stream <LocalDate> stream(LocalDate startInclusive, LocalDate endExclusive)
{
IEnumerator <LocalDate> it = new IteratorAnonymousInnerClass(startInclusive, endExclusive);
long count = endExclusive.toEpochDay() - startInclusive.toEpochDay() + 1;
Spliterator <LocalDate> spliterator = Spliterators.spliterator(it, count, Spliterator.IMMUTABLE | Spliterator.NONNULL | Spliterator.DISTINCT | Spliterator.ORDERED | Spliterator.SORTED | Spliterator.SIZED | Spliterator.SUBSIZED);
return(StreamSupport.stream(spliterator, false));
}
internal virtual Node <T> reduce <P_IN>(PipelineHelper <T> helper, Spliterator <P_IN> spliterator)
{
// If the stream is SORTED then it should also be ORDERED so the following will also
// preserve the sort order
TerminalOp <T, LinkedHashSet <T> > reduceOp = ReduceOps.MakeRef <T, LinkedHashSet <T> >(LinkedHashSet::new, LinkedHashSet::add, LinkedHashSet::addAll);
return(Nodes.Node(reduceOp.evaluateParallel(helper, spliterator)));
}
internal ForEachOrderedTask(ForEachOrderedTask <S, T> parent, Spliterator <S> spliterator, ForEachOrderedTask <S, T> leftPredecessor) : base(parent)
{
this.Helper = parent.Helper;
this.Spliterator = spliterator;
this.TargetSize = parent.TargetSize;
this.CompletionMap = parent.CompletionMap;
this.Action = parent.Action;
this.LeftPredecessor = leftPredecessor;
}
public override Boolean evaluateParallel<S>(PipelineHelper<T> helper, Spliterator<S> spliterator)
{
// Approach for parallel implementation:
// - Decompose as per usual
// - run match on leaf chunks, call result "b"
// - if b == matchKind.shortCircuitOn, complete early and return b
// - else if we complete normally, return !shortCircuitOn
return (new MatchTask<>(this, helper, spliterator)).invoke();
}
protected internal ForEachOrderedTask(PipelineHelper <T> helper, Spliterator <S> spliterator, Sink <T> action) : base(null)
{
this.Helper = helper;
this.Spliterator = spliterator;
this.TargetSize = AbstractTask.SuggestTargetSize(spliterator.EstimateSize());
// Size map to avoid concurrent re-sizes
this.CompletionMap = new ConcurrentDictionary <>(System.Math.Max(16, AbstractTask.LEAF_TARGET << 1));
this.Action = action;
this.LeftPredecessor = null;
}
internal virtual Spliterator <T> UnorderedSkipLimitSpliterator(Spliterator <T> s, long skip, long limit, long sizeIfKnown)
{
if (skip <= sizeIfKnown)
{
// Use just the limit if the number of elements
// to skip is <= the known pipeline size
limit = limit >= 0 ? System.Math.Min(limit, sizeIfKnown - skip) : sizeIfKnown - skip;
skip = 0;
}
return(new StreamSpliterators.UnorderedSliceSpliterator.OfRef <>(s, skip, limit));
}
public virtual Stream <AvailableAnalyzer> ListAvailableAnalyzers()
{
Stream <AnalyzerProvider> stream = Accessor.listAvailableAnalyzers();
return(stream.flatMap(provider =>
{
string description = provider.description();
Spliterator <string> spliterator = provider.Keys.spliterator();
return StreamSupport.stream(spliterator, false).map(name => new AvailableAnalyzer(name, description));
}));
}
/// <summary>
/// Create a stream from the given iterator with given characteristics.
/// <para>
/// <b>Note:</b> returned stream needs to be closed via <seealso cref="Stream.close()"/> if the given iterator implements
/// <seealso cref="Resource"/>.
///
/// </para>
/// </summary>
/// <param name="iterator"> the iterator to convert to stream </param>
/// <param name="characteristics"> the logical OR of characteristics for the underlying <seealso cref="Spliterator"/> </param>
/// @param <T> the type of elements in the given iterator </param>
/// <returns> stream over the iterator elements </returns>
/// <exception cref="NullPointerException"> when the given iterator is {@code null} </exception>
public static Stream <T> Stream <T>(IEnumerator <T> iterator, int characteristics)
{
Objects.requireNonNull(iterator);
Spliterator <T> spliterator = Spliterators.spliteratorUnknownSize(iterator, characteristics);
Stream <T> stream = StreamSupport.stream(spliterator, false);
if (iterator is Resource)
{
return(stream.onClose((( Resource )iterator).close));
}
return(stream);
}
public override Void evaluateParallel <S>(PipelineHelper <T> helper, Spliterator <S> spliterator)
{
if (Ordered)
{
(new ForEachOrderedTask <>(helper, spliterator, this)).Invoke();
}
else
{
(new ForEachTask <>(helper, spliterator, helper.WrapSink(this))).Invoke();
}
return(null);
}
/// <summary>
/// Overrides AbstractTask version to include checks for early
/// exits while splitting or computing.
/// </summary>
public override void Compute()
{
Spliterator <P_IN> rs = spliterator, ls;
long sizeEstimate = rs.EstimateSize();
long sizeThreshold = getTargetSize(sizeEstimate);
bool forkRight = false;
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @SuppressWarnings("unchecked") K task = (K) this;
K task = (K)this;
AtomicReference <R> sr = SharedResult;
R result;
while ((result = sr.Get()) == null)
{
if (task.taskCanceled())
{
result = task.EmptyResult;
break;
}
if (sizeEstimate <= sizeThreshold || (ls = rs.TrySplit()) == null)
{
result = task.doLeaf();
break;
}
K leftChild, rightChild, taskToFork;
task.leftChild = leftChild = task.makeChild(ls);
task.rightChild = rightChild = task.makeChild(rs);
task.PendingCount = 1;
if (forkRight)
{
forkRight = false;
rs = ls;
task = leftChild;
taskToFork = rightChild;
}
else
{
forkRight = true;
task = rightChild;
taskToFork = leftChild;
}
taskToFork.fork();
sizeEstimate = rs.EstimateSize();
}
task.LocalResult = result;
task.tryComplete();
}
// Similar to AbstractTask but doesn't need to track child tasks
public void Compute()
{
Spliterator <S> rightSplit = Spliterator, leftSplit;
long sizeEstimate = rightSplit.EstimateSize(), sizeThreshold;
if ((sizeThreshold = TargetSize) == 0L)
{
TargetSize = sizeThreshold = AbstractTask.SuggestTargetSize(sizeEstimate);
}
bool isShortCircuit = StreamOpFlag.SHORT_CIRCUIT.isKnown(Helper.StreamAndOpFlags);
bool forkRight = false;
Sink <S> taskSink = Sink;
ForEachTask <S, T> task = this;
while (!isShortCircuit || !taskSink.cancellationRequested())
{
if (sizeEstimate <= sizeThreshold || (leftSplit = rightSplit.TrySplit()) == null)
{
task.Helper.CopyInto(taskSink, rightSplit);
break;
}
ForEachTask <S, T> leftTask = new ForEachTask <S, T>(task, leftSplit);
task.AddToPendingCount(1);
ForEachTask <S, T> taskToFork;
if (forkRight)
{
forkRight = false;
rightSplit = leftSplit;
taskToFork = task;
task = leftTask;
}
else
{
forkRight = true;
taskToFork = leftTask;
}
taskToFork.Fork();
sizeEstimate = rightSplit.EstimateSize();
}
task.Spliterator = null;
task.PropagateCompletion();
}
/// <summary>
/// Creates a slice spliterator given a stream shape governing the
/// spliterator type. Requires that the underlying Spliterator
/// be SUBSIZED.
/// </summary>
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @SuppressWarnings("unchecked") private static <P_IN> java.util.Spliterator<P_IN> sliceSpliterator(StreamShape shape, java.util.Spliterator<P_IN> s, long skip, long limit)
private static Spliterator <P_IN> sliceSpliterator <P_IN>(StreamShape shape, Spliterator <P_IN> s, long skip, long limit)
{
Debug.Assert(s.hasCharacteristics(java.util.Spliterator_Fields.SUBSIZED));
long sliceFence = CalcSliceFence(skip, limit);
switch (shape)
{
case java.util.stream.StreamShape.REFERENCE:
return(new StreamSpliterators.SliceSpliterator.OfRef <>(s, skip, sliceFence));
case java.util.stream.StreamShape.INT_VALUE:
return((Spliterator <P_IN>) new StreamSpliterators.SliceSpliterator.OfInt((java.util.Spliterator_OfInt)s, skip, sliceFence));
case java.util.stream.StreamShape.LONG_VALUE:
return((Spliterator <P_IN>) new StreamSpliterators.SliceSpliterator.OfLong((java.util.Spliterator_OfLong)s, skip, sliceFence));
case java.util.stream.StreamShape.DOUBLE_VALUE:
return((Spliterator <P_IN>) new StreamSpliterators.SliceSpliterator.OfDouble((java.util.Spliterator_OfDouble)s, skip, sliceFence));
default:
throw new IllegalStateException("Unknown shape " + shape);
}
}
internal override Spliterator <T> opEvaluateParallelLazy <P_IN>(PipelineHelper <T> helper, Spliterator <P_IN> spliterator)
{
long size = helper.ExactOutputSizeIfKnown(spliterator);
if (size > 0 && spliterator.hasCharacteristics(java.util.Spliterator_Fields.SUBSIZED))
{
return(new StreamSpliterators.SliceSpliterator.OfRef <>(helper.WrapSpliterator(spliterator), Skip, CalcSliceFence(Skip, Limit)));
}
else if (!StreamOpFlag.ORDERED.isKnown(helper.StreamAndOpFlags))
{
return(unorderedSkipLimitSpliterator(helper.WrapSpliterator(spliterator), Skip, Limit, size));
}
else
{
// @@@ OOMEs will occur for LongStream.longs().filter(i -> true).limit(n)
// regardless of the value of n
// Need to adjust the target size of splitting for the
// SliceTask from say (size / k) to say min(size / k, 1 << 14)
// This will limit the size of the buffers created at the leaf nodes
// cancellation will be more aggressive cancelling later tasks
// if the target slice size has been reached from a given task,
// cancellation should also clear local results if any
return((new SliceTask <>(this, helper, spliterator, CastingArray(), Skip, Limit)).invoke().spliterator());
}
}
public override O evaluateParallel <P_IN>(PipelineHelper <T> helper, Spliterator <P_IN> spliterator)
{
return((new FindTask <>(this, helper, spliterator)).invoke());
}
/// <summary>
/// Constructor for root tasks.
/// </summary>
/// <param name="helper"> the {@code PipelineHelper} describing the stream pipeline
/// up to this operation </param>
/// <param name="spliterator"> the {@code Spliterator} describing the source for this
/// pipeline </param>
protected internal AbstractShortCircuitTask(PipelineHelper <P_OUT> helper, Spliterator <P_IN> spliterator) : base(helper, spliterator)
{
SharedResult = new AtomicReference <>(null);
}
/// <summary>
/// Constructor for non-root nodes.
/// </summary>
/// <param name="parent"> parent task in the computation tree </param>
/// <param name="spliterator"> the {@code Spliterator} for the portion of the
/// computation tree described by this task </param>
protected internal AbstractShortCircuitTask(K parent, Spliterator <P_IN> spliterator) : base(parent, spliterator)
{
SharedResult = parent.sharedResult;
}
/// <summary>
/// Constructor for non-root node
/// </summary>
internal MatchTask(MatchTask<P_IN, P_OUT> parent, Spliterator<P_IN> spliterator) : base(parent, spliterator)
{
this.Op = parent.Op;
}
protected internal override MatchTask<P_IN, P_OUT> MakeChild(Spliterator<P_IN> spliterator)
{
return new MatchTask<>(this, spliterator);
}
/// <summary>
/// Constructor for root node
/// </summary>
internal MatchTask(MatchOp<P_OUT> op, PipelineHelper<P_OUT> helper, Spliterator<P_IN> spliterator) : base(helper, spliterator)
{
this.Op = op;
}
internal FindTask(FindOp <P_OUT, O> op, PipelineHelper <P_OUT> helper, Spliterator <P_IN> spliterator) : base(helper, spliterator)
{
this.Op = op;
}
public override Boolean evaluateSequential<S>(PipelineHelper<T> helper, Spliterator<S> spliterator)
{
return helper.WrapAndCopyInto(SinkSupplier.Get(), spliterator).AndClearState;
}
internal FindTask(FindTask <P_IN, P_OUT, O> parent, Spliterator <P_IN> spliterator) : base(parent, spliterator)
{
this.Op = parent.Op;
}
protected internal override FindTask <P_IN, P_OUT, O> MakeChild(Spliterator <P_IN> spliterator)
{
return(new FindTask <>(this, spliterator));
}
internal override Node <T> opEvaluateParallel <P_IN>(PipelineHelper <T> helper, Spliterator <P_IN> spliterator, IntFunction <T[]> generator)
{
if (StreamOpFlag.DISTINCT.isKnown(helper.StreamAndOpFlags))
{
// No-op
return(helper.Evaluate(spliterator, false, generator));
}
else if (StreamOpFlag.ORDERED.isKnown(helper.StreamAndOpFlags))
{
return(reduce(helper, spliterator));
}
else
{
// Holder of null state since ConcurrentHashMap does not support null values
AtomicBoolean seenNull = new AtomicBoolean(false);
ConcurrentDictionary <T, Boolean> map = new ConcurrentDictionary <T, Boolean>();
//JAVA TO C# CONVERTER TODO TASK: Java lambdas satisfy functional interfaces, while .NET lambdas satisfy delegates - change the appropriate interface to a delegate:
TerminalOp <T, Void> forEachOp = ForEachOps.makeRef(Stream_Fields.t =>
{
if (Stream_Fields.t == null)
{
seenNull.Set(Stream_Fields.True);
}
else
{
map.GetOrAdd(Stream_Fields.t, true);
}
}, false);
/// <summary>
/// Returns a stream that wraps this iterator.
/// <para>
/// The stream will process any remaining rows in the CSV file.
/// As such, it is recommended that callers should use this method or the iterator methods and not both.
///
/// </para>
/// </summary>
/// <returns> the stream wrapping this iterator </returns>
public Stream <CsvRow> asStream()
{
Spliterator <CsvRow> spliterator = Spliterators.spliteratorUnknownSize(this, Spliterator.ORDERED | Spliterator.NONNULL);
return(StreamSupport.stream(spliterator, false));
}
