private static long ProgressOf(StageExecution execution)
{
// First see if there's a "progress" stat
Stat progressStat = FindProgressStat(execution.Steps());
if (progressStat != null)
{
return(Weighted(execution.StageName, progressStat.AsLong()));
}
// No, then do the generic progress calculation by looking at "done_batches"
long doneBatches = last(execution.Steps()).stats().stat(Keys.done_batches).asLong();
int batchSize = execution.Config.batchSize();
return(Weighted(execution.StageName, doneBatches * batchSize));
}
private void saturate(StageExecution execution)
{
if (processors == 0)
{
return;
}
Random random = ThreadLocalRandom.current();
int maxThisCheck = random.Next(processors - 1) + 1;
//JAVA TO C# CONVERTER WARNING: Java wildcard generics have no direct equivalent in .NET:
//ORIGINAL LINE: for (Step<?> step : execution.steps())
foreach (Step <object> step in execution.Steps())
{
int before = step.Processors(0);
if (random.nextBoolean() && step.Processors(-1) < before)
{
processors--;
if (--maxThisCheck == 0)
{
return;
}
}
}
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: private void shouldBeAbleToProgressUnderStressfulProcessorChanges(int orderingGuarantees) throws Exception
private void ShouldBeAbleToProgressUnderStressfulProcessorChanges(int orderingGuarantees)
{
// given
int batches = 100;
int processors = Runtime.Runtime.availableProcessors() * 10;
Configuration config = new Configuration_OverriddenAnonymousInnerClass(this, Configuration.DEFAULT, processors);
Stage stage = new StressStage(config, orderingGuarantees, batches);
StageExecution execution = stage.Execute();
//JAVA TO C# CONVERTER WARNING: Java wildcard generics have no direct equivalent in .NET:
//ORIGINAL LINE: java.util.List<Step<?>> steps = asList(execution.steps());
IList <Step <object> > steps = new IList <Step <object> > {
execution.Steps()
};
steps[1].Processors(processors / 3);
// when
ThreadLocalRandom random = ThreadLocalRandom.current();
while (execution.StillExecuting())
{
steps[2].Processors(random.Next(-2, 5));
Thread.Sleep(1);
}
execution.AssertHealthy();
// then
assertEquals(batches, steps[steps.Count - 1].Stats().stat(Keys.done_batches).asLong());
}
private long DoneBatches(StageExecution execution)
{
//JAVA TO C# CONVERTER WARNING: Java wildcard generics have no direct equivalent in .NET:
//ORIGINAL LINE: Step<?> step = org.neo4j.helpers.collection.Iterables.last(execution.steps());
Step <object> step = Iterables.last(execution.Steps());
return(step.Stats().stat(Keys.done_batches).asLong());
}
public static void PrintSpectrum(StringBuilder builder, StageExecution execution, int width, DetailLevel additionalStatsLevel)
{
long[] values = values(execution);
long total = total(values);
// reduce the width with the known extra characters we know we'll print in and around the spectrum
width -= 2 + PROGRESS_WIDTH;
//JAVA TO C# CONVERTER WARNING: Java wildcard generics have no direct equivalent in .NET:
//ORIGINAL LINE: org.neo4j.helpers.collection.Pair<Step<?>,float> bottleNeck = execution.stepsOrderedBy(org.neo4j.unsafe.impl.batchimport.stats.Keys.avg_processing_time, false).iterator().next();
Pair <Step <object>, float> bottleNeck = execution.StepsOrderedBy(Keys.avg_processing_time, false).GetEnumerator().next();
QuantizedProjection projection = new QuantizedProjection(total, width);
long lastDoneBatches = 0;
int stepIndex = 0;
bool hasProgressed = false;
builder.Append('[');
//JAVA TO C# CONVERTER WARNING: Java wildcard generics have no direct equivalent in .NET:
//ORIGINAL LINE: for (Step<?> step : execution.steps())
foreach (Step <object> step in execution.Steps())
{
StepStats stats = step.Stats();
if (!projection.Next(values[stepIndex]))
{
break; // odd though
}
long stepWidth = total == 0 && stepIndex == 0 ? width : projection.Step();
if (stepWidth > 0)
{
if (hasProgressed)
{
stepWidth--;
builder.Append('|');
}
bool isBottleNeck = bottleNeck.First() == step;
string name = (isBottleNeck ? "*" : "") + stats.ToString(additionalStatsLevel) + (step.Processors(0) > 1 ? "(" + step.Processors(0) + ")" : "");
int charIndex = 0; // negative value "delays" the text, i.e. pushes it to the right
char backgroundChar = step.Processors(0) > 1 ? '=' : '-';
for (int i = 0; i < stepWidth; i++, charIndex++)
{
char ch = backgroundChar;
if (charIndex >= 0 && charIndex < name.Length && charIndex < stepWidth)
{
ch = name[charIndex];
}
builder.Append(ch);
}
hasProgressed = true;
}
lastDoneBatches = stats.Stat(Keys.done_batches).asLong();
stepIndex++;
}
long progress = lastDoneBatches * execution.Config.batchSize();
builder.Append("]").Append(FitInProgress(progress));
}
protected internal virtual void RegisterProcessorCount(StageExecution execution)
{
IDictionary <string, int> byStage = new Dictionary <string, int>();
Processors[execution.Name()] = byStage;
//JAVA TO C# CONVERTER WARNING: Java wildcard generics have no direct equivalent in .NET:
//ORIGINAL LINE: for (Step<?> step : execution.steps())
foreach (Step <object> step in execution.Steps())
{
byStage[step.Name()] = step.Processors(0);
}
}
private static long[] Values(StageExecution execution)
{
long[] values = new long[execution.Size()];
int i = 0;
//JAVA TO C# CONVERTER WARNING: Java wildcard generics have no direct equivalent in .NET:
//ORIGINAL LINE: for (Step<?> step : execution.steps())
foreach (Step <object> step in execution.Steps())
{
values[i++] = Avg(step.Stats());
}
return(values);
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void shouldReceiveBatchesInOrder()
public virtual void ShouldReceiveBatchesInOrder()
{
// GIVEN
Configuration config = new Configuration_OverriddenAnonymousInnerClass(this, DEFAULT);
Stage stage = new Stage("Test stage", null, config, ORDER_SEND_DOWNSTREAM);
long batches = 1000;
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final long items = batches * config.batchSize();
long items = batches * config.BatchSize();
stage.Add(new PullingProducerStepAnonymousInnerClass(this, stage.Control(), config, items));
for (int i = 0; i < 3; i++)
{
stage.Add(new ReceiveOrderAssertingStep(stage.Control(), "Step" + i, config, i, false));
}
stage.Add(new ReceiveOrderAssertingStep(stage.Control(), "Final step", config, 0, true));
// WHEN
StageExecution execution = stage.Execute();
//JAVA TO C# CONVERTER WARNING: Java wildcard generics have no direct equivalent in .NET:
//ORIGINAL LINE: for (Step<?> step : execution.steps())
foreach (Step <object> step in execution.Steps())
{
// we start off with two in each step
step.Processors(1);
}
(new ExecutionSupervisor(ExecutionMonitors.Invisible())).supervise(execution);
// THEN
//JAVA TO C# CONVERTER WARNING: Java wildcard generics have no direct equivalent in .NET:
//ORIGINAL LINE: for (Step<?> step : execution.steps())
foreach (Step <object> step in execution.Steps())
{
assertEquals("For " + step, batches, step.Stats().stat(Keys.done_batches).asLong());
}
stage.Close();
}
public override void Check(StageExecution execution)
{
StringBuilder builder = new StringBuilder();
PrintSpectrum(builder, execution, _width, DetailLevel.IMPORTANT);
// add delta
long progress = last(execution.Steps()).stats().stat(Keys.done_batches).asLong() * execution.Config.batchSize();
long currentDelta = progress - _lastProgress;
builder.Append(" ∆").Append(FitInProgress(currentDelta));
// and remember progress to compare with next check
_lastProgress = progress;
// print it (overwriting the previous contents on this console line)
@out.print("\r" + builder);
}
//JAVA TO C# CONVERTER WARNING: 'final' parameters are ignored unless the option to convert to C# 7.2 'in' parameters is selected:
//ORIGINAL LINE: private void saturate(final int availableProcessor, StageExecution execution)
private void saturate(int availableProcessor, StageExecution execution)
{
Random random = ThreadLocalRandom.current();
int processors = availableProcessor;
for (int rounds = 0; rounds < availableProcessor && processors > 0; rounds++)
{
//JAVA TO C# CONVERTER WARNING: Java wildcard generics have no direct equivalent in .NET:
//ORIGINAL LINE: for (Step<?> step : execution.steps())
foreach (Step <object> step in execution.Steps())
{
int before = step.Processors(0);
if (random.nextBoolean() && step.Processors(1) > before && --processors == 0)
{
return;
}
}
}
}
private int CountActiveProcessors(StageExecution execution)
{
float processors = 0;
if (execution.StillExecuting())
{
long highestAverage = Avg(execution.StepsOrderedBy(Keys.avg_processing_time, false).GetEnumerator().next().first());
//JAVA TO C# CONVERTER WARNING: Java wildcard generics have no direct equivalent in .NET:
//ORIGINAL LINE: for (Step<?> step : execution.steps())
foreach (Step <object> step in execution.Steps())
{
// Calculate how active each step is so that a step that is very cheap
// and idles a lot counts for less than 1 processor, so that bottlenecks can
// "steal" some of its processing power.
long avg = avg(step);
float factor = ( float )avg / ( float )highestAverage;
processors += factor * step.Processors(0);
}
}
return(( int )Math.Round(processors, MidpointRounding.AwayFromZero));
}
