internal virtual void StartUp()
{
this.startMockTime = this._enclosing.clock.GetTime();
this.overridingState = null;
this._enclosing.slotsInUse.AddAndGet(this._enclosing.TaskTypeSlots(this.myAttemptID
.GetTaskId().GetTaskType()));
System.Console.Out.WriteLine("TLTRE.MyTaskAttemptImpl.startUp starting " + this.GetID
());
SpeculatorEvent @event = new SpeculatorEvent(this.GetID().GetTaskId(), -1);
this._enclosing.dispatcher.GetEventHandler().Handle(@event);
}
internal virtual void AddAttempt()
{
TaskAttempt taskAttempt = new TestRuntimeEstimators.MyTaskAttemptImpl(this, this.
taskID, this.attempts.Count, this._enclosing.clock);
TaskAttemptId taskAttemptID = taskAttempt.GetID();
this.attempts[taskAttemptID] = taskAttempt;
System.Console.Out.WriteLine("TLTRE.MyTaskImpl.addAttempt " + this.GetID());
SpeculatorEvent @event = new SpeculatorEvent(this.taskID, +1);
this._enclosing.dispatcher.GetEventHandler().Handle(@event);
}
// this has to be at least as much as map slot requirement
// This is a huge kluge. The real implementations have a decent approach
private void CoreTestEstimator(TaskRuntimeEstimator testedEstimator, int expectedSpeculations
)
{
estimator = testedEstimator;
clock = new TestRuntimeEstimators.MockClock();
dispatcher = new AsyncDispatcher();
myJob = null;
slotsInUse.Set(0);
completedMaps.Set(0);
completedReduces.Set(0);
successfulSpeculations.Set(0);
taskTimeSavedBySpeculation.Set(0);
clock.AdvanceTime(1000);
Configuration conf = new Configuration();
myAppContext = new TestRuntimeEstimators.MyAppContext(this, MapTasks, ReduceTasks
);
myJob = myAppContext.GetAllJobs().Values.GetEnumerator().Next();
estimator.Contextualize(conf, myAppContext);
conf.SetLong(MRJobConfig.SpeculativeRetryAfterNoSpeculate, 500L);
conf.SetLong(MRJobConfig.SpeculativeRetryAfterSpeculate, 5000L);
conf.SetDouble(MRJobConfig.SpeculativecapRunningTasks, 0.1);
conf.SetDouble(MRJobConfig.SpeculativecapTotalTasks, 0.001);
conf.SetInt(MRJobConfig.SpeculativeMinimumAllowedTasks, 5);
speculator = new DefaultSpeculator(conf, myAppContext, estimator, clock);
NUnit.Framework.Assert.AreEqual("wrong SPECULATIVE_RETRY_AFTER_NO_SPECULATE value"
, 500L, speculator.GetSoonestRetryAfterNoSpeculate());
NUnit.Framework.Assert.AreEqual("wrong SPECULATIVE_RETRY_AFTER_SPECULATE value",
5000L, speculator.GetSoonestRetryAfterSpeculate());
NUnit.Framework.Assert.AreEqual(speculator.GetProportionRunningTasksSpeculatable(
), 0.1, 0.00001);
NUnit.Framework.Assert.AreEqual(speculator.GetProportionTotalTasksSpeculatable(),
0.001, 0.00001);
NUnit.Framework.Assert.AreEqual("wrong SPECULATIVE_MINIMUM_ALLOWED_TASKS value",
5, speculator.GetMinimumAllowedSpeculativeTasks());
dispatcher.Register(typeof(Speculator.EventType), speculator);
dispatcher.Register(typeof(TaskEventType), new TestRuntimeEstimators.SpeculationRequestEventHandler
(this));
dispatcher.Init(conf);
dispatcher.Start();
speculator.Init(conf);
speculator.Start();
// Now that the plumbing is hooked up, we do the following:
// do until all tasks are finished, ...
// 1: If we have spare capacity, assign as many map tasks as we can, then
// assign as many reduce tasks as we can. Note that an odd reduce
// task might be started while there are still map tasks, because
// map tasks take 3 slots and reduce tasks 2 slots.
// 2: Send a speculation event for every task attempt that's running
// note that new attempts might get started by the speculator
// discover undone tasks
int undoneMaps = MapTasks;
int undoneReduces = ReduceTasks;
// build a task sequence where all the maps precede any of the reduces
IList <Task> allTasksSequence = new List <Task>();
Sharpen.Collections.AddAll(allTasksSequence, myJob.GetTasks(TaskType.Map).Values);
Sharpen.Collections.AddAll(allTasksSequence, myJob.GetTasks(TaskType.Reduce).Values
);
while (undoneMaps + undoneReduces > 0)
{
undoneMaps = 0;
undoneReduces = 0;
// start all attempts which are new but for which there is enough slots
foreach (Task task in allTasksSequence)
{
if (!task.IsFinished())
{
if (task.GetType() == TaskType.Map)
{
++undoneMaps;
}
else
{
++undoneReduces;
}
}
foreach (TaskAttempt attempt in task.GetAttempts().Values)
{
if (attempt.GetState() == TaskAttemptState.New && InitialNumberFreeSlots - slotsInUse
.Get() >= TaskTypeSlots(task.GetType()))
{
TestRuntimeEstimators.MyTaskAttemptImpl attemptImpl = (TestRuntimeEstimators.MyTaskAttemptImpl
)attempt;
SpeculatorEvent @event = new SpeculatorEvent(attempt.GetID(), false, clock.GetTime
());
speculator.Handle(@event);
attemptImpl.StartUp();
}
else
{
// If a task attempt is in progress we should send the news to
// the Speculator.
TaskAttemptStatusUpdateEvent.TaskAttemptStatus status = new TaskAttemptStatusUpdateEvent.TaskAttemptStatus
();
status.id = attempt.GetID();
status.progress = attempt.GetProgress();
status.stateString = attempt.GetState().ToString();
status.taskState = attempt.GetState();
SpeculatorEvent @event = new SpeculatorEvent(status, clock.GetTime());
speculator.Handle(@event);
}
}
}
long startTime = Runtime.CurrentTimeMillis();
// drain the speculator event queue
while (!speculator.EventQueueEmpty())
{
Sharpen.Thread.Yield();
if (Runtime.CurrentTimeMillis() > startTime + 130000)
{
return;
}
}
clock.AdvanceTime(1000L);
if (clock.GetTime() % 10000L == 0L)
{
speculator.ScanForSpeculations();
}
}
NUnit.Framework.Assert.AreEqual("We got the wrong number of successful speculations."
, expectedSpeculations, successfulSpeculations.Get());
}
