public virtual void TestUpdatedNodes()
{
int runCount = 0;
MRApp app = new TestMRApp.MRAppWithHistory(this, 2, 2, false, this.GetType().FullName
, true, ++runCount);
Configuration conf = new Configuration();
// after half of the map completion, reduce will start
conf.SetFloat(MRJobConfig.CompletedMapsForReduceSlowstart, 0.5f);
// uberization forces full slowstart (1.0), so disable that
conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf);
app.WaitForState(job, JobState.Running);
NUnit.Framework.Assert.AreEqual("Num tasks not correct", 4, job.GetTasks().Count);
IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator();
Task mapTask1 = it.Next();
Task mapTask2 = it.Next();
// all maps must be running
app.WaitForState(mapTask1, TaskState.Running);
app.WaitForState(mapTask2, TaskState.Running);
TaskAttempt task1Attempt = mapTask1.GetAttempts().Values.GetEnumerator().Next();
TaskAttempt task2Attempt = mapTask2.GetAttempts().Values.GetEnumerator().Next();
NodeId node1 = task1Attempt.GetNodeId();
NodeId node2 = task2Attempt.GetNodeId();
NUnit.Framework.Assert.AreEqual(node1, node2);
// send the done signal to the task
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task1Attempt.GetID
(), TaskAttemptEventType.TaDone));
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task2Attempt.GetID
(), TaskAttemptEventType.TaDone));
// all maps must be succeeded
app.WaitForState(mapTask1, TaskState.Succeeded);
app.WaitForState(mapTask2, TaskState.Succeeded);
TaskAttemptCompletionEvent[] events = job.GetTaskAttemptCompletionEvents(0, 100);
NUnit.Framework.Assert.AreEqual("Expecting 2 completion events for success", 2, events
.Length);
// send updated nodes info
AList <NodeReport> updatedNodes = new AList <NodeReport>();
NodeReport nr = RecordFactoryProvider.GetRecordFactory(null).NewRecordInstance <NodeReport
>();
nr.SetNodeId(node1);
nr.SetNodeState(NodeState.Unhealthy);
updatedNodes.AddItem(nr);
app.GetContext().GetEventHandler().Handle(new JobUpdatedNodesEvent(job.GetID(), updatedNodes
));
app.WaitForState(task1Attempt, TaskAttemptState.Killed);
app.WaitForState(task2Attempt, TaskAttemptState.Killed);
events = job.GetTaskAttemptCompletionEvents(0, 100);
NUnit.Framework.Assert.AreEqual("Expecting 2 more completion events for killed",
4, events.Length);
// all maps must be back to running
app.WaitForState(mapTask1, TaskState.Running);
app.WaitForState(mapTask2, TaskState.Running);
IEnumerator <TaskAttempt> itr = mapTask1.GetAttempts().Values.GetEnumerator();
itr.Next();
task1Attempt = itr.Next();
// send the done signal to the task
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task1Attempt.GetID
(), TaskAttemptEventType.TaDone));
// map1 must be succeeded. map2 must be running
app.WaitForState(mapTask1, TaskState.Succeeded);
app.WaitForState(mapTask2, TaskState.Running);
events = job.GetTaskAttemptCompletionEvents(0, 100);
NUnit.Framework.Assert.AreEqual("Expecting 1 more completion events for success",
5, events.Length);
// Crash the app again.
app.Stop();
// rerun
// in rerun the 1st map will be recovered from previous run
app = new TestMRApp.MRAppWithHistory(this, 2, 2, false, this.GetType().FullName,
false, ++runCount);
conf = new Configuration();
conf.SetBoolean(MRJobConfig.MrAmJobRecoveryEnable, true);
conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false);
job = app.Submit(conf);
app.WaitForState(job, JobState.Running);
NUnit.Framework.Assert.AreEqual("No of tasks not correct", 4, job.GetTasks().Count
);
it = job.GetTasks().Values.GetEnumerator();
mapTask1 = it.Next();
mapTask2 = it.Next();
Task reduceTask1 = it.Next();
Task reduceTask2 = it.Next();
// map 1 will be recovered, no need to send done
app.WaitForState(mapTask1, TaskState.Succeeded);
app.WaitForState(mapTask2, TaskState.Running);
events = job.GetTaskAttemptCompletionEvents(0, 100);
NUnit.Framework.Assert.AreEqual("Expecting 2 completion events for killed & success of map1"
, 2, events.Length);
task2Attempt = mapTask2.GetAttempts().Values.GetEnumerator().Next();
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task2Attempt.GetID
(), TaskAttemptEventType.TaDone));
app.WaitForState(mapTask2, TaskState.Succeeded);
events = job.GetTaskAttemptCompletionEvents(0, 100);
NUnit.Framework.Assert.AreEqual("Expecting 1 more completion events for success",
3, events.Length);
app.WaitForState(reduceTask1, TaskState.Running);
app.WaitForState(reduceTask2, TaskState.Running);
TaskAttempt task3Attempt = reduceTask1.GetAttempts().Values.GetEnumerator().Next(
);
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task3Attempt.GetID
(), TaskAttemptEventType.TaDone));
app.WaitForState(reduceTask1, TaskState.Succeeded);
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task3Attempt.GetID
(), TaskAttemptEventType.TaKill));
app.WaitForState(reduceTask1, TaskState.Succeeded);
TaskAttempt task4Attempt = reduceTask2.GetAttempts().Values.GetEnumerator().Next(
);
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task4Attempt.GetID
(), TaskAttemptEventType.TaDone));
app.WaitForState(reduceTask2, TaskState.Succeeded);
events = job.GetTaskAttemptCompletionEvents(0, 100);
NUnit.Framework.Assert.AreEqual("Expecting 2 more completion events for reduce success"
, 5, events.Length);
// job succeeds
app.WaitForState(job, JobState.Succeeded);
}
