public static NodeReport NewNodeReport(NodeId nodeId, NodeState nodeState, string httpAddress, string rackName, Resource used, Resource capability, int numContainers , string healthReport, long lastHealthReportTime, ICollection <string> nodeLabels ) { NodeReport nodeReport = recordFactory.NewRecordInstance <NodeReport>(); nodeReport.SetNodeId(nodeId); nodeReport.SetNodeState(nodeState); nodeReport.SetHttpAddress(httpAddress); nodeReport.SetRackName(rackName); nodeReport.SetUsed(used); nodeReport.SetCapability(capability); nodeReport.SetNumContainers(numContainers); nodeReport.SetHealthReport(healthReport); nodeReport.SetLastHealthReportTime(lastHealthReportTime); nodeReport.SetNodeLabels(nodeLabels); return(nodeReport); }
public virtual void TestUpdatedNodes() { int runCount = 0; MRApp app = new TestMRApp.MRAppWithHistory(this, 2, 2, false, this.GetType().FullName , true, ++runCount); Configuration conf = new Configuration(); // after half of the map completion, reduce will start conf.SetFloat(MRJobConfig.CompletedMapsForReduceSlowstart, 0.5f); // uberization forces full slowstart (1.0), so disable that conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf); app.WaitForState(job, JobState.Running); NUnit.Framework.Assert.AreEqual("Num tasks not correct", 4, job.GetTasks().Count); IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator(); Task mapTask1 = it.Next(); Task mapTask2 = it.Next(); // all maps must be running app.WaitForState(mapTask1, TaskState.Running); app.WaitForState(mapTask2, TaskState.Running); TaskAttempt task1Attempt = mapTask1.GetAttempts().Values.GetEnumerator().Next(); TaskAttempt task2Attempt = mapTask2.GetAttempts().Values.GetEnumerator().Next(); NodeId node1 = task1Attempt.GetNodeId(); NodeId node2 = task2Attempt.GetNodeId(); NUnit.Framework.Assert.AreEqual(node1, node2); // send the done signal to the task app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task1Attempt.GetID (), TaskAttemptEventType.TaDone)); app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task2Attempt.GetID (), TaskAttemptEventType.TaDone)); // all maps must be succeeded app.WaitForState(mapTask1, TaskState.Succeeded); app.WaitForState(mapTask2, TaskState.Succeeded); TaskAttemptCompletionEvent[] events = job.GetTaskAttemptCompletionEvents(0, 100); NUnit.Framework.Assert.AreEqual("Expecting 2 completion events for success", 2, events .Length); // send updated nodes info AList <NodeReport> updatedNodes = new AList <NodeReport>(); NodeReport nr = RecordFactoryProvider.GetRecordFactory(null).NewRecordInstance <NodeReport >(); nr.SetNodeId(node1); nr.SetNodeState(NodeState.Unhealthy); updatedNodes.AddItem(nr); app.GetContext().GetEventHandler().Handle(new JobUpdatedNodesEvent(job.GetID(), updatedNodes )); app.WaitForState(task1Attempt, TaskAttemptState.Killed); app.WaitForState(task2Attempt, TaskAttemptState.Killed); events = job.GetTaskAttemptCompletionEvents(0, 100); NUnit.Framework.Assert.AreEqual("Expecting 2 more completion events for killed", 4, events.Length); // all maps must be back to running app.WaitForState(mapTask1, TaskState.Running); app.WaitForState(mapTask2, TaskState.Running); IEnumerator <TaskAttempt> itr = mapTask1.GetAttempts().Values.GetEnumerator(); itr.Next(); task1Attempt = itr.Next(); // send the done signal to the task app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task1Attempt.GetID (), TaskAttemptEventType.TaDone)); // map1 must be succeeded. map2 must be running app.WaitForState(mapTask1, TaskState.Succeeded); app.WaitForState(mapTask2, TaskState.Running); events = job.GetTaskAttemptCompletionEvents(0, 100); NUnit.Framework.Assert.AreEqual("Expecting 1 more completion events for success", 5, events.Length); // Crash the app again. app.Stop(); // rerun // in rerun the 1st map will be recovered from previous run app = new TestMRApp.MRAppWithHistory(this, 2, 2, false, this.GetType().FullName, false, ++runCount); conf = new Configuration(); conf.SetBoolean(MRJobConfig.MrAmJobRecoveryEnable, true); conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false); job = app.Submit(conf); app.WaitForState(job, JobState.Running); NUnit.Framework.Assert.AreEqual("No of tasks not correct", 4, job.GetTasks().Count ); it = job.GetTasks().Values.GetEnumerator(); mapTask1 = it.Next(); mapTask2 = it.Next(); Task reduceTask1 = it.Next(); Task reduceTask2 = it.Next(); // map 1 will be recovered, no need to send done app.WaitForState(mapTask1, TaskState.Succeeded); app.WaitForState(mapTask2, TaskState.Running); events = job.GetTaskAttemptCompletionEvents(0, 100); NUnit.Framework.Assert.AreEqual("Expecting 2 completion events for killed & success of map1" , 2, events.Length); task2Attempt = mapTask2.GetAttempts().Values.GetEnumerator().Next(); app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task2Attempt.GetID (), TaskAttemptEventType.TaDone)); app.WaitForState(mapTask2, TaskState.Succeeded); events = job.GetTaskAttemptCompletionEvents(0, 100); NUnit.Framework.Assert.AreEqual("Expecting 1 more completion events for success", 3, events.Length); app.WaitForState(reduceTask1, TaskState.Running); app.WaitForState(reduceTask2, TaskState.Running); TaskAttempt task3Attempt = reduceTask1.GetAttempts().Values.GetEnumerator().Next( ); app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task3Attempt.GetID (), TaskAttemptEventType.TaDone)); app.WaitForState(reduceTask1, TaskState.Succeeded); app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task3Attempt.GetID (), TaskAttemptEventType.TaKill)); app.WaitForState(reduceTask1, TaskState.Succeeded); TaskAttempt task4Attempt = reduceTask2.GetAttempts().Values.GetEnumerator().Next( ); app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task4Attempt.GetID (), TaskAttemptEventType.TaDone)); app.WaitForState(reduceTask2, TaskState.Succeeded); events = job.GetTaskAttemptCompletionEvents(0, 100); NUnit.Framework.Assert.AreEqual("Expecting 2 more completion events for reduce success" , 5, events.Length); // job succeeds app.WaitForState(job, JobState.Succeeded); }