/// <exception cref="System.IO.IOException"/> public virtual void TestGetMapCompletionEvents() { TaskAttemptCompletionEvent[] empty = new TaskAttemptCompletionEvent[] { }; TaskAttemptCompletionEvent[] taskEvents = new TaskAttemptCompletionEvent[] { CreateTce (0, true, TaskAttemptCompletionEventStatus.Obsolete), CreateTce(1, false, TaskAttemptCompletionEventStatus .Failed), CreateTce(2, true, TaskAttemptCompletionEventStatus.Succeeded), CreateTce (3, false, TaskAttemptCompletionEventStatus.Failed) }; TaskAttemptCompletionEvent[] mapEvents = new TaskAttemptCompletionEvent[] { taskEvents [0], taskEvents[2] }; Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job mockJob = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job >(); Org.Mockito.Mockito.When(mockJob.GetTaskAttemptCompletionEvents(0, 100)).ThenReturn (taskEvents); Org.Mockito.Mockito.When(mockJob.GetTaskAttemptCompletionEvents(0, 2)).ThenReturn (Arrays.CopyOfRange(taskEvents, 0, 2)); Org.Mockito.Mockito.When(mockJob.GetTaskAttemptCompletionEvents(2, 100)).ThenReturn (Arrays.CopyOfRange(taskEvents, 2, 4)); Org.Mockito.Mockito.When(mockJob.GetMapAttemptCompletionEvents(0, 100)).ThenReturn (TypeConverter.FromYarn(mapEvents)); Org.Mockito.Mockito.When(mockJob.GetMapAttemptCompletionEvents(0, 2)).ThenReturn( TypeConverter.FromYarn(mapEvents)); Org.Mockito.Mockito.When(mockJob.GetMapAttemptCompletionEvents(2, 100)).ThenReturn (TypeConverter.FromYarn(empty)); AppContext appCtx = Org.Mockito.Mockito.Mock <AppContext>(); Org.Mockito.Mockito.When(appCtx.GetJob(Matchers.Any <JobId>())).ThenReturn(mockJob ); JobTokenSecretManager secret = Org.Mockito.Mockito.Mock <JobTokenSecretManager>(); RMHeartbeatHandler rmHeartbeatHandler = Org.Mockito.Mockito.Mock <RMHeartbeatHandler >(); TaskHeartbeatHandler hbHandler = Org.Mockito.Mockito.Mock <TaskHeartbeatHandler>(); TaskAttemptListenerImpl listener = new _MockTaskAttemptListenerImpl_200(hbHandler , appCtx, secret, rmHeartbeatHandler); Configuration conf = new Configuration(); listener.Init(conf); listener.Start(); JobID jid = new JobID("12345", 1); TaskAttemptID tid = new TaskAttemptID("12345", 1, TaskType.Reduce, 1, 0); MapTaskCompletionEventsUpdate update = listener.GetMapCompletionEvents(jid, 0, 100 , tid); NUnit.Framework.Assert.AreEqual(2, update.events.Length); update = listener.GetMapCompletionEvents(jid, 0, 2, tid); NUnit.Framework.Assert.AreEqual(2, update.events.Length); update = listener.GetMapCompletionEvents(jid, 2, 100, tid); NUnit.Framework.Assert.AreEqual(0, update.events.Length); }
public virtual void TestFetchFailureMultipleReduces() { MRApp app = new MRApp(1, 3, false, this.GetType().FullName, true); Configuration conf = new Configuration(); // map -> reduce -> fetch-failure -> map retry is incompatible with // sequential, single-task-attempt approach in uber-AM, so disable: conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf); app.WaitForState(job, JobState.Running); //all maps would be running NUnit.Framework.Assert.AreEqual("Num tasks not correct", 4, job.GetTasks().Count); IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator(); Task mapTask = it.Next(); Task reduceTask = it.Next(); Task reduceTask2 = it.Next(); Task reduceTask3 = it.Next(); //wait for Task state move to RUNNING app.WaitForState(mapTask, TaskState.Running); TaskAttempt mapAttempt1 = mapTask.GetAttempts().Values.GetEnumerator().Next(); app.WaitForState(mapAttempt1, TaskAttemptState.Running); //send the done signal to the map attempt app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(mapAttempt1.GetID( ), TaskAttemptEventType.TaDone)); // wait for map success app.WaitForState(mapTask, TaskState.Succeeded); TaskAttemptCompletionEvent[] events = job.GetTaskAttemptCompletionEvents(0, 100); NUnit.Framework.Assert.AreEqual("Num completion events not correct", 1, events.Length ); NUnit.Framework.Assert.AreEqual("Event status not correct", TaskAttemptCompletionEventStatus .Succeeded, events[0].GetStatus()); // wait for reduce to start running app.WaitForState(reduceTask, TaskState.Running); app.WaitForState(reduceTask2, TaskState.Running); app.WaitForState(reduceTask3, TaskState.Running); TaskAttempt reduceAttempt = reduceTask.GetAttempts().Values.GetEnumerator().Next( ); app.WaitForState(reduceAttempt, TaskAttemptState.Running); UpdateStatus(app, reduceAttempt, Phase.Shuffle); TaskAttempt reduceAttempt2 = reduceTask2.GetAttempts().Values.GetEnumerator().Next (); app.WaitForState(reduceAttempt2, TaskAttemptState.Running); UpdateStatus(app, reduceAttempt2, Phase.Shuffle); TaskAttempt reduceAttempt3 = reduceTask3.GetAttempts().Values.GetEnumerator().Next (); app.WaitForState(reduceAttempt3, TaskAttemptState.Running); UpdateStatus(app, reduceAttempt3, Phase.Shuffle); //send 2 fetch failures from reduce to prepare for map re execution SendFetchFailure(app, reduceAttempt, mapAttempt1); SendFetchFailure(app, reduceAttempt, mapAttempt1); //We should not re-launch the map task yet NUnit.Framework.Assert.AreEqual(TaskState.Succeeded, mapTask.GetState()); UpdateStatus(app, reduceAttempt2, Phase.Reduce); UpdateStatus(app, reduceAttempt3, Phase.Reduce); //send 3rd fetch failures from reduce to trigger map re execution SendFetchFailure(app, reduceAttempt, mapAttempt1); //wait for map Task state move back to RUNNING app.WaitForState(mapTask, TaskState.Running); //map attempt must have become FAILED NUnit.Framework.Assert.AreEqual("Map TaskAttempt state not correct", TaskAttemptState .Failed, mapAttempt1.GetState()); NUnit.Framework.Assert.AreEqual("Num attempts in Map Task not correct", 2, mapTask .GetAttempts().Count); IEnumerator <TaskAttempt> atIt = mapTask.GetAttempts().Values.GetEnumerator(); atIt.Next(); TaskAttempt mapAttempt2 = atIt.Next(); app.WaitForState(mapAttempt2, TaskAttemptState.Running); //send the done signal to the second map attempt app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(mapAttempt2.GetID( ), TaskAttemptEventType.TaDone)); // wait for map success app.WaitForState(mapTask, TaskState.Succeeded); //send done to reduce app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(reduceAttempt.GetID (), TaskAttemptEventType.TaDone)); //send done to reduce app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(reduceAttempt2.GetID (), TaskAttemptEventType.TaDone)); //send done to reduce app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(reduceAttempt3.GetID (), TaskAttemptEventType.TaDone)); app.WaitForState(job, JobState.Succeeded); //previous completion event now becomes obsolete NUnit.Framework.Assert.AreEqual("Event status not correct", TaskAttemptCompletionEventStatus .Obsolete, events[0].GetStatus()); events = job.GetTaskAttemptCompletionEvents(0, 100); NUnit.Framework.Assert.AreEqual("Num completion events not correct", 6, events.Length ); NUnit.Framework.Assert.AreEqual("Event map attempt id not correct", mapAttempt1.GetID (), events[0].GetAttemptId()); NUnit.Framework.Assert.AreEqual("Event map attempt id not correct", mapAttempt1.GetID (), events[1].GetAttemptId()); NUnit.Framework.Assert.AreEqual("Event map attempt id not correct", mapAttempt2.GetID (), events[2].GetAttemptId()); NUnit.Framework.Assert.AreEqual("Event reduce attempt id not correct", reduceAttempt .GetID(), events[3].GetAttemptId()); NUnit.Framework.Assert.AreEqual("Event status not correct for map attempt1", TaskAttemptCompletionEventStatus .Obsolete, events[0].GetStatus()); NUnit.Framework.Assert.AreEqual("Event status not correct for map attempt1", TaskAttemptCompletionEventStatus .Failed, events[1].GetStatus()); NUnit.Framework.Assert.AreEqual("Event status not correct for map attempt2", TaskAttemptCompletionEventStatus .Succeeded, events[2].GetStatus()); NUnit.Framework.Assert.AreEqual("Event status not correct for reduce attempt1", TaskAttemptCompletionEventStatus .Succeeded, events[3].GetStatus()); TaskCompletionEvent[] mapEvents = job.GetMapAttemptCompletionEvents(0, 2); TaskCompletionEvent[] convertedEvents = TypeConverter.FromYarn(events); NUnit.Framework.Assert.AreEqual("Incorrect number of map events", 2, mapEvents.Length ); Assert.AssertArrayEquals("Unexpected map events", Arrays.CopyOfRange(convertedEvents , 0, 2), mapEvents); mapEvents = job.GetMapAttemptCompletionEvents(2, 200); NUnit.Framework.Assert.AreEqual("Incorrect number of map events", 1, mapEvents.Length ); NUnit.Framework.Assert.AreEqual("Unexpected map event", convertedEvents[2], mapEvents [0]); }
public override TaskCompletionEvent[] GetMapAttemptCompletionEvents(int startIndex , int maxEvents) { return(job.GetMapAttemptCompletionEvents(startIndex, maxEvents)); }
public virtual TaskCompletionEvent[] GetMapAttemptCompletionEvents(int startIndex , int maxEvents) { return(mockJob.GetMapAttemptCompletionEvents(startIndex, maxEvents)); }