/// <exception cref="System.IO.IOException"/>
public virtual void TestGetMapCompletionEvents()
{
TaskAttemptCompletionEvent[] empty = new TaskAttemptCompletionEvent[] { };
TaskAttemptCompletionEvent[] taskEvents = new TaskAttemptCompletionEvent[] { CreateTce
(0, true, TaskAttemptCompletionEventStatus.Obsolete), CreateTce(1, false, TaskAttemptCompletionEventStatus
.Failed), CreateTce(2, true, TaskAttemptCompletionEventStatus.Succeeded), CreateTce
(3, false, TaskAttemptCompletionEventStatus.Failed) };
TaskAttemptCompletionEvent[] mapEvents = new TaskAttemptCompletionEvent[] { taskEvents
[0], taskEvents[2] };
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job mockJob = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
>();
Org.Mockito.Mockito.When(mockJob.GetTaskAttemptCompletionEvents(0, 100)).ThenReturn
(taskEvents);
Org.Mockito.Mockito.When(mockJob.GetTaskAttemptCompletionEvents(0, 2)).ThenReturn
(Arrays.CopyOfRange(taskEvents, 0, 2));
Org.Mockito.Mockito.When(mockJob.GetTaskAttemptCompletionEvents(2, 100)).ThenReturn
(Arrays.CopyOfRange(taskEvents, 2, 4));
Org.Mockito.Mockito.When(mockJob.GetMapAttemptCompletionEvents(0, 100)).ThenReturn
(TypeConverter.FromYarn(mapEvents));
Org.Mockito.Mockito.When(mockJob.GetMapAttemptCompletionEvents(0, 2)).ThenReturn(
TypeConverter.FromYarn(mapEvents));
Org.Mockito.Mockito.When(mockJob.GetMapAttemptCompletionEvents(2, 100)).ThenReturn
(TypeConverter.FromYarn(empty));
AppContext appCtx = Org.Mockito.Mockito.Mock <AppContext>();
Org.Mockito.Mockito.When(appCtx.GetJob(Matchers.Any <JobId>())).ThenReturn(mockJob
);
JobTokenSecretManager secret = Org.Mockito.Mockito.Mock <JobTokenSecretManager>();
RMHeartbeatHandler rmHeartbeatHandler = Org.Mockito.Mockito.Mock <RMHeartbeatHandler
>();
TaskHeartbeatHandler hbHandler = Org.Mockito.Mockito.Mock <TaskHeartbeatHandler>();
TaskAttemptListenerImpl listener = new _MockTaskAttemptListenerImpl_200(hbHandler
, appCtx, secret, rmHeartbeatHandler);
Configuration conf = new Configuration();
listener.Init(conf);
listener.Start();
JobID jid = new JobID("12345", 1);
TaskAttemptID tid = new TaskAttemptID("12345", 1, TaskType.Reduce, 1, 0);
MapTaskCompletionEventsUpdate update = listener.GetMapCompletionEvents(jid, 0, 100
, tid);
NUnit.Framework.Assert.AreEqual(2, update.events.Length);
update = listener.GetMapCompletionEvents(jid, 0, 2, tid);
NUnit.Framework.Assert.AreEqual(2, update.events.Length);
update = listener.GetMapCompletionEvents(jid, 2, 100, tid);
NUnit.Framework.Assert.AreEqual(0, update.events.Length);
}
public virtual void TestFetchFailureMultipleReduces()
{
MRApp app = new MRApp(1, 3, false, this.GetType().FullName, true);
Configuration conf = new Configuration();
// map -> reduce -> fetch-failure -> map retry is incompatible with
// sequential, single-task-attempt approach in uber-AM, so disable:
conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf);
app.WaitForState(job, JobState.Running);
//all maps would be running
NUnit.Framework.Assert.AreEqual("Num tasks not correct", 4, job.GetTasks().Count);
IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator();
Task mapTask = it.Next();
Task reduceTask = it.Next();
Task reduceTask2 = it.Next();
Task reduceTask3 = it.Next();
//wait for Task state move to RUNNING
app.WaitForState(mapTask, TaskState.Running);
TaskAttempt mapAttempt1 = mapTask.GetAttempts().Values.GetEnumerator().Next();
app.WaitForState(mapAttempt1, TaskAttemptState.Running);
//send the done signal to the map attempt
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(mapAttempt1.GetID(
), TaskAttemptEventType.TaDone));
// wait for map success
app.WaitForState(mapTask, TaskState.Succeeded);
TaskAttemptCompletionEvent[] events = job.GetTaskAttemptCompletionEvents(0, 100);
NUnit.Framework.Assert.AreEqual("Num completion events not correct", 1, events.Length
);
NUnit.Framework.Assert.AreEqual("Event status not correct", TaskAttemptCompletionEventStatus
.Succeeded, events[0].GetStatus());
// wait for reduce to start running
app.WaitForState(reduceTask, TaskState.Running);
app.WaitForState(reduceTask2, TaskState.Running);
app.WaitForState(reduceTask3, TaskState.Running);
TaskAttempt reduceAttempt = reduceTask.GetAttempts().Values.GetEnumerator().Next(
);
app.WaitForState(reduceAttempt, TaskAttemptState.Running);
UpdateStatus(app, reduceAttempt, Phase.Shuffle);
TaskAttempt reduceAttempt2 = reduceTask2.GetAttempts().Values.GetEnumerator().Next
();
app.WaitForState(reduceAttempt2, TaskAttemptState.Running);
UpdateStatus(app, reduceAttempt2, Phase.Shuffle);
TaskAttempt reduceAttempt3 = reduceTask3.GetAttempts().Values.GetEnumerator().Next
();
app.WaitForState(reduceAttempt3, TaskAttemptState.Running);
UpdateStatus(app, reduceAttempt3, Phase.Shuffle);
//send 2 fetch failures from reduce to prepare for map re execution
SendFetchFailure(app, reduceAttempt, mapAttempt1);
SendFetchFailure(app, reduceAttempt, mapAttempt1);
//We should not re-launch the map task yet
NUnit.Framework.Assert.AreEqual(TaskState.Succeeded, mapTask.GetState());
UpdateStatus(app, reduceAttempt2, Phase.Reduce);
UpdateStatus(app, reduceAttempt3, Phase.Reduce);
//send 3rd fetch failures from reduce to trigger map re execution
SendFetchFailure(app, reduceAttempt, mapAttempt1);
//wait for map Task state move back to RUNNING
app.WaitForState(mapTask, TaskState.Running);
//map attempt must have become FAILED
NUnit.Framework.Assert.AreEqual("Map TaskAttempt state not correct", TaskAttemptState
.Failed, mapAttempt1.GetState());
NUnit.Framework.Assert.AreEqual("Num attempts in Map Task not correct", 2, mapTask
.GetAttempts().Count);
IEnumerator <TaskAttempt> atIt = mapTask.GetAttempts().Values.GetEnumerator();
atIt.Next();
TaskAttempt mapAttempt2 = atIt.Next();
app.WaitForState(mapAttempt2, TaskAttemptState.Running);
//send the done signal to the second map attempt
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(mapAttempt2.GetID(
), TaskAttemptEventType.TaDone));
// wait for map success
app.WaitForState(mapTask, TaskState.Succeeded);
//send done to reduce
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(reduceAttempt.GetID
(), TaskAttemptEventType.TaDone));
//send done to reduce
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(reduceAttempt2.GetID
(), TaskAttemptEventType.TaDone));
//send done to reduce
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(reduceAttempt3.GetID
(), TaskAttemptEventType.TaDone));
app.WaitForState(job, JobState.Succeeded);
//previous completion event now becomes obsolete
NUnit.Framework.Assert.AreEqual("Event status not correct", TaskAttemptCompletionEventStatus
.Obsolete, events[0].GetStatus());
events = job.GetTaskAttemptCompletionEvents(0, 100);
NUnit.Framework.Assert.AreEqual("Num completion events not correct", 6, events.Length
);
NUnit.Framework.Assert.AreEqual("Event map attempt id not correct", mapAttempt1.GetID
(), events[0].GetAttemptId());
NUnit.Framework.Assert.AreEqual("Event map attempt id not correct", mapAttempt1.GetID
(), events[1].GetAttemptId());
NUnit.Framework.Assert.AreEqual("Event map attempt id not correct", mapAttempt2.GetID
(), events[2].GetAttemptId());
NUnit.Framework.Assert.AreEqual("Event reduce attempt id not correct", reduceAttempt
.GetID(), events[3].GetAttemptId());
NUnit.Framework.Assert.AreEqual("Event status not correct for map attempt1", TaskAttemptCompletionEventStatus
.Obsolete, events[0].GetStatus());
NUnit.Framework.Assert.AreEqual("Event status not correct for map attempt1", TaskAttemptCompletionEventStatus
.Failed, events[1].GetStatus());
NUnit.Framework.Assert.AreEqual("Event status not correct for map attempt2", TaskAttemptCompletionEventStatus
.Succeeded, events[2].GetStatus());
NUnit.Framework.Assert.AreEqual("Event status not correct for reduce attempt1", TaskAttemptCompletionEventStatus
.Succeeded, events[3].GetStatus());
TaskCompletionEvent[] mapEvents = job.GetMapAttemptCompletionEvents(0, 2);
TaskCompletionEvent[] convertedEvents = TypeConverter.FromYarn(events);
NUnit.Framework.Assert.AreEqual("Incorrect number of map events", 2, mapEvents.Length
);
Assert.AssertArrayEquals("Unexpected map events", Arrays.CopyOfRange(convertedEvents
, 0, 2), mapEvents);
mapEvents = job.GetMapAttemptCompletionEvents(2, 200);
NUnit.Framework.Assert.AreEqual("Incorrect number of map events", 1, mapEvents.Length
);
NUnit.Framework.Assert.AreEqual("Unexpected map event", convertedEvents[2], mapEvents
[0]);
}
public override TaskCompletionEvent[] GetMapAttemptCompletionEvents(int startIndex
, int maxEvents)
{
return(job.GetMapAttemptCompletionEvents(startIndex, maxEvents));
}
public virtual TaskCompletionEvent[] GetMapAttemptCompletionEvents(int startIndex
, int maxEvents)
{
return(mockJob.GetMapAttemptCompletionEvents(startIndex, maxEvents));
}
