protected internal virtual void RegisterHeartbeatHandler(Configuration conf)
{
taskHeartbeatHandler = new TaskHeartbeatHandler(context.GetEventHandler(), context
.GetClock(), conf.GetInt(MRJobConfig.MrAmTaskListenerThreadCount, MRJobConfig.DefaultMrAmTaskListenerThreadCount
));
AddService(taskHeartbeatHandler);
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestGetMapCompletionEvents()
{
TaskAttemptCompletionEvent[] empty = new TaskAttemptCompletionEvent[] { };
TaskAttemptCompletionEvent[] taskEvents = new TaskAttemptCompletionEvent[] { CreateTce
(0, true, TaskAttemptCompletionEventStatus.Obsolete), CreateTce(1, false, TaskAttemptCompletionEventStatus
.Failed), CreateTce(2, true, TaskAttemptCompletionEventStatus.Succeeded), CreateTce
(3, false, TaskAttemptCompletionEventStatus.Failed) };
TaskAttemptCompletionEvent[] mapEvents = new TaskAttemptCompletionEvent[] { taskEvents
[0], taskEvents[2] };
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job mockJob = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
>();
Org.Mockito.Mockito.When(mockJob.GetTaskAttemptCompletionEvents(0, 100)).ThenReturn
(taskEvents);
Org.Mockito.Mockito.When(mockJob.GetTaskAttemptCompletionEvents(0, 2)).ThenReturn
(Arrays.CopyOfRange(taskEvents, 0, 2));
Org.Mockito.Mockito.When(mockJob.GetTaskAttemptCompletionEvents(2, 100)).ThenReturn
(Arrays.CopyOfRange(taskEvents, 2, 4));
Org.Mockito.Mockito.When(mockJob.GetMapAttemptCompletionEvents(0, 100)).ThenReturn
(TypeConverter.FromYarn(mapEvents));
Org.Mockito.Mockito.When(mockJob.GetMapAttemptCompletionEvents(0, 2)).ThenReturn(
TypeConverter.FromYarn(mapEvents));
Org.Mockito.Mockito.When(mockJob.GetMapAttemptCompletionEvents(2, 100)).ThenReturn
(TypeConverter.FromYarn(empty));
AppContext appCtx = Org.Mockito.Mockito.Mock <AppContext>();
Org.Mockito.Mockito.When(appCtx.GetJob(Matchers.Any <JobId>())).ThenReturn(mockJob
);
JobTokenSecretManager secret = Org.Mockito.Mockito.Mock <JobTokenSecretManager>();
RMHeartbeatHandler rmHeartbeatHandler = Org.Mockito.Mockito.Mock <RMHeartbeatHandler
>();
TaskHeartbeatHandler hbHandler = Org.Mockito.Mockito.Mock <TaskHeartbeatHandler>();
TaskAttemptListenerImpl listener = new _MockTaskAttemptListenerImpl_200(hbHandler
, appCtx, secret, rmHeartbeatHandler);
Configuration conf = new Configuration();
listener.Init(conf);
listener.Start();
JobID jid = new JobID("12345", 1);
TaskAttemptID tid = new TaskAttemptID("12345", 1, TaskType.Reduce, 1, 0);
MapTaskCompletionEventsUpdate update = listener.GetMapCompletionEvents(jid, 0, 100
, tid);
NUnit.Framework.Assert.AreEqual(2, update.events.Length);
update = listener.GetMapCompletionEvents(jid, 0, 2, tid);
NUnit.Framework.Assert.AreEqual(2, update.events.Length);
update = listener.GetMapCompletionEvents(jid, 2, 100, tid);
NUnit.Framework.Assert.AreEqual(0, update.events.Length);
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestCommitWindow()
{
SystemClock clock = new SystemClock();
Task mockTask = Org.Mockito.Mockito.Mock <Task>();
Org.Mockito.Mockito.When(mockTask.CanCommit(Matchers.Any <TaskAttemptId>())).ThenReturn
(true);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job mockJob = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
>();
Org.Mockito.Mockito.When(mockJob.GetTask(Matchers.Any <TaskId>())).ThenReturn(mockTask
);
AppContext appCtx = Org.Mockito.Mockito.Mock <AppContext>();
Org.Mockito.Mockito.When(appCtx.GetJob(Matchers.Any <JobId>())).ThenReturn(mockJob
);
Org.Mockito.Mockito.When(appCtx.GetClock()).ThenReturn(clock);
JobTokenSecretManager secret = Org.Mockito.Mockito.Mock <JobTokenSecretManager>();
RMHeartbeatHandler rmHeartbeatHandler = Org.Mockito.Mockito.Mock <RMHeartbeatHandler
>();
TaskHeartbeatHandler hbHandler = Org.Mockito.Mockito.Mock <TaskHeartbeatHandler>();
TaskAttemptListenerImpl listener = new _MockTaskAttemptListenerImpl_254(hbHandler
, appCtx, secret, rmHeartbeatHandler);
Configuration conf = new Configuration();
listener.Init(conf);
listener.Start();
// verify commit not allowed when RM heartbeat has not occurred recently
TaskAttemptID tid = new TaskAttemptID("12345", 1, TaskType.Reduce, 1, 0);
bool canCommit = listener.CanCommit(tid);
NUnit.Framework.Assert.IsFalse(canCommit);
Org.Mockito.Mockito.Verify(mockTask, Org.Mockito.Mockito.Never()).CanCommit(Matchers.Any
<TaskAttemptId>());
// verify commit allowed when RM heartbeat is recent
Org.Mockito.Mockito.When(rmHeartbeatHandler.GetLastHeartbeatTime()).ThenReturn(clock
.GetTime());
canCommit = listener.CanCommit(tid);
NUnit.Framework.Assert.IsTrue(canCommit);
Org.Mockito.Mockito.Verify(mockTask, Org.Mockito.Mockito.Times(1)).CanCommit(Matchers.Any
<TaskAttemptId>());
listener.Stop();
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestGetTask()
{
AppContext appCtx = Org.Mockito.Mockito.Mock <AppContext>();
JobTokenSecretManager secret = Org.Mockito.Mockito.Mock <JobTokenSecretManager>();
RMHeartbeatHandler rmHeartbeatHandler = Org.Mockito.Mockito.Mock <RMHeartbeatHandler
>();
TaskHeartbeatHandler hbHandler = Org.Mockito.Mockito.Mock <TaskHeartbeatHandler>();
TestTaskAttemptListenerImpl.MockTaskAttemptListenerImpl listener = new TestTaskAttemptListenerImpl.MockTaskAttemptListenerImpl
(appCtx, secret, rmHeartbeatHandler, hbHandler);
Configuration conf = new Configuration();
listener.Init(conf);
listener.Start();
JVMId id = new JVMId("foo", 1, true, 1);
WrappedJvmID wid = new WrappedJvmID(id.GetJobId(), id.isMap, id.GetId());
// Verify ask before registration.
//The JVM ID has not been registered yet so we should kill it.
JvmContext context = new JvmContext();
context.jvmId = id;
JvmTask result = listener.GetTask(context);
NUnit.Framework.Assert.IsNotNull(result);
NUnit.Framework.Assert.IsTrue(result.shouldDie);
// Verify ask after registration but before launch.
// Don't kill, should be null.
TaskAttemptId attemptID = Org.Mockito.Mockito.Mock <TaskAttemptId>();
Task task = Org.Mockito.Mockito.Mock <Task>();
//Now put a task with the ID
listener.RegisterPendingTask(task, wid);
result = listener.GetTask(context);
NUnit.Framework.Assert.IsNull(result);
// Unregister for more testing.
listener.Unregister(attemptID, wid);
// Verify ask after registration and launch
//Now put a task with the ID
listener.RegisterPendingTask(task, wid);
listener.RegisterLaunchedTask(attemptID, wid);
Org.Mockito.Mockito.Verify(hbHandler).Register(attemptID);
result = listener.GetTask(context);
NUnit.Framework.Assert.IsNotNull(result);
NUnit.Framework.Assert.IsFalse(result.shouldDie);
// Don't unregister yet for more testing.
//Verify that if we call it again a second time we are told to die.
result = listener.GetTask(context);
NUnit.Framework.Assert.IsNotNull(result);
NUnit.Framework.Assert.IsTrue(result.shouldDie);
listener.Unregister(attemptID, wid);
// Verify after unregistration.
result = listener.GetTask(context);
NUnit.Framework.Assert.IsNotNull(result);
NUnit.Framework.Assert.IsTrue(result.shouldDie);
listener.Stop();
// test JVMID
JVMId jvmid = JVMId.ForName("jvm_001_002_m_004");
NUnit.Framework.Assert.IsNotNull(jvmid);
try
{
JVMId.ForName("jvm_001_002_m_004_006");
Assert.Fail();
}
catch (ArgumentException e)
{
NUnit.Framework.Assert.AreEqual(e.Message, "TaskId string : jvm_001_002_m_004_006 is not properly formed"
);
}
}
public MockTaskAttemptListenerImpl(AppContext context, JobTokenSecretManager jobTokenSecretManager
, RMHeartbeatHandler rmHeartbeatHandler, TaskHeartbeatHandler hbHandler)
: base(context, jobTokenSecretManager, rmHeartbeatHandler, null)
{
this.taskHeartbeatHandler = hbHandler;
}
public _MockTaskAttemptListenerImpl_254(TaskHeartbeatHandler hbHandler, AppContext
baseArg1, JobTokenSecretManager baseArg2, RMHeartbeatHandler baseArg3)
: base(baseArg1, baseArg2, baseArg3)
{
this.hbHandler = hbHandler;
}
