public _Runnable_228(EventHandler _enclosing, ContainerRemoteLaunchEvent launchEv
, IDictionary <TaskAttemptID, MapOutputFile> localMapFiles)
{
this._enclosing = _enclosing;
this.launchEv = launchEv;
this.localMapFiles = localMapFiles;
}
/*
* Uber-AM lifecycle/ordering ("normal" case):
*
* - [somebody] sends TA_ASSIGNED
* - handled by ContainerAssignedTransition (TaskAttemptImpl.java)
* - creates "remoteTask" for us == real Task
* - sends CONTAINER_REMOTE_LAUNCH
* - TA: UNASSIGNED -> ASSIGNED
* - CONTAINER_REMOTE_LAUNCH handled by LocalContainerLauncher (us)
* - sucks "remoteTask" out of TaskAttemptImpl via getRemoteTask()
* - sends TA_CONTAINER_LAUNCHED
* [[ elsewhere...
* - TA_CONTAINER_LAUNCHED handled by LaunchedContainerTransition
* - registers "remoteTask" with TaskAttemptListener (== umbilical)
* - NUKES "remoteTask"
* - sends T_ATTEMPT_LAUNCHED (Task: SCHEDULED -> RUNNING)
* - TA: ASSIGNED -> RUNNING
* ]]
* - runs Task (runSubMap() or runSubReduce())
* - TA can safely send TA_UPDATE since in RUNNING state
*/
// doneWithMaps and finishedSubMaps are accessed from only
// one thread. Therefore, no need to make them volatile.
public virtual void Run()
{
ContainerLauncherEvent @event = null;
// Collect locations of map outputs to give to reduces
IDictionary <TaskAttemptID, MapOutputFile> localMapFiles = new Dictionary <TaskAttemptID
, MapOutputFile>();
// _must_ either run subtasks sequentially or accept expense of new JVMs
// (i.e., fork()), else will get weird failures when maps try to create/
// write same dirname or filename: no chdir() in Java
while (!Sharpen.Thread.CurrentThread().IsInterrupted())
{
try
{
@event = this._enclosing.eventQueue.Take();
}
catch (Exception e)
{
// mostly via T_KILL? JOB_KILL?
LocalContainerLauncher.Log.Error("Returning, interrupted : " + e);
break;
}
LocalContainerLauncher.Log.Info("Processing the event " + @event.ToString());
if (@event.GetType() == ContainerLauncher.EventType.ContainerRemoteLaunch)
{
ContainerRemoteLaunchEvent launchEv = (ContainerRemoteLaunchEvent)@event;
// execute the task on a separate thread
Future <object> future = this._enclosing.taskRunner.Submit(new _Runnable_228(this,
launchEv, localMapFiles));
// remember the current attempt
this.futures[@event.GetTaskAttemptID()] = future;
}
else
{
if (@event.GetType() == ContainerLauncher.EventType.ContainerRemoteCleanup)
{
// cancel (and interrupt) the current running task associated with the
// event
TaskAttemptId taId = @event.GetTaskAttemptID();
Future <object> future = Sharpen.Collections.Remove(this.futures, taId);
if (future != null)
{
LocalContainerLauncher.Log.Info("canceling the task attempt " + taId);
future.Cancel(true);
}
// send "cleaned" event to task attempt to move us from
// SUCCESS_CONTAINER_CLEANUP to SUCCEEDED state (or
// {FAIL|KILL}_CONTAINER_CLEANUP to {FAIL|KILL}_TASK_CLEANUP)
this._enclosing.context.GetEventHandler().Handle(new TaskAttemptEvent(taId, TaskAttemptEventType
.TaContainerCleaned));
}
else
{
LocalContainerLauncher.Log.Warn("Ignoring unexpected event " + @event.ToString());
}
}
}
}
public override void Handle(ContainerLauncherEvent @event)
{
if (@event.GetType() == ContainerLauncher.EventType.ContainerRemoteLaunch)
{
ContainerRemoteLaunchEvent launchEvent = (ContainerRemoteLaunchEvent)@event;
ContainerLaunchContext launchContext = launchEvent.GetContainerLaunchContext();
string cmdString = launchContext.GetCommands().ToString();
TestMapReduceChildJVM.Log.Info("launchContext " + cmdString);
this._enclosing.myCommandLine = cmdString;
this._enclosing.cmdEnvironment = launchContext.GetEnvironment();
}
base.Handle(@event);
}
private void RunTask(ContainerRemoteLaunchEvent launchEv, IDictionary <TaskAttemptID
, MapOutputFile> localMapFiles)
{
TaskAttemptId attemptID = launchEv.GetTaskAttemptID();
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = this._enclosing.context.GetAllJobs
()[attemptID.GetTaskId().GetJobId()];
int numMapTasks = job.GetTotalMaps();
int numReduceTasks = job.GetTotalReduces();
// YARN (tracking) Task:
Task ytask = job.GetTask(attemptID.GetTaskId());
// classic mapred Task:
Task remoteTask = launchEv.GetRemoteTask();
// after "launching," send launched event to task attempt to move
// state from ASSIGNED to RUNNING (also nukes "remoteTask", so must
// do getRemoteTask() call first)
//There is no port number because we are not really talking to a task
// tracker. The shuffle is just done through local files. So the
// port number is set to -1 in this case.
this._enclosing.context.GetEventHandler().Handle(new TaskAttemptContainerLaunchedEvent
(attemptID, -1));
if (numMapTasks == 0)
{
this.doneWithMaps = true;
}
try
{
if (remoteTask.IsMapOrReduce())
{
JobCounterUpdateEvent jce = new JobCounterUpdateEvent(attemptID.GetTaskId().GetJobId
());
jce.AddCounterUpdate(JobCounter.TotalLaunchedUbertasks, 1);
if (remoteTask.IsMapTask())
{
jce.AddCounterUpdate(JobCounter.NumUberSubmaps, 1);
}
else
{
jce.AddCounterUpdate(JobCounter.NumUberSubreduces, 1);
}
this._enclosing.context.GetEventHandler().Handle(jce);
}
this.RunSubtask(remoteTask, ytask.GetType(), attemptID, numMapTasks, (numReduceTasks
> 0), localMapFiles);
}
catch (RuntimeException)
{
JobCounterUpdateEvent jce = new JobCounterUpdateEvent(attemptID.GetTaskId().GetJobId
());
jce.AddCounterUpdate(JobCounter.NumFailedUbertasks, 1);
this._enclosing.context.GetEventHandler().Handle(jce);
// this is our signal that the subtask failed in some way, so
// simulate a failed JVM/container and send a container-completed
// event to task attempt (i.e., move state machine from RUNNING
// to FAIL_CONTAINER_CLEANUP [and ultimately to FAILED])
this._enclosing.context.GetEventHandler().Handle(new TaskAttemptEvent(attemptID,
TaskAttemptEventType.TaContainerCompleted));
}
catch (IOException ioe)
{
// if umbilical itself barfs (in error-handler of runSubMap()),
// we're pretty much hosed, so do what YarnChild main() does
// (i.e., exit clumsily--but can never happen, so no worries!)
LocalContainerLauncher.Log.Fatal("oopsie... this can never happen: " + StringUtils
.StringifyException(ioe));
ExitUtil.Terminate(-1);
}
finally
{
// remove my future
if (Sharpen.Collections.Remove(this.futures, attemptID) != null)
{
LocalContainerLauncher.Log.Info("removed attempt " + attemptID + " from the futures to keep track of"
);
}
}
}
/// <exception cref="System.Exception"/>
public virtual void TestKillJob()
{
JobConf conf = new JobConf();
AppContext context = Org.Mockito.Mockito.Mock <AppContext>();
// a simple event handler solely to detect the container cleaned event
CountDownLatch isDone = new CountDownLatch(1);
EventHandler handler = new _EventHandler_106(isDone);
Org.Mockito.Mockito.When(context.GetEventHandler()).ThenReturn(handler);
// create and start the launcher
LocalContainerLauncher launcher = new LocalContainerLauncher(context, Org.Mockito.Mockito.Mock
<TaskUmbilicalProtocol>());
launcher.Init(conf);
launcher.Start();
// create mocked job, task, and task attempt
// a single-mapper job
JobId jobId = MRBuilderUtils.NewJobId(Runtime.CurrentTimeMillis(), 1, 1);
TaskId taskId = MRBuilderUtils.NewTaskId(jobId, 1, TaskType.Map);
TaskAttemptId taId = MRBuilderUtils.NewTaskAttemptId(taskId, 0);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
>();
Org.Mockito.Mockito.When(job.GetTotalMaps()).ThenReturn(1);
Org.Mockito.Mockito.When(job.GetTotalReduces()).ThenReturn(0);
IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> jobs = new Dictionary
<JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job>();
jobs[jobId] = job;
// app context returns the one and only job
Org.Mockito.Mockito.When(context.GetAllJobs()).ThenReturn(jobs);
Task ytask = Org.Mockito.Mockito.Mock <Task>();
Org.Mockito.Mockito.When(ytask.GetType()).ThenReturn(TaskType.Map);
Org.Mockito.Mockito.When(job.GetTask(taskId)).ThenReturn(ytask);
// create a sleeping mapper that runs beyond the test timeout
MapTask mapTask = Org.Mockito.Mockito.Mock <MapTask>();
Org.Mockito.Mockito.When(mapTask.IsMapOrReduce()).ThenReturn(true);
Org.Mockito.Mockito.When(mapTask.IsMapTask()).ThenReturn(true);
TaskAttemptID taskID = TypeConverter.FromYarn(taId);
Org.Mockito.Mockito.When(mapTask.GetTaskID()).ThenReturn(taskID);
Org.Mockito.Mockito.When(mapTask.GetJobID()).ThenReturn(((JobID)taskID.GetJobID()
));
Org.Mockito.Mockito.DoAnswer(new _Answer_152()).When(mapTask).Run(Matchers.IsA <JobConf
>(), Matchers.IsA <TaskUmbilicalProtocol>());
// sleep for a long time
// pump in a task attempt launch event
ContainerLauncherEvent launchEvent = new ContainerRemoteLaunchEvent(taId, null, CreateMockContainer
(), mapTask);
launcher.Handle(launchEvent);
Sharpen.Thread.Sleep(200);
// now pump in a container clean-up event
ContainerLauncherEvent cleanupEvent = new ContainerLauncherEvent(taId, null, null
, null, ContainerLauncher.EventType.ContainerRemoteCleanup);
launcher.Handle(cleanupEvent);
// wait for the event to fire: this should be received promptly
isDone.Await();
launcher.Close();
}
