/* * Uber-AM lifecycle/ordering ("normal" case): * * - [somebody] sends TA_ASSIGNED * - handled by ContainerAssignedTransition (TaskAttemptImpl.java) * - creates "remoteTask" for us == real Task * - sends CONTAINER_REMOTE_LAUNCH * - TA: UNASSIGNED -> ASSIGNED * - CONTAINER_REMOTE_LAUNCH handled by LocalContainerLauncher (us) * - sucks "remoteTask" out of TaskAttemptImpl via getRemoteTask() * - sends TA_CONTAINER_LAUNCHED * [[ elsewhere... * - TA_CONTAINER_LAUNCHED handled by LaunchedContainerTransition * - registers "remoteTask" with TaskAttemptListener (== umbilical) * - NUKES "remoteTask" * - sends T_ATTEMPT_LAUNCHED (Task: SCHEDULED -> RUNNING) * - TA: ASSIGNED -> RUNNING * ]] * - runs Task (runSubMap() or runSubReduce()) * - TA can safely send TA_UPDATE since in RUNNING state */ // doneWithMaps and finishedSubMaps are accessed from only // one thread. Therefore, no need to make them volatile. public virtual void Run() { ContainerLauncherEvent @event = null; // Collect locations of map outputs to give to reduces IDictionary <TaskAttemptID, MapOutputFile> localMapFiles = new Dictionary <TaskAttemptID , MapOutputFile>(); // _must_ either run subtasks sequentially or accept expense of new JVMs // (i.e., fork()), else will get weird failures when maps try to create/ // write same dirname or filename: no chdir() in Java while (!Sharpen.Thread.CurrentThread().IsInterrupted()) { try { @event = this._enclosing.eventQueue.Take(); } catch (Exception e) { // mostly via T_KILL? JOB_KILL? LocalContainerLauncher.Log.Error("Returning, interrupted : " + e); break; } LocalContainerLauncher.Log.Info("Processing the event " + @event.ToString()); if (@event.GetType() == ContainerLauncher.EventType.ContainerRemoteLaunch) { ContainerRemoteLaunchEvent launchEv = (ContainerRemoteLaunchEvent)@event; // execute the task on a separate thread Future <object> future = this._enclosing.taskRunner.Submit(new _Runnable_228(this, launchEv, localMapFiles)); // remember the current attempt this.futures[@event.GetTaskAttemptID()] = future; } else { if (@event.GetType() == ContainerLauncher.EventType.ContainerRemoteCleanup) { // cancel (and interrupt) the current running task associated with the // event TaskAttemptId taId = @event.GetTaskAttemptID(); Future <object> future = Sharpen.Collections.Remove(this.futures, taId); if (future != null) { LocalContainerLauncher.Log.Info("canceling the task attempt " + taId); future.Cancel(true); } // send "cleaned" event to task attempt to move us from // SUCCESS_CONTAINER_CLEANUP to SUCCEEDED state (or // {FAIL|KILL}_CONTAINER_CLEANUP to {FAIL|KILL}_TASK_CLEANUP) this._enclosing.context.GetEventHandler().Handle(new TaskAttemptEvent(taId, TaskAttemptEventType .TaContainerCleaned)); } else { LocalContainerLauncher.Log.Warn("Ignoring unexpected event " + @event.ToString()); } } } }
public override void Handle(ContainerLauncherEvent @event) { if (@event is ContainerRemoteLaunchEvent) { this._enclosing._enclosing.containerObtainedByContainerLauncher = ((ContainerRemoteLaunchEvent )@event).GetAllocatedContainer(); } base.Handle(@event); }
public virtual void Handle(ContainerLauncherEvent @event) { try { eventQueue.Put(@event); } catch (Exception e) { throw new YarnRuntimeException(e); } }
public override void Handle(ContainerLauncherEvent @event) { if (@event.GetType() == ContainerLauncher.EventType.ContainerRemoteLaunch) { ContainerRemoteLaunchEvent launchEvent = (ContainerRemoteLaunchEvent)@event; ContainerLaunchContext launchContext = launchEvent.GetContainerLaunchContext(); string cmdString = launchContext.GetCommands().ToString(); TestMapReduceChildJVM.Log.Info("launchContext " + cmdString); this._enclosing.myCommandLine = cmdString; this._enclosing.cmdEnvironment = launchContext.GetEnvironment(); } base.Handle(@event); }
public override void Handle(ContainerLauncherEvent @event) { switch (@event.GetType()) { case ContainerLauncher.EventType.ContainerRemoteLaunch: { base.Handle(@event); // Unused event and container. break; } case ContainerLauncher.EventType.ContainerRemoteCleanup: { this._enclosing.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(@event .GetTaskAttemptID(), TaskAttemptEventType.TaContainerCleaned)); break; } } }
//We are running locally so set the shuffle port to -1 public virtual void Handle(ContainerLauncherEvent @event) { switch (@event.GetType()) { case ContainerLauncher.EventType.ContainerRemoteLaunch: { this._enclosing.ContainerLaunched(@event.GetTaskAttemptID(), this.shufflePort); this._enclosing.AttemptLaunched(@event.GetTaskAttemptID()); break; } case ContainerLauncher.EventType.ContainerRemoteCleanup: { this._enclosing.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(@event .GetTaskAttemptID(), TaskAttemptEventType.TaContainerCleaned)); break; } } }
/// <exception cref="System.Exception"/> public virtual void TestKillJob() { JobConf conf = new JobConf(); AppContext context = Org.Mockito.Mockito.Mock <AppContext>(); // a simple event handler solely to detect the container cleaned event CountDownLatch isDone = new CountDownLatch(1); EventHandler handler = new _EventHandler_106(isDone); Org.Mockito.Mockito.When(context.GetEventHandler()).ThenReturn(handler); // create and start the launcher LocalContainerLauncher launcher = new LocalContainerLauncher(context, Org.Mockito.Mockito.Mock <TaskUmbilicalProtocol>()); launcher.Init(conf); launcher.Start(); // create mocked job, task, and task attempt // a single-mapper job JobId jobId = MRBuilderUtils.NewJobId(Runtime.CurrentTimeMillis(), 1, 1); TaskId taskId = MRBuilderUtils.NewTaskId(jobId, 1, TaskType.Map); TaskAttemptId taId = MRBuilderUtils.NewTaskAttemptId(taskId, 0); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job >(); Org.Mockito.Mockito.When(job.GetTotalMaps()).ThenReturn(1); Org.Mockito.Mockito.When(job.GetTotalReduces()).ThenReturn(0); IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> jobs = new Dictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job>(); jobs[jobId] = job; // app context returns the one and only job Org.Mockito.Mockito.When(context.GetAllJobs()).ThenReturn(jobs); Task ytask = Org.Mockito.Mockito.Mock <Task>(); Org.Mockito.Mockito.When(ytask.GetType()).ThenReturn(TaskType.Map); Org.Mockito.Mockito.When(job.GetTask(taskId)).ThenReturn(ytask); // create a sleeping mapper that runs beyond the test timeout MapTask mapTask = Org.Mockito.Mockito.Mock <MapTask>(); Org.Mockito.Mockito.When(mapTask.IsMapOrReduce()).ThenReturn(true); Org.Mockito.Mockito.When(mapTask.IsMapTask()).ThenReturn(true); TaskAttemptID taskID = TypeConverter.FromYarn(taId); Org.Mockito.Mockito.When(mapTask.GetTaskID()).ThenReturn(taskID); Org.Mockito.Mockito.When(mapTask.GetJobID()).ThenReturn(((JobID)taskID.GetJobID() )); Org.Mockito.Mockito.DoAnswer(new _Answer_152()).When(mapTask).Run(Matchers.IsA <JobConf >(), Matchers.IsA <TaskUmbilicalProtocol>()); // sleep for a long time // pump in a task attempt launch event ContainerLauncherEvent launchEvent = new ContainerRemoteLaunchEvent(taId, null, CreateMockContainer (), mapTask); launcher.Handle(launchEvent); Sharpen.Thread.Sleep(200); // now pump in a container clean-up event ContainerLauncherEvent cleanupEvent = new ContainerLauncherEvent(taId, null, null , null, ContainerLauncher.EventType.ContainerRemoteCleanup); launcher.Handle(cleanupEvent); // wait for the event to fire: this should be received promptly isDone.Await(); launcher.Close(); }