public virtual void TestRenameMapOutputForReduce()
{
JobConf conf = new JobConf();
MROutputFiles mrOutputFiles = new MROutputFiles();
mrOutputFiles.SetConf(conf);
// make sure both dirs are distinct
//
conf.Set(MRConfig.LocalDir, localDirs[0].ToString());
Path mapOut = mrOutputFiles.GetOutputFileForWrite(1);
conf.Set(MRConfig.LocalDir, localDirs[1].ToString());
Path mapOutIdx = mrOutputFiles.GetOutputIndexFileForWrite(1);
Assert.AssertNotEquals("Paths must be different!", mapOut.GetParent(), mapOutIdx.
GetParent());
// make both dirs part of LOCAL_DIR
conf.SetStrings(MRConfig.LocalDir, localDirs);
FileContext lfc = FileContext.GetLocalFSFileContext(conf);
lfc.Create(mapOut, EnumSet.Of(CreateFlag.Create)).Close();
lfc.Create(mapOutIdx, EnumSet.Of(CreateFlag.Create)).Close();
JobId jobId = MRBuilderUtils.NewJobId(12345L, 1, 2);
TaskId tid = MRBuilderUtils.NewTaskId(jobId, 0, TaskType.Map);
TaskAttemptId taid = MRBuilderUtils.NewTaskAttemptId(tid, 0);
LocalContainerLauncher.RenameMapOutputForReduce(conf, taid, mrOutputFiles);
}
/// <exception cref="Sharpen.RuntimeException"/>
/// <exception cref="System.IO.IOException"/>
private void RunSubtask(Task task, TaskType taskType, TaskAttemptId attemptID, int
numMapTasks, bool renameOutputs, IDictionary <TaskAttemptID, MapOutputFile> localMapFiles
)
{
TaskAttemptID classicAttemptID = TypeConverter.FromYarn(attemptID);
try
{
JobConf conf = new JobConf(this._enclosing.GetConfig());
conf.Set(JobContext.TaskId, task.GetTaskID().ToString());
conf.Set(JobContext.TaskAttemptId, classicAttemptID.ToString());
conf.SetBoolean(JobContext.TaskIsmap, (taskType == TaskType.Map));
conf.SetInt(JobContext.TaskPartition, task.GetPartition());
conf.Set(JobContext.Id, task.GetJobID().ToString());
// Use the AM's local dir env to generate the intermediate step
// output files
string[] localSysDirs = StringUtils.GetTrimmedStrings(Runtime.Getenv(ApplicationConstants.Environment
.LocalDirs.ToString()));
conf.SetStrings(MRConfig.LocalDir, localSysDirs);
LocalContainerLauncher.Log.Info(MRConfig.LocalDir + " for uber task: " + conf.Get
(MRConfig.LocalDir));
// mark this as an uberized subtask so it can set task counter
// (longer-term/FIXME: could redefine as job counter and send
// "JobCounterEvent" to JobImpl on [successful] completion of subtask;
// will need new Job state-machine transition and JobImpl jobCounters
// map to handle)
conf.SetBoolean("mapreduce.task.uberized", true);
// Check and handle Encrypted spill key
task.SetEncryptedSpillKey(this._enclosing.encryptedSpillKey);
YarnChild.SetEncryptedSpillKeyIfRequired(task);
// META-FIXME: do we want the extra sanity-checking (doneWithMaps,
// etc.), or just assume/hope the state machine(s) and uber-AM work
// as expected?
if (taskType == TaskType.Map)
{
if (this.doneWithMaps)
{
LocalContainerLauncher.Log.Error("CONTAINER_REMOTE_LAUNCH contains a map task ("
+ attemptID + "), but should be finished with maps");
throw new RuntimeException();
}
MapTask map = (MapTask)task;
map.SetConf(conf);
map.Run(conf, this._enclosing.umbilical);
if (renameOutputs)
{
MapOutputFile renamed = LocalContainerLauncher.RenameMapOutputForReduce(conf, attemptID
, map.GetMapOutputFile());
localMapFiles[classicAttemptID] = renamed;
}
this.Relocalize();
if (++this.finishedSubMaps == numMapTasks)
{
this.doneWithMaps = true;
}
}
else
{
/* TaskType.REDUCE */
if (!this.doneWithMaps)
{
// check if event-queue empty? whole idea of counting maps vs.
// checking event queue is a tad wacky...but could enforce ordering
// (assuming no "lost events") at LocalMRAppMaster [CURRENT BUG(?):
// doesn't send reduce event until maps all done]
LocalContainerLauncher.Log.Error("CONTAINER_REMOTE_LAUNCH contains a reduce task ("
+ attemptID + "), but not yet finished with maps");
throw new RuntimeException();
}
// a.k.a. "mapreduce.jobtracker.address" in LocalJobRunner:
// set framework name to local to make task local
conf.Set(MRConfig.FrameworkName, MRConfig.LocalFrameworkName);
conf.Set(MRConfig.MasterAddress, "local");
// bypass shuffle
ReduceTask reduce = (ReduceTask)task;
reduce.SetLocalMapFiles(localMapFiles);
reduce.SetConf(conf);
reduce.Run(conf, this._enclosing.umbilical);
this.Relocalize();
}
}
catch (FSError e)
{
LocalContainerLauncher.Log.Fatal("FSError from child", e);
// umbilical: MRAppMaster creates (taskAttemptListener), passes to us
if (!ShutdownHookManager.Get().IsShutdownInProgress())
{
this._enclosing.umbilical.FsError(classicAttemptID, e.Message);
}
throw new RuntimeException();
}
catch (Exception exception)
{
LocalContainerLauncher.Log.Warn("Exception running local (uberized) 'child' : " +
StringUtils.StringifyException(exception));
try
{
if (task != null)
{
// do cleanup for the task
task.TaskCleanup(this._enclosing.umbilical);
}
}
catch (Exception e)
{
LocalContainerLauncher.Log.Info("Exception cleaning up: " + StringUtils.StringifyException
(e));
}
// Report back any failures, for diagnostic purposes
this._enclosing.umbilical.ReportDiagnosticInfo(classicAttemptID, StringUtils.StringifyException
(exception));
throw new RuntimeException();
}
catch (Exception throwable)
{
LocalContainerLauncher.Log.Fatal("Error running local (uberized) 'child' : " + StringUtils
.StringifyException(throwable));
if (!ShutdownHookManager.Get().IsShutdownInProgress())
{
Exception tCause = throwable.InnerException;
string cause = (tCause == null) ? throwable.Message : StringUtils.StringifyException
(tCause);
this._enclosing.umbilical.FatalError(classicAttemptID, cause);
}
throw new RuntimeException();
}
}
internal EventHandler(LocalContainerLauncher _enclosing)
{
this._enclosing = _enclosing;
}
/// <exception cref="System.Exception"/>
public virtual void TestKillJob()
{
JobConf conf = new JobConf();
AppContext context = Org.Mockito.Mockito.Mock <AppContext>();
// a simple event handler solely to detect the container cleaned event
CountDownLatch isDone = new CountDownLatch(1);
EventHandler handler = new _EventHandler_106(isDone);
Org.Mockito.Mockito.When(context.GetEventHandler()).ThenReturn(handler);
// create and start the launcher
LocalContainerLauncher launcher = new LocalContainerLauncher(context, Org.Mockito.Mockito.Mock
<TaskUmbilicalProtocol>());
launcher.Init(conf);
launcher.Start();
// create mocked job, task, and task attempt
// a single-mapper job
JobId jobId = MRBuilderUtils.NewJobId(Runtime.CurrentTimeMillis(), 1, 1);
TaskId taskId = MRBuilderUtils.NewTaskId(jobId, 1, TaskType.Map);
TaskAttemptId taId = MRBuilderUtils.NewTaskAttemptId(taskId, 0);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
>();
Org.Mockito.Mockito.When(job.GetTotalMaps()).ThenReturn(1);
Org.Mockito.Mockito.When(job.GetTotalReduces()).ThenReturn(0);
IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> jobs = new Dictionary
<JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job>();
jobs[jobId] = job;
// app context returns the one and only job
Org.Mockito.Mockito.When(context.GetAllJobs()).ThenReturn(jobs);
Task ytask = Org.Mockito.Mockito.Mock <Task>();
Org.Mockito.Mockito.When(ytask.GetType()).ThenReturn(TaskType.Map);
Org.Mockito.Mockito.When(job.GetTask(taskId)).ThenReturn(ytask);
// create a sleeping mapper that runs beyond the test timeout
MapTask mapTask = Org.Mockito.Mockito.Mock <MapTask>();
Org.Mockito.Mockito.When(mapTask.IsMapOrReduce()).ThenReturn(true);
Org.Mockito.Mockito.When(mapTask.IsMapTask()).ThenReturn(true);
TaskAttemptID taskID = TypeConverter.FromYarn(taId);
Org.Mockito.Mockito.When(mapTask.GetTaskID()).ThenReturn(taskID);
Org.Mockito.Mockito.When(mapTask.GetJobID()).ThenReturn(((JobID)taskID.GetJobID()
));
Org.Mockito.Mockito.DoAnswer(new _Answer_152()).When(mapTask).Run(Matchers.IsA <JobConf
>(), Matchers.IsA <TaskUmbilicalProtocol>());
// sleep for a long time
// pump in a task attempt launch event
ContainerLauncherEvent launchEvent = new ContainerRemoteLaunchEvent(taId, null, CreateMockContainer
(), mapTask);
launcher.Handle(launchEvent);
Sharpen.Thread.Sleep(200);
// now pump in a container clean-up event
ContainerLauncherEvent cleanupEvent = new ContainerLauncherEvent(taId, null, null
, null, ContainerLauncher.EventType.ContainerRemoteCleanup);
launcher.Handle(cleanupEvent);
// wait for the event to fire: this should be received promptly
isDone.Await();
launcher.Close();
}