internal static FilePath GetAttemptDir(TaskAttemptID taskid, bool isCleanup)
{
string cleanupSuffix = isCleanup ? ".cleanup" : string.Empty;
return(new FilePath(GetJobDir(((JobID)taskid.GetJobID())), taskid + cleanupSuffix
));
}
public virtual void LaunchMap(TaskAttemptID taskAttemptID)
{
lock (this)
{
++numMapTasksLaunched;
DecWaitingMaps(((JobID)taskAttemptID.GetJobID()), 1);
}
}
public virtual void LaunchReduce(TaskAttemptID taskAttemptID)
{
lock (this)
{
++numReduceTasksLaunched;
DecWaitingReduces(((JobID)taskAttemptID.GetJobID()), 1);
}
}
/// <summary>
/// A test that mimics a failed task to ensure that it does
/// not get into the COMMIT_PENDING state, by using a fake
/// UmbilicalProtocol's implementation that fails if the commit.
/// </summary>
/// <remarks>
/// A test that mimics a failed task to ensure that it does
/// not get into the COMMIT_PENDING state, by using a fake
/// UmbilicalProtocol's implementation that fails if the commit.
/// protocol is played.
/// The test mocks the various steps in a failed task's
/// life-cycle using a special OutputCommitter and UmbilicalProtocol
/// implementation.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestTaskCleanupDoesNotCommit()
{
// Mimic a job with a special committer that does not cleanup
// files when a task fails.
JobConf job = new JobConf();
job.SetOutputCommitter(typeof(TestTaskCommit.CommitterWithoutCleanup));
Path outDir = new Path(rootDir, "output");
FileOutputFormat.SetOutputPath(job, outDir);
// Mimic job setup
string dummyAttemptID = "attempt_200707121733_0001_m_000000_0";
TaskAttemptID attemptID = ((TaskAttemptID)TaskAttemptID.ForName(dummyAttemptID));
OutputCommitter committer = new TestTaskCommit.CommitterWithoutCleanup();
JobContext jContext = new JobContextImpl(job, ((JobID)attemptID.GetJobID()));
committer.SetupJob(jContext);
// Mimic a map task
dummyAttemptID = "attempt_200707121733_0001_m_000001_0";
attemptID = ((TaskAttemptID)TaskAttemptID.ForName(dummyAttemptID));
Task task = new MapTask(null, attemptID, 0, null, 1);
task.SetConf(job);
task.LocalizeConfiguration(job);
task.Initialize(job, ((JobID)attemptID.GetJobID()), Reporter.Null, false);
// Mimic the map task writing some output.
string file = "test.txt";
FileSystem localFs = FileSystem.GetLocal(job);
TextOutputFormat <Text, Text> theOutputFormat = new TextOutputFormat <Text, Text>();
RecordWriter <Text, Text> theRecordWriter = theOutputFormat.GetRecordWriter(localFs
, job, file, Reporter.Null);
theRecordWriter.Write(new Text("key"), new Text("value"));
theRecordWriter.Close(Reporter.Null);
// Mimic a task failure; setting up the task for cleanup simulates
// the abort protocol to be played.
// Without checks in the framework, this will fail
// as the committer will cause a COMMIT to happen for
// the cleanup task.
task.SetTaskCleanupTask();
TestTaskCommit.MyUmbilical umbilical = new TestTaskCommit.MyUmbilical(this);
task.Run(job, umbilical);
NUnit.Framework.Assert.IsTrue("Task did not succeed", umbilical.taskDone);
}
/// <summary>Obtain the owner of the log dir.</summary>
/// <remarks>
/// Obtain the owner of the log dir. This is
/// determined by checking the job's log directory.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
internal static string ObtainLogDirOwner(TaskAttemptID taskid)
{
Configuration conf = new Configuration();
FileSystem raw = FileSystem.GetLocal(conf).GetRaw();
Path jobLogDir = new Path(GetJobDir(((JobID)taskid.GetJobID())).GetAbsolutePath()
);
FileStatus jobStat = raw.GetFileStatus(jobLogDir);
return(jobStat.GetOwner());
}
/// <exception cref="System.Exception"/>
public virtual void TestCommitter()
{
JobConf job = new JobConf();
SetConfForFileOutputCommitter(job);
JobContext jContext = new JobContextImpl(job, ((JobID)taskID.GetJobID()));
TaskAttemptContext tContext = new TaskAttemptContextImpl(job, taskID);
FileOutputCommitter committer = new FileOutputCommitter();
FileOutputFormat.SetWorkOutputPath(job, committer.GetTaskAttemptPath(tContext));
committer.SetupJob(jContext);
committer.SetupTask(tContext);
string file = "test.txt";
// A reporter that does nothing
Reporter reporter = Reporter.Null;
// write output
FileSystem localFs = FileSystem.GetLocal(job);
TextOutputFormat theOutputFormat = new TextOutputFormat();
RecordWriter theRecordWriter = theOutputFormat.GetRecordWriter(localFs, job, file
, reporter);
WriteOutput(theRecordWriter, reporter);
// do commit
committer.CommitTask(tContext);
committer.CommitJob(jContext);
// validate output
FilePath expectedFile = new FilePath(new Path(outDir, file).ToString());
StringBuilder expectedOutput = new StringBuilder();
expectedOutput.Append(key1).Append('\t').Append(val1).Append("\n");
expectedOutput.Append(val1).Append("\n");
expectedOutput.Append(val2).Append("\n");
expectedOutput.Append(key2).Append("\n");
expectedOutput.Append(key1).Append("\n");
expectedOutput.Append(key2).Append('\t').Append(val2).Append("\n");
string output = UtilsForTests.Slurp(expectedFile);
NUnit.Framework.Assert.AreEqual(output, expectedOutput.ToString());
FileUtil.FullyDelete(new FilePath(outDir.ToString()));
}
/// <summary>test without TASK_LOG_DIR</summary>
/// <exception cref="System.IO.IOException"/>
public virtual void TestTaskLogWithoutTaskLogDir()
{
// TaskLog tasklog= new TaskLog();
Runtime.ClearProperty(YarnConfiguration.YarnAppContainerLogDir);
// test TaskLog
NUnit.Framework.Assert.AreEqual(TaskLog.GetMRv2LogDir(), null);
TaskAttemptID taid = Org.Mockito.Mockito.Mock <TaskAttemptID>();
JobID jid = new JobID("job", 1);
Org.Mockito.Mockito.When(((JobID)taid.GetJobID())).ThenReturn(jid);
Org.Mockito.Mockito.When(taid.ToString()).ThenReturn("JobId");
FilePath f = TaskLog.GetTaskLogFile(taid, true, TaskLog.LogName.Stdout);
NUnit.Framework.Assert.IsTrue(f.GetAbsolutePath().EndsWith("stdout"));
}
/// <summary>test TaskAttemptID</summary>
/// <exception cref="System.IO.IOException"/>
public virtual void TestTaskLog()
{
// test TaskLog
Runtime.SetProperty(YarnConfiguration.YarnAppContainerLogDir, "testString");
NUnit.Framework.Assert.AreEqual(TaskLog.GetMRv2LogDir(), "testString");
TaskAttemptID taid = Org.Mockito.Mockito.Mock <TaskAttemptID>();
JobID jid = new JobID("job", 1);
Org.Mockito.Mockito.When(((JobID)taid.GetJobID())).ThenReturn(jid);
Org.Mockito.Mockito.When(taid.ToString()).ThenReturn("JobId");
FilePath f = TaskLog.GetTaskLogFile(taid, true, TaskLog.LogName.Stdout);
NUnit.Framework.Assert.IsTrue(f.GetAbsolutePath().EndsWith("testString" + FilePath
.separatorChar + "stdout"));
// test getRealTaskLogFileLocation
FilePath indexFile = TaskLog.GetIndexFile(taid, true);
if (!indexFile.GetParentFile().Exists())
{
indexFile.GetParentFile().Mkdirs();
}
indexFile.Delete();
indexFile.CreateNewFile();
TaskLog.SyncLogs("location", taid, true);
NUnit.Framework.Assert.IsTrue(indexFile.GetAbsolutePath().EndsWith("userlogs" + FilePath
.separatorChar + "job_job_0001" + FilePath.separatorChar + "JobId.cleanup" + FilePath
.separatorChar + "log.index"));
f = TaskLog.GetRealTaskLogFileLocation(taid, true, TaskLog.LogName.Debugout);
if (f != null)
{
NUnit.Framework.Assert.IsTrue(f.GetAbsolutePath().EndsWith("location" + FilePath.
separatorChar + "debugout"));
FileUtils.CopyFile(indexFile, f);
}
// test obtainLogDirOwner
NUnit.Framework.Assert.IsTrue(TaskLog.ObtainLogDirOwner(taid).Length > 0);
// test TaskLog.Reader
NUnit.Framework.Assert.IsTrue(ReadTaskLog(TaskLog.LogName.Debugout, taid, true).Length
> 0);
}
/// <summary>
/// Validates map phase progress after each record is processed by map task
/// using custom task reporter.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestMapProgress()
{
JobConf job = new JobConf();
fs = FileSystem.GetLocal(job);
Path rootDir = new Path(TestRootDir);
CreateInputFile(rootDir);
job.SetNumReduceTasks(0);
TaskAttemptID taskId = ((TaskAttemptID)TaskAttemptID.ForName("attempt_200907082313_0424_m_000000_0"
));
job.SetClass("mapreduce.job.outputformat.class", typeof(NullOutputFormat), typeof(
OutputFormat));
job.Set(FileInputFormat.InputDir, TestRootDir);
jobId = ((JobID)taskId.GetJobID());
JobContext jContext = new JobContextImpl(job, jobId);
InputFormat <object, object> input = ReflectionUtils.NewInstance(jContext.GetInputFormatClass
(), job);
IList <InputSplit> splits = input.GetSplits(jContext);
JobSplitWriter.CreateSplitFiles(new Path(TestRootDir), job, new Path(TestRootDir)
.GetFileSystem(job), splits);
JobSplit.TaskSplitMetaInfo[] splitMetaInfo = SplitMetaInfoReader.ReadSplitMetaInfo
(jobId, fs, job, new Path(TestRootDir));
job.SetUseNewMapper(true);
// use new api
for (int i = 0; i < splitMetaInfo.Length; i++)
{
// rawSplits.length is 1
map = new TestMapProgress.TestMapTask(this, job.Get(JTConfig.JtSystemDir, "/tmp/hadoop/mapred/system"
) + jobId + "job.xml", taskId, i, splitMetaInfo[i].GetSplitIndex(), 1);
JobConf localConf = new JobConf(job);
map.LocalizeConfiguration(localConf);
map.SetConf(localConf);
map.Run(localConf, fakeUmbilical);
}
// clean up
fs.Delete(rootDir, true);
}
/// <exception cref="System.Exception"/>
public static void Main(string[] args)
{
Sharpen.Thread.SetDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler
());
Log.Debug("Child starting");
JobConf job = new JobConf(MRJobConfig.JobConfFile);
// Initing with our JobConf allows us to avoid loading confs twice
Limits.Init(job);
UserGroupInformation.SetConfiguration(job);
string host = args[0];
int port = System.Convert.ToInt32(args[1]);
IPEndPoint address = NetUtils.CreateSocketAddrForHost(host, port);
TaskAttemptID firstTaskid = ((TaskAttemptID)TaskAttemptID.ForName(args[2]));
long jvmIdLong = long.Parse(args[3]);
JVMId jvmId = new JVMId(((JobID)firstTaskid.GetJobID()), firstTaskid.GetTaskType(
) == TaskType.Map, jvmIdLong);
// initialize metrics
DefaultMetricsSystem.Initialize(StringUtils.Camelize(firstTaskid.GetTaskType().ToString
()) + "Task");
// Security framework already loaded the tokens into current ugi
Credentials credentials = UserGroupInformation.GetCurrentUser().GetCredentials();
Log.Info("Executing with tokens:");
foreach (Org.Apache.Hadoop.Security.Token.Token <object> token in credentials.GetAllTokens
())
{
Log.Info(token);
}
// Create TaskUmbilicalProtocol as actual task owner.
UserGroupInformation taskOwner = UserGroupInformation.CreateRemoteUser(((JobID)firstTaskid
.GetJobID()).ToString());
Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> jt = TokenCache.GetJobToken
(credentials);
SecurityUtil.SetTokenService(jt, address);
taskOwner.AddToken(jt);
TaskUmbilicalProtocol umbilical = taskOwner.DoAs(new _PrivilegedExceptionAction_108
(address, job));
// report non-pid to application master
JvmContext context = new JvmContext(jvmId, "-1000");
Log.Debug("PID: " + Sharpen.Runtime.GetEnv()["JVM_PID"]);
Task task = null;
UserGroupInformation childUGI = null;
ScheduledExecutorService logSyncer = null;
try
{
int idleLoopCount = 0;
JvmTask myTask = null;
// poll for new task
for (int idle = 0; null == myTask; ++idle)
{
long sleepTimeMilliSecs = Math.Min(idle * 500, 1500);
Log.Info("Sleeping for " + sleepTimeMilliSecs + "ms before retrying again. Got null now."
);
TimeUnit.Milliseconds.Sleep(sleepTimeMilliSecs);
myTask = umbilical.GetTask(context);
}
if (myTask.ShouldDie())
{
return;
}
task = myTask.GetTask();
YarnChild.taskid = task.GetTaskID();
// Create the job-conf and set credentials
ConfigureTask(job, task, credentials, jt);
// Initiate Java VM metrics
JvmMetrics.InitSingleton(jvmId.ToString(), job.GetSessionId());
childUGI = UserGroupInformation.CreateRemoteUser(Runtime.Getenv(ApplicationConstants.Environment
.User.ToString()));
// Add tokens to new user so that it may execute its task correctly.
childUGI.AddCredentials(credentials);
// set job classloader if configured before invoking the task
MRApps.SetJobClassLoader(job);
logSyncer = TaskLog.CreateLogSyncer();
// Create a final reference to the task for the doAs block
Task taskFinal = task;
childUGI.DoAs(new _PrivilegedExceptionAction_158(taskFinal, job, umbilical));
}
catch (FSError e)
{
// use job-specified working directory
// run the task
Log.Fatal("FSError from child", e);
if (!ShutdownHookManager.Get().IsShutdownInProgress())
{
umbilical.FsError(taskid, e.Message);
}
}
catch (Exception exception)
{
Log.Warn("Exception running child : " + StringUtils.StringifyException(exception)
);
try
{
if (task != null)
{
// do cleanup for the task
if (childUGI == null)
{
// no need to job into doAs block
task.TaskCleanup(umbilical);
}
else
{
Task taskFinal = task;
childUGI.DoAs(new _PrivilegedExceptionAction_183(taskFinal, umbilical));
}
}
}
catch (Exception e)
{
Log.Info("Exception cleaning up: " + StringUtils.StringifyException(e));
}
// Report back any failures, for diagnostic purposes
if (taskid != null)
{
if (!ShutdownHookManager.Get().IsShutdownInProgress())
{
umbilical.FatalError(taskid, StringUtils.StringifyException(exception));
}
}
}
catch (Exception throwable)
{
Log.Fatal("Error running child : " + StringUtils.StringifyException(throwable));
if (taskid != null)
{
if (!ShutdownHookManager.Get().IsShutdownInProgress())
{
Exception tCause = throwable.InnerException;
string cause = tCause == null ? throwable.Message : StringUtils.StringifyException
(tCause);
umbilical.FatalError(taskid, cause);
}
}
}
finally
{
RPC.StopProxy(umbilical);
DefaultMetricsSystem.Shutdown();
TaskLog.SyncLogsShutdown(logSyncer);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public override bool KillTask(TaskAttemptID arg0, bool arg1)
{
return(clientCache.GetClient(arg0.GetJobID()).KillTask(arg0, arg1));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public override string[] GetTaskDiagnostics(TaskAttemptID arg0)
{
return(clientCache.GetClient(arg0.GetJobID()).GetTaskDiagnostics(arg0));
}
/// <exception cref="System.Exception"/>
public virtual void TestKillJob()
{
JobConf conf = new JobConf();
AppContext context = Org.Mockito.Mockito.Mock <AppContext>();
// a simple event handler solely to detect the container cleaned event
CountDownLatch isDone = new CountDownLatch(1);
EventHandler handler = new _EventHandler_106(isDone);
Org.Mockito.Mockito.When(context.GetEventHandler()).ThenReturn(handler);
// create and start the launcher
LocalContainerLauncher launcher = new LocalContainerLauncher(context, Org.Mockito.Mockito.Mock
<TaskUmbilicalProtocol>());
launcher.Init(conf);
launcher.Start();
// create mocked job, task, and task attempt
// a single-mapper job
JobId jobId = MRBuilderUtils.NewJobId(Runtime.CurrentTimeMillis(), 1, 1);
TaskId taskId = MRBuilderUtils.NewTaskId(jobId, 1, TaskType.Map);
TaskAttemptId taId = MRBuilderUtils.NewTaskAttemptId(taskId, 0);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
>();
Org.Mockito.Mockito.When(job.GetTotalMaps()).ThenReturn(1);
Org.Mockito.Mockito.When(job.GetTotalReduces()).ThenReturn(0);
IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> jobs = new Dictionary
<JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job>();
jobs[jobId] = job;
// app context returns the one and only job
Org.Mockito.Mockito.When(context.GetAllJobs()).ThenReturn(jobs);
Task ytask = Org.Mockito.Mockito.Mock <Task>();
Org.Mockito.Mockito.When(ytask.GetType()).ThenReturn(TaskType.Map);
Org.Mockito.Mockito.When(job.GetTask(taskId)).ThenReturn(ytask);
// create a sleeping mapper that runs beyond the test timeout
MapTask mapTask = Org.Mockito.Mockito.Mock <MapTask>();
Org.Mockito.Mockito.When(mapTask.IsMapOrReduce()).ThenReturn(true);
Org.Mockito.Mockito.When(mapTask.IsMapTask()).ThenReturn(true);
TaskAttemptID taskID = TypeConverter.FromYarn(taId);
Org.Mockito.Mockito.When(mapTask.GetTaskID()).ThenReturn(taskID);
Org.Mockito.Mockito.When(mapTask.GetJobID()).ThenReturn(((JobID)taskID.GetJobID()
));
Org.Mockito.Mockito.DoAnswer(new _Answer_152()).When(mapTask).Run(Matchers.IsA <JobConf
>(), Matchers.IsA <TaskUmbilicalProtocol>());
// sleep for a long time
// pump in a task attempt launch event
ContainerLauncherEvent launchEvent = new ContainerRemoteLaunchEvent(taId, null, CreateMockContainer
(), mapTask);
launcher.Handle(launchEvent);
Sharpen.Thread.Sleep(200);
// now pump in a container clean-up event
ContainerLauncherEvent cleanupEvent = new ContainerLauncherEvent(taId, null, null
, null, ContainerLauncher.EventType.ContainerRemoteCleanup);
launcher.Handle(cleanupEvent);
// wait for the event to fire: this should be received promptly
isDone.Await();
launcher.Close();
}
/// <exception cref="System.Exception"/>
private void TestRecoveryInternal(int commitVersion, int recoveryVersion)
{
JobConf conf = new JobConf();
FileOutputFormat.SetOutputPath(conf, outDir);
conf.Set(JobContext.TaskAttemptId, attempt);
conf.SetInt(MRConstants.ApplicationAttemptId, 1);
conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, commitVersion
);
JobContext jContext = new JobContextImpl(conf, ((JobID)taskID.GetJobID()));
TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID);
FileOutputCommitter committer = new FileOutputCommitter();
// setup
committer.SetupJob(jContext);
committer.SetupTask(tContext);
// write output
TextOutputFormat theOutputFormat = new TextOutputFormat();
RecordWriter theRecordWriter = theOutputFormat.GetRecordWriter(null, conf, partFile
, null);
WriteOutput(theRecordWriter, tContext);
// do commit
if (committer.NeedsTaskCommit(tContext))
{
committer.CommitTask(tContext);
}
Path jobTempDir1 = committer.GetCommittedTaskPath(tContext);
FilePath jtd1 = new FilePath(jobTempDir1.ToUri().GetPath());
if (commitVersion == 1)
{
NUnit.Framework.Assert.IsTrue("Version 1 commits to temporary dir " + jtd1, jtd1.
Exists());
ValidateContent(jobTempDir1);
}
else
{
NUnit.Framework.Assert.IsFalse("Version 2 commits to output dir " + jtd1, jtd1.Exists
());
}
//now while running the second app attempt,
//recover the task output from first attempt
JobConf conf2 = new JobConf(conf);
conf2.Set(JobContext.TaskAttemptId, attempt);
conf2.SetInt(MRConstants.ApplicationAttemptId, 2);
conf2.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, recoveryVersion
);
JobContext jContext2 = new JobContextImpl(conf2, ((JobID)taskID.GetJobID()));
TaskAttemptContext tContext2 = new TaskAttemptContextImpl(conf2, taskID);
FileOutputCommitter committer2 = new FileOutputCommitter();
committer2.SetupJob(jContext2);
committer2.RecoverTask(tContext2);
Path jobTempDir2 = committer2.GetCommittedTaskPath(tContext2);
FilePath jtd2 = new FilePath(jobTempDir2.ToUri().GetPath());
if (recoveryVersion == 1)
{
NUnit.Framework.Assert.IsTrue("Version 1 recovers to " + jtd2, jtd2.Exists());
ValidateContent(jobTempDir2);
}
else
{
NUnit.Framework.Assert.IsFalse("Version 2 commits to output dir " + jtd2, jtd2.Exists
());
if (commitVersion == 1)
{
NUnit.Framework.Assert.IsTrue("Version 2 recovery moves to output dir from " + jtd1
, jtd1.List().Length == 0);
}
}
committer2.CommitJob(jContext2);
ValidateContent(outDir);
FileUtil.FullyDelete(new FilePath(outDir.ToString()));
}