/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
private void TestFailAbortInternal(int version)
{
JobConf conf = new JobConf();
conf.Set(FileSystem.FsDefaultNameKey, "faildel:///");
conf.SetClass("fs.faildel.impl", typeof(TestFileOutputCommitter.FakeFileSystem),
typeof(FileSystem));
conf.Set(JobContext.TaskAttemptId, attempt);
conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, version);
conf.SetInt(MRConstants.ApplicationAttemptId, 1);
FileOutputFormat.SetOutputPath(conf, outDir);
JobContext jContext = new JobContextImpl(conf, ((JobID)taskID.GetJobID()));
TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID);
FileOutputCommitter committer = new FileOutputCommitter();
// do setup
committer.SetupJob(jContext);
committer.SetupTask(tContext);
// write output
FilePath jobTmpDir = new FilePath(new Path(outDir, FileOutputCommitter.TempDirName
+ Path.Separator + conf.GetInt(MRConstants.ApplicationAttemptId, 0) + Path.Separator
+ FileOutputCommitter.TempDirName).ToString());
FilePath taskTmpDir = new FilePath(jobTmpDir, "_" + taskID);
FilePath expectedFile = new FilePath(taskTmpDir, partFile);
TextOutputFormat <object, object> theOutputFormat = new TextOutputFormat();
RecordWriter <object, object> theRecordWriter = theOutputFormat.GetRecordWriter(null
, conf, expectedFile.GetAbsolutePath(), null);
WriteOutput(theRecordWriter, tContext);
// do abort
Exception th = null;
try
{
committer.AbortTask(tContext);
}
catch (IOException ie)
{
th = ie;
}
NUnit.Framework.Assert.IsNotNull(th);
NUnit.Framework.Assert.IsTrue(th is IOException);
NUnit.Framework.Assert.IsTrue(th.Message.Contains("fake delete failed"));
NUnit.Framework.Assert.IsTrue(expectedFile + " does not exists", expectedFile.Exists
());
th = null;
try
{
committer.AbortJob(jContext, JobStatus.State.Failed);
}
catch (IOException ie)
{
th = ie;
}
NUnit.Framework.Assert.IsNotNull(th);
NUnit.Framework.Assert.IsTrue(th is IOException);
NUnit.Framework.Assert.IsTrue(th.Message.Contains("fake delete failed"));
NUnit.Framework.Assert.IsTrue("job temp dir does not exists", jobTmpDir.Exists());
FileUtil.FullyDelete(new FilePath(outDir.ToString()));
}
/// <exception cref="System.Exception"/>
private void TestMapFileOutputCommitterInternal(int version)
{
JobConf conf = new JobConf();
FileOutputFormat.SetOutputPath(conf, outDir);
conf.Set(JobContext.TaskAttemptId, attempt);
conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, version);
JobContext jContext = new JobContextImpl(conf, ((JobID)taskID.GetJobID()));
TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID);
FileOutputCommitter committer = new FileOutputCommitter();
// setup
committer.SetupJob(jContext);
committer.SetupTask(tContext);
// write output
MapFileOutputFormat theOutputFormat = new MapFileOutputFormat();
RecordWriter theRecordWriter = theOutputFormat.GetRecordWriter(null, conf, partFile
, null);
WriteMapFileOutput(theRecordWriter, tContext);
// do commit
if (committer.NeedsTaskCommit(tContext))
{
committer.CommitTask(tContext);
}
committer.CommitJob(jContext);
// validate output
ValidateMapFileOutputContent(FileSystem.Get(conf), outDir);
FileUtil.FullyDelete(new FilePath(outDir.ToString()));
}
/// <summary>
/// A test that mimics a failed task to ensure that it does
/// not get into the COMMIT_PENDING state, by using a fake
/// UmbilicalProtocol's implementation that fails if the commit.
/// </summary>
/// <remarks>
/// A test that mimics a failed task to ensure that it does
/// not get into the COMMIT_PENDING state, by using a fake
/// UmbilicalProtocol's implementation that fails if the commit.
/// protocol is played.
/// The test mocks the various steps in a failed task's
/// life-cycle using a special OutputCommitter and UmbilicalProtocol
/// implementation.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestTaskCleanupDoesNotCommit()
{
// Mimic a job with a special committer that does not cleanup
// files when a task fails.
JobConf job = new JobConf();
job.SetOutputCommitter(typeof(TestTaskCommit.CommitterWithoutCleanup));
Path outDir = new Path(rootDir, "output");
FileOutputFormat.SetOutputPath(job, outDir);
// Mimic job setup
string dummyAttemptID = "attempt_200707121733_0001_m_000000_0";
TaskAttemptID attemptID = ((TaskAttemptID)TaskAttemptID.ForName(dummyAttemptID));
OutputCommitter committer = new TestTaskCommit.CommitterWithoutCleanup();
JobContext jContext = new JobContextImpl(job, ((JobID)attemptID.GetJobID()));
committer.SetupJob(jContext);
// Mimic a map task
dummyAttemptID = "attempt_200707121733_0001_m_000001_0";
attemptID = ((TaskAttemptID)TaskAttemptID.ForName(dummyAttemptID));
Task task = new MapTask(null, attemptID, 0, null, 1);
task.SetConf(job);
task.LocalizeConfiguration(job);
task.Initialize(job, ((JobID)attemptID.GetJobID()), Reporter.Null, false);
// Mimic the map task writing some output.
string file = "test.txt";
FileSystem localFs = FileSystem.GetLocal(job);
TextOutputFormat <Text, Text> theOutputFormat = new TextOutputFormat <Text, Text>();
RecordWriter <Text, Text> theRecordWriter = theOutputFormat.GetRecordWriter(localFs
, job, file, Reporter.Null);
theRecordWriter.Write(new Text("key"), new Text("value"));
theRecordWriter.Close(Reporter.Null);
// Mimic a task failure; setting up the task for cleanup simulates
// the abort protocol to be played.
// Without checks in the framework, this will fail
// as the committer will cause a COMMIT to happen for
// the cleanup task.
task.SetTaskCleanupTask();
TestTaskCommit.MyUmbilical umbilical = new TestTaskCommit.MyUmbilical(this);
task.Run(job, umbilical);
NUnit.Framework.Assert.IsTrue("Task did not succeed", umbilical.taskDone);
}
/// <exception cref="System.Exception"/>
public virtual void TestCommitter()
{
JobConf job = new JobConf();
SetConfForFileOutputCommitter(job);
JobContext jContext = new JobContextImpl(job, ((JobID)taskID.GetJobID()));
TaskAttemptContext tContext = new TaskAttemptContextImpl(job, taskID);
FileOutputCommitter committer = new FileOutputCommitter();
FileOutputFormat.SetWorkOutputPath(job, committer.GetTaskAttemptPath(tContext));
committer.SetupJob(jContext);
committer.SetupTask(tContext);
string file = "test.txt";
// A reporter that does nothing
Reporter reporter = Reporter.Null;
// write output
FileSystem localFs = FileSystem.GetLocal(job);
TextOutputFormat theOutputFormat = new TextOutputFormat();
RecordWriter theRecordWriter = theOutputFormat.GetRecordWriter(localFs, job, file
, reporter);
WriteOutput(theRecordWriter, reporter);
// do commit
committer.CommitTask(tContext);
committer.CommitJob(jContext);
// validate output
FilePath expectedFile = new FilePath(new Path(outDir, file).ToString());
StringBuilder expectedOutput = new StringBuilder();
expectedOutput.Append(key1).Append('\t').Append(val1).Append("\n");
expectedOutput.Append(val1).Append("\n");
expectedOutput.Append(val2).Append("\n");
expectedOutput.Append(key2).Append("\n");
expectedOutput.Append(key1).Append("\n");
expectedOutput.Append(key2).Append('\t').Append(val2).Append("\n");
string output = UtilsForTests.Slurp(expectedFile);
NUnit.Framework.Assert.AreEqual(output, expectedOutput.ToString());
FileUtil.FullyDelete(new FilePath(outDir.ToString()));
}
/// <summary>
/// Validates map phase progress after each record is processed by map task
/// using custom task reporter.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestMapProgress()
{
JobConf job = new JobConf();
fs = FileSystem.GetLocal(job);
Path rootDir = new Path(TestRootDir);
CreateInputFile(rootDir);
job.SetNumReduceTasks(0);
TaskAttemptID taskId = ((TaskAttemptID)TaskAttemptID.ForName("attempt_200907082313_0424_m_000000_0"
));
job.SetClass("mapreduce.job.outputformat.class", typeof(NullOutputFormat), typeof(
OutputFormat));
job.Set(FileInputFormat.InputDir, TestRootDir);
jobId = ((JobID)taskId.GetJobID());
JobContext jContext = new JobContextImpl(job, jobId);
InputFormat <object, object> input = ReflectionUtils.NewInstance(jContext.GetInputFormatClass
(), job);
IList <InputSplit> splits = input.GetSplits(jContext);
JobSplitWriter.CreateSplitFiles(new Path(TestRootDir), job, new Path(TestRootDir)
.GetFileSystem(job), splits);
JobSplit.TaskSplitMetaInfo[] splitMetaInfo = SplitMetaInfoReader.ReadSplitMetaInfo
(jobId, fs, job, new Path(TestRootDir));
job.SetUseNewMapper(true);
// use new api
for (int i = 0; i < splitMetaInfo.Length; i++)
{
// rawSplits.length is 1
map = new TestMapProgress.TestMapTask(this, job.Get(JTConfig.JtSystemDir, "/tmp/hadoop/mapred/system"
) + jobId + "job.xml", taskId, i, splitMetaInfo[i].GetSplitIndex(), 1);
JobConf localConf = new JobConf(job);
map.LocalizeConfiguration(localConf);
map.SetConf(localConf);
map.Run(localConf, fakeUmbilical);
}
// clean up
fs.Delete(rootDir, true);
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestAbort()
{
JobConf job = new JobConf();
SetConfForFileOutputCommitter(job);
JobContext jContext = new JobContextImpl(job, ((JobID)taskID.GetJobID()));
TaskAttemptContext tContext = new TaskAttemptContextImpl(job, taskID);
FileOutputCommitter committer = new FileOutputCommitter();
FileOutputFormat.SetWorkOutputPath(job, committer.GetTaskAttemptPath(tContext));
// do setup
committer.SetupJob(jContext);
committer.SetupTask(tContext);
string file = "test.txt";
// A reporter that does nothing
Reporter reporter = Reporter.Null;
// write output
FileSystem localFs = FileSystem.GetLocal(job);
TextOutputFormat theOutputFormat = new TextOutputFormat();
RecordWriter theRecordWriter = theOutputFormat.GetRecordWriter(localFs, job, file
, reporter);
WriteOutput(theRecordWriter, reporter);
// do abort
committer.AbortTask(tContext);
FilePath expectedFile = new FilePath(new Path(committer.GetTaskAttemptPath(tContext
), file).ToString());
NUnit.Framework.Assert.IsFalse("task temp dir still exists", expectedFile.Exists(
));
committer.AbortJob(jContext, JobStatus.State.Failed);
expectedFile = new FilePath(new Path(outDir, FileOutputCommitter.TempDirName).ToString
());
NUnit.Framework.Assert.IsFalse("job temp dir " + expectedFile + " still exists",
expectedFile.Exists());
NUnit.Framework.Assert.AreEqual("Output directory not empty", 0, new FilePath(outDir
.ToString()).ListFiles().Length);
FileUtil.FullyDelete(new FilePath(outDir.ToString()));
}
/// <exception cref="System.Exception"/>
private void TestMapOnlyNoOutputInternal(int version)
{
JobConf conf = new JobConf();
//This is not set on purpose. FileOutputFormat.setOutputPath(conf, outDir);
conf.Set(JobContext.TaskAttemptId, attempt);
conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, version);
JobContext jContext = new JobContextImpl(conf, ((JobID)taskID.GetJobID()));
TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID);
FileOutputCommitter committer = new FileOutputCommitter();
// setup
committer.SetupJob(jContext);
committer.SetupTask(tContext);
if (committer.NeedsTaskCommit(tContext))
{
// do commit
committer.CommitTask(tContext);
}
committer.CommitJob(jContext);
// validate output
FileUtil.FullyDelete(new FilePath(outDir.ToString()));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
private void TestAbortInternal(int version)
{
JobConf conf = new JobConf();
FileOutputFormat.SetOutputPath(conf, outDir);
conf.Set(JobContext.TaskAttemptId, attempt);
conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, version);
JobContext jContext = new JobContextImpl(conf, ((JobID)taskID.GetJobID()));
TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID);
FileOutputCommitter committer = new FileOutputCommitter();
// do setup
committer.SetupJob(jContext);
committer.SetupTask(tContext);
// write output
TextOutputFormat theOutputFormat = new TextOutputFormat();
RecordWriter theRecordWriter = theOutputFormat.GetRecordWriter(null, conf, partFile
, null);
WriteOutput(theRecordWriter, tContext);
// do abort
committer.AbortTask(tContext);
FilePath @out = new FilePath(outDir.ToUri().GetPath());
Path workPath = committer.GetWorkPath(tContext, outDir);
FilePath wp = new FilePath(workPath.ToUri().GetPath());
FilePath expectedFile = new FilePath(wp, partFile);
NUnit.Framework.Assert.IsFalse("task temp dir still exists", expectedFile.Exists(
));
committer.AbortJob(jContext, JobStatus.State.Failed);
expectedFile = new FilePath(@out, FileOutputCommitter.TempDirName);
NUnit.Framework.Assert.IsFalse("job temp dir still exists", expectedFile.Exists()
);
NUnit.Framework.Assert.AreEqual("Output directory not empty", 0, @out.ListFiles()
.Length);
FileUtil.FullyDelete(@out);
}
public override void Run()
{
JobID jobId = this.profile.GetJobID();
JobContext jContext = new JobContextImpl(this.job, jobId);
OutputCommitter outputCommitter = null;
try
{
outputCommitter = this.CreateOutputCommitter(this._enclosing.conf.GetUseNewMapper
(), jobId, this._enclosing.conf);
}
catch (Exception e)
{
LocalJobRunner.Log.Info("Failed to createOutputCommitter", e);
return;
}
try
{
JobSplit.TaskSplitMetaInfo[] taskSplitMetaInfos = SplitMetaInfoReader.ReadSplitMetaInfo
(jobId, this.localFs, this._enclosing.conf, this.systemJobDir);
int numReduceTasks = this.job.GetNumReduceTasks();
outputCommitter.SetupJob(jContext);
this.status.SetSetupProgress(1.0f);
IDictionary <TaskAttemptID, MapOutputFile> mapOutputFiles = Sharpen.Collections.SynchronizedMap
(new Dictionary <TaskAttemptID, MapOutputFile>());
IList <LocalJobRunner.Job.RunnableWithThrowable> mapRunnables = this.GetMapTaskRunnables
(taskSplitMetaInfos, jobId, mapOutputFiles);
this.InitCounters(mapRunnables.Count, numReduceTasks);
ExecutorService mapService = this.CreateMapExecutor();
this.RunTasks(mapRunnables, mapService, "map");
try
{
if (numReduceTasks > 0)
{
IList <LocalJobRunner.Job.RunnableWithThrowable> reduceRunnables = this.GetReduceTaskRunnables
(jobId, mapOutputFiles);
ExecutorService reduceService = this.CreateReduceExecutor();
this.RunTasks(reduceRunnables, reduceService, "reduce");
}
}
finally
{
foreach (MapOutputFile output in mapOutputFiles.Values)
{
output.RemoveAll();
}
}
// delete the temporary directory in output directory
outputCommitter.CommitJob(jContext);
this.status.SetCleanupProgress(1.0f);
if (this.killed)
{
this.status.SetRunState(JobStatus.Killed);
}
else
{
this.status.SetRunState(JobStatus.Succeeded);
}
JobEndNotifier.LocalRunnerNotification(this.job, this.status);
}
catch (Exception t)
{
try
{
outputCommitter.AbortJob(jContext, JobStatus.State.Failed);
}
catch (IOException)
{
LocalJobRunner.Log.Info("Error cleaning up job:" + this.id);
}
this.status.SetCleanupProgress(1.0f);
if (this.killed)
{
this.status.SetRunState(JobStatus.Killed);
}
else
{
this.status.SetRunState(JobStatus.Failed);
}
LocalJobRunner.Log.Warn(this.id, t);
JobEndNotifier.LocalRunnerNotification(this.job, this.status);
}
finally
{
try
{
this._enclosing.fs.Delete(this.systemJobFile.GetParent(), true);
// delete submit dir
this.localFs.Delete(this.localJobFile, true);
// delete local copy
// Cleanup distributed cache
this.localDistributedCacheManager.Close();
}
catch (IOException e)
{
LocalJobRunner.Log.Warn("Error cleaning up " + this.id + ": " + e);
}
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestFailAbort()
{
JobConf job = new JobConf();
job.Set(FileSystem.FsDefaultNameKey, "faildel:///");
job.SetClass("fs.faildel.impl", typeof(TestMRCJCFileOutputCommitter.FakeFileSystem
), typeof(FileSystem));
SetConfForFileOutputCommitter(job);
JobContext jContext = new JobContextImpl(job, ((JobID)taskID.GetJobID()));
TaskAttemptContext tContext = new TaskAttemptContextImpl(job, taskID);
FileOutputCommitter committer = new FileOutputCommitter();
FileOutputFormat.SetWorkOutputPath(job, committer.GetTaskAttemptPath(tContext));
// do setup
committer.SetupJob(jContext);
committer.SetupTask(tContext);
string file = "test.txt";
FilePath jobTmpDir = new FilePath(committer.GetJobAttemptPath(jContext).ToUri().GetPath
());
FilePath taskTmpDir = new FilePath(committer.GetTaskAttemptPath(tContext).ToUri()
.GetPath());
FilePath expectedFile = new FilePath(taskTmpDir, file);
// A reporter that does nothing
Reporter reporter = Reporter.Null;
// write output
FileSystem localFs = new TestMRCJCFileOutputCommitter.FakeFileSystem();
TextOutputFormat theOutputFormat = new TextOutputFormat();
RecordWriter theRecordWriter = theOutputFormat.GetRecordWriter(localFs, job, expectedFile
.GetAbsolutePath(), reporter);
WriteOutput(theRecordWriter, reporter);
// do abort
Exception th = null;
try
{
committer.AbortTask(tContext);
}
catch (IOException ie)
{
th = ie;
}
NUnit.Framework.Assert.IsNotNull(th);
NUnit.Framework.Assert.IsTrue(th is IOException);
NUnit.Framework.Assert.IsTrue(th.Message.Contains("fake delete failed"));
NUnit.Framework.Assert.IsTrue(expectedFile + " does not exists", expectedFile.Exists
());
th = null;
try
{
committer.AbortJob(jContext, JobStatus.State.Failed);
}
catch (IOException ie)
{
th = ie;
}
NUnit.Framework.Assert.IsNotNull(th);
NUnit.Framework.Assert.IsTrue(th is IOException);
NUnit.Framework.Assert.IsTrue(th.Message.Contains("fake delete failed"));
NUnit.Framework.Assert.IsTrue("job temp dir does not exists", jobTmpDir.Exists());
}
/// <exception cref="System.Exception"/>
private void TestRecoveryInternal(int commitVersion, int recoveryVersion)
{
JobConf conf = new JobConf();
FileOutputFormat.SetOutputPath(conf, outDir);
conf.Set(JobContext.TaskAttemptId, attempt);
conf.SetInt(MRConstants.ApplicationAttemptId, 1);
conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, commitVersion
);
JobContext jContext = new JobContextImpl(conf, ((JobID)taskID.GetJobID()));
TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID);
FileOutputCommitter committer = new FileOutputCommitter();
// setup
committer.SetupJob(jContext);
committer.SetupTask(tContext);
// write output
TextOutputFormat theOutputFormat = new TextOutputFormat();
RecordWriter theRecordWriter = theOutputFormat.GetRecordWriter(null, conf, partFile
, null);
WriteOutput(theRecordWriter, tContext);
// do commit
if (committer.NeedsTaskCommit(tContext))
{
committer.CommitTask(tContext);
}
Path jobTempDir1 = committer.GetCommittedTaskPath(tContext);
FilePath jtd1 = new FilePath(jobTempDir1.ToUri().GetPath());
if (commitVersion == 1)
{
NUnit.Framework.Assert.IsTrue("Version 1 commits to temporary dir " + jtd1, jtd1.
Exists());
ValidateContent(jobTempDir1);
}
else
{
NUnit.Framework.Assert.IsFalse("Version 2 commits to output dir " + jtd1, jtd1.Exists
());
}
//now while running the second app attempt,
//recover the task output from first attempt
JobConf conf2 = new JobConf(conf);
conf2.Set(JobContext.TaskAttemptId, attempt);
conf2.SetInt(MRConstants.ApplicationAttemptId, 2);
conf2.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, recoveryVersion
);
JobContext jContext2 = new JobContextImpl(conf2, ((JobID)taskID.GetJobID()));
TaskAttemptContext tContext2 = new TaskAttemptContextImpl(conf2, taskID);
FileOutputCommitter committer2 = new FileOutputCommitter();
committer2.SetupJob(jContext2);
committer2.RecoverTask(tContext2);
Path jobTempDir2 = committer2.GetCommittedTaskPath(tContext2);
FilePath jtd2 = new FilePath(jobTempDir2.ToUri().GetPath());
if (recoveryVersion == 1)
{
NUnit.Framework.Assert.IsTrue("Version 1 recovers to " + jtd2, jtd2.Exists());
ValidateContent(jobTempDir2);
}
else
{
NUnit.Framework.Assert.IsFalse("Version 2 commits to output dir " + jtd2, jtd2.Exists
());
if (commitVersion == 1)
{
NUnit.Framework.Assert.IsTrue("Version 2 recovery moves to output dir from " + jtd1
, jtd1.List().Length == 0);
}
}
committer2.CommitJob(jContext2);
ValidateContent(outDir);
FileUtil.FullyDelete(new FilePath(outDir.ToString()));
}
