/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
private void TestFailAbortInternal(int version)
{
JobConf conf = new JobConf();
conf.Set(FileSystem.FsDefaultNameKey, "faildel:///");
conf.SetClass("fs.faildel.impl", typeof(TestFileOutputCommitter.FakeFileSystem),
typeof(FileSystem));
conf.Set(JobContext.TaskAttemptId, attempt);
conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, version);
conf.SetInt(MRConstants.ApplicationAttemptId, 1);
FileOutputFormat.SetOutputPath(conf, outDir);
JobContext jContext = new JobContextImpl(conf, ((JobID)taskID.GetJobID()));
TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID);
FileOutputCommitter committer = new FileOutputCommitter();
// do setup
committer.SetupJob(jContext);
committer.SetupTask(tContext);
// write output
FilePath jobTmpDir = new FilePath(new Path(outDir, FileOutputCommitter.TempDirName
+ Path.Separator + conf.GetInt(MRConstants.ApplicationAttemptId, 0) + Path.Separator
+ FileOutputCommitter.TempDirName).ToString());
FilePath taskTmpDir = new FilePath(jobTmpDir, "_" + taskID);
FilePath expectedFile = new FilePath(taskTmpDir, partFile);
TextOutputFormat <object, object> theOutputFormat = new TextOutputFormat();
RecordWriter <object, object> theRecordWriter = theOutputFormat.GetRecordWriter(null
, conf, expectedFile.GetAbsolutePath(), null);
WriteOutput(theRecordWriter, tContext);
// do abort
Exception th = null;
try
{
committer.AbortTask(tContext);
}
catch (IOException ie)
{
th = ie;
}
NUnit.Framework.Assert.IsNotNull(th);
NUnit.Framework.Assert.IsTrue(th is IOException);
NUnit.Framework.Assert.IsTrue(th.Message.Contains("fake delete failed"));
NUnit.Framework.Assert.IsTrue(expectedFile + " does not exists", expectedFile.Exists
());
th = null;
try
{
committer.AbortJob(jContext, JobStatus.State.Failed);
}
catch (IOException ie)
{
th = ie;
}
NUnit.Framework.Assert.IsNotNull(th);
NUnit.Framework.Assert.IsTrue(th is IOException);
NUnit.Framework.Assert.IsTrue(th.Message.Contains("fake delete failed"));
NUnit.Framework.Assert.IsTrue("job temp dir does not exists", jobTmpDir.Exists());
FileUtil.FullyDelete(new FilePath(outDir.ToString()));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="System.TypeLoadException"/>
private void RunNewReducer <Inkey, Invalue, Outkey, Outvalue>(JobConf job, TaskUmbilicalProtocol
umbilical, Task.TaskReporter reporter, RawKeyValueIterator rIter, RawComparator
<INKEY> comparator)
{
System.Type keyClass = typeof(INKEY);
System.Type valueClass = typeof(INVALUE);
// wrap value iterator to report progress.
RawKeyValueIterator rawIter = rIter;
rIter = new _RawKeyValueIterator_587(rawIter, reporter);
// make a task context so we can get the classes
TaskAttemptContext taskContext = new TaskAttemptContextImpl(job, GetTaskID(), reporter
);
// make a reducer
Reducer <INKEY, INVALUE, OUTKEY, OUTVALUE> reducer = (Reducer <INKEY, INVALUE, OUTKEY
, OUTVALUE>)ReflectionUtils.NewInstance(taskContext.GetReducerClass(), job);
RecordWriter <OUTKEY, OUTVALUE> trackedRW = new ReduceTask.NewTrackingRecordWriter
<OUTKEY, OUTVALUE>(this, taskContext);
job.SetBoolean("mapred.skip.on", IsSkipping());
job.SetBoolean(JobContext.SkipRecords, IsSkipping());
Reducer.Context reducerContext = CreateReduceContext(reducer, job, GetTaskID(), rIter
, reduceInputKeyCounter, reduceInputValueCounter, trackedRW, committer, reporter
, comparator, keyClass, valueClass);
try
{
reducer.Run(reducerContext);
}
finally
{
trackedRW.Close(reducerContext);
}
}
/// <exception cref="System.Exception"/>
private void TestMapFileOutputCommitterInternal(int version)
{
JobConf conf = new JobConf();
FileOutputFormat.SetOutputPath(conf, outDir);
conf.Set(JobContext.TaskAttemptId, attempt);
conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, version);
JobContext jContext = new JobContextImpl(conf, ((JobID)taskID.GetJobID()));
TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID);
FileOutputCommitter committer = new FileOutputCommitter();
// setup
committer.SetupJob(jContext);
committer.SetupTask(tContext);
// write output
MapFileOutputFormat theOutputFormat = new MapFileOutputFormat();
RecordWriter theRecordWriter = theOutputFormat.GetRecordWriter(null, conf, partFile
, null);
WriteMapFileOutput(theRecordWriter, tContext);
// do commit
if (committer.NeedsTaskCommit(tContext))
{
committer.CommitTask(tContext);
}
committer.CommitJob(jContext);
// validate output
ValidateMapFileOutputContent(FileSystem.Get(conf), outDir);
FileUtil.FullyDelete(new FilePath(outDir.ToString()));
}
/// <summary>
/// Helper function to create the task's temporary output directory and
/// return the path to the task's output file.
/// </summary>
/// <param name="conf">job-configuration</param>
/// <param name="name">temporary task-output filename</param>
/// <returns>path to the task's temporary output file</returns>
/// <exception cref="System.IO.IOException"/>
public static Path GetTaskOutputPath(JobConf conf, string name)
{
// ${mapred.out.dir}
Path outputPath = GetOutputPath(conf);
if (outputPath == null)
{
throw new IOException("Undefined job output-path");
}
OutputCommitter committer = conf.GetOutputCommitter();
Path workPath = outputPath;
TaskAttemptContext context = new TaskAttemptContextImpl(conf, ((TaskAttemptID)TaskAttemptID
.ForName(conf.Get(JobContext.TaskAttemptId))));
if (committer is FileOutputCommitter)
{
workPath = ((FileOutputCommitter)committer).GetWorkPath(context, outputPath);
}
// ${mapred.out.dir}/_temporary/_${taskid}/${name}
return(new Path(workPath, name));
}
/// <exception cref="System.Exception"/>
public virtual void TestCommitter()
{
JobConf job = new JobConf();
SetConfForFileOutputCommitter(job);
JobContext jContext = new JobContextImpl(job, ((JobID)taskID.GetJobID()));
TaskAttemptContext tContext = new TaskAttemptContextImpl(job, taskID);
FileOutputCommitter committer = new FileOutputCommitter();
FileOutputFormat.SetWorkOutputPath(job, committer.GetTaskAttemptPath(tContext));
committer.SetupJob(jContext);
committer.SetupTask(tContext);
string file = "test.txt";
// A reporter that does nothing
Reporter reporter = Reporter.Null;
// write output
FileSystem localFs = FileSystem.GetLocal(job);
TextOutputFormat theOutputFormat = new TextOutputFormat();
RecordWriter theRecordWriter = theOutputFormat.GetRecordWriter(localFs, job, file
, reporter);
WriteOutput(theRecordWriter, reporter);
// do commit
committer.CommitTask(tContext);
committer.CommitJob(jContext);
// validate output
FilePath expectedFile = new FilePath(new Path(outDir, file).ToString());
StringBuilder expectedOutput = new StringBuilder();
expectedOutput.Append(key1).Append('\t').Append(val1).Append("\n");
expectedOutput.Append(val1).Append("\n");
expectedOutput.Append(val2).Append("\n");
expectedOutput.Append(key2).Append("\n");
expectedOutput.Append(key1).Append("\n");
expectedOutput.Append(key2).Append('\t').Append(val2).Append("\n");
string output = UtilsForTests.Slurp(expectedFile);
NUnit.Framework.Assert.AreEqual(output, expectedOutput.ToString());
FileUtil.FullyDelete(new FilePath(outDir.ToString()));
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestAbort()
{
JobConf job = new JobConf();
SetConfForFileOutputCommitter(job);
JobContext jContext = new JobContextImpl(job, ((JobID)taskID.GetJobID()));
TaskAttemptContext tContext = new TaskAttemptContextImpl(job, taskID);
FileOutputCommitter committer = new FileOutputCommitter();
FileOutputFormat.SetWorkOutputPath(job, committer.GetTaskAttemptPath(tContext));
// do setup
committer.SetupJob(jContext);
committer.SetupTask(tContext);
string file = "test.txt";
// A reporter that does nothing
Reporter reporter = Reporter.Null;
// write output
FileSystem localFs = FileSystem.GetLocal(job);
TextOutputFormat theOutputFormat = new TextOutputFormat();
RecordWriter theRecordWriter = theOutputFormat.GetRecordWriter(localFs, job, file
, reporter);
WriteOutput(theRecordWriter, reporter);
// do abort
committer.AbortTask(tContext);
FilePath expectedFile = new FilePath(new Path(committer.GetTaskAttemptPath(tContext
), file).ToString());
NUnit.Framework.Assert.IsFalse("task temp dir still exists", expectedFile.Exists(
));
committer.AbortJob(jContext, JobStatus.State.Failed);
expectedFile = new FilePath(new Path(outDir, FileOutputCommitter.TempDirName).ToString
());
NUnit.Framework.Assert.IsFalse("job temp dir " + expectedFile + " still exists",
expectedFile.Exists());
NUnit.Framework.Assert.AreEqual("Output directory not empty", 0, new FilePath(outDir
.ToString()).ListFiles().Length);
FileUtil.FullyDelete(new FilePath(outDir.ToString()));
}
/// <exception cref="System.Exception"/>
private void TestMapOnlyNoOutputInternal(int version)
{
JobConf conf = new JobConf();
//This is not set on purpose. FileOutputFormat.setOutputPath(conf, outDir);
conf.Set(JobContext.TaskAttemptId, attempt);
conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, version);
JobContext jContext = new JobContextImpl(conf, ((JobID)taskID.GetJobID()));
TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID);
FileOutputCommitter committer = new FileOutputCommitter();
// setup
committer.SetupJob(jContext);
committer.SetupTask(tContext);
if (committer.NeedsTaskCommit(tContext))
{
// do commit
committer.CommitTask(tContext);
}
committer.CommitJob(jContext);
// validate output
FileUtil.FullyDelete(new FilePath(outDir.ToString()));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
private void TestAbortInternal(int version)
{
JobConf conf = new JobConf();
FileOutputFormat.SetOutputPath(conf, outDir);
conf.Set(JobContext.TaskAttemptId, attempt);
conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, version);
JobContext jContext = new JobContextImpl(conf, ((JobID)taskID.GetJobID()));
TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID);
FileOutputCommitter committer = new FileOutputCommitter();
// do setup
committer.SetupJob(jContext);
committer.SetupTask(tContext);
// write output
TextOutputFormat theOutputFormat = new TextOutputFormat();
RecordWriter theRecordWriter = theOutputFormat.GetRecordWriter(null, conf, partFile
, null);
WriteOutput(theRecordWriter, tContext);
// do abort
committer.AbortTask(tContext);
FilePath @out = new FilePath(outDir.ToUri().GetPath());
Path workPath = committer.GetWorkPath(tContext, outDir);
FilePath wp = new FilePath(workPath.ToUri().GetPath());
FilePath expectedFile = new FilePath(wp, partFile);
NUnit.Framework.Assert.IsFalse("task temp dir still exists", expectedFile.Exists(
));
committer.AbortJob(jContext, JobStatus.State.Failed);
expectedFile = new FilePath(@out, FileOutputCommitter.TempDirName);
NUnit.Framework.Assert.IsFalse("job temp dir still exists", expectedFile.Exists()
);
NUnit.Framework.Assert.AreEqual("Output directory not empty", 0, @out.ListFiles()
.Length);
FileUtil.FullyDelete(@out);
}
/// <exception cref="System.Exception"/>
private OutputCommitter CreateOutputCommitter(bool newApiCommitter, JobID jobId,
Configuration conf)
{
OutputCommitter committer = null;
LocalJobRunner.Log.Info("OutputCommitter set in config " + conf.Get("mapred.output.committer.class"
));
if (newApiCommitter)
{
TaskID taskId = new TaskID(jobId, TaskType.Map, 0);
TaskAttemptID taskAttemptID = new TaskAttemptID(taskId, 0);
TaskAttemptContext taskContext = new TaskAttemptContextImpl(conf, taskAttemptID);
OutputFormat outputFormat = ReflectionUtils.NewInstance(taskContext.GetOutputFormatClass
(), conf);
committer = outputFormat.GetOutputCommitter(taskContext);
}
else
{
committer = ReflectionUtils.NewInstance(conf.GetClass <OutputCommitter>("mapred.output.committer.class"
, typeof(FileOutputCommitter)), conf);
}
LocalJobRunner.Log.Info("OutputCommitter is " + committer.GetType().FullName);
return(committer);
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestFailAbort()
{
JobConf job = new JobConf();
job.Set(FileSystem.FsDefaultNameKey, "faildel:///");
job.SetClass("fs.faildel.impl", typeof(TestMRCJCFileOutputCommitter.FakeFileSystem
), typeof(FileSystem));
SetConfForFileOutputCommitter(job);
JobContext jContext = new JobContextImpl(job, ((JobID)taskID.GetJobID()));
TaskAttemptContext tContext = new TaskAttemptContextImpl(job, taskID);
FileOutputCommitter committer = new FileOutputCommitter();
FileOutputFormat.SetWorkOutputPath(job, committer.GetTaskAttemptPath(tContext));
// do setup
committer.SetupJob(jContext);
committer.SetupTask(tContext);
string file = "test.txt";
FilePath jobTmpDir = new FilePath(committer.GetJobAttemptPath(jContext).ToUri().GetPath
());
FilePath taskTmpDir = new FilePath(committer.GetTaskAttemptPath(tContext).ToUri()
.GetPath());
FilePath expectedFile = new FilePath(taskTmpDir, file);
// A reporter that does nothing
Reporter reporter = Reporter.Null;
// write output
FileSystem localFs = new TestMRCJCFileOutputCommitter.FakeFileSystem();
TextOutputFormat theOutputFormat = new TextOutputFormat();
RecordWriter theRecordWriter = theOutputFormat.GetRecordWriter(localFs, job, expectedFile
.GetAbsolutePath(), reporter);
WriteOutput(theRecordWriter, reporter);
// do abort
Exception th = null;
try
{
committer.AbortTask(tContext);
}
catch (IOException ie)
{
th = ie;
}
NUnit.Framework.Assert.IsNotNull(th);
NUnit.Framework.Assert.IsTrue(th is IOException);
NUnit.Framework.Assert.IsTrue(th.Message.Contains("fake delete failed"));
NUnit.Framework.Assert.IsTrue(expectedFile + " does not exists", expectedFile.Exists
());
th = null;
try
{
committer.AbortJob(jContext, JobStatus.State.Failed);
}
catch (IOException ie)
{
th = ie;
}
NUnit.Framework.Assert.IsNotNull(th);
NUnit.Framework.Assert.IsTrue(th is IOException);
NUnit.Framework.Assert.IsTrue(th.Message.Contains("fake delete failed"));
NUnit.Framework.Assert.IsTrue("job temp dir does not exists", jobTmpDir.Exists());
}
/// <exception cref="System.Exception"/>
private void TestRecoveryInternal(int commitVersion, int recoveryVersion)
{
JobConf conf = new JobConf();
FileOutputFormat.SetOutputPath(conf, outDir);
conf.Set(JobContext.TaskAttemptId, attempt);
conf.SetInt(MRConstants.ApplicationAttemptId, 1);
conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, commitVersion
);
JobContext jContext = new JobContextImpl(conf, ((JobID)taskID.GetJobID()));
TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID);
FileOutputCommitter committer = new FileOutputCommitter();
// setup
committer.SetupJob(jContext);
committer.SetupTask(tContext);
// write output
TextOutputFormat theOutputFormat = new TextOutputFormat();
RecordWriter theRecordWriter = theOutputFormat.GetRecordWriter(null, conf, partFile
, null);
WriteOutput(theRecordWriter, tContext);
// do commit
if (committer.NeedsTaskCommit(tContext))
{
committer.CommitTask(tContext);
}
Path jobTempDir1 = committer.GetCommittedTaskPath(tContext);
FilePath jtd1 = new FilePath(jobTempDir1.ToUri().GetPath());
if (commitVersion == 1)
{
NUnit.Framework.Assert.IsTrue("Version 1 commits to temporary dir " + jtd1, jtd1.
Exists());
ValidateContent(jobTempDir1);
}
else
{
NUnit.Framework.Assert.IsFalse("Version 2 commits to output dir " + jtd1, jtd1.Exists
());
}
//now while running the second app attempt,
//recover the task output from first attempt
JobConf conf2 = new JobConf(conf);
conf2.Set(JobContext.TaskAttemptId, attempt);
conf2.SetInt(MRConstants.ApplicationAttemptId, 2);
conf2.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, recoveryVersion
);
JobContext jContext2 = new JobContextImpl(conf2, ((JobID)taskID.GetJobID()));
TaskAttemptContext tContext2 = new TaskAttemptContextImpl(conf2, taskID);
FileOutputCommitter committer2 = new FileOutputCommitter();
committer2.SetupJob(jContext2);
committer2.RecoverTask(tContext2);
Path jobTempDir2 = committer2.GetCommittedTaskPath(tContext2);
FilePath jtd2 = new FilePath(jobTempDir2.ToUri().GetPath());
if (recoveryVersion == 1)
{
NUnit.Framework.Assert.IsTrue("Version 1 recovers to " + jtd2, jtd2.Exists());
ValidateContent(jobTempDir2);
}
else
{
NUnit.Framework.Assert.IsFalse("Version 2 commits to output dir " + jtd2, jtd2.Exists
());
if (commitVersion == 1)
{
NUnit.Framework.Assert.IsTrue("Version 2 recovery moves to output dir from " + jtd1
, jtd1.List().Length == 0);
}
}
committer2.CommitJob(jContext2);
ValidateContent(outDir);
FileUtil.FullyDelete(new FilePath(outDir.ToString()));
}