internal virtual Path GetCommittedTaskPath(TaskAttemptContext context)
{
Path @out = GetOutputPath(context);
return(@out == null ? null : FileOutputCommitter.GetCommittedTaskPath(context, @out
));
}
/// <exception cref="System.Exception"/>
private void TestRecoveryInternal(int commitVersion, int recoveryVersion)
{
JobConf conf = new JobConf();
FileOutputFormat.SetOutputPath(conf, outDir);
conf.Set(JobContext.TaskAttemptId, attempt);
conf.SetInt(MRConstants.ApplicationAttemptId, 1);
conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, commitVersion
);
JobContext jContext = new JobContextImpl(conf, ((JobID)taskID.GetJobID()));
TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID);
FileOutputCommitter committer = new FileOutputCommitter();
// setup
committer.SetupJob(jContext);
committer.SetupTask(tContext);
// write output
TextOutputFormat theOutputFormat = new TextOutputFormat();
RecordWriter theRecordWriter = theOutputFormat.GetRecordWriter(null, conf, partFile
, null);
WriteOutput(theRecordWriter, tContext);
// do commit
if (committer.NeedsTaskCommit(tContext))
{
committer.CommitTask(tContext);
}
Path jobTempDir1 = committer.GetCommittedTaskPath(tContext);
FilePath jtd1 = new FilePath(jobTempDir1.ToUri().GetPath());
if (commitVersion == 1)
{
NUnit.Framework.Assert.IsTrue("Version 1 commits to temporary dir " + jtd1, jtd1.
Exists());
ValidateContent(jobTempDir1);
}
else
{
NUnit.Framework.Assert.IsFalse("Version 2 commits to output dir " + jtd1, jtd1.Exists
());
}
//now while running the second app attempt,
//recover the task output from first attempt
JobConf conf2 = new JobConf(conf);
conf2.Set(JobContext.TaskAttemptId, attempt);
conf2.SetInt(MRConstants.ApplicationAttemptId, 2);
conf2.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, recoveryVersion
);
JobContext jContext2 = new JobContextImpl(conf2, ((JobID)taskID.GetJobID()));
TaskAttemptContext tContext2 = new TaskAttemptContextImpl(conf2, taskID);
FileOutputCommitter committer2 = new FileOutputCommitter();
committer2.SetupJob(jContext2);
committer2.RecoverTask(tContext2);
Path jobTempDir2 = committer2.GetCommittedTaskPath(tContext2);
FilePath jtd2 = new FilePath(jobTempDir2.ToUri().GetPath());
if (recoveryVersion == 1)
{
NUnit.Framework.Assert.IsTrue("Version 1 recovers to " + jtd2, jtd2.Exists());
ValidateContent(jobTempDir2);
}
else
{
NUnit.Framework.Assert.IsFalse("Version 2 commits to output dir " + jtd2, jtd2.Exists
());
if (commitVersion == 1)
{
NUnit.Framework.Assert.IsTrue("Version 2 recovery moves to output dir from " + jtd1
, jtd1.List().Length == 0);
}
}
committer2.CommitJob(jContext2);
ValidateContent(outDir);
FileUtil.FullyDelete(new FilePath(outDir.ToString()));
}