/// <summary>Reads the output file into a string</summary>
/// <param name="conf"/>
/// <returns/>
/// <exception cref="System.IO.IOException"/>
public virtual string ReadOutputFile(Configuration conf)
{
FileSystem localFs = FileSystem.GetLocal(conf);
Path file = new Path(outputDir, "part-r-00000");
return(UtilsForTests.SlurpHadoop(file, localFs));
}
// run a job which gets stuck in mapper and kill it.
/// <exception cref="System.Exception"/>
private void TestKilledJob(string fileName, Type output, string[] exclude)
{
Path outDir = GetNewOutputDir();
Job job = MapReduceTestUtil.CreateKillJob(conf, outDir, inDir);
job.SetOutputFormatClass(output);
job.Submit();
// wait for the setup to be completed
while (job.SetupProgress() != 1.0f)
{
UtilsForTests.WaitFor(100);
}
job.KillJob();
// kill the job
NUnit.Framework.Assert.IsFalse("Job did not get kill", job.WaitForCompletion(true
));
if (fileName != null)
{
Path testFile = new Path(outDir, fileName);
NUnit.Framework.Assert.IsTrue("File " + testFile + " missing for job " + job.GetJobID
(), fs.Exists(testFile));
}
// check if the files from the missing set exists
foreach (string ex in exclude)
{
Path file = new Path(outDir, ex);
NUnit.Framework.Assert.IsFalse("File " + file + " should not be present for killed job "
+ job.GetJobID(), fs.Exists(file));
}
}
/// <exception cref="System.Exception"/>
public virtual void TestCommitter()
{
Job job = Job.GetInstance();
FileOutputFormat.SetOutputPath(job, outDir);
Configuration conf = job.GetConfiguration();
conf.Set(MRJobConfig.TaskAttemptId, attempt);
JobContext jContext = new JobContextImpl(conf, taskID.GetJobID());
TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID);
FileOutputCommitter committer = new FileOutputCommitter(outDir, tContext);
// setup
committer.SetupJob(jContext);
committer.SetupTask(tContext);
// write output
TextOutputFormat theOutputFormat = new TextOutputFormat();
RecordWriter theRecordWriter = theOutputFormat.GetRecordWriter(tContext);
WriteOutput(theRecordWriter, tContext);
// do commit
committer.CommitTask(tContext);
committer.CommitJob(jContext);
// validate output
FilePath expectedFile = new FilePath(new Path(outDir, partFile).ToString());
StringBuilder expectedOutput = new StringBuilder();
expectedOutput.Append(key1).Append('\t').Append(val1).Append("\n");
expectedOutput.Append(val1).Append("\n");
expectedOutput.Append(val2).Append("\n");
expectedOutput.Append(key2).Append("\n");
expectedOutput.Append(key1).Append("\n");
expectedOutput.Append(key2).Append('\t').Append(val2).Append("\n");
string output = UtilsForTests.Slurp(expectedFile);
NUnit.Framework.Assert.AreEqual(output, expectedOutput.ToString());
FileUtil.FullyDelete(new FilePath(outDir.ToString()));
}
