/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> private void TestFailAbortInternal(int version) { JobConf conf = new JobConf(); conf.Set(FileSystem.FsDefaultNameKey, "faildel:///"); conf.SetClass("fs.faildel.impl", typeof(TestFileOutputCommitter.FakeFileSystem), typeof(FileSystem)); conf.Set(JobContext.TaskAttemptId, attempt); conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, version); conf.SetInt(MRConstants.ApplicationAttemptId, 1); FileOutputFormat.SetOutputPath(conf, outDir); JobContext jContext = new JobContextImpl(conf, ((JobID)taskID.GetJobID())); TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID); FileOutputCommitter committer = new FileOutputCommitter(); // do setup committer.SetupJob(jContext); committer.SetupTask(tContext); // write output FilePath jobTmpDir = new FilePath(new Path(outDir, FileOutputCommitter.TempDirName + Path.Separator + conf.GetInt(MRConstants.ApplicationAttemptId, 0) + Path.Separator + FileOutputCommitter.TempDirName).ToString()); FilePath taskTmpDir = new FilePath(jobTmpDir, "_" + taskID); FilePath expectedFile = new FilePath(taskTmpDir, partFile); TextOutputFormat <object, object> theOutputFormat = new TextOutputFormat(); RecordWriter <object, object> theRecordWriter = theOutputFormat.GetRecordWriter(null , conf, expectedFile.GetAbsolutePath(), null); WriteOutput(theRecordWriter, tContext); // do abort Exception th = null; try { committer.AbortTask(tContext); } catch (IOException ie) { th = ie; } NUnit.Framework.Assert.IsNotNull(th); NUnit.Framework.Assert.IsTrue(th is IOException); NUnit.Framework.Assert.IsTrue(th.Message.Contains("fake delete failed")); NUnit.Framework.Assert.IsTrue(expectedFile + " does not exists", expectedFile.Exists ()); th = null; try { committer.AbortJob(jContext, JobStatus.State.Failed); } catch (IOException ie) { th = ie; } NUnit.Framework.Assert.IsNotNull(th); NUnit.Framework.Assert.IsTrue(th is IOException); NUnit.Framework.Assert.IsTrue(th.Message.Contains("fake delete failed")); NUnit.Framework.Assert.IsTrue("job temp dir does not exists", jobTmpDir.Exists()); FileUtil.FullyDelete(new FilePath(outDir.ToString())); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> /// <exception cref="System.TypeLoadException"/> private void RunNewReducer <Inkey, Invalue, Outkey, Outvalue>(JobConf job, TaskUmbilicalProtocol umbilical, Task.TaskReporter reporter, RawKeyValueIterator rIter, RawComparator <INKEY> comparator) { System.Type keyClass = typeof(INKEY); System.Type valueClass = typeof(INVALUE); // wrap value iterator to report progress. RawKeyValueIterator rawIter = rIter; rIter = new _RawKeyValueIterator_587(rawIter, reporter); // make a task context so we can get the classes TaskAttemptContext taskContext = new TaskAttemptContextImpl(job, GetTaskID(), reporter ); // make a reducer Reducer <INKEY, INVALUE, OUTKEY, OUTVALUE> reducer = (Reducer <INKEY, INVALUE, OUTKEY , OUTVALUE>)ReflectionUtils.NewInstance(taskContext.GetReducerClass(), job); RecordWriter <OUTKEY, OUTVALUE> trackedRW = new ReduceTask.NewTrackingRecordWriter <OUTKEY, OUTVALUE>(this, taskContext); job.SetBoolean("mapred.skip.on", IsSkipping()); job.SetBoolean(JobContext.SkipRecords, IsSkipping()); Reducer.Context reducerContext = CreateReduceContext(reducer, job, GetTaskID(), rIter , reduceInputKeyCounter, reduceInputValueCounter, trackedRW, committer, reporter , comparator, keyClass, valueClass); try { reducer.Run(reducerContext); } finally { trackedRW.Close(reducerContext); } }
/// <exception cref="System.Exception"/> private void TestMapFileOutputCommitterInternal(int version) { JobConf conf = new JobConf(); FileOutputFormat.SetOutputPath(conf, outDir); conf.Set(JobContext.TaskAttemptId, attempt); conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, version); JobContext jContext = new JobContextImpl(conf, ((JobID)taskID.GetJobID())); TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID); FileOutputCommitter committer = new FileOutputCommitter(); // setup committer.SetupJob(jContext); committer.SetupTask(tContext); // write output MapFileOutputFormat theOutputFormat = new MapFileOutputFormat(); RecordWriter theRecordWriter = theOutputFormat.GetRecordWriter(null, conf, partFile , null); WriteMapFileOutput(theRecordWriter, tContext); // do commit if (committer.NeedsTaskCommit(tContext)) { committer.CommitTask(tContext); } committer.CommitJob(jContext); // validate output ValidateMapFileOutputContent(FileSystem.Get(conf), outDir); FileUtil.FullyDelete(new FilePath(outDir.ToString())); }
/// <summary> /// Helper function to create the task's temporary output directory and /// return the path to the task's output file. /// </summary> /// <param name="conf">job-configuration</param> /// <param name="name">temporary task-output filename</param> /// <returns>path to the task's temporary output file</returns> /// <exception cref="System.IO.IOException"/> public static Path GetTaskOutputPath(JobConf conf, string name) { // ${mapred.out.dir} Path outputPath = GetOutputPath(conf); if (outputPath == null) { throw new IOException("Undefined job output-path"); } OutputCommitter committer = conf.GetOutputCommitter(); Path workPath = outputPath; TaskAttemptContext context = new TaskAttemptContextImpl(conf, ((TaskAttemptID)TaskAttemptID .ForName(conf.Get(JobContext.TaskAttemptId)))); if (committer is FileOutputCommitter) { workPath = ((FileOutputCommitter)committer).GetWorkPath(context, outputPath); } // ${mapred.out.dir}/_temporary/_${taskid}/${name} return(new Path(workPath, name)); }
/// <exception cref="System.Exception"/> public virtual void TestCommitter() { JobConf job = new JobConf(); SetConfForFileOutputCommitter(job); JobContext jContext = new JobContextImpl(job, ((JobID)taskID.GetJobID())); TaskAttemptContext tContext = new TaskAttemptContextImpl(job, taskID); FileOutputCommitter committer = new FileOutputCommitter(); FileOutputFormat.SetWorkOutputPath(job, committer.GetTaskAttemptPath(tContext)); committer.SetupJob(jContext); committer.SetupTask(tContext); string file = "test.txt"; // A reporter that does nothing Reporter reporter = Reporter.Null; // write output FileSystem localFs = FileSystem.GetLocal(job); TextOutputFormat theOutputFormat = new TextOutputFormat(); RecordWriter theRecordWriter = theOutputFormat.GetRecordWriter(localFs, job, file , reporter); WriteOutput(theRecordWriter, reporter); // do commit committer.CommitTask(tContext); committer.CommitJob(jContext); // validate output FilePath expectedFile = new FilePath(new Path(outDir, file).ToString()); StringBuilder expectedOutput = new StringBuilder(); expectedOutput.Append(key1).Append('\t').Append(val1).Append("\n"); expectedOutput.Append(val1).Append("\n"); expectedOutput.Append(val2).Append("\n"); expectedOutput.Append(key2).Append("\n"); expectedOutput.Append(key1).Append("\n"); expectedOutput.Append(key2).Append('\t').Append(val2).Append("\n"); string output = UtilsForTests.Slurp(expectedFile); NUnit.Framework.Assert.AreEqual(output, expectedOutput.ToString()); FileUtil.FullyDelete(new FilePath(outDir.ToString())); }
/// <exception cref="System.IO.IOException"/> public virtual void TestAbort() { JobConf job = new JobConf(); SetConfForFileOutputCommitter(job); JobContext jContext = new JobContextImpl(job, ((JobID)taskID.GetJobID())); TaskAttemptContext tContext = new TaskAttemptContextImpl(job, taskID); FileOutputCommitter committer = new FileOutputCommitter(); FileOutputFormat.SetWorkOutputPath(job, committer.GetTaskAttemptPath(tContext)); // do setup committer.SetupJob(jContext); committer.SetupTask(tContext); string file = "test.txt"; // A reporter that does nothing Reporter reporter = Reporter.Null; // write output FileSystem localFs = FileSystem.GetLocal(job); TextOutputFormat theOutputFormat = new TextOutputFormat(); RecordWriter theRecordWriter = theOutputFormat.GetRecordWriter(localFs, job, file , reporter); WriteOutput(theRecordWriter, reporter); // do abort committer.AbortTask(tContext); FilePath expectedFile = new FilePath(new Path(committer.GetTaskAttemptPath(tContext ), file).ToString()); NUnit.Framework.Assert.IsFalse("task temp dir still exists", expectedFile.Exists( )); committer.AbortJob(jContext, JobStatus.State.Failed); expectedFile = new FilePath(new Path(outDir, FileOutputCommitter.TempDirName).ToString ()); NUnit.Framework.Assert.IsFalse("job temp dir " + expectedFile + " still exists", expectedFile.Exists()); NUnit.Framework.Assert.AreEqual("Output directory not empty", 0, new FilePath(outDir .ToString()).ListFiles().Length); FileUtil.FullyDelete(new FilePath(outDir.ToString())); }
/// <exception cref="System.Exception"/> private void TestMapOnlyNoOutputInternal(int version) { JobConf conf = new JobConf(); //This is not set on purpose. FileOutputFormat.setOutputPath(conf, outDir); conf.Set(JobContext.TaskAttemptId, attempt); conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, version); JobContext jContext = new JobContextImpl(conf, ((JobID)taskID.GetJobID())); TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID); FileOutputCommitter committer = new FileOutputCommitter(); // setup committer.SetupJob(jContext); committer.SetupTask(tContext); if (committer.NeedsTaskCommit(tContext)) { // do commit committer.CommitTask(tContext); } committer.CommitJob(jContext); // validate output FileUtil.FullyDelete(new FilePath(outDir.ToString())); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> private void TestAbortInternal(int version) { JobConf conf = new JobConf(); FileOutputFormat.SetOutputPath(conf, outDir); conf.Set(JobContext.TaskAttemptId, attempt); conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, version); JobContext jContext = new JobContextImpl(conf, ((JobID)taskID.GetJobID())); TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID); FileOutputCommitter committer = new FileOutputCommitter(); // do setup committer.SetupJob(jContext); committer.SetupTask(tContext); // write output TextOutputFormat theOutputFormat = new TextOutputFormat(); RecordWriter theRecordWriter = theOutputFormat.GetRecordWriter(null, conf, partFile , null); WriteOutput(theRecordWriter, tContext); // do abort committer.AbortTask(tContext); FilePath @out = new FilePath(outDir.ToUri().GetPath()); Path workPath = committer.GetWorkPath(tContext, outDir); FilePath wp = new FilePath(workPath.ToUri().GetPath()); FilePath expectedFile = new FilePath(wp, partFile); NUnit.Framework.Assert.IsFalse("task temp dir still exists", expectedFile.Exists( )); committer.AbortJob(jContext, JobStatus.State.Failed); expectedFile = new FilePath(@out, FileOutputCommitter.TempDirName); NUnit.Framework.Assert.IsFalse("job temp dir still exists", expectedFile.Exists() ); NUnit.Framework.Assert.AreEqual("Output directory not empty", 0, @out.ListFiles() .Length); FileUtil.FullyDelete(@out); }
/// <exception cref="System.Exception"/> private OutputCommitter CreateOutputCommitter(bool newApiCommitter, JobID jobId, Configuration conf) { OutputCommitter committer = null; LocalJobRunner.Log.Info("OutputCommitter set in config " + conf.Get("mapred.output.committer.class" )); if (newApiCommitter) { TaskID taskId = new TaskID(jobId, TaskType.Map, 0); TaskAttemptID taskAttemptID = new TaskAttemptID(taskId, 0); TaskAttemptContext taskContext = new TaskAttemptContextImpl(conf, taskAttemptID); OutputFormat outputFormat = ReflectionUtils.NewInstance(taskContext.GetOutputFormatClass (), conf); committer = outputFormat.GetOutputCommitter(taskContext); } else { committer = ReflectionUtils.NewInstance(conf.GetClass <OutputCommitter>("mapred.output.committer.class" , typeof(FileOutputCommitter)), conf); } LocalJobRunner.Log.Info("OutputCommitter is " + committer.GetType().FullName); return(committer); }
/// <exception cref="System.IO.IOException"/> public virtual void TestFailAbort() { JobConf job = new JobConf(); job.Set(FileSystem.FsDefaultNameKey, "faildel:///"); job.SetClass("fs.faildel.impl", typeof(TestMRCJCFileOutputCommitter.FakeFileSystem ), typeof(FileSystem)); SetConfForFileOutputCommitter(job); JobContext jContext = new JobContextImpl(job, ((JobID)taskID.GetJobID())); TaskAttemptContext tContext = new TaskAttemptContextImpl(job, taskID); FileOutputCommitter committer = new FileOutputCommitter(); FileOutputFormat.SetWorkOutputPath(job, committer.GetTaskAttemptPath(tContext)); // do setup committer.SetupJob(jContext); committer.SetupTask(tContext); string file = "test.txt"; FilePath jobTmpDir = new FilePath(committer.GetJobAttemptPath(jContext).ToUri().GetPath ()); FilePath taskTmpDir = new FilePath(committer.GetTaskAttemptPath(tContext).ToUri() .GetPath()); FilePath expectedFile = new FilePath(taskTmpDir, file); // A reporter that does nothing Reporter reporter = Reporter.Null; // write output FileSystem localFs = new TestMRCJCFileOutputCommitter.FakeFileSystem(); TextOutputFormat theOutputFormat = new TextOutputFormat(); RecordWriter theRecordWriter = theOutputFormat.GetRecordWriter(localFs, job, expectedFile .GetAbsolutePath(), reporter); WriteOutput(theRecordWriter, reporter); // do abort Exception th = null; try { committer.AbortTask(tContext); } catch (IOException ie) { th = ie; } NUnit.Framework.Assert.IsNotNull(th); NUnit.Framework.Assert.IsTrue(th is IOException); NUnit.Framework.Assert.IsTrue(th.Message.Contains("fake delete failed")); NUnit.Framework.Assert.IsTrue(expectedFile + " does not exists", expectedFile.Exists ()); th = null; try { committer.AbortJob(jContext, JobStatus.State.Failed); } catch (IOException ie) { th = ie; } NUnit.Framework.Assert.IsNotNull(th); NUnit.Framework.Assert.IsTrue(th is IOException); NUnit.Framework.Assert.IsTrue(th.Message.Contains("fake delete failed")); NUnit.Framework.Assert.IsTrue("job temp dir does not exists", jobTmpDir.Exists()); }
/// <exception cref="System.Exception"/> private void TestRecoveryInternal(int commitVersion, int recoveryVersion) { JobConf conf = new JobConf(); FileOutputFormat.SetOutputPath(conf, outDir); conf.Set(JobContext.TaskAttemptId, attempt); conf.SetInt(MRConstants.ApplicationAttemptId, 1); conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, commitVersion ); JobContext jContext = new JobContextImpl(conf, ((JobID)taskID.GetJobID())); TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID); FileOutputCommitter committer = new FileOutputCommitter(); // setup committer.SetupJob(jContext); committer.SetupTask(tContext); // write output TextOutputFormat theOutputFormat = new TextOutputFormat(); RecordWriter theRecordWriter = theOutputFormat.GetRecordWriter(null, conf, partFile , null); WriteOutput(theRecordWriter, tContext); // do commit if (committer.NeedsTaskCommit(tContext)) { committer.CommitTask(tContext); } Path jobTempDir1 = committer.GetCommittedTaskPath(tContext); FilePath jtd1 = new FilePath(jobTempDir1.ToUri().GetPath()); if (commitVersion == 1) { NUnit.Framework.Assert.IsTrue("Version 1 commits to temporary dir " + jtd1, jtd1. Exists()); ValidateContent(jobTempDir1); } else { NUnit.Framework.Assert.IsFalse("Version 2 commits to output dir " + jtd1, jtd1.Exists ()); } //now while running the second app attempt, //recover the task output from first attempt JobConf conf2 = new JobConf(conf); conf2.Set(JobContext.TaskAttemptId, attempt); conf2.SetInt(MRConstants.ApplicationAttemptId, 2); conf2.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, recoveryVersion ); JobContext jContext2 = new JobContextImpl(conf2, ((JobID)taskID.GetJobID())); TaskAttemptContext tContext2 = new TaskAttemptContextImpl(conf2, taskID); FileOutputCommitter committer2 = new FileOutputCommitter(); committer2.SetupJob(jContext2); committer2.RecoverTask(tContext2); Path jobTempDir2 = committer2.GetCommittedTaskPath(tContext2); FilePath jtd2 = new FilePath(jobTempDir2.ToUri().GetPath()); if (recoveryVersion == 1) { NUnit.Framework.Assert.IsTrue("Version 1 recovers to " + jtd2, jtd2.Exists()); ValidateContent(jobTempDir2); } else { NUnit.Framework.Assert.IsFalse("Version 2 commits to output dir " + jtd2, jtd2.Exists ()); if (commitVersion == 1) { NUnit.Framework.Assert.IsTrue("Version 2 recovery moves to output dir from " + jtd1 , jtd1.List().Length == 0); } } committer2.CommitJob(jContext2); ValidateContent(outDir); FileUtil.FullyDelete(new FilePath(outDir.ToString())); }