/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> private void TestFailAbortInternal(int version) { JobConf conf = new JobConf(); conf.Set(FileSystem.FsDefaultNameKey, "faildel:///"); conf.SetClass("fs.faildel.impl", typeof(TestFileOutputCommitter.FakeFileSystem), typeof(FileSystem)); conf.Set(JobContext.TaskAttemptId, attempt); conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, version); conf.SetInt(MRConstants.ApplicationAttemptId, 1); FileOutputFormat.SetOutputPath(conf, outDir); JobContext jContext = new JobContextImpl(conf, ((JobID)taskID.GetJobID())); TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID); FileOutputCommitter committer = new FileOutputCommitter(); // do setup committer.SetupJob(jContext); committer.SetupTask(tContext); // write output FilePath jobTmpDir = new FilePath(new Path(outDir, FileOutputCommitter.TempDirName + Path.Separator + conf.GetInt(MRConstants.ApplicationAttemptId, 0) + Path.Separator + FileOutputCommitter.TempDirName).ToString()); FilePath taskTmpDir = new FilePath(jobTmpDir, "_" + taskID); FilePath expectedFile = new FilePath(taskTmpDir, partFile); TextOutputFormat <object, object> theOutputFormat = new TextOutputFormat(); RecordWriter <object, object> theRecordWriter = theOutputFormat.GetRecordWriter(null , conf, expectedFile.GetAbsolutePath(), null); WriteOutput(theRecordWriter, tContext); // do abort Exception th = null; try { committer.AbortTask(tContext); } catch (IOException ie) { th = ie; } NUnit.Framework.Assert.IsNotNull(th); NUnit.Framework.Assert.IsTrue(th is IOException); NUnit.Framework.Assert.IsTrue(th.Message.Contains("fake delete failed")); NUnit.Framework.Assert.IsTrue(expectedFile + " does not exists", expectedFile.Exists ()); th = null; try { committer.AbortJob(jContext, JobStatus.State.Failed); } catch (IOException ie) { th = ie; } NUnit.Framework.Assert.IsNotNull(th); NUnit.Framework.Assert.IsTrue(th is IOException); NUnit.Framework.Assert.IsTrue(th.Message.Contains("fake delete failed")); NUnit.Framework.Assert.IsTrue("job temp dir does not exists", jobTmpDir.Exists()); FileUtil.FullyDelete(new FilePath(outDir.ToString())); }
/// <exception cref="System.Exception"/> private void TestMapFileOutputCommitterInternal(int version) { JobConf conf = new JobConf(); FileOutputFormat.SetOutputPath(conf, outDir); conf.Set(JobContext.TaskAttemptId, attempt); conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, version); JobContext jContext = new JobContextImpl(conf, ((JobID)taskID.GetJobID())); TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID); FileOutputCommitter committer = new FileOutputCommitter(); // setup committer.SetupJob(jContext); committer.SetupTask(tContext); // write output MapFileOutputFormat theOutputFormat = new MapFileOutputFormat(); RecordWriter theRecordWriter = theOutputFormat.GetRecordWriter(null, conf, partFile , null); WriteMapFileOutput(theRecordWriter, tContext); // do commit if (committer.NeedsTaskCommit(tContext)) { committer.CommitTask(tContext); } committer.CommitJob(jContext); // validate output ValidateMapFileOutputContent(FileSystem.Get(conf), outDir); FileUtil.FullyDelete(new FilePath(outDir.ToString())); }
/// <summary> /// A test that mimics a failed task to ensure that it does /// not get into the COMMIT_PENDING state, by using a fake /// UmbilicalProtocol's implementation that fails if the commit. /// </summary> /// <remarks> /// A test that mimics a failed task to ensure that it does /// not get into the COMMIT_PENDING state, by using a fake /// UmbilicalProtocol's implementation that fails if the commit. /// protocol is played. /// The test mocks the various steps in a failed task's /// life-cycle using a special OutputCommitter and UmbilicalProtocol /// implementation. /// </remarks> /// <exception cref="System.Exception"/> public virtual void TestTaskCleanupDoesNotCommit() { // Mimic a job with a special committer that does not cleanup // files when a task fails. JobConf job = new JobConf(); job.SetOutputCommitter(typeof(TestTaskCommit.CommitterWithoutCleanup)); Path outDir = new Path(rootDir, "output"); FileOutputFormat.SetOutputPath(job, outDir); // Mimic job setup string dummyAttemptID = "attempt_200707121733_0001_m_000000_0"; TaskAttemptID attemptID = ((TaskAttemptID)TaskAttemptID.ForName(dummyAttemptID)); OutputCommitter committer = new TestTaskCommit.CommitterWithoutCleanup(); JobContext jContext = new JobContextImpl(job, ((JobID)attemptID.GetJobID())); committer.SetupJob(jContext); // Mimic a map task dummyAttemptID = "attempt_200707121733_0001_m_000001_0"; attemptID = ((TaskAttemptID)TaskAttemptID.ForName(dummyAttemptID)); Task task = new MapTask(null, attemptID, 0, null, 1); task.SetConf(job); task.LocalizeConfiguration(job); task.Initialize(job, ((JobID)attemptID.GetJobID()), Reporter.Null, false); // Mimic the map task writing some output. string file = "test.txt"; FileSystem localFs = FileSystem.GetLocal(job); TextOutputFormat <Text, Text> theOutputFormat = new TextOutputFormat <Text, Text>(); RecordWriter <Text, Text> theRecordWriter = theOutputFormat.GetRecordWriter(localFs , job, file, Reporter.Null); theRecordWriter.Write(new Text("key"), new Text("value")); theRecordWriter.Close(Reporter.Null); // Mimic a task failure; setting up the task for cleanup simulates // the abort protocol to be played. // Without checks in the framework, this will fail // as the committer will cause a COMMIT to happen for // the cleanup task. task.SetTaskCleanupTask(); TestTaskCommit.MyUmbilical umbilical = new TestTaskCommit.MyUmbilical(this); task.Run(job, umbilical); NUnit.Framework.Assert.IsTrue("Task did not succeed", umbilical.taskDone); }
/// <exception cref="System.Exception"/> public virtual void TestCommitter() { JobConf job = new JobConf(); SetConfForFileOutputCommitter(job); JobContext jContext = new JobContextImpl(job, ((JobID)taskID.GetJobID())); TaskAttemptContext tContext = new TaskAttemptContextImpl(job, taskID); FileOutputCommitter committer = new FileOutputCommitter(); FileOutputFormat.SetWorkOutputPath(job, committer.GetTaskAttemptPath(tContext)); committer.SetupJob(jContext); committer.SetupTask(tContext); string file = "test.txt"; // A reporter that does nothing Reporter reporter = Reporter.Null; // write output FileSystem localFs = FileSystem.GetLocal(job); TextOutputFormat theOutputFormat = new TextOutputFormat(); RecordWriter theRecordWriter = theOutputFormat.GetRecordWriter(localFs, job, file , reporter); WriteOutput(theRecordWriter, reporter); // do commit committer.CommitTask(tContext); committer.CommitJob(jContext); // validate output FilePath expectedFile = new FilePath(new Path(outDir, file).ToString()); StringBuilder expectedOutput = new StringBuilder(); expectedOutput.Append(key1).Append('\t').Append(val1).Append("\n"); expectedOutput.Append(val1).Append("\n"); expectedOutput.Append(val2).Append("\n"); expectedOutput.Append(key2).Append("\n"); expectedOutput.Append(key1).Append("\n"); expectedOutput.Append(key2).Append('\t').Append(val2).Append("\n"); string output = UtilsForTests.Slurp(expectedFile); NUnit.Framework.Assert.AreEqual(output, expectedOutput.ToString()); FileUtil.FullyDelete(new FilePath(outDir.ToString())); }
/// <summary> /// Validates map phase progress after each record is processed by map task /// using custom task reporter. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestMapProgress() { JobConf job = new JobConf(); fs = FileSystem.GetLocal(job); Path rootDir = new Path(TestRootDir); CreateInputFile(rootDir); job.SetNumReduceTasks(0); TaskAttemptID taskId = ((TaskAttemptID)TaskAttemptID.ForName("attempt_200907082313_0424_m_000000_0" )); job.SetClass("mapreduce.job.outputformat.class", typeof(NullOutputFormat), typeof( OutputFormat)); job.Set(FileInputFormat.InputDir, TestRootDir); jobId = ((JobID)taskId.GetJobID()); JobContext jContext = new JobContextImpl(job, jobId); InputFormat <object, object> input = ReflectionUtils.NewInstance(jContext.GetInputFormatClass (), job); IList <InputSplit> splits = input.GetSplits(jContext); JobSplitWriter.CreateSplitFiles(new Path(TestRootDir), job, new Path(TestRootDir) .GetFileSystem(job), splits); JobSplit.TaskSplitMetaInfo[] splitMetaInfo = SplitMetaInfoReader.ReadSplitMetaInfo (jobId, fs, job, new Path(TestRootDir)); job.SetUseNewMapper(true); // use new api for (int i = 0; i < splitMetaInfo.Length; i++) { // rawSplits.length is 1 map = new TestMapProgress.TestMapTask(this, job.Get(JTConfig.JtSystemDir, "/tmp/hadoop/mapred/system" ) + jobId + "job.xml", taskId, i, splitMetaInfo[i].GetSplitIndex(), 1); JobConf localConf = new JobConf(job); map.LocalizeConfiguration(localConf); map.SetConf(localConf); map.Run(localConf, fakeUmbilical); } // clean up fs.Delete(rootDir, true); }
/// <exception cref="System.IO.IOException"/> public virtual void TestAbort() { JobConf job = new JobConf(); SetConfForFileOutputCommitter(job); JobContext jContext = new JobContextImpl(job, ((JobID)taskID.GetJobID())); TaskAttemptContext tContext = new TaskAttemptContextImpl(job, taskID); FileOutputCommitter committer = new FileOutputCommitter(); FileOutputFormat.SetWorkOutputPath(job, committer.GetTaskAttemptPath(tContext)); // do setup committer.SetupJob(jContext); committer.SetupTask(tContext); string file = "test.txt"; // A reporter that does nothing Reporter reporter = Reporter.Null; // write output FileSystem localFs = FileSystem.GetLocal(job); TextOutputFormat theOutputFormat = new TextOutputFormat(); RecordWriter theRecordWriter = theOutputFormat.GetRecordWriter(localFs, job, file , reporter); WriteOutput(theRecordWriter, reporter); // do abort committer.AbortTask(tContext); FilePath expectedFile = new FilePath(new Path(committer.GetTaskAttemptPath(tContext ), file).ToString()); NUnit.Framework.Assert.IsFalse("task temp dir still exists", expectedFile.Exists( )); committer.AbortJob(jContext, JobStatus.State.Failed); expectedFile = new FilePath(new Path(outDir, FileOutputCommitter.TempDirName).ToString ()); NUnit.Framework.Assert.IsFalse("job temp dir " + expectedFile + " still exists", expectedFile.Exists()); NUnit.Framework.Assert.AreEqual("Output directory not empty", 0, new FilePath(outDir .ToString()).ListFiles().Length); FileUtil.FullyDelete(new FilePath(outDir.ToString())); }
/// <exception cref="System.Exception"/> private void TestMapOnlyNoOutputInternal(int version) { JobConf conf = new JobConf(); //This is not set on purpose. FileOutputFormat.setOutputPath(conf, outDir); conf.Set(JobContext.TaskAttemptId, attempt); conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, version); JobContext jContext = new JobContextImpl(conf, ((JobID)taskID.GetJobID())); TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID); FileOutputCommitter committer = new FileOutputCommitter(); // setup committer.SetupJob(jContext); committer.SetupTask(tContext); if (committer.NeedsTaskCommit(tContext)) { // do commit committer.CommitTask(tContext); } committer.CommitJob(jContext); // validate output FileUtil.FullyDelete(new FilePath(outDir.ToString())); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> private void TestAbortInternal(int version) { JobConf conf = new JobConf(); FileOutputFormat.SetOutputPath(conf, outDir); conf.Set(JobContext.TaskAttemptId, attempt); conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, version); JobContext jContext = new JobContextImpl(conf, ((JobID)taskID.GetJobID())); TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID); FileOutputCommitter committer = new FileOutputCommitter(); // do setup committer.SetupJob(jContext); committer.SetupTask(tContext); // write output TextOutputFormat theOutputFormat = new TextOutputFormat(); RecordWriter theRecordWriter = theOutputFormat.GetRecordWriter(null, conf, partFile , null); WriteOutput(theRecordWriter, tContext); // do abort committer.AbortTask(tContext); FilePath @out = new FilePath(outDir.ToUri().GetPath()); Path workPath = committer.GetWorkPath(tContext, outDir); FilePath wp = new FilePath(workPath.ToUri().GetPath()); FilePath expectedFile = new FilePath(wp, partFile); NUnit.Framework.Assert.IsFalse("task temp dir still exists", expectedFile.Exists( )); committer.AbortJob(jContext, JobStatus.State.Failed); expectedFile = new FilePath(@out, FileOutputCommitter.TempDirName); NUnit.Framework.Assert.IsFalse("job temp dir still exists", expectedFile.Exists() ); NUnit.Framework.Assert.AreEqual("Output directory not empty", 0, @out.ListFiles() .Length); FileUtil.FullyDelete(@out); }
public override void Run() { JobID jobId = this.profile.GetJobID(); JobContext jContext = new JobContextImpl(this.job, jobId); OutputCommitter outputCommitter = null; try { outputCommitter = this.CreateOutputCommitter(this._enclosing.conf.GetUseNewMapper (), jobId, this._enclosing.conf); } catch (Exception e) { LocalJobRunner.Log.Info("Failed to createOutputCommitter", e); return; } try { JobSplit.TaskSplitMetaInfo[] taskSplitMetaInfos = SplitMetaInfoReader.ReadSplitMetaInfo (jobId, this.localFs, this._enclosing.conf, this.systemJobDir); int numReduceTasks = this.job.GetNumReduceTasks(); outputCommitter.SetupJob(jContext); this.status.SetSetupProgress(1.0f); IDictionary <TaskAttemptID, MapOutputFile> mapOutputFiles = Sharpen.Collections.SynchronizedMap (new Dictionary <TaskAttemptID, MapOutputFile>()); IList <LocalJobRunner.Job.RunnableWithThrowable> mapRunnables = this.GetMapTaskRunnables (taskSplitMetaInfos, jobId, mapOutputFiles); this.InitCounters(mapRunnables.Count, numReduceTasks); ExecutorService mapService = this.CreateMapExecutor(); this.RunTasks(mapRunnables, mapService, "map"); try { if (numReduceTasks > 0) { IList <LocalJobRunner.Job.RunnableWithThrowable> reduceRunnables = this.GetReduceTaskRunnables (jobId, mapOutputFiles); ExecutorService reduceService = this.CreateReduceExecutor(); this.RunTasks(reduceRunnables, reduceService, "reduce"); } } finally { foreach (MapOutputFile output in mapOutputFiles.Values) { output.RemoveAll(); } } // delete the temporary directory in output directory outputCommitter.CommitJob(jContext); this.status.SetCleanupProgress(1.0f); if (this.killed) { this.status.SetRunState(JobStatus.Killed); } else { this.status.SetRunState(JobStatus.Succeeded); } JobEndNotifier.LocalRunnerNotification(this.job, this.status); } catch (Exception t) { try { outputCommitter.AbortJob(jContext, JobStatus.State.Failed); } catch (IOException) { LocalJobRunner.Log.Info("Error cleaning up job:" + this.id); } this.status.SetCleanupProgress(1.0f); if (this.killed) { this.status.SetRunState(JobStatus.Killed); } else { this.status.SetRunState(JobStatus.Failed); } LocalJobRunner.Log.Warn(this.id, t); JobEndNotifier.LocalRunnerNotification(this.job, this.status); } finally { try { this._enclosing.fs.Delete(this.systemJobFile.GetParent(), true); // delete submit dir this.localFs.Delete(this.localJobFile, true); // delete local copy // Cleanup distributed cache this.localDistributedCacheManager.Close(); } catch (IOException e) { LocalJobRunner.Log.Warn("Error cleaning up " + this.id + ": " + e); } } }
/// <exception cref="System.IO.IOException"/> public virtual void TestFailAbort() { JobConf job = new JobConf(); job.Set(FileSystem.FsDefaultNameKey, "faildel:///"); job.SetClass("fs.faildel.impl", typeof(TestMRCJCFileOutputCommitter.FakeFileSystem ), typeof(FileSystem)); SetConfForFileOutputCommitter(job); JobContext jContext = new JobContextImpl(job, ((JobID)taskID.GetJobID())); TaskAttemptContext tContext = new TaskAttemptContextImpl(job, taskID); FileOutputCommitter committer = new FileOutputCommitter(); FileOutputFormat.SetWorkOutputPath(job, committer.GetTaskAttemptPath(tContext)); // do setup committer.SetupJob(jContext); committer.SetupTask(tContext); string file = "test.txt"; FilePath jobTmpDir = new FilePath(committer.GetJobAttemptPath(jContext).ToUri().GetPath ()); FilePath taskTmpDir = new FilePath(committer.GetTaskAttemptPath(tContext).ToUri() .GetPath()); FilePath expectedFile = new FilePath(taskTmpDir, file); // A reporter that does nothing Reporter reporter = Reporter.Null; // write output FileSystem localFs = new TestMRCJCFileOutputCommitter.FakeFileSystem(); TextOutputFormat theOutputFormat = new TextOutputFormat(); RecordWriter theRecordWriter = theOutputFormat.GetRecordWriter(localFs, job, expectedFile .GetAbsolutePath(), reporter); WriteOutput(theRecordWriter, reporter); // do abort Exception th = null; try { committer.AbortTask(tContext); } catch (IOException ie) { th = ie; } NUnit.Framework.Assert.IsNotNull(th); NUnit.Framework.Assert.IsTrue(th is IOException); NUnit.Framework.Assert.IsTrue(th.Message.Contains("fake delete failed")); NUnit.Framework.Assert.IsTrue(expectedFile + " does not exists", expectedFile.Exists ()); th = null; try { committer.AbortJob(jContext, JobStatus.State.Failed); } catch (IOException ie) { th = ie; } NUnit.Framework.Assert.IsNotNull(th); NUnit.Framework.Assert.IsTrue(th is IOException); NUnit.Framework.Assert.IsTrue(th.Message.Contains("fake delete failed")); NUnit.Framework.Assert.IsTrue("job temp dir does not exists", jobTmpDir.Exists()); }
/// <exception cref="System.Exception"/> private void TestRecoveryInternal(int commitVersion, int recoveryVersion) { JobConf conf = new JobConf(); FileOutputFormat.SetOutputPath(conf, outDir); conf.Set(JobContext.TaskAttemptId, attempt); conf.SetInt(MRConstants.ApplicationAttemptId, 1); conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, commitVersion ); JobContext jContext = new JobContextImpl(conf, ((JobID)taskID.GetJobID())); TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID); FileOutputCommitter committer = new FileOutputCommitter(); // setup committer.SetupJob(jContext); committer.SetupTask(tContext); // write output TextOutputFormat theOutputFormat = new TextOutputFormat(); RecordWriter theRecordWriter = theOutputFormat.GetRecordWriter(null, conf, partFile , null); WriteOutput(theRecordWriter, tContext); // do commit if (committer.NeedsTaskCommit(tContext)) { committer.CommitTask(tContext); } Path jobTempDir1 = committer.GetCommittedTaskPath(tContext); FilePath jtd1 = new FilePath(jobTempDir1.ToUri().GetPath()); if (commitVersion == 1) { NUnit.Framework.Assert.IsTrue("Version 1 commits to temporary dir " + jtd1, jtd1. Exists()); ValidateContent(jobTempDir1); } else { NUnit.Framework.Assert.IsFalse("Version 2 commits to output dir " + jtd1, jtd1.Exists ()); } //now while running the second app attempt, //recover the task output from first attempt JobConf conf2 = new JobConf(conf); conf2.Set(JobContext.TaskAttemptId, attempt); conf2.SetInt(MRConstants.ApplicationAttemptId, 2); conf2.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, recoveryVersion ); JobContext jContext2 = new JobContextImpl(conf2, ((JobID)taskID.GetJobID())); TaskAttemptContext tContext2 = new TaskAttemptContextImpl(conf2, taskID); FileOutputCommitter committer2 = new FileOutputCommitter(); committer2.SetupJob(jContext2); committer2.RecoverTask(tContext2); Path jobTempDir2 = committer2.GetCommittedTaskPath(tContext2); FilePath jtd2 = new FilePath(jobTempDir2.ToUri().GetPath()); if (recoveryVersion == 1) { NUnit.Framework.Assert.IsTrue("Version 1 recovers to " + jtd2, jtd2.Exists()); ValidateContent(jobTempDir2); } else { NUnit.Framework.Assert.IsFalse("Version 2 commits to output dir " + jtd2, jtd2.Exists ()); if (commitVersion == 1) { NUnit.Framework.Assert.IsTrue("Version 2 recovery moves to output dir from " + jtd1 , jtd1.List().Length == 0); } } committer2.CommitJob(jContext2); ValidateContent(outDir); FileUtil.FullyDelete(new FilePath(outDir.ToString())); }