/// <exception cref="System.Exception"/> private void TestMapFileOutputCommitterInternal(int version) { Job job = Job.GetInstance(); FileOutputFormat.SetOutputPath(job, outDir); Configuration conf = job.GetConfiguration(); conf.Set(MRJobConfig.TaskAttemptId, attempt); conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, version); JobContext jContext = new JobContextImpl(conf, taskID.GetJobID()); TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID); FileOutputCommitter committer = new FileOutputCommitter(outDir, tContext); // setup committer.SetupJob(jContext); committer.SetupTask(tContext); // write output MapFileOutputFormat theOutputFormat = new MapFileOutputFormat(); RecordWriter theRecordWriter = theOutputFormat.GetRecordWriter(tContext); WriteMapFileOutput(theRecordWriter, tContext); // do commit committer.CommitTask(tContext); committer.CommitJob(jContext); // validate output ValidateMapFileOutputContent(FileSystem.Get(job.GetConfiguration()), outDir); FileUtil.FullyDelete(new FilePath(outDir.ToString())); }
/// <exception cref="System.Exception"/> private void TestConcurrentCommitTaskWithSubDir(int version) { Job job = Job.GetInstance(); FileOutputFormat.SetOutputPath(job, outDir); Configuration conf = job.GetConfiguration(); conf.Set(MRJobConfig.TaskAttemptId, attempt); conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, version); conf.SetClass("fs.file.impl", typeof(TestFileOutputCommitter.RLFS), typeof(FileSystem )); FileSystem.CloseAll(); JobContext jContext = new JobContextImpl(conf, taskID.GetJobID()); FileOutputCommitter amCommitter = new FileOutputCommitter(outDir, jContext); amCommitter.SetupJob(jContext); TaskAttemptContext[] taCtx = new TaskAttemptContextImpl[2]; taCtx[0] = new TaskAttemptContextImpl(conf, taskID); taCtx[1] = new TaskAttemptContextImpl(conf, taskID1); TextOutputFormat[] tof = new TextOutputFormat[2]; for (int i = 0; i < tof.Length; i++) { tof[i] = new _TextOutputFormat_508(this); } ExecutorService executor = Executors.NewFixedThreadPool(2); try { for (int i_1 = 0; i_1 < taCtx.Length; i_1++) { int taskIdx = i_1; executor.Submit(new _Callable_524(this, tof, taskIdx, taCtx)); } } finally { executor.Shutdown(); while (!executor.AwaitTermination(1, TimeUnit.Seconds)) { Log.Info("Awaiting thread termination!"); } } amCommitter.CommitJob(jContext); RawLocalFileSystem lfs = new RawLocalFileSystem(); lfs.SetConf(conf); NUnit.Framework.Assert.IsFalse("Must not end up with sub_dir/sub_dir", lfs.Exists (new Path(OutSubDir, SubDir))); // validate output ValidateContent(OutSubDir); FileUtil.FullyDelete(new FilePath(outDir.ToString())); }
/// <exception cref="System.Exception"/> public virtual void TestEmptyOutput() { Job job = Job.GetInstance(); FileOutputFormat.SetOutputPath(job, outDir); Configuration conf = job.GetConfiguration(); conf.Set(MRJobConfig.TaskAttemptId, attempt); JobContext jContext = new JobContextImpl(conf, taskID.GetJobID()); TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID); FileOutputCommitter committer = new FileOutputCommitter(outDir, tContext); // setup committer.SetupJob(jContext); committer.SetupTask(tContext); // Do not write any output // do commit committer.CommitTask(tContext); committer.CommitJob(jContext); FileUtil.FullyDelete(new FilePath(outDir.ToString())); }
/// <exception cref="System.Exception"/> public virtual void TestCommitter() { Job job = Job.GetInstance(); FileOutputFormat.SetOutputPath(job, outDir); Configuration conf = job.GetConfiguration(); conf.Set(MRJobConfig.TaskAttemptId, attempt); JobContext jContext = new JobContextImpl(conf, taskID.GetJobID()); TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID); FileOutputCommitter committer = new FileOutputCommitter(outDir, tContext); // setup committer.SetupJob(jContext); committer.SetupTask(tContext); // write output TextOutputFormat theOutputFormat = new TextOutputFormat(); RecordWriter theRecordWriter = theOutputFormat.GetRecordWriter(tContext); WriteOutput(theRecordWriter, tContext); // do commit committer.CommitTask(tContext); committer.CommitJob(jContext); // validate output FilePath expectedFile = new FilePath(new Path(outDir, partFile).ToString()); StringBuilder expectedOutput = new StringBuilder(); expectedOutput.Append(key1).Append('\t').Append(val1).Append("\n"); expectedOutput.Append(val1).Append("\n"); expectedOutput.Append(val2).Append("\n"); expectedOutput.Append(key2).Append("\n"); expectedOutput.Append(key1).Append("\n"); expectedOutput.Append(key2).Append('\t').Append(val2).Append("\n"); string output = UtilsForTests.Slurp(expectedFile); NUnit.Framework.Assert.AreEqual(output, expectedOutput.ToString()); FileUtil.FullyDelete(new FilePath(outDir.ToString())); }
/// <exception cref="System.Exception"/> private void TestRecoveryInternal(int commitVersion, int recoveryVersion) { Job job = Job.GetInstance(); FileOutputFormat.SetOutputPath(job, outDir); Configuration conf = job.GetConfiguration(); conf.Set(MRJobConfig.TaskAttemptId, attempt); conf.SetInt(MRJobConfig.ApplicationAttemptId, 1); conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, commitVersion ); JobContext jContext = new JobContextImpl(conf, taskID.GetJobID()); TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID); FileOutputCommitter committer = new FileOutputCommitter(outDir, tContext); // setup committer.SetupJob(jContext); committer.SetupTask(tContext); // write output TextOutputFormat theOutputFormat = new TextOutputFormat(); RecordWriter theRecordWriter = theOutputFormat.GetRecordWriter(tContext); WriteOutput(theRecordWriter, tContext); // do commit committer.CommitTask(tContext); Path jobTempDir1 = committer.GetCommittedTaskPath(tContext); FilePath jtd = new FilePath(jobTempDir1.ToUri().GetPath()); if (commitVersion == 1) { NUnit.Framework.Assert.IsTrue("Version 1 commits to temporary dir " + jtd, jtd.Exists ()); ValidateContent(jtd); } else { NUnit.Framework.Assert.IsFalse("Version 2 commits to output dir " + jtd, jtd.Exists ()); } //now while running the second app attempt, //recover the task output from first attempt Configuration conf2 = job.GetConfiguration(); conf2.Set(MRJobConfig.TaskAttemptId, attempt); conf2.SetInt(MRJobConfig.ApplicationAttemptId, 2); conf2.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, recoveryVersion ); JobContext jContext2 = new JobContextImpl(conf2, taskID.GetJobID()); TaskAttemptContext tContext2 = new TaskAttemptContextImpl(conf2, taskID); FileOutputCommitter committer2 = new FileOutputCommitter(outDir, tContext2); committer2.SetupJob(tContext2); Path jobTempDir2 = committer2.GetCommittedTaskPath(tContext2); FilePath jtd2 = new FilePath(jobTempDir2.ToUri().GetPath()); committer2.RecoverTask(tContext2); if (recoveryVersion == 1) { NUnit.Framework.Assert.IsTrue("Version 1 recovers to " + jtd2, jtd2.Exists()); ValidateContent(jtd2); } else { NUnit.Framework.Assert.IsFalse("Version 2 commits to output dir " + jtd2, jtd2.Exists ()); if (commitVersion == 1) { NUnit.Framework.Assert.IsTrue("Version 2 recovery moves to output dir from " + jtd , jtd.List().Length == 0); } } committer2.CommitJob(jContext2); ValidateContent(outDir); FileUtil.FullyDelete(new FilePath(outDir.ToString())); }