/// <exception cref="System.IO.IOException"/>
        /// <exception cref="System.Exception"/>
        private void TestFailAbortInternal(int version)
        {
            JobConf conf = new JobConf();

            conf.Set(FileSystem.FsDefaultNameKey, "faildel:///");
            conf.SetClass("fs.faildel.impl", typeof(TestFileOutputCommitter.FakeFileSystem),
                          typeof(FileSystem));
            conf.Set(JobContext.TaskAttemptId, attempt);
            conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, version);
            conf.SetInt(MRConstants.ApplicationAttemptId, 1);
            FileOutputFormat.SetOutputPath(conf, outDir);
            JobContext          jContext  = new JobContextImpl(conf, ((JobID)taskID.GetJobID()));
            TaskAttemptContext  tContext  = new TaskAttemptContextImpl(conf, taskID);
            FileOutputCommitter committer = new FileOutputCommitter();

            // do setup
            committer.SetupJob(jContext);
            committer.SetupTask(tContext);
            // write output
            FilePath jobTmpDir = new FilePath(new Path(outDir, FileOutputCommitter.TempDirName
                                                       + Path.Separator + conf.GetInt(MRConstants.ApplicationAttemptId, 0) + Path.Separator
                                                       + FileOutputCommitter.TempDirName).ToString());
            FilePath taskTmpDir   = new FilePath(jobTmpDir, "_" + taskID);
            FilePath expectedFile = new FilePath(taskTmpDir, partFile);
            TextOutputFormat <object, object> theOutputFormat = new TextOutputFormat();
            RecordWriter <object, object>     theRecordWriter = theOutputFormat.GetRecordWriter(null
                                                                                                , conf, expectedFile.GetAbsolutePath(), null);

            WriteOutput(theRecordWriter, tContext);
            // do abort
            Exception th = null;

            try
            {
                committer.AbortTask(tContext);
            }
            catch (IOException ie)
            {
                th = ie;
            }
            NUnit.Framework.Assert.IsNotNull(th);
            NUnit.Framework.Assert.IsTrue(th is IOException);
            NUnit.Framework.Assert.IsTrue(th.Message.Contains("fake delete failed"));
            NUnit.Framework.Assert.IsTrue(expectedFile + " does not exists", expectedFile.Exists
                                              ());
            th = null;
            try
            {
                committer.AbortJob(jContext, JobStatus.State.Failed);
            }
            catch (IOException ie)
            {
                th = ie;
            }
            NUnit.Framework.Assert.IsNotNull(th);
            NUnit.Framework.Assert.IsTrue(th is IOException);
            NUnit.Framework.Assert.IsTrue(th.Message.Contains("fake delete failed"));
            NUnit.Framework.Assert.IsTrue("job temp dir does not exists", jobTmpDir.Exists());
            FileUtil.FullyDelete(new FilePath(outDir.ToString()));
        }
        /// <exception cref="System.Exception"/>
        private void TestMapFileOutputCommitterInternal(int version)
        {
            JobConf conf = new JobConf();

            FileOutputFormat.SetOutputPath(conf, outDir);
            conf.Set(JobContext.TaskAttemptId, attempt);
            conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, version);
            JobContext          jContext  = new JobContextImpl(conf, ((JobID)taskID.GetJobID()));
            TaskAttemptContext  tContext  = new TaskAttemptContextImpl(conf, taskID);
            FileOutputCommitter committer = new FileOutputCommitter();

            // setup
            committer.SetupJob(jContext);
            committer.SetupTask(tContext);
            // write output
            MapFileOutputFormat theOutputFormat = new MapFileOutputFormat();
            RecordWriter        theRecordWriter = theOutputFormat.GetRecordWriter(null, conf, partFile
                                                                                  , null);

            WriteMapFileOutput(theRecordWriter, tContext);
            // do commit
            if (committer.NeedsTaskCommit(tContext))
            {
                committer.CommitTask(tContext);
            }
            committer.CommitJob(jContext);
            // validate output
            ValidateMapFileOutputContent(FileSystem.Get(conf), outDir);
            FileUtil.FullyDelete(new FilePath(outDir.ToString()));
        }
        /// <exception cref="System.Exception"/>
        public virtual void TestCommitter()
        {
            JobConf job = new JobConf();

            SetConfForFileOutputCommitter(job);
            JobContext          jContext  = new JobContextImpl(job, ((JobID)taskID.GetJobID()));
            TaskAttemptContext  tContext  = new TaskAttemptContextImpl(job, taskID);
            FileOutputCommitter committer = new FileOutputCommitter();

            FileOutputFormat.SetWorkOutputPath(job, committer.GetTaskAttemptPath(tContext));
            committer.SetupJob(jContext);
            committer.SetupTask(tContext);
            string file = "test.txt";
            // A reporter that does nothing
            Reporter reporter = Reporter.Null;
            // write output
            FileSystem       localFs         = FileSystem.GetLocal(job);
            TextOutputFormat theOutputFormat = new TextOutputFormat();
            RecordWriter     theRecordWriter = theOutputFormat.GetRecordWriter(localFs, job, file
                                                                               , reporter);

            WriteOutput(theRecordWriter, reporter);
            // do commit
            committer.CommitTask(tContext);
            committer.CommitJob(jContext);
            // validate output
            FilePath      expectedFile   = new FilePath(new Path(outDir, file).ToString());
            StringBuilder expectedOutput = new StringBuilder();

            expectedOutput.Append(key1).Append('\t').Append(val1).Append("\n");
            expectedOutput.Append(val1).Append("\n");
            expectedOutput.Append(val2).Append("\n");
            expectedOutput.Append(key2).Append("\n");
            expectedOutput.Append(key1).Append("\n");
            expectedOutput.Append(key2).Append('\t').Append(val2).Append("\n");
            string output = UtilsForTests.Slurp(expectedFile);

            NUnit.Framework.Assert.AreEqual(output, expectedOutput.ToString());
            FileUtil.FullyDelete(new FilePath(outDir.ToString()));
        }
        /// <exception cref="System.IO.IOException"/>
        public virtual void TestAbort()
        {
            JobConf job = new JobConf();

            SetConfForFileOutputCommitter(job);
            JobContext          jContext  = new JobContextImpl(job, ((JobID)taskID.GetJobID()));
            TaskAttemptContext  tContext  = new TaskAttemptContextImpl(job, taskID);
            FileOutputCommitter committer = new FileOutputCommitter();

            FileOutputFormat.SetWorkOutputPath(job, committer.GetTaskAttemptPath(tContext));
            // do setup
            committer.SetupJob(jContext);
            committer.SetupTask(tContext);
            string file = "test.txt";
            // A reporter that does nothing
            Reporter reporter = Reporter.Null;
            // write output
            FileSystem       localFs         = FileSystem.GetLocal(job);
            TextOutputFormat theOutputFormat = new TextOutputFormat();
            RecordWriter     theRecordWriter = theOutputFormat.GetRecordWriter(localFs, job, file
                                                                               , reporter);

            WriteOutput(theRecordWriter, reporter);
            // do abort
            committer.AbortTask(tContext);
            FilePath expectedFile = new FilePath(new Path(committer.GetTaskAttemptPath(tContext
                                                                                       ), file).ToString());

            NUnit.Framework.Assert.IsFalse("task temp dir still exists", expectedFile.Exists(
                                               ));
            committer.AbortJob(jContext, JobStatus.State.Failed);
            expectedFile = new FilePath(new Path(outDir, FileOutputCommitter.TempDirName).ToString
                                            ());
            NUnit.Framework.Assert.IsFalse("job temp dir " + expectedFile + " still exists",
                                           expectedFile.Exists());
            NUnit.Framework.Assert.AreEqual("Output directory not empty", 0, new FilePath(outDir
                                                                                          .ToString()).ListFiles().Length);
            FileUtil.FullyDelete(new FilePath(outDir.ToString()));
        }
        /// <exception cref="System.Exception"/>
        private void TestMapOnlyNoOutputInternal(int version)
        {
            JobConf conf = new JobConf();

            //This is not set on purpose. FileOutputFormat.setOutputPath(conf, outDir);
            conf.Set(JobContext.TaskAttemptId, attempt);
            conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, version);
            JobContext          jContext  = new JobContextImpl(conf, ((JobID)taskID.GetJobID()));
            TaskAttemptContext  tContext  = new TaskAttemptContextImpl(conf, taskID);
            FileOutputCommitter committer = new FileOutputCommitter();

            // setup
            committer.SetupJob(jContext);
            committer.SetupTask(tContext);
            if (committer.NeedsTaskCommit(tContext))
            {
                // do commit
                committer.CommitTask(tContext);
            }
            committer.CommitJob(jContext);
            // validate output
            FileUtil.FullyDelete(new FilePath(outDir.ToString()));
        }
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="System.Exception"/>
        private void TestAbortInternal(int version)
        {
            JobConf conf = new JobConf();

            FileOutputFormat.SetOutputPath(conf, outDir);
            conf.Set(JobContext.TaskAttemptId, attempt);
            conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, version);
            JobContext          jContext  = new JobContextImpl(conf, ((JobID)taskID.GetJobID()));
            TaskAttemptContext  tContext  = new TaskAttemptContextImpl(conf, taskID);
            FileOutputCommitter committer = new FileOutputCommitter();

            // do setup
            committer.SetupJob(jContext);
            committer.SetupTask(tContext);
            // write output
            TextOutputFormat theOutputFormat = new TextOutputFormat();
            RecordWriter     theRecordWriter = theOutputFormat.GetRecordWriter(null, conf, partFile
                                                                               , null);

            WriteOutput(theRecordWriter, tContext);
            // do abort
            committer.AbortTask(tContext);
            FilePath @out         = new FilePath(outDir.ToUri().GetPath());
            Path     workPath     = committer.GetWorkPath(tContext, outDir);
            FilePath wp           = new FilePath(workPath.ToUri().GetPath());
            FilePath expectedFile = new FilePath(wp, partFile);

            NUnit.Framework.Assert.IsFalse("task temp dir still exists", expectedFile.Exists(
                                               ));
            committer.AbortJob(jContext, JobStatus.State.Failed);
            expectedFile = new FilePath(@out, FileOutputCommitter.TempDirName);
            NUnit.Framework.Assert.IsFalse("job temp dir still exists", expectedFile.Exists()
                                           );
            NUnit.Framework.Assert.AreEqual("Output directory not empty", 0, @out.ListFiles()
                                            .Length);
            FileUtil.FullyDelete(@out);
        }
        /// <exception cref="System.IO.IOException"/>
        public virtual void TestFailAbort()
        {
            JobConf job = new JobConf();

            job.Set(FileSystem.FsDefaultNameKey, "faildel:///");
            job.SetClass("fs.faildel.impl", typeof(TestMRCJCFileOutputCommitter.FakeFileSystem
                                                   ), typeof(FileSystem));
            SetConfForFileOutputCommitter(job);
            JobContext          jContext  = new JobContextImpl(job, ((JobID)taskID.GetJobID()));
            TaskAttemptContext  tContext  = new TaskAttemptContextImpl(job, taskID);
            FileOutputCommitter committer = new FileOutputCommitter();

            FileOutputFormat.SetWorkOutputPath(job, committer.GetTaskAttemptPath(tContext));
            // do setup
            committer.SetupJob(jContext);
            committer.SetupTask(tContext);
            string   file      = "test.txt";
            FilePath jobTmpDir = new FilePath(committer.GetJobAttemptPath(jContext).ToUri().GetPath
                                                  ());
            FilePath taskTmpDir = new FilePath(committer.GetTaskAttemptPath(tContext).ToUri()
                                               .GetPath());
            FilePath expectedFile = new FilePath(taskTmpDir, file);
            // A reporter that does nothing
            Reporter reporter = Reporter.Null;
            // write output
            FileSystem       localFs         = new TestMRCJCFileOutputCommitter.FakeFileSystem();
            TextOutputFormat theOutputFormat = new TextOutputFormat();
            RecordWriter     theRecordWriter = theOutputFormat.GetRecordWriter(localFs, job, expectedFile
                                                                               .GetAbsolutePath(), reporter);

            WriteOutput(theRecordWriter, reporter);
            // do abort
            Exception th = null;

            try
            {
                committer.AbortTask(tContext);
            }
            catch (IOException ie)
            {
                th = ie;
            }
            NUnit.Framework.Assert.IsNotNull(th);
            NUnit.Framework.Assert.IsTrue(th is IOException);
            NUnit.Framework.Assert.IsTrue(th.Message.Contains("fake delete failed"));
            NUnit.Framework.Assert.IsTrue(expectedFile + " does not exists", expectedFile.Exists
                                              ());
            th = null;
            try
            {
                committer.AbortJob(jContext, JobStatus.State.Failed);
            }
            catch (IOException ie)
            {
                th = ie;
            }
            NUnit.Framework.Assert.IsNotNull(th);
            NUnit.Framework.Assert.IsTrue(th is IOException);
            NUnit.Framework.Assert.IsTrue(th.Message.Contains("fake delete failed"));
            NUnit.Framework.Assert.IsTrue("job temp dir does not exists", jobTmpDir.Exists());
        }
        /// <exception cref="System.Exception"/>
        private void TestRecoveryInternal(int commitVersion, int recoveryVersion)
        {
            JobConf conf = new JobConf();

            FileOutputFormat.SetOutputPath(conf, outDir);
            conf.Set(JobContext.TaskAttemptId, attempt);
            conf.SetInt(MRConstants.ApplicationAttemptId, 1);
            conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, commitVersion
                        );
            JobContext          jContext  = new JobContextImpl(conf, ((JobID)taskID.GetJobID()));
            TaskAttemptContext  tContext  = new TaskAttemptContextImpl(conf, taskID);
            FileOutputCommitter committer = new FileOutputCommitter();

            // setup
            committer.SetupJob(jContext);
            committer.SetupTask(tContext);
            // write output
            TextOutputFormat theOutputFormat = new TextOutputFormat();
            RecordWriter     theRecordWriter = theOutputFormat.GetRecordWriter(null, conf, partFile
                                                                               , null);

            WriteOutput(theRecordWriter, tContext);
            // do commit
            if (committer.NeedsTaskCommit(tContext))
            {
                committer.CommitTask(tContext);
            }
            Path     jobTempDir1 = committer.GetCommittedTaskPath(tContext);
            FilePath jtd1        = new FilePath(jobTempDir1.ToUri().GetPath());

            if (commitVersion == 1)
            {
                NUnit.Framework.Assert.IsTrue("Version 1 commits to temporary dir " + jtd1, jtd1.
                                              Exists());
                ValidateContent(jobTempDir1);
            }
            else
            {
                NUnit.Framework.Assert.IsFalse("Version 2 commits to output dir " + jtd1, jtd1.Exists
                                                   ());
            }
            //now while running the second app attempt,
            //recover the task output from first attempt
            JobConf conf2 = new JobConf(conf);

            conf2.Set(JobContext.TaskAttemptId, attempt);
            conf2.SetInt(MRConstants.ApplicationAttemptId, 2);
            conf2.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, recoveryVersion
                         );
            JobContext          jContext2  = new JobContextImpl(conf2, ((JobID)taskID.GetJobID()));
            TaskAttemptContext  tContext2  = new TaskAttemptContextImpl(conf2, taskID);
            FileOutputCommitter committer2 = new FileOutputCommitter();

            committer2.SetupJob(jContext2);
            committer2.RecoverTask(tContext2);
            Path     jobTempDir2 = committer2.GetCommittedTaskPath(tContext2);
            FilePath jtd2        = new FilePath(jobTempDir2.ToUri().GetPath());

            if (recoveryVersion == 1)
            {
                NUnit.Framework.Assert.IsTrue("Version 1 recovers to " + jtd2, jtd2.Exists());
                ValidateContent(jobTempDir2);
            }
            else
            {
                NUnit.Framework.Assert.IsFalse("Version 2 commits to output dir " + jtd2, jtd2.Exists
                                                   ());
                if (commitVersion == 1)
                {
                    NUnit.Framework.Assert.IsTrue("Version 2  recovery moves to output dir from " + jtd1
                                                  , jtd1.List().Length == 0);
                }
            }
            committer2.CommitJob(jContext2);
            ValidateContent(outDir);
            FileUtil.FullyDelete(new FilePath(outDir.ToString()));
        }