/// <exception cref="System.Exception"/>
        private void TestCommitterInternal(int version)
        {
            Job job = Job.GetInstance();

            FileOutputFormat.SetOutputPath(job, outDir);
            Configuration conf = job.GetConfiguration();

            conf.Set(MRJobConfig.TaskAttemptId, attempt);
            conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, version);
            JobContext          jContext  = new JobContextImpl(conf, taskID.GetJobID());
            TaskAttemptContext  tContext  = new TaskAttemptContextImpl(conf, taskID);
            FileOutputCommitter committer = new FileOutputCommitter(outDir, tContext);

            // setup
            committer.SetupJob(jContext);
            committer.SetupTask(tContext);
            // write output
            TextOutputFormat theOutputFormat = new TextOutputFormat();
            RecordWriter     theRecordWriter = theOutputFormat.GetRecordWriter(tContext);

            WriteOutput(theRecordWriter, tContext);
            // do commit
            committer.CommitTask(tContext);
            committer.CommitJob(jContext);
            // validate output
            ValidateContent(outDir);
            FileUtil.FullyDelete(new FilePath(outDir.ToString()));
        }
        /// <exception cref="System.Exception"/>
        private void TestConcurrentCommitTaskWithSubDir(int version)
        {
            Job job = Job.GetInstance();

            FileOutputFormat.SetOutputPath(job, outDir);
            Configuration conf = job.GetConfiguration();

            conf.Set(MRJobConfig.TaskAttemptId, attempt);
            conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, version);
            conf.SetClass("fs.file.impl", typeof(TestFileOutputCommitter.RLFS), typeof(FileSystem
                                                                                       ));
            FileSystem.CloseAll();
            JobContext          jContext    = new JobContextImpl(conf, taskID.GetJobID());
            FileOutputCommitter amCommitter = new FileOutputCommitter(outDir, jContext);

            amCommitter.SetupJob(jContext);
            TaskAttemptContext[] taCtx = new TaskAttemptContextImpl[2];
            taCtx[0] = new TaskAttemptContextImpl(conf, taskID);
            taCtx[1] = new TaskAttemptContextImpl(conf, taskID1);
            TextOutputFormat[] tof = new TextOutputFormat[2];
            for (int i = 0; i < tof.Length; i++)
            {
                tof[i] = new _TextOutputFormat_508(this);
            }
            ExecutorService executor = Executors.NewFixedThreadPool(2);

            try
            {
                for (int i_1 = 0; i_1 < taCtx.Length; i_1++)
                {
                    int taskIdx = i_1;
                    executor.Submit(new _Callable_524(this, tof, taskIdx, taCtx));
                }
            }
            finally
            {
                executor.Shutdown();
                while (!executor.AwaitTermination(1, TimeUnit.Seconds))
                {
                    Log.Info("Awaiting thread termination!");
                }
            }
            amCommitter.CommitJob(jContext);
            RawLocalFileSystem lfs = new RawLocalFileSystem();

            lfs.SetConf(conf);
            NUnit.Framework.Assert.IsFalse("Must not end up with sub_dir/sub_dir", lfs.Exists
                                               (new Path(OutSubDir, SubDir)));
            // validate output
            ValidateContent(OutSubDir);
            FileUtil.FullyDelete(new FilePath(outDir.ToString()));
        }
예제 #3
0
        /// <exception cref="System.Exception"/>
        public virtual void TestCommitter()
        {
            Job job = Job.GetInstance();

            FileOutputFormat.SetOutputPath(job, outDir);
            Configuration conf = job.GetConfiguration();

            conf.Set(MRJobConfig.TaskAttemptId, attempt);
            JobContext          jContext  = new JobContextImpl(conf, taskID.GetJobID());
            TaskAttemptContext  tContext  = new TaskAttemptContextImpl(conf, taskID);
            FileOutputCommitter committer = new FileOutputCommitter(outDir, tContext);

            // setup
            committer.SetupJob(jContext);
            committer.SetupTask(tContext);
            // write output
            TextOutputFormat theOutputFormat = new TextOutputFormat();
            RecordWriter     theRecordWriter = theOutputFormat.GetRecordWriter(tContext);

            WriteOutput(theRecordWriter, tContext);
            // do commit
            committer.CommitTask(tContext);
            committer.CommitJob(jContext);
            // validate output
            FilePath      expectedFile   = new FilePath(new Path(outDir, partFile).ToString());
            StringBuilder expectedOutput = new StringBuilder();

            expectedOutput.Append(key1).Append('\t').Append(val1).Append("\n");
            expectedOutput.Append(val1).Append("\n");
            expectedOutput.Append(val2).Append("\n");
            expectedOutput.Append(key2).Append("\n");
            expectedOutput.Append(key1).Append("\n");
            expectedOutput.Append(key2).Append('\t').Append(val2).Append("\n");
            string output = UtilsForTests.Slurp(expectedFile);

            NUnit.Framework.Assert.AreEqual(output, expectedOutput.ToString());
            FileUtil.FullyDelete(new FilePath(outDir.ToString()));
        }
        //test passed
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="System.Exception"/>
        private void TestAbortInternal(int version)
        {
            Job job = Job.GetInstance();

            FileOutputFormat.SetOutputPath(job, outDir);
            Configuration conf = job.GetConfiguration();

            conf.Set(MRJobConfig.TaskAttemptId, attempt);
            conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, version);
            JobContext          jContext  = new JobContextImpl(conf, taskID.GetJobID());
            TaskAttemptContext  tContext  = new TaskAttemptContextImpl(conf, taskID);
            FileOutputCommitter committer = new FileOutputCommitter(outDir, tContext);

            // do setup
            committer.SetupJob(jContext);
            committer.SetupTask(tContext);
            // write output
            TextOutputFormat theOutputFormat = new TextOutputFormat();
            RecordWriter     theRecordWriter = theOutputFormat.GetRecordWriter(tContext);

            WriteOutput(theRecordWriter, tContext);
            // do abort
            committer.AbortTask(tContext);
            FilePath expectedFile = new FilePath(new Path(committer.GetWorkPath(), partFile).
                                                 ToString());

            NUnit.Framework.Assert.IsFalse("task temp dir still exists", expectedFile.Exists(
                                               ));
            committer.AbortJob(jContext, JobStatus.State.Failed);
            expectedFile = new FilePath(new Path(outDir, FileOutputCommitter.PendingDirName).
                                        ToString());
            NUnit.Framework.Assert.IsFalse("job temp dir still exists", expectedFile.Exists()
                                           );
            NUnit.Framework.Assert.AreEqual("Output directory not empty", 0, new FilePath(outDir
                                                                                          .ToString()).ListFiles().Length);
            FileUtil.FullyDelete(new FilePath(outDir.ToString()));
        }
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="System.Exception"/>
        private void TestFailAbortInternal(int version)
        {
            Job           job  = Job.GetInstance();
            Configuration conf = job.GetConfiguration();

            conf.Set(FileSystem.FsDefaultNameKey, "faildel:///");
            conf.SetClass("fs.faildel.impl", typeof(TestFileOutputCommitter.FakeFileSystem),
                          typeof(FileSystem));
            conf.Set(MRJobConfig.TaskAttemptId, attempt);
            conf.SetInt(MRJobConfig.ApplicationAttemptId, 1);
            conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, version);
            FileOutputFormat.SetOutputPath(job, outDir);
            JobContext          jContext  = new JobContextImpl(conf, taskID.GetJobID());
            TaskAttemptContext  tContext  = new TaskAttemptContextImpl(conf, taskID);
            FileOutputCommitter committer = new FileOutputCommitter(outDir, tContext);

            // do setup
            committer.SetupJob(jContext);
            committer.SetupTask(tContext);
            // write output
            TextOutputFormat <object, object> theOutputFormat = new TextOutputFormat();
            RecordWriter <object, object>     theRecordWriter = theOutputFormat.GetRecordWriter(tContext
                                                                                                );

            WriteOutput(theRecordWriter, tContext);
            // do abort
            Exception th = null;

            try
            {
                committer.AbortTask(tContext);
            }
            catch (IOException ie)
            {
                th = ie;
            }
            NUnit.Framework.Assert.IsNotNull(th);
            NUnit.Framework.Assert.IsTrue(th is IOException);
            NUnit.Framework.Assert.IsTrue(th.Message.Contains("fake delete failed"));
            Path     jtd          = committer.GetJobAttemptPath(jContext);
            FilePath jobTmpDir    = new FilePath(jtd.ToUri().GetPath());
            Path     ttd          = committer.GetTaskAttemptPath(tContext);
            FilePath taskTmpDir   = new FilePath(ttd.ToUri().GetPath());
            FilePath expectedFile = new FilePath(taskTmpDir, partFile);

            NUnit.Framework.Assert.IsTrue(expectedFile + " does not exists", expectedFile.Exists
                                              ());
            th = null;
            try
            {
                committer.AbortJob(jContext, JobStatus.State.Failed);
            }
            catch (IOException ie)
            {
                th = ie;
            }
            NUnit.Framework.Assert.IsNotNull(th);
            NUnit.Framework.Assert.IsTrue(th is IOException);
            NUnit.Framework.Assert.IsTrue(th.Message.Contains("fake delete failed"));
            NUnit.Framework.Assert.IsTrue("job temp dir does not exists", jobTmpDir.Exists());
            FileUtil.FullyDelete(new FilePath(outDir.ToString()));
        }
        /// <exception cref="System.Exception"/>
        private void TestRecoveryInternal(int commitVersion, int recoveryVersion)
        {
            Job job = Job.GetInstance();

            FileOutputFormat.SetOutputPath(job, outDir);
            Configuration conf = job.GetConfiguration();

            conf.Set(MRJobConfig.TaskAttemptId, attempt);
            conf.SetInt(MRJobConfig.ApplicationAttemptId, 1);
            conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, commitVersion
                        );
            JobContext          jContext  = new JobContextImpl(conf, taskID.GetJobID());
            TaskAttemptContext  tContext  = new TaskAttemptContextImpl(conf, taskID);
            FileOutputCommitter committer = new FileOutputCommitter(outDir, tContext);

            // setup
            committer.SetupJob(jContext);
            committer.SetupTask(tContext);
            // write output
            TextOutputFormat theOutputFormat = new TextOutputFormat();
            RecordWriter     theRecordWriter = theOutputFormat.GetRecordWriter(tContext);

            WriteOutput(theRecordWriter, tContext);
            // do commit
            committer.CommitTask(tContext);
            Path     jobTempDir1 = committer.GetCommittedTaskPath(tContext);
            FilePath jtd         = new FilePath(jobTempDir1.ToUri().GetPath());

            if (commitVersion == 1)
            {
                NUnit.Framework.Assert.IsTrue("Version 1 commits to temporary dir " + jtd, jtd.Exists
                                                  ());
                ValidateContent(jtd);
            }
            else
            {
                NUnit.Framework.Assert.IsFalse("Version 2 commits to output dir " + jtd, jtd.Exists
                                                   ());
            }
            //now while running the second app attempt,
            //recover the task output from first attempt
            Configuration conf2 = job.GetConfiguration();

            conf2.Set(MRJobConfig.TaskAttemptId, attempt);
            conf2.SetInt(MRJobConfig.ApplicationAttemptId, 2);
            conf2.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, recoveryVersion
                         );
            JobContext          jContext2  = new JobContextImpl(conf2, taskID.GetJobID());
            TaskAttemptContext  tContext2  = new TaskAttemptContextImpl(conf2, taskID);
            FileOutputCommitter committer2 = new FileOutputCommitter(outDir, tContext2);

            committer2.SetupJob(tContext2);
            Path     jobTempDir2 = committer2.GetCommittedTaskPath(tContext2);
            FilePath jtd2        = new FilePath(jobTempDir2.ToUri().GetPath());

            committer2.RecoverTask(tContext2);
            if (recoveryVersion == 1)
            {
                NUnit.Framework.Assert.IsTrue("Version 1 recovers to " + jtd2, jtd2.Exists());
                ValidateContent(jtd2);
            }
            else
            {
                NUnit.Framework.Assert.IsFalse("Version 2 commits to output dir " + jtd2, jtd2.Exists
                                                   ());
                if (commitVersion == 1)
                {
                    NUnit.Framework.Assert.IsTrue("Version 2  recovery moves to output dir from " + jtd
                                                  , jtd.List().Length == 0);
                }
            }
            committer2.CommitJob(jContext2);
            ValidateContent(outDir);
            FileUtil.FullyDelete(new FilePath(outDir.ToString()));
        }