コード例 #1
0
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="System.Exception"/>
        private void TestFailAbortInternal(int version)
        {
            JobConf conf = new JobConf();

            conf.Set(FileSystem.FsDefaultNameKey, "faildel:///");
            conf.SetClass("fs.faildel.impl", typeof(TestFileOutputCommitter.FakeFileSystem),
                          typeof(FileSystem));
            conf.Set(JobContext.TaskAttemptId, attempt);
            conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, version);
            conf.SetInt(MRConstants.ApplicationAttemptId, 1);
            FileOutputFormat.SetOutputPath(conf, outDir);
            JobContext          jContext  = new JobContextImpl(conf, ((JobID)taskID.GetJobID()));
            TaskAttemptContext  tContext  = new TaskAttemptContextImpl(conf, taskID);
            FileOutputCommitter committer = new FileOutputCommitter();

            // do setup
            committer.SetupJob(jContext);
            committer.SetupTask(tContext);
            // write output
            FilePath jobTmpDir = new FilePath(new Path(outDir, FileOutputCommitter.TempDirName
                                                       + Path.Separator + conf.GetInt(MRConstants.ApplicationAttemptId, 0) + Path.Separator
                                                       + FileOutputCommitter.TempDirName).ToString());
            FilePath taskTmpDir   = new FilePath(jobTmpDir, "_" + taskID);
            FilePath expectedFile = new FilePath(taskTmpDir, partFile);
            TextOutputFormat <object, object> theOutputFormat = new TextOutputFormat();
            RecordWriter <object, object>     theRecordWriter = theOutputFormat.GetRecordWriter(null
                                                                                                , conf, expectedFile.GetAbsolutePath(), null);

            WriteOutput(theRecordWriter, tContext);
            // do abort
            Exception th = null;

            try
            {
                committer.AbortTask(tContext);
            }
            catch (IOException ie)
            {
                th = ie;
            }
            NUnit.Framework.Assert.IsNotNull(th);
            NUnit.Framework.Assert.IsTrue(th is IOException);
            NUnit.Framework.Assert.IsTrue(th.Message.Contains("fake delete failed"));
            NUnit.Framework.Assert.IsTrue(expectedFile + " does not exists", expectedFile.Exists
                                              ());
            th = null;
            try
            {
                committer.AbortJob(jContext, JobStatus.State.Failed);
            }
            catch (IOException ie)
            {
                th = ie;
            }
            NUnit.Framework.Assert.IsNotNull(th);
            NUnit.Framework.Assert.IsTrue(th is IOException);
            NUnit.Framework.Assert.IsTrue(th.Message.Contains("fake delete failed"));
            NUnit.Framework.Assert.IsTrue("job temp dir does not exists", jobTmpDir.Exists());
            FileUtil.FullyDelete(new FilePath(outDir.ToString()));
        }
コード例 #2
0
        /// <exception cref="System.IO.IOException"/>
        public virtual void TestAbort()
        {
            JobConf job = new JobConf();

            SetConfForFileOutputCommitter(job);
            JobContext          jContext  = new JobContextImpl(job, ((JobID)taskID.GetJobID()));
            TaskAttemptContext  tContext  = new TaskAttemptContextImpl(job, taskID);
            FileOutputCommitter committer = new FileOutputCommitter();

            FileOutputFormat.SetWorkOutputPath(job, committer.GetTaskAttemptPath(tContext));
            // do setup
            committer.SetupJob(jContext);
            committer.SetupTask(tContext);
            string file = "test.txt";
            // A reporter that does nothing
            Reporter reporter = Reporter.Null;
            // write output
            FileSystem       localFs         = FileSystem.GetLocal(job);
            TextOutputFormat theOutputFormat = new TextOutputFormat();
            RecordWriter     theRecordWriter = theOutputFormat.GetRecordWriter(localFs, job, file
                                                                               , reporter);

            WriteOutput(theRecordWriter, reporter);
            // do abort
            committer.AbortTask(tContext);
            FilePath expectedFile = new FilePath(new Path(committer.GetTaskAttemptPath(tContext
                                                                                       ), file).ToString());

            NUnit.Framework.Assert.IsFalse("task temp dir still exists", expectedFile.Exists(
                                               ));
            committer.AbortJob(jContext, JobStatus.State.Failed);
            expectedFile = new FilePath(new Path(outDir, FileOutputCommitter.TempDirName).ToString
                                            ());
            NUnit.Framework.Assert.IsFalse("job temp dir " + expectedFile + " still exists",
                                           expectedFile.Exists());
            NUnit.Framework.Assert.AreEqual("Output directory not empty", 0, new FilePath(outDir
                                                                                          .ToString()).ListFiles().Length);
            FileUtil.FullyDelete(new FilePath(outDir.ToString()));
        }
コード例 #3
0
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="System.Exception"/>
        private void TestAbortInternal(int version)
        {
            JobConf conf = new JobConf();

            FileOutputFormat.SetOutputPath(conf, outDir);
            conf.Set(JobContext.TaskAttemptId, attempt);
            conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, version);
            JobContext          jContext  = new JobContextImpl(conf, ((JobID)taskID.GetJobID()));
            TaskAttemptContext  tContext  = new TaskAttemptContextImpl(conf, taskID);
            FileOutputCommitter committer = new FileOutputCommitter();

            // do setup
            committer.SetupJob(jContext);
            committer.SetupTask(tContext);
            // write output
            TextOutputFormat theOutputFormat = new TextOutputFormat();
            RecordWriter     theRecordWriter = theOutputFormat.GetRecordWriter(null, conf, partFile
                                                                               , null);

            WriteOutput(theRecordWriter, tContext);
            // do abort
            committer.AbortTask(tContext);
            FilePath @out         = new FilePath(outDir.ToUri().GetPath());
            Path     workPath     = committer.GetWorkPath(tContext, outDir);
            FilePath wp           = new FilePath(workPath.ToUri().GetPath());
            FilePath expectedFile = new FilePath(wp, partFile);

            NUnit.Framework.Assert.IsFalse("task temp dir still exists", expectedFile.Exists(
                                               ));
            committer.AbortJob(jContext, JobStatus.State.Failed);
            expectedFile = new FilePath(@out, FileOutputCommitter.TempDirName);
            NUnit.Framework.Assert.IsFalse("job temp dir still exists", expectedFile.Exists()
                                           );
            NUnit.Framework.Assert.AreEqual("Output directory not empty", 0, @out.ListFiles()
                                            .Length);
            FileUtil.FullyDelete(@out);
        }
コード例 #4
0
        /// <exception cref="System.IO.IOException"/>
        public virtual void TestFailAbort()
        {
            JobConf job = new JobConf();

            job.Set(FileSystem.FsDefaultNameKey, "faildel:///");
            job.SetClass("fs.faildel.impl", typeof(TestMRCJCFileOutputCommitter.FakeFileSystem
                                                   ), typeof(FileSystem));
            SetConfForFileOutputCommitter(job);
            JobContext          jContext  = new JobContextImpl(job, ((JobID)taskID.GetJobID()));
            TaskAttemptContext  tContext  = new TaskAttemptContextImpl(job, taskID);
            FileOutputCommitter committer = new FileOutputCommitter();

            FileOutputFormat.SetWorkOutputPath(job, committer.GetTaskAttemptPath(tContext));
            // do setup
            committer.SetupJob(jContext);
            committer.SetupTask(tContext);
            string   file      = "test.txt";
            FilePath jobTmpDir = new FilePath(committer.GetJobAttemptPath(jContext).ToUri().GetPath
                                                  ());
            FilePath taskTmpDir = new FilePath(committer.GetTaskAttemptPath(tContext).ToUri()
                                               .GetPath());
            FilePath expectedFile = new FilePath(taskTmpDir, file);
            // A reporter that does nothing
            Reporter reporter = Reporter.Null;
            // write output
            FileSystem       localFs         = new TestMRCJCFileOutputCommitter.FakeFileSystem();
            TextOutputFormat theOutputFormat = new TextOutputFormat();
            RecordWriter     theRecordWriter = theOutputFormat.GetRecordWriter(localFs, job, expectedFile
                                                                               .GetAbsolutePath(), reporter);

            WriteOutput(theRecordWriter, reporter);
            // do abort
            Exception th = null;

            try
            {
                committer.AbortTask(tContext);
            }
            catch (IOException ie)
            {
                th = ie;
            }
            NUnit.Framework.Assert.IsNotNull(th);
            NUnit.Framework.Assert.IsTrue(th is IOException);
            NUnit.Framework.Assert.IsTrue(th.Message.Contains("fake delete failed"));
            NUnit.Framework.Assert.IsTrue(expectedFile + " does not exists", expectedFile.Exists
                                              ());
            th = null;
            try
            {
                committer.AbortJob(jContext, JobStatus.State.Failed);
            }
            catch (IOException ie)
            {
                th = ie;
            }
            NUnit.Framework.Assert.IsNotNull(th);
            NUnit.Framework.Assert.IsTrue(th is IOException);
            NUnit.Framework.Assert.IsTrue(th.Message.Contains("fake delete failed"));
            NUnit.Framework.Assert.IsTrue("job temp dir does not exists", jobTmpDir.Exists());
        }