예제 #1
0
        // Run a job that will be failed and wait until it completes
        /// <exception cref="System.IO.IOException"/>
        public static RunningJob RunJobFail(JobConf conf, Path inDir, Path outDir)
        {
            conf.SetJobName("test-job-fail");
            conf.SetMapperClass(typeof(UtilsForTests.FailMapper));
            conf.SetReducerClass(typeof(IdentityReducer));
            conf.SetMaxMapAttempts(1);
            RunningJob job        = UtilsForTests.RunJob(conf, inDir, outDir);
            long       sleepCount = 0;

            while (!job.IsComplete())
            {
                try
                {
                    if (sleepCount > 300)
                    {
                        // 30 seconds
                        throw new IOException("Job didn't finish in 30 seconds");
                    }
                    Sharpen.Thread.Sleep(100);
                    sleepCount++;
                }
                catch (Exception)
                {
                    break;
                }
            }
            return(job);
        }
예제 #2
0
        /// <exception cref="System.IO.IOException"/>
        public virtual void TestCommitFail()
        {
            Path    inDir   = new Path(rootDir, "./input");
            Path    outDir  = new Path(rootDir, "./output");
            JobConf jobConf = CreateJobConf();

            jobConf.SetMaxMapAttempts(1);
            jobConf.SetOutputCommitter(typeof(TestTaskCommit.CommitterWithCommitFail));
            RunningJob rJob = UtilsForTests.RunJob(jobConf, inDir, outDir, 1, 0);

            rJob.WaitForCompletion();
            NUnit.Framework.Assert.AreEqual(JobStatus.Failed, rJob.GetJobState());
        }
예제 #3
0
        public virtual void TestReporterProgressForMapOnlyJob()
        {
            Path    test = new Path(testRootTempDir, "testReporterProgressForMapOnlyJob");
            JobConf conf = new JobConf();

            conf.SetMapperClass(typeof(TestReporter.ProgressTesterMapper));
            conf.SetMapOutputKeyClass(typeof(Org.Apache.Hadoop.IO.Text));
            // fail early
            conf.SetMaxMapAttempts(1);
            conf.SetMaxReduceAttempts(0);
            RunningJob job = UtilsForTests.RunJob(conf, new Path(test, "in"), new Path(test,
                                                                                       "out"), 1, 0, Input);

            job.WaitForCompletion();
            NUnit.Framework.Assert.IsTrue("Job failed", job.IsSuccessful());
        }
예제 #4
0
        // Run a job that will be killed and wait until it completes
        /// <exception cref="System.IO.IOException"/>
        public static RunningJob RunJobKill(JobConf conf, Path inDir, Path outDir)
        {
            conf.SetJobName("test-job-kill");
            conf.SetMapperClass(typeof(UtilsForTests.KillMapper));
            conf.SetReducerClass(typeof(IdentityReducer));
            RunningJob job        = UtilsForTests.RunJob(conf, inDir, outDir);
            long       sleepCount = 0;

            while (job.GetJobState() != JobStatus.Running)
            {
                try
                {
                    if (sleepCount > 300)
                    {
                        // 30 seconds
                        throw new IOException("Job didn't finish in 30 seconds");
                    }
                    Sharpen.Thread.Sleep(100);
                    sleepCount++;
                }
                catch (Exception)
                {
                    break;
                }
            }
            job.KillJob();
            sleepCount = 0;
            while (job.CleanupProgress() == 0.0f)
            {
                try
                {
                    if (sleepCount > 2000)
                    {
                        // 20 seconds
                        throw new IOException("Job cleanup didn't start in 20 seconds");
                    }
                    Sharpen.Thread.Sleep(10);
                    sleepCount++;
                }
                catch (Exception)
                {
                    break;
                }
            }
            return(job);
        }