예제 #1
0
        /// <summary>Validate job.end.retry.attempts for the localJobRunner.</summary>
        /// <exception cref="System.Exception"/>
        public virtual void TestLocalJobRunnerRetryCount()
        {
            int       retryAttempts = 3;
            JobStatus jobStatus     = CreateTestJobStatus("job_20130313155005308_0001", JobStatus
                                                          .Succeeded);
            JobConf jobConf = CreateTestJobConf(new Configuration(), retryAttempts, baseUrl +
                                                "fail");

            JobEndNotifier.LocalRunnerNotification(jobConf, jobStatus);
            // Validate params
            NUnit.Framework.Assert.AreEqual(retryAttempts + 1, TestJobEndNotifier.FailServlet
                                            .calledTimes);
        }
예제 #2
0
        /// <summary>Basic validation for localRunnerNotification.</summary>
        /// <exception cref="System.Exception"/>
        public virtual void TestLocalJobRunnerUriSubstitution()
        {
            JobStatus jobStatus = CreateTestJobStatus("job_20130313155005308_0001", JobStatus
                                                      .Succeeded);
            JobConf jobConf = CreateTestJobConf(new Configuration(), 0, baseUrl + "jobend?jobid=$jobId&status=$jobStatus"
                                                );

            JobEndNotifier.LocalRunnerNotification(jobConf, jobStatus);
            // No need to wait for the notification to go thru since calls are
            // synchronous
            // Validate params
            NUnit.Framework.Assert.AreEqual(1, TestJobEndNotifier.JobEndServlet.calledTimes);
            NUnit.Framework.Assert.AreEqual("jobid=job_20130313155005308_0001&status=SUCCEEDED"
                                            , TestJobEndNotifier.JobEndServlet.requestUri.GetQuery());
        }
예제 #3
0
        /// <summary>
        /// Validate that the notification times out after reaching
        /// mapreduce.job.end-notification.timeout.
        /// </summary>
        /// <exception cref="System.Exception"/>
        public virtual void TestNotificationTimeout()
        {
            Configuration conf = new Configuration();

            // Reduce the timeout to 1 second
            conf.SetInt("mapreduce.job.end-notification.timeout", 1000);
            JobStatus jobStatus = CreateTestJobStatus("job_20130313155005308_0001", JobStatus
                                                      .Succeeded);
            JobConf jobConf   = CreateTestJobConf(conf, 0, baseUrl + "delay");
            long    startTime = Runtime.CurrentTimeMillis();

            JobEndNotifier.LocalRunnerNotification(jobConf, jobStatus);
            long elapsedTime = Runtime.CurrentTimeMillis() - startTime;

            // Validate params
            NUnit.Framework.Assert.AreEqual(1, TestJobEndNotifier.DelayServlet.calledTimes);
            // Make sure we timed out with time slightly above 1 second
            // (default timeout is in terms of minutes, so we'll catch the problem)
            NUnit.Framework.Assert.IsTrue(elapsedTime < 2000);
        }
예제 #4
0
            public override void Run()
            {
                JobID           jobId           = this.profile.GetJobID();
                JobContext      jContext        = new JobContextImpl(this.job, jobId);
                OutputCommitter outputCommitter = null;

                try
                {
                    outputCommitter = this.CreateOutputCommitter(this._enclosing.conf.GetUseNewMapper
                                                                     (), jobId, this._enclosing.conf);
                }
                catch (Exception e)
                {
                    LocalJobRunner.Log.Info("Failed to createOutputCommitter", e);
                    return;
                }
                try
                {
                    JobSplit.TaskSplitMetaInfo[] taskSplitMetaInfos = SplitMetaInfoReader.ReadSplitMetaInfo
                                                                          (jobId, this.localFs, this._enclosing.conf, this.systemJobDir);
                    int numReduceTasks = this.job.GetNumReduceTasks();
                    outputCommitter.SetupJob(jContext);
                    this.status.SetSetupProgress(1.0f);
                    IDictionary <TaskAttemptID, MapOutputFile> mapOutputFiles = Sharpen.Collections.SynchronizedMap
                                                                                    (new Dictionary <TaskAttemptID, MapOutputFile>());
                    IList <LocalJobRunner.Job.RunnableWithThrowable> mapRunnables = this.GetMapTaskRunnables
                                                                                        (taskSplitMetaInfos, jobId, mapOutputFiles);
                    this.InitCounters(mapRunnables.Count, numReduceTasks);
                    ExecutorService mapService = this.CreateMapExecutor();
                    this.RunTasks(mapRunnables, mapService, "map");
                    try
                    {
                        if (numReduceTasks > 0)
                        {
                            IList <LocalJobRunner.Job.RunnableWithThrowable> reduceRunnables = this.GetReduceTaskRunnables
                                                                                                   (jobId, mapOutputFiles);
                            ExecutorService reduceService = this.CreateReduceExecutor();
                            this.RunTasks(reduceRunnables, reduceService, "reduce");
                        }
                    }
                    finally
                    {
                        foreach (MapOutputFile output in mapOutputFiles.Values)
                        {
                            output.RemoveAll();
                        }
                    }
                    // delete the temporary directory in output directory
                    outputCommitter.CommitJob(jContext);
                    this.status.SetCleanupProgress(1.0f);
                    if (this.killed)
                    {
                        this.status.SetRunState(JobStatus.Killed);
                    }
                    else
                    {
                        this.status.SetRunState(JobStatus.Succeeded);
                    }
                    JobEndNotifier.LocalRunnerNotification(this.job, this.status);
                }
                catch (Exception t)
                {
                    try
                    {
                        outputCommitter.AbortJob(jContext, JobStatus.State.Failed);
                    }
                    catch (IOException)
                    {
                        LocalJobRunner.Log.Info("Error cleaning up job:" + this.id);
                    }
                    this.status.SetCleanupProgress(1.0f);
                    if (this.killed)
                    {
                        this.status.SetRunState(JobStatus.Killed);
                    }
                    else
                    {
                        this.status.SetRunState(JobStatus.Failed);
                    }
                    LocalJobRunner.Log.Warn(this.id, t);
                    JobEndNotifier.LocalRunnerNotification(this.job, this.status);
                }
                finally
                {
                    try
                    {
                        this._enclosing.fs.Delete(this.systemJobFile.GetParent(), true);
                        // delete submit dir
                        this.localFs.Delete(this.localJobFile, true);
                        // delete local copy
                        // Cleanup distributed cache
                        this.localDistributedCacheManager.Close();
                    }
                    catch (IOException e)
                    {
                        LocalJobRunner.Log.Warn("Error cleaning up " + this.id + ": " + e);
                    }
                }
            }