예제 #1
0
        /// <summary>
        /// Validates map phase progress after each record is processed by map task
        /// using custom task reporter.
        /// </summary>
        /// <exception cref="System.Exception"/>
        public virtual void TestMapProgress()
        {
            JobConf job = new JobConf();

            fs = FileSystem.GetLocal(job);
            Path rootDir = new Path(TestRootDir);

            CreateInputFile(rootDir);
            job.SetNumReduceTasks(0);
            TaskAttemptID taskId = ((TaskAttemptID)TaskAttemptID.ForName("attempt_200907082313_0424_m_000000_0"
                                                                         ));

            job.SetClass("mapreduce.job.outputformat.class", typeof(NullOutputFormat), typeof(
                             OutputFormat));
            job.Set(FileInputFormat.InputDir, TestRootDir);
            jobId = ((JobID)taskId.GetJobID());
            JobContext jContext = new JobContextImpl(job, jobId);
            InputFormat <object, object> input = ReflectionUtils.NewInstance(jContext.GetInputFormatClass
                                                                                 (), job);
            IList <InputSplit> splits = input.GetSplits(jContext);

            JobSplitWriter.CreateSplitFiles(new Path(TestRootDir), job, new Path(TestRootDir)
                                            .GetFileSystem(job), splits);
            JobSplit.TaskSplitMetaInfo[] splitMetaInfo = SplitMetaInfoReader.ReadSplitMetaInfo
                                                             (jobId, fs, job, new Path(TestRootDir));
            job.SetUseNewMapper(true);
            // use new api
            for (int i = 0; i < splitMetaInfo.Length; i++)
            {
                // rawSplits.length is 1
                map = new TestMapProgress.TestMapTask(this, job.Get(JTConfig.JtSystemDir, "/tmp/hadoop/mapred/system"
                                                                    ) + jobId + "job.xml", taskId, i, splitMetaInfo[i].GetSplitIndex(), 1);
                JobConf localConf = new JobConf(job);
                map.LocalizeConfiguration(localConf);
                map.SetConf(localConf);
                map.Run(localConf, fakeUmbilical);
            }
            // clean up
            fs.Delete(rootDir, true);
        }
예제 #2
0
            public override void Run()
            {
                JobID           jobId           = this.profile.GetJobID();
                JobContext      jContext        = new JobContextImpl(this.job, jobId);
                OutputCommitter outputCommitter = null;

                try
                {
                    outputCommitter = this.CreateOutputCommitter(this._enclosing.conf.GetUseNewMapper
                                                                     (), jobId, this._enclosing.conf);
                }
                catch (Exception e)
                {
                    LocalJobRunner.Log.Info("Failed to createOutputCommitter", e);
                    return;
                }
                try
                {
                    JobSplit.TaskSplitMetaInfo[] taskSplitMetaInfos = SplitMetaInfoReader.ReadSplitMetaInfo
                                                                          (jobId, this.localFs, this._enclosing.conf, this.systemJobDir);
                    int numReduceTasks = this.job.GetNumReduceTasks();
                    outputCommitter.SetupJob(jContext);
                    this.status.SetSetupProgress(1.0f);
                    IDictionary <TaskAttemptID, MapOutputFile> mapOutputFiles = Sharpen.Collections.SynchronizedMap
                                                                                    (new Dictionary <TaskAttemptID, MapOutputFile>());
                    IList <LocalJobRunner.Job.RunnableWithThrowable> mapRunnables = this.GetMapTaskRunnables
                                                                                        (taskSplitMetaInfos, jobId, mapOutputFiles);
                    this.InitCounters(mapRunnables.Count, numReduceTasks);
                    ExecutorService mapService = this.CreateMapExecutor();
                    this.RunTasks(mapRunnables, mapService, "map");
                    try
                    {
                        if (numReduceTasks > 0)
                        {
                            IList <LocalJobRunner.Job.RunnableWithThrowable> reduceRunnables = this.GetReduceTaskRunnables
                                                                                                   (jobId, mapOutputFiles);
                            ExecutorService reduceService = this.CreateReduceExecutor();
                            this.RunTasks(reduceRunnables, reduceService, "reduce");
                        }
                    }
                    finally
                    {
                        foreach (MapOutputFile output in mapOutputFiles.Values)
                        {
                            output.RemoveAll();
                        }
                    }
                    // delete the temporary directory in output directory
                    outputCommitter.CommitJob(jContext);
                    this.status.SetCleanupProgress(1.0f);
                    if (this.killed)
                    {
                        this.status.SetRunState(JobStatus.Killed);
                    }
                    else
                    {
                        this.status.SetRunState(JobStatus.Succeeded);
                    }
                    JobEndNotifier.LocalRunnerNotification(this.job, this.status);
                }
                catch (Exception t)
                {
                    try
                    {
                        outputCommitter.AbortJob(jContext, JobStatus.State.Failed);
                    }
                    catch (IOException)
                    {
                        LocalJobRunner.Log.Info("Error cleaning up job:" + this.id);
                    }
                    this.status.SetCleanupProgress(1.0f);
                    if (this.killed)
                    {
                        this.status.SetRunState(JobStatus.Killed);
                    }
                    else
                    {
                        this.status.SetRunState(JobStatus.Failed);
                    }
                    LocalJobRunner.Log.Warn(this.id, t);
                    JobEndNotifier.LocalRunnerNotification(this.job, this.status);
                }
                finally
                {
                    try
                    {
                        this._enclosing.fs.Delete(this.systemJobFile.GetParent(), true);
                        // delete submit dir
                        this.localFs.Delete(this.localJobFile, true);
                        // delete local copy
                        // Cleanup distributed cache
                        this.localDistributedCacheManager.Close();
                    }
                    catch (IOException e)
                    {
                        LocalJobRunner.Log.Warn("Error cleaning up " + this.id + ": " + e);
                    }
                }
            }