Beispiel #1
0
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="System.Exception"/>
        /// <exception cref="System.TypeLoadException"/>
        public override void Run(JobConf job, TaskUmbilicalProtocol umbilical)
        {
            job.SetBoolean(JobContext.SkipRecords, IsSkipping());
            if (IsMapOrReduce())
            {
                copyPhase   = GetProgress().AddPhase("copy");
                sortPhase   = GetProgress().AddPhase("sort");
                reducePhase = GetProgress().AddPhase("reduce");
            }
            // start thread that will handle communication with parent
            Task.TaskReporter reporter = StartReporter(umbilical);
            bool useNewApi             = job.GetUseNewReducer();

            Initialize(job, GetJobID(), reporter, useNewApi);
            // check if it is a cleanupJobTask
            if (jobCleanup)
            {
                RunJobCleanupTask(umbilical, reporter);
                return;
            }
            if (jobSetup)
            {
                RunJobSetupTask(umbilical, reporter);
                return;
            }
            if (taskCleanup)
            {
                RunTaskCleanupTask(umbilical, reporter);
                return;
            }
            // Initialize the codec
            codec = InitCodec();
            RawKeyValueIterator   rIter = null;
            ShuffleConsumerPlugin shuffleConsumerPlugin = null;
            Type combinerClass = conf.GetCombinerClass();

            Task.CombineOutputCollector combineCollector = (null != combinerClass) ? new Task.CombineOutputCollector
                                                               (reduceCombineOutputCounter, reporter, conf) : null;
            Type clazz = job.GetClass <ShuffleConsumerPlugin>(MRConfig.ShuffleConsumerPlugin,
                                                              typeof(Shuffle));

            shuffleConsumerPlugin = ReflectionUtils.NewInstance(clazz, job);
            Log.Info("Using ShuffleConsumerPlugin: " + shuffleConsumerPlugin);
            ShuffleConsumerPlugin.Context shuffleContext = new ShuffleConsumerPlugin.Context(
                GetTaskID(), job, FileSystem.GetLocal(job), umbilical, base.lDirAlloc, reporter,
                codec, combinerClass, combineCollector, spilledRecordsCounter, reduceCombineInputCounter
                , shuffledMapsCounter, reduceShuffleBytes, failedShuffleCounter, mergedMapOutputsCounter
                , taskStatus, copyPhase, sortPhase, this, mapOutputFile, localMapFiles);
            shuffleConsumerPlugin.Init(shuffleContext);
            rIter = shuffleConsumerPlugin.Run();
            // free up the data structures
            mapOutputFilesOnDisk.Clear();
            sortPhase.Complete();
            // sort is complete
            SetPhase(TaskStatus.Phase.Reduce);
            StatusUpdate(umbilical);
            Type          keyClass   = job.GetMapOutputKeyClass();
            Type          valueClass = job.GetMapOutputValueClass();
            RawComparator comparator = job.GetOutputValueGroupingComparator();

            if (useNewApi)
            {
                RunNewReducer(job, umbilical, reporter, rIter, comparator, keyClass, valueClass);
            }
            else
            {
                RunOldReducer(job, umbilical, reporter, rIter, comparator, keyClass, valueClass);
            }
            shuffleConsumerPlugin.Close();
            Done(umbilical, reporter);
        }
 public abstract void Init(ShuffleConsumerPlugin.Context <K, V> context);