/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> /// <exception cref="System.TypeLoadException"/> public override void Run(JobConf job, TaskUmbilicalProtocol umbilical) { job.SetBoolean(JobContext.SkipRecords, IsSkipping()); if (IsMapOrReduce()) { copyPhase = GetProgress().AddPhase("copy"); sortPhase = GetProgress().AddPhase("sort"); reducePhase = GetProgress().AddPhase("reduce"); } // start thread that will handle communication with parent Task.TaskReporter reporter = StartReporter(umbilical); bool useNewApi = job.GetUseNewReducer(); Initialize(job, GetJobID(), reporter, useNewApi); // check if it is a cleanupJobTask if (jobCleanup) { RunJobCleanupTask(umbilical, reporter); return; } if (jobSetup) { RunJobSetupTask(umbilical, reporter); return; } if (taskCleanup) { RunTaskCleanupTask(umbilical, reporter); return; } // Initialize the codec codec = InitCodec(); RawKeyValueIterator rIter = null; ShuffleConsumerPlugin shuffleConsumerPlugin = null; Type combinerClass = conf.GetCombinerClass(); Task.CombineOutputCollector combineCollector = (null != combinerClass) ? new Task.CombineOutputCollector (reduceCombineOutputCounter, reporter, conf) : null; Type clazz = job.GetClass <ShuffleConsumerPlugin>(MRConfig.ShuffleConsumerPlugin, typeof(Shuffle)); shuffleConsumerPlugin = ReflectionUtils.NewInstance(clazz, job); Log.Info("Using ShuffleConsumerPlugin: " + shuffleConsumerPlugin); ShuffleConsumerPlugin.Context shuffleContext = new ShuffleConsumerPlugin.Context( GetTaskID(), job, FileSystem.GetLocal(job), umbilical, base.lDirAlloc, reporter, codec, combinerClass, combineCollector, spilledRecordsCounter, reduceCombineInputCounter , shuffledMapsCounter, reduceShuffleBytes, failedShuffleCounter, mergedMapOutputsCounter , taskStatus, copyPhase, sortPhase, this, mapOutputFile, localMapFiles); shuffleConsumerPlugin.Init(shuffleContext); rIter = shuffleConsumerPlugin.Run(); // free up the data structures mapOutputFilesOnDisk.Clear(); sortPhase.Complete(); // sort is complete SetPhase(TaskStatus.Phase.Reduce); StatusUpdate(umbilical); Type keyClass = job.GetMapOutputKeyClass(); Type valueClass = job.GetMapOutputValueClass(); RawComparator comparator = job.GetOutputValueGroupingComparator(); if (useNewApi) { RunNewReducer(job, umbilical, reporter, rIter, comparator, keyClass, valueClass); } else { RunOldReducer(job, umbilical, reporter, rIter, comparator, keyClass, valueClass); } shuffleConsumerPlugin.Close(); Done(umbilical, reporter); }
public abstract void Init(ShuffleConsumerPlugin.Context <K, V> context);