Пример #1
0
            /// <exception cref="System.IO.IOException"/>
            /// <exception cref="System.Exception"/>
            public override void Close()
            {
                long totalSize = 0;

                for (int i = 0; i < numberOfPartitions; i++)
                {
                    recordWriters[i].Close();
                    outStreams[i].Close();
                    totalSize += outStreams[i].Size();
                }
                MapOutputFile mapOutputFile = mapTask.GetMapOutputFile();
                Path          finalOutput   = mapOutputFile.GetOutputFileForWrite(totalSize);
                Path          indexPath     = mapOutputFile.GetOutputIndexFileForWrite(numberOfPartitions * mapTask
                                                                                       .MapOutputIndexRecordLength);

                // Copy partitions to final map output.
                CopyPartitions(finalOutput, indexPath);
            }
Пример #2
0
            /// <exception cref="Sharpen.RuntimeException"/>
            /// <exception cref="System.IO.IOException"/>
            private void RunSubtask(Task task, TaskType taskType, TaskAttemptId attemptID, int
                                    numMapTasks, bool renameOutputs, IDictionary <TaskAttemptID, MapOutputFile> localMapFiles
                                    )
            {
                TaskAttemptID classicAttemptID = TypeConverter.FromYarn(attemptID);

                try
                {
                    JobConf conf = new JobConf(this._enclosing.GetConfig());
                    conf.Set(JobContext.TaskId, task.GetTaskID().ToString());
                    conf.Set(JobContext.TaskAttemptId, classicAttemptID.ToString());
                    conf.SetBoolean(JobContext.TaskIsmap, (taskType == TaskType.Map));
                    conf.SetInt(JobContext.TaskPartition, task.GetPartition());
                    conf.Set(JobContext.Id, task.GetJobID().ToString());
                    // Use the AM's local dir env to generate the intermediate step
                    // output files
                    string[] localSysDirs = StringUtils.GetTrimmedStrings(Runtime.Getenv(ApplicationConstants.Environment
                                                                                         .LocalDirs.ToString()));
                    conf.SetStrings(MRConfig.LocalDir, localSysDirs);
                    LocalContainerLauncher.Log.Info(MRConfig.LocalDir + " for uber task: " + conf.Get
                                                        (MRConfig.LocalDir));
                    // mark this as an uberized subtask so it can set task counter
                    // (longer-term/FIXME:  could redefine as job counter and send
                    // "JobCounterEvent" to JobImpl on [successful] completion of subtask;
                    // will need new Job state-machine transition and JobImpl jobCounters
                    // map to handle)
                    conf.SetBoolean("mapreduce.task.uberized", true);
                    // Check and handle Encrypted spill key
                    task.SetEncryptedSpillKey(this._enclosing.encryptedSpillKey);
                    YarnChild.SetEncryptedSpillKeyIfRequired(task);
                    // META-FIXME: do we want the extra sanity-checking (doneWithMaps,
                    // etc.), or just assume/hope the state machine(s) and uber-AM work
                    // as expected?
                    if (taskType == TaskType.Map)
                    {
                        if (this.doneWithMaps)
                        {
                            LocalContainerLauncher.Log.Error("CONTAINER_REMOTE_LAUNCH contains a map task ("
                                                             + attemptID + "), but should be finished with maps");
                            throw new RuntimeException();
                        }
                        MapTask map = (MapTask)task;
                        map.SetConf(conf);
                        map.Run(conf, this._enclosing.umbilical);
                        if (renameOutputs)
                        {
                            MapOutputFile renamed = LocalContainerLauncher.RenameMapOutputForReduce(conf, attemptID
                                                                                                    , map.GetMapOutputFile());
                            localMapFiles[classicAttemptID] = renamed;
                        }
                        this.Relocalize();
                        if (++this.finishedSubMaps == numMapTasks)
                        {
                            this.doneWithMaps = true;
                        }
                    }
                    else
                    {
                        /* TaskType.REDUCE */
                        if (!this.doneWithMaps)
                        {
                            // check if event-queue empty?  whole idea of counting maps vs.
                            // checking event queue is a tad wacky...but could enforce ordering
                            // (assuming no "lost events") at LocalMRAppMaster [CURRENT BUG(?):
                            // doesn't send reduce event until maps all done]
                            LocalContainerLauncher.Log.Error("CONTAINER_REMOTE_LAUNCH contains a reduce task ("
                                                             + attemptID + "), but not yet finished with maps");
                            throw new RuntimeException();
                        }
                        // a.k.a. "mapreduce.jobtracker.address" in LocalJobRunner:
                        // set framework name to local to make task local
                        conf.Set(MRConfig.FrameworkName, MRConfig.LocalFrameworkName);
                        conf.Set(MRConfig.MasterAddress, "local");
                        // bypass shuffle
                        ReduceTask reduce = (ReduceTask)task;
                        reduce.SetLocalMapFiles(localMapFiles);
                        reduce.SetConf(conf);
                        reduce.Run(conf, this._enclosing.umbilical);
                        this.Relocalize();
                    }
                }
                catch (FSError e)
                {
                    LocalContainerLauncher.Log.Fatal("FSError from child", e);
                    // umbilical:  MRAppMaster creates (taskAttemptListener), passes to us
                    if (!ShutdownHookManager.Get().IsShutdownInProgress())
                    {
                        this._enclosing.umbilical.FsError(classicAttemptID, e.Message);
                    }
                    throw new RuntimeException();
                }
                catch (Exception exception)
                {
                    LocalContainerLauncher.Log.Warn("Exception running local (uberized) 'child' : " +
                                                    StringUtils.StringifyException(exception));
                    try
                    {
                        if (task != null)
                        {
                            // do cleanup for the task
                            task.TaskCleanup(this._enclosing.umbilical);
                        }
                    }
                    catch (Exception e)
                    {
                        LocalContainerLauncher.Log.Info("Exception cleaning up: " + StringUtils.StringifyException
                                                            (e));
                    }
                    // Report back any failures, for diagnostic purposes
                    this._enclosing.umbilical.ReportDiagnosticInfo(classicAttemptID, StringUtils.StringifyException
                                                                       (exception));
                    throw new RuntimeException();
                }
                catch (Exception throwable)
                {
                    LocalContainerLauncher.Log.Fatal("Error running local (uberized) 'child' : " + StringUtils
                                                     .StringifyException(throwable));
                    if (!ShutdownHookManager.Get().IsShutdownInProgress())
                    {
                        Exception tCause = throwable.InnerException;
                        string    cause  = (tCause == null) ? throwable.Message : StringUtils.StringifyException
                                               (tCause);
                        this._enclosing.umbilical.FatalError(classicAttemptID, cause);
                    }
                    throw new RuntimeException();
                }
            }