public virtual void TestRenameMapOutputForReduce() { JobConf conf = new JobConf(); MROutputFiles mrOutputFiles = new MROutputFiles(); mrOutputFiles.SetConf(conf); // make sure both dirs are distinct // conf.Set(MRConfig.LocalDir, localDirs[0].ToString()); Path mapOut = mrOutputFiles.GetOutputFileForWrite(1); conf.Set(MRConfig.LocalDir, localDirs[1].ToString()); Path mapOutIdx = mrOutputFiles.GetOutputIndexFileForWrite(1); Assert.AssertNotEquals("Paths must be different!", mapOut.GetParent(), mapOutIdx. GetParent()); // make both dirs part of LOCAL_DIR conf.SetStrings(MRConfig.LocalDir, localDirs); FileContext lfc = FileContext.GetLocalFSFileContext(conf); lfc.Create(mapOut, EnumSet.Of(CreateFlag.Create)).Close(); lfc.Create(mapOutIdx, EnumSet.Of(CreateFlag.Create)).Close(); JobId jobId = MRBuilderUtils.NewJobId(12345L, 1, 2); TaskId tid = MRBuilderUtils.NewTaskId(jobId, 0, TaskType.Map); TaskAttemptId taid = MRBuilderUtils.NewTaskAttemptId(tid, 0); LocalContainerLauncher.RenameMapOutputForReduce(conf, taid, mrOutputFiles); }
/// <exception cref="Sharpen.RuntimeException"/> /// <exception cref="System.IO.IOException"/> private void RunSubtask(Task task, TaskType taskType, TaskAttemptId attemptID, int numMapTasks, bool renameOutputs, IDictionary <TaskAttemptID, MapOutputFile> localMapFiles ) { TaskAttemptID classicAttemptID = TypeConverter.FromYarn(attemptID); try { JobConf conf = new JobConf(this._enclosing.GetConfig()); conf.Set(JobContext.TaskId, task.GetTaskID().ToString()); conf.Set(JobContext.TaskAttemptId, classicAttemptID.ToString()); conf.SetBoolean(JobContext.TaskIsmap, (taskType == TaskType.Map)); conf.SetInt(JobContext.TaskPartition, task.GetPartition()); conf.Set(JobContext.Id, task.GetJobID().ToString()); // Use the AM's local dir env to generate the intermediate step // output files string[] localSysDirs = StringUtils.GetTrimmedStrings(Runtime.Getenv(ApplicationConstants.Environment .LocalDirs.ToString())); conf.SetStrings(MRConfig.LocalDir, localSysDirs); LocalContainerLauncher.Log.Info(MRConfig.LocalDir + " for uber task: " + conf.Get (MRConfig.LocalDir)); // mark this as an uberized subtask so it can set task counter // (longer-term/FIXME: could redefine as job counter and send // "JobCounterEvent" to JobImpl on [successful] completion of subtask; // will need new Job state-machine transition and JobImpl jobCounters // map to handle) conf.SetBoolean("mapreduce.task.uberized", true); // Check and handle Encrypted spill key task.SetEncryptedSpillKey(this._enclosing.encryptedSpillKey); YarnChild.SetEncryptedSpillKeyIfRequired(task); // META-FIXME: do we want the extra sanity-checking (doneWithMaps, // etc.), or just assume/hope the state machine(s) and uber-AM work // as expected? if (taskType == TaskType.Map) { if (this.doneWithMaps) { LocalContainerLauncher.Log.Error("CONTAINER_REMOTE_LAUNCH contains a map task (" + attemptID + "), but should be finished with maps"); throw new RuntimeException(); } MapTask map = (MapTask)task; map.SetConf(conf); map.Run(conf, this._enclosing.umbilical); if (renameOutputs) { MapOutputFile renamed = LocalContainerLauncher.RenameMapOutputForReduce(conf, attemptID , map.GetMapOutputFile()); localMapFiles[classicAttemptID] = renamed; } this.Relocalize(); if (++this.finishedSubMaps == numMapTasks) { this.doneWithMaps = true; } } else { /* TaskType.REDUCE */ if (!this.doneWithMaps) { // check if event-queue empty? whole idea of counting maps vs. // checking event queue is a tad wacky...but could enforce ordering // (assuming no "lost events") at LocalMRAppMaster [CURRENT BUG(?): // doesn't send reduce event until maps all done] LocalContainerLauncher.Log.Error("CONTAINER_REMOTE_LAUNCH contains a reduce task (" + attemptID + "), but not yet finished with maps"); throw new RuntimeException(); } // a.k.a. "mapreduce.jobtracker.address" in LocalJobRunner: // set framework name to local to make task local conf.Set(MRConfig.FrameworkName, MRConfig.LocalFrameworkName); conf.Set(MRConfig.MasterAddress, "local"); // bypass shuffle ReduceTask reduce = (ReduceTask)task; reduce.SetLocalMapFiles(localMapFiles); reduce.SetConf(conf); reduce.Run(conf, this._enclosing.umbilical); this.Relocalize(); } } catch (FSError e) { LocalContainerLauncher.Log.Fatal("FSError from child", e); // umbilical: MRAppMaster creates (taskAttemptListener), passes to us if (!ShutdownHookManager.Get().IsShutdownInProgress()) { this._enclosing.umbilical.FsError(classicAttemptID, e.Message); } throw new RuntimeException(); } catch (Exception exception) { LocalContainerLauncher.Log.Warn("Exception running local (uberized) 'child' : " + StringUtils.StringifyException(exception)); try { if (task != null) { // do cleanup for the task task.TaskCleanup(this._enclosing.umbilical); } } catch (Exception e) { LocalContainerLauncher.Log.Info("Exception cleaning up: " + StringUtils.StringifyException (e)); } // Report back any failures, for diagnostic purposes this._enclosing.umbilical.ReportDiagnosticInfo(classicAttemptID, StringUtils.StringifyException (exception)); throw new RuntimeException(); } catch (Exception throwable) { LocalContainerLauncher.Log.Fatal("Error running local (uberized) 'child' : " + StringUtils .StringifyException(throwable)); if (!ShutdownHookManager.Get().IsShutdownInProgress()) { Exception tCause = throwable.InnerException; string cause = (tCause == null) ? throwable.Message : StringUtils.StringifyException (tCause); this._enclosing.umbilical.FatalError(classicAttemptID, cause); } throw new RuntimeException(); } }