public LocalContainerLauncher(AppContext context, TaskUmbilicalProtocol umbilical ) : base(typeof(Org.Apache.Hadoop.Mapred.LocalContainerLauncher).FullName) { this.context = context; this.umbilical = umbilical; // umbilical: MRAppMaster creates (taskAttemptListener), passes to us // (TODO/FIXME: pointless to use RPC to talk to self; should create // LocalTaskAttemptListener or similar: implement umbilical protocol // but skip RPC stuff) try { curFC = FileContext.GetFileContext(curDir.ToURI()); } catch (UnsupportedFileSystemException) { Log.Error("Local filesystem " + curDir.ToURI().ToString() + " is unsupported?? (should never happen)" ); } // Save list of files/dirs that are supposed to be present so can delete // any extras created by one task before starting subsequent task. Note // that there's no protection against deleted or renamed localization; // users who do that get what they deserve (and will have to disable // uberization in order to run correctly). FilePath[] curLocalFiles = curDir.ListFiles(); localizedFiles = new HashSet <FilePath>(curLocalFiles.Length); for (int j = 0; j < curLocalFiles.Length; ++j) { localizedFiles.AddItem(curLocalFiles[j]); } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> /// <exception cref="System.TypeLoadException"/> private void RunNewReducer <Inkey, Invalue, Outkey, Outvalue>(JobConf job, TaskUmbilicalProtocol umbilical, Task.TaskReporter reporter, RawKeyValueIterator rIter, RawComparator <INKEY> comparator) { System.Type keyClass = typeof(INKEY); System.Type valueClass = typeof(INVALUE); // wrap value iterator to report progress. RawKeyValueIterator rawIter = rIter; rIter = new _RawKeyValueIterator_587(rawIter, reporter); // make a task context so we can get the classes TaskAttemptContext taskContext = new TaskAttemptContextImpl(job, GetTaskID(), reporter ); // make a reducer Reducer <INKEY, INVALUE, OUTKEY, OUTVALUE> reducer = (Reducer <INKEY, INVALUE, OUTKEY , OUTVALUE>)ReflectionUtils.NewInstance(taskContext.GetReducerClass(), job); RecordWriter <OUTKEY, OUTVALUE> trackedRW = new ReduceTask.NewTrackingRecordWriter <OUTKEY, OUTVALUE>(this, taskContext); job.SetBoolean("mapred.skip.on", IsSkipping()); job.SetBoolean(JobContext.SkipRecords, IsSkipping()); Reducer.Context reducerContext = CreateReduceContext(reducer, job, GetTaskID(), rIter , reduceInputKeyCounter, reduceInputValueCounter, trackedRW, committer, reporter , comparator, keyClass, valueClass); try { reducer.Run(reducerContext); } finally { trackedRW.Close(reducerContext); } }
public _PrivilegedExceptionAction_158(Task taskFinal, JobConf job, TaskUmbilicalProtocol umbilical) { this.taskFinal = taskFinal; this.job = job; this.umbilical = umbilical; }
public Context(TaskAttemptID reduceId, JobConf jobConf, FileSystem localFS, TaskUmbilicalProtocol umbilical, LocalDirAllocator localDirAllocator, Reporter reporter, CompressionCodec codec, Type combinerClass, Task.CombineOutputCollector <K, V> combineCollector, Counters.Counter spilledRecordsCounter, Counters.Counter reduceCombineInputCounter , Counters.Counter shuffledMapsCounter, Counters.Counter reduceShuffleBytes, Counters.Counter failedShuffleCounter, Counters.Counter mergedMapOutputsCounter, TaskStatus status , Progress copyPhase, Progress mergePhase, Task reduceTask, MapOutputFile mapOutputFile , IDictionary <TaskAttemptID, MapOutputFile> localMapFiles) { this.reduceId = reduceId; this.jobConf = jobConf; this.localFS = localFS; this.umbilical = umbilical; this.localDirAllocator = localDirAllocator; this.reporter = reporter; this.codec = codec; this.combinerClass = combinerClass; this.combineCollector = combineCollector; this.spilledRecordsCounter = spilledRecordsCounter; this.reduceCombineInputCounter = reduceCombineInputCounter; this.shuffledMapsCounter = shuffledMapsCounter; this.reduceShuffleBytes = reduceShuffleBytes; this.failedShuffleCounter = failedShuffleCounter; this.mergedMapOutputsCounter = mergedMapOutputsCounter; this.status = status; this.copyPhase = copyPhase; this.mergePhase = mergePhase; this.reduceTask = reduceTask; this.mapOutputFile = mapOutputFile; this.localMapFiles = localMapFiles; }
/// <summary>Create a TestTaskReporter and use it for validating map phase progress</summary> internal override Task.TaskReporter StartReporter(TaskUmbilicalProtocol umbilical ) { // start thread that will handle communication with parent Task.TaskReporter reporter = new TestMapProgress.TestTaskReporter(this, this._enclosing .map); return(reporter); }
/// <exception cref="System.IO.IOException"/> private void RunOldReducer <Inkey, Invalue, Outkey, Outvalue>(JobConf job, TaskUmbilicalProtocol umbilical, Task.TaskReporter reporter, RawKeyValueIterator rIter, RawComparator <INKEY> comparator) { System.Type keyClass = typeof(INKEY); System.Type valueClass = typeof(INVALUE); Reducer <INKEY, INVALUE, OUTKEY, OUTVALUE> reducer = ReflectionUtils.NewInstance(job .GetReducerClass(), job); // make output collector string finalName = GetOutputName(GetPartition()); RecordWriter <OUTKEY, OUTVALUE> @out = new ReduceTask.OldTrackingRecordWriter <OUTKEY , OUTVALUE>(this, job, reporter, finalName); RecordWriter <OUTKEY, OUTVALUE> finalOut = @out; OutputCollector <OUTKEY, OUTVALUE> collector = new _OutputCollector_419(finalOut, reporter); // indicate that progress update needs to be sent // apply reduce function try { //increment processed counter only if skipping feature is enabled bool incrProcCount = SkipBadRecords.GetReducerMaxSkipGroups(job) > 0 && SkipBadRecords .GetAutoIncrReducerProcCount(job); ReduceTask.ReduceValuesIterator <INKEY, INVALUE> values = IsSkipping() ? new ReduceTask.SkippingReduceValuesIterator <INKEY, INVALUE>(this, rIter, comparator, keyClass, valueClass, job, reporter, umbilical ) : new ReduceTask.ReduceValuesIterator <INKEY, INVALUE>(this, rIter, job.GetOutputValueGroupingComparator (), keyClass, valueClass, job, reporter); values.InformReduceProgress(); while (values.More()) { reduceInputKeyCounter.Increment(1); reducer.Reduce(values.GetKey(), values, collector, reporter); if (incrProcCount) { reporter.IncrCounter(SkipBadRecords.CounterGroup, SkipBadRecords.CounterReduceProcessedGroups , 1); } values.NextKey(); values.InformReduceProgress(); } reducer.Close(); reducer = null; @out.Close(reporter); @out = null; } finally { IOUtils.Cleanup(Log, reducer); CloseQuietly(@out, reporter); } }
/// <exception cref="System.IO.IOException"/> public SkippingReduceValuesIterator(ReduceTask _enclosing, RawKeyValueIterator @in , RawComparator <KEY> comparator, Type keyClass, Type valClass, Configuration conf , Task.TaskReporter reporter, TaskUmbilicalProtocol umbilical) : base(_enclosing) { this._enclosing = _enclosing; this.umbilical = umbilical; this.skipGroupCounter = ((Counters.Counter)reporter.GetCounter(TaskCounter.ReduceSkippedGroups )); this.skipRecCounter = ((Counters.Counter)reporter.GetCounter(TaskCounter.ReduceSkippedRecords )); this.toWriteSkipRecs = this._enclosing.ToWriteSkipRecs() && SkipBadRecords.GetSkipOutputPath (conf) != null; this.keyClass = keyClass; this.valClass = valClass; this.reporter = reporter; this.skipIt = this._enclosing.GetSkipRanges().SkipRangeIterator(); this.MayBeSkip(); }
/// <exception cref="System.Exception"/> public static void Main(string[] args) { Sharpen.Thread.SetDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler ()); Log.Debug("Child starting"); JobConf job = new JobConf(MRJobConfig.JobConfFile); // Initing with our JobConf allows us to avoid loading confs twice Limits.Init(job); UserGroupInformation.SetConfiguration(job); string host = args[0]; int port = System.Convert.ToInt32(args[1]); IPEndPoint address = NetUtils.CreateSocketAddrForHost(host, port); TaskAttemptID firstTaskid = ((TaskAttemptID)TaskAttemptID.ForName(args[2])); long jvmIdLong = long.Parse(args[3]); JVMId jvmId = new JVMId(((JobID)firstTaskid.GetJobID()), firstTaskid.GetTaskType( ) == TaskType.Map, jvmIdLong); // initialize metrics DefaultMetricsSystem.Initialize(StringUtils.Camelize(firstTaskid.GetTaskType().ToString ()) + "Task"); // Security framework already loaded the tokens into current ugi Credentials credentials = UserGroupInformation.GetCurrentUser().GetCredentials(); Log.Info("Executing with tokens:"); foreach (Org.Apache.Hadoop.Security.Token.Token <object> token in credentials.GetAllTokens ()) { Log.Info(token); } // Create TaskUmbilicalProtocol as actual task owner. UserGroupInformation taskOwner = UserGroupInformation.CreateRemoteUser(((JobID)firstTaskid .GetJobID()).ToString()); Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> jt = TokenCache.GetJobToken (credentials); SecurityUtil.SetTokenService(jt, address); taskOwner.AddToken(jt); TaskUmbilicalProtocol umbilical = taskOwner.DoAs(new _PrivilegedExceptionAction_108 (address, job)); // report non-pid to application master JvmContext context = new JvmContext(jvmId, "-1000"); Log.Debug("PID: " + Sharpen.Runtime.GetEnv()["JVM_PID"]); Task task = null; UserGroupInformation childUGI = null; ScheduledExecutorService logSyncer = null; try { int idleLoopCount = 0; JvmTask myTask = null; // poll for new task for (int idle = 0; null == myTask; ++idle) { long sleepTimeMilliSecs = Math.Min(idle * 500, 1500); Log.Info("Sleeping for " + sleepTimeMilliSecs + "ms before retrying again. Got null now." ); TimeUnit.Milliseconds.Sleep(sleepTimeMilliSecs); myTask = umbilical.GetTask(context); } if (myTask.ShouldDie()) { return; } task = myTask.GetTask(); YarnChild.taskid = task.GetTaskID(); // Create the job-conf and set credentials ConfigureTask(job, task, credentials, jt); // Initiate Java VM metrics JvmMetrics.InitSingleton(jvmId.ToString(), job.GetSessionId()); childUGI = UserGroupInformation.CreateRemoteUser(Runtime.Getenv(ApplicationConstants.Environment .User.ToString())); // Add tokens to new user so that it may execute its task correctly. childUGI.AddCredentials(credentials); // set job classloader if configured before invoking the task MRApps.SetJobClassLoader(job); logSyncer = TaskLog.CreateLogSyncer(); // Create a final reference to the task for the doAs block Task taskFinal = task; childUGI.DoAs(new _PrivilegedExceptionAction_158(taskFinal, job, umbilical)); } catch (FSError e) { // use job-specified working directory // run the task Log.Fatal("FSError from child", e); if (!ShutdownHookManager.Get().IsShutdownInProgress()) { umbilical.FsError(taskid, e.Message); } } catch (Exception exception) { Log.Warn("Exception running child : " + StringUtils.StringifyException(exception) ); try { if (task != null) { // do cleanup for the task if (childUGI == null) { // no need to job into doAs block task.TaskCleanup(umbilical); } else { Task taskFinal = task; childUGI.DoAs(new _PrivilegedExceptionAction_183(taskFinal, umbilical)); } } } catch (Exception e) { Log.Info("Exception cleaning up: " + StringUtils.StringifyException(e)); } // Report back any failures, for diagnostic purposes if (taskid != null) { if (!ShutdownHookManager.Get().IsShutdownInProgress()) { umbilical.FatalError(taskid, StringUtils.StringifyException(exception)); } } } catch (Exception throwable) { Log.Fatal("Error running child : " + StringUtils.StringifyException(throwable)); if (taskid != null) { if (!ShutdownHookManager.Get().IsShutdownInProgress()) { Exception tCause = throwable.InnerException; string cause = tCause == null ? throwable.Message : StringUtils.StringifyException (tCause); umbilical.FatalError(taskid, cause); } } } finally { RPC.StopProxy(umbilical); DefaultMetricsSystem.Shutdown(); TaskLog.SyncLogsShutdown(logSyncer); } }
public _PrivilegedExceptionAction_183(Task taskFinal, TaskUmbilicalProtocol umbilical ) { this.taskFinal = taskFinal; this.umbilical = umbilical; }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> /// <exception cref="System.TypeLoadException"/> public override void Run(JobConf job, TaskUmbilicalProtocol umbilical) { job.SetBoolean(JobContext.SkipRecords, IsSkipping()); if (IsMapOrReduce()) { copyPhase = GetProgress().AddPhase("copy"); sortPhase = GetProgress().AddPhase("sort"); reducePhase = GetProgress().AddPhase("reduce"); } // start thread that will handle communication with parent Task.TaskReporter reporter = StartReporter(umbilical); bool useNewApi = job.GetUseNewReducer(); Initialize(job, GetJobID(), reporter, useNewApi); // check if it is a cleanupJobTask if (jobCleanup) { RunJobCleanupTask(umbilical, reporter); return; } if (jobSetup) { RunJobSetupTask(umbilical, reporter); return; } if (taskCleanup) { RunTaskCleanupTask(umbilical, reporter); return; } // Initialize the codec codec = InitCodec(); RawKeyValueIterator rIter = null; ShuffleConsumerPlugin shuffleConsumerPlugin = null; Type combinerClass = conf.GetCombinerClass(); Task.CombineOutputCollector combineCollector = (null != combinerClass) ? new Task.CombineOutputCollector (reduceCombineOutputCounter, reporter, conf) : null; Type clazz = job.GetClass <ShuffleConsumerPlugin>(MRConfig.ShuffleConsumerPlugin, typeof(Shuffle)); shuffleConsumerPlugin = ReflectionUtils.NewInstance(clazz, job); Log.Info("Using ShuffleConsumerPlugin: " + shuffleConsumerPlugin); ShuffleConsumerPlugin.Context shuffleContext = new ShuffleConsumerPlugin.Context( GetTaskID(), job, FileSystem.GetLocal(job), umbilical, base.lDirAlloc, reporter, codec, combinerClass, combineCollector, spilledRecordsCounter, reduceCombineInputCounter , shuffledMapsCounter, reduceShuffleBytes, failedShuffleCounter, mergedMapOutputsCounter , taskStatus, copyPhase, sortPhase, this, mapOutputFile, localMapFiles); shuffleConsumerPlugin.Init(shuffleContext); rIter = shuffleConsumerPlugin.Run(); // free up the data structures mapOutputFilesOnDisk.Clear(); sortPhase.Complete(); // sort is complete SetPhase(TaskStatus.Phase.Reduce); StatusUpdate(umbilical); Type keyClass = job.GetMapOutputKeyClass(); Type valueClass = job.GetMapOutputValueClass(); RawComparator comparator = job.GetOutputValueGroupingComparator(); if (useNewApi) { RunNewReducer(job, umbilical, reporter, rIter, comparator, keyClass, valueClass); } else { RunOldReducer(job, umbilical, reporter, rIter, comparator, keyClass, valueClass); } shuffleConsumerPlugin.Close(); Done(umbilical, reporter); }