public LocalContainerLauncher(AppContext context, TaskUmbilicalProtocol umbilical
)
: base(typeof(Org.Apache.Hadoop.Mapred.LocalContainerLauncher).FullName)
{
this.context = context;
this.umbilical = umbilical;
// umbilical: MRAppMaster creates (taskAttemptListener), passes to us
// (TODO/FIXME: pointless to use RPC to talk to self; should create
// LocalTaskAttemptListener or similar: implement umbilical protocol
// but skip RPC stuff)
try
{
curFC = FileContext.GetFileContext(curDir.ToURI());
}
catch (UnsupportedFileSystemException)
{
Log.Error("Local filesystem " + curDir.ToURI().ToString() + " is unsupported?? (should never happen)"
);
}
// Save list of files/dirs that are supposed to be present so can delete
// any extras created by one task before starting subsequent task. Note
// that there's no protection against deleted or renamed localization;
// users who do that get what they deserve (and will have to disable
// uberization in order to run correctly).
FilePath[] curLocalFiles = curDir.ListFiles();
localizedFiles = new HashSet <FilePath>(curLocalFiles.Length);
for (int j = 0; j < curLocalFiles.Length; ++j)
{
localizedFiles.AddItem(curLocalFiles[j]);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="System.TypeLoadException"/>
private void RunNewReducer <Inkey, Invalue, Outkey, Outvalue>(JobConf job, TaskUmbilicalProtocol
umbilical, Task.TaskReporter reporter, RawKeyValueIterator rIter, RawComparator
<INKEY> comparator)
{
System.Type keyClass = typeof(INKEY);
System.Type valueClass = typeof(INVALUE);
// wrap value iterator to report progress.
RawKeyValueIterator rawIter = rIter;
rIter = new _RawKeyValueIterator_587(rawIter, reporter);
// make a task context so we can get the classes
TaskAttemptContext taskContext = new TaskAttemptContextImpl(job, GetTaskID(), reporter
);
// make a reducer
Reducer <INKEY, INVALUE, OUTKEY, OUTVALUE> reducer = (Reducer <INKEY, INVALUE, OUTKEY
, OUTVALUE>)ReflectionUtils.NewInstance(taskContext.GetReducerClass(), job);
RecordWriter <OUTKEY, OUTVALUE> trackedRW = new ReduceTask.NewTrackingRecordWriter
<OUTKEY, OUTVALUE>(this, taskContext);
job.SetBoolean("mapred.skip.on", IsSkipping());
job.SetBoolean(JobContext.SkipRecords, IsSkipping());
Reducer.Context reducerContext = CreateReduceContext(reducer, job, GetTaskID(), rIter
, reduceInputKeyCounter, reduceInputValueCounter, trackedRW, committer, reporter
, comparator, keyClass, valueClass);
try
{
reducer.Run(reducerContext);
}
finally
{
trackedRW.Close(reducerContext);
}
}
public _PrivilegedExceptionAction_158(Task taskFinal, JobConf job, TaskUmbilicalProtocol
umbilical)
{
this.taskFinal = taskFinal;
this.job = job;
this.umbilical = umbilical;
}
public Context(TaskAttemptID reduceId, JobConf jobConf, FileSystem localFS, TaskUmbilicalProtocol
umbilical, LocalDirAllocator localDirAllocator, Reporter reporter, CompressionCodec
codec, Type combinerClass, Task.CombineOutputCollector <K, V> combineCollector,
Counters.Counter spilledRecordsCounter, Counters.Counter reduceCombineInputCounter
, Counters.Counter shuffledMapsCounter, Counters.Counter reduceShuffleBytes, Counters.Counter
failedShuffleCounter, Counters.Counter mergedMapOutputsCounter, TaskStatus status
, Progress copyPhase, Progress mergePhase, Task reduceTask, MapOutputFile mapOutputFile
, IDictionary <TaskAttemptID, MapOutputFile> localMapFiles)
{
this.reduceId = reduceId;
this.jobConf = jobConf;
this.localFS = localFS;
this.umbilical = umbilical;
this.localDirAllocator = localDirAllocator;
this.reporter = reporter;
this.codec = codec;
this.combinerClass = combinerClass;
this.combineCollector = combineCollector;
this.spilledRecordsCounter = spilledRecordsCounter;
this.reduceCombineInputCounter = reduceCombineInputCounter;
this.shuffledMapsCounter = shuffledMapsCounter;
this.reduceShuffleBytes = reduceShuffleBytes;
this.failedShuffleCounter = failedShuffleCounter;
this.mergedMapOutputsCounter = mergedMapOutputsCounter;
this.status = status;
this.copyPhase = copyPhase;
this.mergePhase = mergePhase;
this.reduceTask = reduceTask;
this.mapOutputFile = mapOutputFile;
this.localMapFiles = localMapFiles;
}
/// <summary>Create a TestTaskReporter and use it for validating map phase progress</summary>
internal override Task.TaskReporter StartReporter(TaskUmbilicalProtocol umbilical
)
{
// start thread that will handle communication with parent
Task.TaskReporter reporter = new TestMapProgress.TestTaskReporter(this, this._enclosing
.map);
return(reporter);
}
/// <exception cref="System.IO.IOException"/>
private void RunOldReducer <Inkey, Invalue, Outkey, Outvalue>(JobConf job, TaskUmbilicalProtocol
umbilical, Task.TaskReporter reporter, RawKeyValueIterator rIter, RawComparator
<INKEY> comparator)
{
System.Type keyClass = typeof(INKEY);
System.Type valueClass = typeof(INVALUE);
Reducer <INKEY, INVALUE, OUTKEY, OUTVALUE> reducer = ReflectionUtils.NewInstance(job
.GetReducerClass(), job);
// make output collector
string finalName = GetOutputName(GetPartition());
RecordWriter <OUTKEY, OUTVALUE> @out = new ReduceTask.OldTrackingRecordWriter <OUTKEY
, OUTVALUE>(this, job, reporter, finalName);
RecordWriter <OUTKEY, OUTVALUE> finalOut = @out;
OutputCollector <OUTKEY, OUTVALUE> collector = new _OutputCollector_419(finalOut,
reporter);
// indicate that progress update needs to be sent
// apply reduce function
try
{
//increment processed counter only if skipping feature is enabled
bool incrProcCount = SkipBadRecords.GetReducerMaxSkipGroups(job) > 0 && SkipBadRecords
.GetAutoIncrReducerProcCount(job);
ReduceTask.ReduceValuesIterator <INKEY, INVALUE> values = IsSkipping() ? new ReduceTask.SkippingReduceValuesIterator
<INKEY, INVALUE>(this, rIter, comparator, keyClass, valueClass, job, reporter, umbilical
) : new ReduceTask.ReduceValuesIterator <INKEY, INVALUE>(this, rIter, job.GetOutputValueGroupingComparator
(), keyClass, valueClass, job, reporter);
values.InformReduceProgress();
while (values.More())
{
reduceInputKeyCounter.Increment(1);
reducer.Reduce(values.GetKey(), values, collector, reporter);
if (incrProcCount)
{
reporter.IncrCounter(SkipBadRecords.CounterGroup, SkipBadRecords.CounterReduceProcessedGroups
, 1);
}
values.NextKey();
values.InformReduceProgress();
}
reducer.Close();
reducer = null;
@out.Close(reporter);
@out = null;
}
finally
{
IOUtils.Cleanup(Log, reducer);
CloseQuietly(@out, reporter);
}
}
/// <exception cref="System.IO.IOException"/>
public SkippingReduceValuesIterator(ReduceTask _enclosing, RawKeyValueIterator @in
, RawComparator <KEY> comparator, Type keyClass, Type valClass, Configuration conf
, Task.TaskReporter reporter, TaskUmbilicalProtocol umbilical)
: base(_enclosing)
{
this._enclosing = _enclosing;
this.umbilical = umbilical;
this.skipGroupCounter = ((Counters.Counter)reporter.GetCounter(TaskCounter.ReduceSkippedGroups
));
this.skipRecCounter = ((Counters.Counter)reporter.GetCounter(TaskCounter.ReduceSkippedRecords
));
this.toWriteSkipRecs = this._enclosing.ToWriteSkipRecs() && SkipBadRecords.GetSkipOutputPath
(conf) != null;
this.keyClass = keyClass;
this.valClass = valClass;
this.reporter = reporter;
this.skipIt = this._enclosing.GetSkipRanges().SkipRangeIterator();
this.MayBeSkip();
}
/// <exception cref="System.Exception"/>
public static void Main(string[] args)
{
Sharpen.Thread.SetDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler
());
Log.Debug("Child starting");
JobConf job = new JobConf(MRJobConfig.JobConfFile);
// Initing with our JobConf allows us to avoid loading confs twice
Limits.Init(job);
UserGroupInformation.SetConfiguration(job);
string host = args[0];
int port = System.Convert.ToInt32(args[1]);
IPEndPoint address = NetUtils.CreateSocketAddrForHost(host, port);
TaskAttemptID firstTaskid = ((TaskAttemptID)TaskAttemptID.ForName(args[2]));
long jvmIdLong = long.Parse(args[3]);
JVMId jvmId = new JVMId(((JobID)firstTaskid.GetJobID()), firstTaskid.GetTaskType(
) == TaskType.Map, jvmIdLong);
// initialize metrics
DefaultMetricsSystem.Initialize(StringUtils.Camelize(firstTaskid.GetTaskType().ToString
()) + "Task");
// Security framework already loaded the tokens into current ugi
Credentials credentials = UserGroupInformation.GetCurrentUser().GetCredentials();
Log.Info("Executing with tokens:");
foreach (Org.Apache.Hadoop.Security.Token.Token <object> token in credentials.GetAllTokens
())
{
Log.Info(token);
}
// Create TaskUmbilicalProtocol as actual task owner.
UserGroupInformation taskOwner = UserGroupInformation.CreateRemoteUser(((JobID)firstTaskid
.GetJobID()).ToString());
Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> jt = TokenCache.GetJobToken
(credentials);
SecurityUtil.SetTokenService(jt, address);
taskOwner.AddToken(jt);
TaskUmbilicalProtocol umbilical = taskOwner.DoAs(new _PrivilegedExceptionAction_108
(address, job));
// report non-pid to application master
JvmContext context = new JvmContext(jvmId, "-1000");
Log.Debug("PID: " + Sharpen.Runtime.GetEnv()["JVM_PID"]);
Task task = null;
UserGroupInformation childUGI = null;
ScheduledExecutorService logSyncer = null;
try
{
int idleLoopCount = 0;
JvmTask myTask = null;
// poll for new task
for (int idle = 0; null == myTask; ++idle)
{
long sleepTimeMilliSecs = Math.Min(idle * 500, 1500);
Log.Info("Sleeping for " + sleepTimeMilliSecs + "ms before retrying again. Got null now."
);
TimeUnit.Milliseconds.Sleep(sleepTimeMilliSecs);
myTask = umbilical.GetTask(context);
}
if (myTask.ShouldDie())
{
return;
}
task = myTask.GetTask();
YarnChild.taskid = task.GetTaskID();
// Create the job-conf and set credentials
ConfigureTask(job, task, credentials, jt);
// Initiate Java VM metrics
JvmMetrics.InitSingleton(jvmId.ToString(), job.GetSessionId());
childUGI = UserGroupInformation.CreateRemoteUser(Runtime.Getenv(ApplicationConstants.Environment
.User.ToString()));
// Add tokens to new user so that it may execute its task correctly.
childUGI.AddCredentials(credentials);
// set job classloader if configured before invoking the task
MRApps.SetJobClassLoader(job);
logSyncer = TaskLog.CreateLogSyncer();
// Create a final reference to the task for the doAs block
Task taskFinal = task;
childUGI.DoAs(new _PrivilegedExceptionAction_158(taskFinal, job, umbilical));
}
catch (FSError e)
{
// use job-specified working directory
// run the task
Log.Fatal("FSError from child", e);
if (!ShutdownHookManager.Get().IsShutdownInProgress())
{
umbilical.FsError(taskid, e.Message);
}
}
catch (Exception exception)
{
Log.Warn("Exception running child : " + StringUtils.StringifyException(exception)
);
try
{
if (task != null)
{
// do cleanup for the task
if (childUGI == null)
{
// no need to job into doAs block
task.TaskCleanup(umbilical);
}
else
{
Task taskFinal = task;
childUGI.DoAs(new _PrivilegedExceptionAction_183(taskFinal, umbilical));
}
}
}
catch (Exception e)
{
Log.Info("Exception cleaning up: " + StringUtils.StringifyException(e));
}
// Report back any failures, for diagnostic purposes
if (taskid != null)
{
if (!ShutdownHookManager.Get().IsShutdownInProgress())
{
umbilical.FatalError(taskid, StringUtils.StringifyException(exception));
}
}
}
catch (Exception throwable)
{
Log.Fatal("Error running child : " + StringUtils.StringifyException(throwable));
if (taskid != null)
{
if (!ShutdownHookManager.Get().IsShutdownInProgress())
{
Exception tCause = throwable.InnerException;
string cause = tCause == null ? throwable.Message : StringUtils.StringifyException
(tCause);
umbilical.FatalError(taskid, cause);
}
}
}
finally
{
RPC.StopProxy(umbilical);
DefaultMetricsSystem.Shutdown();
TaskLog.SyncLogsShutdown(logSyncer);
}
}
public _PrivilegedExceptionAction_183(Task taskFinal, TaskUmbilicalProtocol umbilical
)
{
this.taskFinal = taskFinal;
this.umbilical = umbilical;
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="System.TypeLoadException"/>
public override void Run(JobConf job, TaskUmbilicalProtocol umbilical)
{
job.SetBoolean(JobContext.SkipRecords, IsSkipping());
if (IsMapOrReduce())
{
copyPhase = GetProgress().AddPhase("copy");
sortPhase = GetProgress().AddPhase("sort");
reducePhase = GetProgress().AddPhase("reduce");
}
// start thread that will handle communication with parent
Task.TaskReporter reporter = StartReporter(umbilical);
bool useNewApi = job.GetUseNewReducer();
Initialize(job, GetJobID(), reporter, useNewApi);
// check if it is a cleanupJobTask
if (jobCleanup)
{
RunJobCleanupTask(umbilical, reporter);
return;
}
if (jobSetup)
{
RunJobSetupTask(umbilical, reporter);
return;
}
if (taskCleanup)
{
RunTaskCleanupTask(umbilical, reporter);
return;
}
// Initialize the codec
codec = InitCodec();
RawKeyValueIterator rIter = null;
ShuffleConsumerPlugin shuffleConsumerPlugin = null;
Type combinerClass = conf.GetCombinerClass();
Task.CombineOutputCollector combineCollector = (null != combinerClass) ? new Task.CombineOutputCollector
(reduceCombineOutputCounter, reporter, conf) : null;
Type clazz = job.GetClass <ShuffleConsumerPlugin>(MRConfig.ShuffleConsumerPlugin,
typeof(Shuffle));
shuffleConsumerPlugin = ReflectionUtils.NewInstance(clazz, job);
Log.Info("Using ShuffleConsumerPlugin: " + shuffleConsumerPlugin);
ShuffleConsumerPlugin.Context shuffleContext = new ShuffleConsumerPlugin.Context(
GetTaskID(), job, FileSystem.GetLocal(job), umbilical, base.lDirAlloc, reporter,
codec, combinerClass, combineCollector, spilledRecordsCounter, reduceCombineInputCounter
, shuffledMapsCounter, reduceShuffleBytes, failedShuffleCounter, mergedMapOutputsCounter
, taskStatus, copyPhase, sortPhase, this, mapOutputFile, localMapFiles);
shuffleConsumerPlugin.Init(shuffleContext);
rIter = shuffleConsumerPlugin.Run();
// free up the data structures
mapOutputFilesOnDisk.Clear();
sortPhase.Complete();
// sort is complete
SetPhase(TaskStatus.Phase.Reduce);
StatusUpdate(umbilical);
Type keyClass = job.GetMapOutputKeyClass();
Type valueClass = job.GetMapOutputValueClass();
RawComparator comparator = job.GetOutputValueGroupingComparator();
if (useNewApi)
{
RunNewReducer(job, umbilical, reporter, rIter, comparator, keyClass, valueClass);
}
else
{
RunOldReducer(job, umbilical, reporter, rIter, comparator, keyClass, valueClass);
}
shuffleConsumerPlugin.Close();
Done(umbilical, reporter);
}