/// <exception cref="System.IO.IOException"/>
public ReduceValuesIterator(ReduceTask _enclosing, RawKeyValueIterator @in, RawComparator
<KEY> comparator, Type keyClass, Type valClass, Configuration conf, Progressable
reporter)
: base(@in, comparator, keyClass, valClass, conf, reporter)
{
this._enclosing = _enclosing;
}
protected internal override Task CreateRemoteTask()
{
//job file name is set in TaskAttempt, setting it null here
ReduceTask reduceTask = new ReduceTask(string.Empty, TypeConverter.FromYarn(GetID
()), partition, numMapTasks, 1);
// YARN doesn't have the concept of slots per task, set it as 1.
reduceTask.SetUser(conf.Get(MRJobConfig.UserName));
reduceTask.SetConf(conf);
return(reduceTask);
}
/// <exception cref="System.IO.IOException"/>
public SkippingReduceValuesIterator(ReduceTask _enclosing, RawKeyValueIterator @in
, RawComparator <KEY> comparator, Type keyClass, Type valClass, Configuration conf
, Task.TaskReporter reporter, TaskUmbilicalProtocol umbilical)
: base(_enclosing)
{
this._enclosing = _enclosing;
this.umbilical = umbilical;
this.skipGroupCounter = ((Counters.Counter)reporter.GetCounter(TaskCounter.ReduceSkippedGroups
));
this.skipRecCounter = ((Counters.Counter)reporter.GetCounter(TaskCounter.ReduceSkippedRecords
));
this.toWriteSkipRecs = this._enclosing.ToWriteSkipRecs() && SkipBadRecords.GetSkipOutputPath
(conf) != null;
this.keyClass = keyClass;
this.valClass = valClass;
this.reporter = reporter;
this.skipIt = this._enclosing.GetSkipRanges().SkipRangeIterator();
this.MayBeSkip();
}
public override void Run()
{
try
{
TaskAttemptID reduceId = new TaskAttemptID(new TaskID(this.jobId, TaskType.Reduce
, this.taskId), 0);
LocalJobRunner.Log.Info("Starting task: " + reduceId);
ReduceTask reduce = new ReduceTask(this._enclosing.systemJobFile.ToString(), reduceId
, this.taskId, this._enclosing.mapIds.Count, 1);
reduce.SetUser(UserGroupInformation.GetCurrentUser().GetShortUserName());
LocalJobRunner.SetupChildMapredLocalDirs(reduce, this.localConf);
reduce.SetLocalMapFiles(this.mapOutputFiles);
if (!this._enclosing.IsInterrupted())
{
reduce.SetJobFile(this._enclosing.localJobFile.ToString());
this.localConf.SetUser(reduce.GetUser());
reduce.LocalizeConfiguration(this.localConf);
reduce.SetConf(this.localConf);
try
{
this._enclosing._enclosing.reduce_tasks.GetAndIncrement();
this._enclosing._enclosing.myMetrics.LaunchReduce(reduce.GetTaskID());
reduce.Run(this.localConf, this._enclosing);
this._enclosing._enclosing.myMetrics.CompleteReduce(reduce.GetTaskID());
}
finally
{
this._enclosing._enclosing.reduce_tasks.GetAndDecrement();
}
LocalJobRunner.Log.Info("Finishing task: " + reduceId);
}
else
{
throw new Exception();
}
}
catch (Exception t)
{
// store this to be rethrown in the initial thread context.
this.storedException = t;
}
}
/// <exception cref="System.Exception"/>
/// <exception cref="System.IO.IOException"/>
internal NewTrackingRecordWriter(ReduceTask reduce, TaskAttemptContext taskContext
)
{
this.outputRecordCounter = reduce.reduceOutputCounter;
this.fileOutputByteCounter = reduce.fileOutputByteCounter;
IList <FileSystem.Statistics> matchedStats = null;
if (reduce.outputFormat is FileOutputFormat)
{
matchedStats = GetFsStatistics(FileOutputFormat.GetOutputPath(taskContext), taskContext
.GetConfiguration());
}
fsStats = matchedStats;
long bytesOutPrev = GetOutputBytes(fsStats);
this.real = (RecordWriter <K, V>)reduce.outputFormat.GetRecordWriter(taskContext);
long bytesOutCurr = GetOutputBytes(fsStats);
fileOutputByteCounter.Increment(bytesOutCurr - bytesOutPrev);
}
/// <exception cref="System.IO.IOException"/>
public OldTrackingRecordWriter(ReduceTask reduce, JobConf job, Task.TaskReporter
reporter, string finalName)
{
this.reduceOutputCounter = reduce.reduceOutputCounter;
this.fileOutputByteCounter = reduce.fileOutputByteCounter;
IList <FileSystem.Statistics> matchedStats = null;
if (job.GetOutputFormat() is FileOutputFormat)
{
matchedStats = GetFsStatistics(FileOutputFormat.GetOutputPath(job), job);
}
fsStats = matchedStats;
FileSystem fs = FileSystem.Get(job);
long bytesOutPrev = GetOutputBytes(fsStats);
this.real = job.GetOutputFormat().GetRecordWriter(fs, job, finalName, reporter);
long bytesOutCurr = GetOutputBytes(fsStats);
fileOutputByteCounter.Increment(bytesOutCurr - bytesOutPrev);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.TypeLoadException"/>
/// <exception cref="System.Exception"/>
private Task CreateDummyTask(TaskType type)
{
JobConf conf = new JobConf();
conf.SetOutputCommitter(typeof(TestTaskCommit.CommitterThatAlwaysRequiresCommit));
Path outDir = new Path(rootDir, "output");
FileOutputFormat.SetOutputPath(conf, outDir);
JobID jobId = ((JobID)JobID.ForName("job_201002121132_0001"));
Task testTask;
if (type == TaskType.Map)
{
testTask = new MapTask();
}
else
{
testTask = new ReduceTask();
}
testTask.SetConf(conf);
testTask.Initialize(conf, jobId, Reporter.Null, false);
return(testTask);
}
/// <exception cref="Sharpen.RuntimeException"/>
/// <exception cref="System.IO.IOException"/>
private void RunSubtask(Task task, TaskType taskType, TaskAttemptId attemptID, int
numMapTasks, bool renameOutputs, IDictionary <TaskAttemptID, MapOutputFile> localMapFiles
)
{
TaskAttemptID classicAttemptID = TypeConverter.FromYarn(attemptID);
try
{
JobConf conf = new JobConf(this._enclosing.GetConfig());
conf.Set(JobContext.TaskId, task.GetTaskID().ToString());
conf.Set(JobContext.TaskAttemptId, classicAttemptID.ToString());
conf.SetBoolean(JobContext.TaskIsmap, (taskType == TaskType.Map));
conf.SetInt(JobContext.TaskPartition, task.GetPartition());
conf.Set(JobContext.Id, task.GetJobID().ToString());
// Use the AM's local dir env to generate the intermediate step
// output files
string[] localSysDirs = StringUtils.GetTrimmedStrings(Runtime.Getenv(ApplicationConstants.Environment
.LocalDirs.ToString()));
conf.SetStrings(MRConfig.LocalDir, localSysDirs);
LocalContainerLauncher.Log.Info(MRConfig.LocalDir + " for uber task: " + conf.Get
(MRConfig.LocalDir));
// mark this as an uberized subtask so it can set task counter
// (longer-term/FIXME: could redefine as job counter and send
// "JobCounterEvent" to JobImpl on [successful] completion of subtask;
// will need new Job state-machine transition and JobImpl jobCounters
// map to handle)
conf.SetBoolean("mapreduce.task.uberized", true);
// Check and handle Encrypted spill key
task.SetEncryptedSpillKey(this._enclosing.encryptedSpillKey);
YarnChild.SetEncryptedSpillKeyIfRequired(task);
// META-FIXME: do we want the extra sanity-checking (doneWithMaps,
// etc.), or just assume/hope the state machine(s) and uber-AM work
// as expected?
if (taskType == TaskType.Map)
{
if (this.doneWithMaps)
{
LocalContainerLauncher.Log.Error("CONTAINER_REMOTE_LAUNCH contains a map task ("
+ attemptID + "), but should be finished with maps");
throw new RuntimeException();
}
MapTask map = (MapTask)task;
map.SetConf(conf);
map.Run(conf, this._enclosing.umbilical);
if (renameOutputs)
{
MapOutputFile renamed = LocalContainerLauncher.RenameMapOutputForReduce(conf, attemptID
, map.GetMapOutputFile());
localMapFiles[classicAttemptID] = renamed;
}
this.Relocalize();
if (++this.finishedSubMaps == numMapTasks)
{
this.doneWithMaps = true;
}
}
else
{
/* TaskType.REDUCE */
if (!this.doneWithMaps)
{
// check if event-queue empty? whole idea of counting maps vs.
// checking event queue is a tad wacky...but could enforce ordering
// (assuming no "lost events") at LocalMRAppMaster [CURRENT BUG(?):
// doesn't send reduce event until maps all done]
LocalContainerLauncher.Log.Error("CONTAINER_REMOTE_LAUNCH contains a reduce task ("
+ attemptID + "), but not yet finished with maps");
throw new RuntimeException();
}
// a.k.a. "mapreduce.jobtracker.address" in LocalJobRunner:
// set framework name to local to make task local
conf.Set(MRConfig.FrameworkName, MRConfig.LocalFrameworkName);
conf.Set(MRConfig.MasterAddress, "local");
// bypass shuffle
ReduceTask reduce = (ReduceTask)task;
reduce.SetLocalMapFiles(localMapFiles);
reduce.SetConf(conf);
reduce.Run(conf, this._enclosing.umbilical);
this.Relocalize();
}
}
catch (FSError e)
{
LocalContainerLauncher.Log.Fatal("FSError from child", e);
// umbilical: MRAppMaster creates (taskAttemptListener), passes to us
if (!ShutdownHookManager.Get().IsShutdownInProgress())
{
this._enclosing.umbilical.FsError(classicAttemptID, e.Message);
}
throw new RuntimeException();
}
catch (Exception exception)
{
LocalContainerLauncher.Log.Warn("Exception running local (uberized) 'child' : " +
StringUtils.StringifyException(exception));
try
{
if (task != null)
{
// do cleanup for the task
task.TaskCleanup(this._enclosing.umbilical);
}
}
catch (Exception e)
{
LocalContainerLauncher.Log.Info("Exception cleaning up: " + StringUtils.StringifyException
(e));
}
// Report back any failures, for diagnostic purposes
this._enclosing.umbilical.ReportDiagnosticInfo(classicAttemptID, StringUtils.StringifyException
(exception));
throw new RuntimeException();
}
catch (Exception throwable)
{
LocalContainerLauncher.Log.Fatal("Error running local (uberized) 'child' : " + StringUtils
.StringifyException(throwable));
if (!ShutdownHookManager.Get().IsShutdownInProgress())
{
Exception tCause = throwable.InnerException;
string cause = (tCause == null) ? throwable.Message : StringUtils.StringifyException
(tCause);
this._enclosing.umbilical.FatalError(classicAttemptID, cause);
}
throw new RuntimeException();
}
}
