public virtual void TestRenameMapOutputForReduce()
{
JobConf conf = new JobConf();
MROutputFiles mrOutputFiles = new MROutputFiles();
mrOutputFiles.SetConf(conf);
// make sure both dirs are distinct
//
conf.Set(MRConfig.LocalDir, localDirs[0].ToString());
Path mapOut = mrOutputFiles.GetOutputFileForWrite(1);
conf.Set(MRConfig.LocalDir, localDirs[1].ToString());
Path mapOutIdx = mrOutputFiles.GetOutputIndexFileForWrite(1);
Assert.AssertNotEquals("Paths must be different!", mapOut.GetParent(), mapOutIdx.
GetParent());
// make both dirs part of LOCAL_DIR
conf.SetStrings(MRConfig.LocalDir, localDirs);
FileContext lfc = FileContext.GetLocalFSFileContext(conf);
lfc.Create(mapOut, EnumSet.Of(CreateFlag.Create)).Close();
lfc.Create(mapOutIdx, EnumSet.Of(CreateFlag.Create)).Close();
JobId jobId = MRBuilderUtils.NewJobId(12345L, 1, 2);
TaskId tid = MRBuilderUtils.NewTaskId(jobId, 0, TaskType.Map);
TaskAttemptId taid = MRBuilderUtils.NewTaskAttemptId(tid, 0);
LocalContainerLauncher.RenameMapOutputForReduce(conf, taid, mrOutputFiles);
}
/// <exception cref="System.IO.IOException"/>
public virtual void Reduce(WritableComparable key, IEnumerator values, OutputCollector
output, Reporter reporter)
{
if (first)
{
first = false;
MapOutputFile mapOutputFile = new MROutputFiles();
mapOutputFile.SetConf(conf);
Path input = mapOutputFile.GetInputFile(0);
FileSystem fs = FileSystem.Get(conf);
NUnit.Framework.Assert.IsTrue("reduce input exists " + input, fs.Exists(input));
SequenceFile.Reader rdr = new SequenceFile.Reader(fs, input, conf);
NUnit.Framework.Assert.AreEqual("is reduce input compressed " + input, compressInput
, rdr.IsCompressed());
rdr.Close();
}
}
public override void Run()
{
try
{
TaskAttemptID mapId = new TaskAttemptID(new TaskID(this.jobId, TaskType.Map, this
.taskId), 0);
LocalJobRunner.Log.Info("Starting task: " + mapId);
this._enclosing.mapIds.AddItem(mapId);
MapTask map = new MapTask(this._enclosing.systemJobFile.ToString(), mapId, this.taskId
, this.info.GetSplitIndex(), 1);
map.SetUser(UserGroupInformation.GetCurrentUser().GetShortUserName());
LocalJobRunner.SetupChildMapredLocalDirs(map, this.localConf);
MapOutputFile mapOutput = new MROutputFiles();
mapOutput.SetConf(this.localConf);
this.mapOutputFiles[mapId] = mapOutput;
map.SetJobFile(this._enclosing.localJobFile.ToString());
this.localConf.SetUser(map.GetUser());
map.LocalizeConfiguration(this.localConf);
map.SetConf(this.localConf);
try
{
this._enclosing._enclosing.map_tasks.GetAndIncrement();
this._enclosing._enclosing.myMetrics.LaunchMap(mapId);
map.Run(this.localConf, this._enclosing);
this._enclosing._enclosing.myMetrics.CompleteMap(mapId);
}
finally
{
this._enclosing._enclosing.map_tasks.GetAndDecrement();
}
LocalJobRunner.Log.Info("Finishing task: " + mapId);
}
catch (Exception e)
{
this.storedException = e;
}
}