public virtual void TestRenameMapOutputForReduce() { JobConf conf = new JobConf(); MROutputFiles mrOutputFiles = new MROutputFiles(); mrOutputFiles.SetConf(conf); // make sure both dirs are distinct // conf.Set(MRConfig.LocalDir, localDirs[0].ToString()); Path mapOut = mrOutputFiles.GetOutputFileForWrite(1); conf.Set(MRConfig.LocalDir, localDirs[1].ToString()); Path mapOutIdx = mrOutputFiles.GetOutputIndexFileForWrite(1); Assert.AssertNotEquals("Paths must be different!", mapOut.GetParent(), mapOutIdx. GetParent()); // make both dirs part of LOCAL_DIR conf.SetStrings(MRConfig.LocalDir, localDirs); FileContext lfc = FileContext.GetLocalFSFileContext(conf); lfc.Create(mapOut, EnumSet.Of(CreateFlag.Create)).Close(); lfc.Create(mapOutIdx, EnumSet.Of(CreateFlag.Create)).Close(); JobId jobId = MRBuilderUtils.NewJobId(12345L, 1, 2); TaskId tid = MRBuilderUtils.NewTaskId(jobId, 0, TaskType.Map); TaskAttemptId taid = MRBuilderUtils.NewTaskAttemptId(tid, 0); LocalContainerLauncher.RenameMapOutputForReduce(conf, taid, mrOutputFiles); }
/// <exception cref="System.IO.IOException"/> public virtual void Reduce(WritableComparable key, IEnumerator values, OutputCollector output, Reporter reporter) { if (first) { first = false; MapOutputFile mapOutputFile = new MROutputFiles(); mapOutputFile.SetConf(conf); Path input = mapOutputFile.GetInputFile(0); FileSystem fs = FileSystem.Get(conf); NUnit.Framework.Assert.IsTrue("reduce input exists " + input, fs.Exists(input)); SequenceFile.Reader rdr = new SequenceFile.Reader(fs, input, conf); NUnit.Framework.Assert.AreEqual("is reduce input compressed " + input, compressInput , rdr.IsCompressed()); rdr.Close(); } }
public override void Run() { try { TaskAttemptID mapId = new TaskAttemptID(new TaskID(this.jobId, TaskType.Map, this .taskId), 0); LocalJobRunner.Log.Info("Starting task: " + mapId); this._enclosing.mapIds.AddItem(mapId); MapTask map = new MapTask(this._enclosing.systemJobFile.ToString(), mapId, this.taskId , this.info.GetSplitIndex(), 1); map.SetUser(UserGroupInformation.GetCurrentUser().GetShortUserName()); LocalJobRunner.SetupChildMapredLocalDirs(map, this.localConf); MapOutputFile mapOutput = new MROutputFiles(); mapOutput.SetConf(this.localConf); this.mapOutputFiles[mapId] = mapOutput; map.SetJobFile(this._enclosing.localJobFile.ToString()); this.localConf.SetUser(map.GetUser()); map.LocalizeConfiguration(this.localConf); map.SetConf(this.localConf); try { this._enclosing._enclosing.map_tasks.GetAndIncrement(); this._enclosing._enclosing.myMetrics.LaunchMap(mapId); map.Run(this.localConf, this._enclosing); this._enclosing._enclosing.myMetrics.CompleteMap(mapId); } finally { this._enclosing._enclosing.map_tasks.GetAndDecrement(); } LocalJobRunner.Log.Info("Finishing task: " + mapId); } catch (Exception e) { this.storedException = e; } }