Exemple #1
0
        public virtual void TestRenameMapOutputForReduce()
        {
            JobConf       conf          = new JobConf();
            MROutputFiles mrOutputFiles = new MROutputFiles();

            mrOutputFiles.SetConf(conf);
            // make sure both dirs are distinct
            //
            conf.Set(MRConfig.LocalDir, localDirs[0].ToString());
            Path mapOut = mrOutputFiles.GetOutputFileForWrite(1);

            conf.Set(MRConfig.LocalDir, localDirs[1].ToString());
            Path mapOutIdx = mrOutputFiles.GetOutputIndexFileForWrite(1);

            Assert.AssertNotEquals("Paths must be different!", mapOut.GetParent(), mapOutIdx.
                                   GetParent());
            // make both dirs part of LOCAL_DIR
            conf.SetStrings(MRConfig.LocalDir, localDirs);
            FileContext lfc = FileContext.GetLocalFSFileContext(conf);

            lfc.Create(mapOut, EnumSet.Of(CreateFlag.Create)).Close();
            lfc.Create(mapOutIdx, EnumSet.Of(CreateFlag.Create)).Close();
            JobId         jobId = MRBuilderUtils.NewJobId(12345L, 1, 2);
            TaskId        tid   = MRBuilderUtils.NewTaskId(jobId, 0, TaskType.Map);
            TaskAttemptId taid  = MRBuilderUtils.NewTaskAttemptId(tid, 0);

            LocalContainerLauncher.RenameMapOutputForReduce(conf, taid, mrOutputFiles);
        }
Exemple #2
0
 /// <exception cref="System.IO.IOException"/>
 public virtual void Reduce(WritableComparable key, IEnumerator values, OutputCollector
                            output, Reporter reporter)
 {
     if (first)
     {
         first = false;
         MapOutputFile mapOutputFile = new MROutputFiles();
         mapOutputFile.SetConf(conf);
         Path       input = mapOutputFile.GetInputFile(0);
         FileSystem fs    = FileSystem.Get(conf);
         NUnit.Framework.Assert.IsTrue("reduce input exists " + input, fs.Exists(input));
         SequenceFile.Reader rdr = new SequenceFile.Reader(fs, input, conf);
         NUnit.Framework.Assert.AreEqual("is reduce input compressed " + input, compressInput
                                         , rdr.IsCompressed());
         rdr.Close();
     }
 }
 public override void Run()
 {
     try
     {
         TaskAttemptID mapId = new TaskAttemptID(new TaskID(this.jobId, TaskType.Map, this
                                                            .taskId), 0);
         LocalJobRunner.Log.Info("Starting task: " + mapId);
         this._enclosing.mapIds.AddItem(mapId);
         MapTask map = new MapTask(this._enclosing.systemJobFile.ToString(), mapId, this.taskId
                                   , this.info.GetSplitIndex(), 1);
         map.SetUser(UserGroupInformation.GetCurrentUser().GetShortUserName());
         LocalJobRunner.SetupChildMapredLocalDirs(map, this.localConf);
         MapOutputFile mapOutput = new MROutputFiles();
         mapOutput.SetConf(this.localConf);
         this.mapOutputFiles[mapId] = mapOutput;
         map.SetJobFile(this._enclosing.localJobFile.ToString());
         this.localConf.SetUser(map.GetUser());
         map.LocalizeConfiguration(this.localConf);
         map.SetConf(this.localConf);
         try
         {
             this._enclosing._enclosing.map_tasks.GetAndIncrement();
             this._enclosing._enclosing.myMetrics.LaunchMap(mapId);
             map.Run(this.localConf, this._enclosing);
             this._enclosing._enclosing.myMetrics.CompleteMap(mapId);
         }
         finally
         {
             this._enclosing._enclosing.map_tasks.GetAndDecrement();
         }
         LocalJobRunner.Log.Info("Finishing task: " + mapId);
     }
     catch (Exception e)
     {
         this.storedException = e;
     }
 }