예제 #1
0
            /// <exception cref="System.IO.IOException"/>
            public Job(LocalJobRunner _enclosing, JobID jobid, string jobSubmitDir)
            {
                this._enclosing    = _enclosing;
                this.systemJobDir  = new Path(jobSubmitDir);
                this.systemJobFile = new Path(this.systemJobDir, "job.xml");
                this.id            = jobid;
                JobConf conf = new JobConf(this.systemJobFile);

                this.localFs = FileSystem.GetLocal(conf);
                string user = UserGroupInformation.GetCurrentUser().GetShortUserName();

                this.localJobDir = this.localFs.MakeQualified(new Path(new Path(conf.GetLocalPath
                                                                                    (LocalJobRunner.jobDir), user), jobid.ToString()));
                this.localJobFile = new Path(this.localJobDir, this.id + ".xml");
                // Manage the distributed cache.  If there are files to be copied,
                // this will trigger localFile to be re-written again.
                this.localDistributedCacheManager = new LocalDistributedCacheManager();
                this.localDistributedCacheManager.Setup(conf);
                // Write out configuration file.  Instead of copying it from
                // systemJobFile, we re-write it, since setup(), above, may have
                // updated it.
                OutputStream @out = this.localFs.Create(this.localJobFile);

                try
                {
                    conf.WriteXml(@out);
                }
                finally
                {
                    @out.Close();
                }
                this.job = new JobConf(this.localJobFile);
                // Job (the current object) is a Thread, so we wrap its class loader.
                if (this.localDistributedCacheManager.HasLocalClasspaths())
                {
                    this.SetContextClassLoader(this.localDistributedCacheManager.MakeClassLoader(this
                                                                                                 .GetContextClassLoader()));
                }
                this.profile = new JobProfile(this.job.GetUser(), this.id, this.systemJobFile.ToString
                                                  (), "http://localhost:8080/", this.job.GetJobName());
                this.status = new JobStatus(this.id, 0.0f, 0.0f, JobStatus.Running, this.profile.
                                            GetUser(), this.profile.GetJobName(), this.profile.GetJobFile(), this.profile.GetURL
                                                ().ToString());
                this._enclosing.jobs[this.id] = this;
                this.Start();
            }
예제 #2
0
 public override void Run()
 {
     try
     {
         TaskAttemptID reduceId = new TaskAttemptID(new TaskID(this.jobId, TaskType.Reduce
                                                               , this.taskId), 0);
         LocalJobRunner.Log.Info("Starting task: " + reduceId);
         ReduceTask reduce = new ReduceTask(this._enclosing.systemJobFile.ToString(), reduceId
                                            , this.taskId, this._enclosing.mapIds.Count, 1);
         reduce.SetUser(UserGroupInformation.GetCurrentUser().GetShortUserName());
         LocalJobRunner.SetupChildMapredLocalDirs(reduce, this.localConf);
         reduce.SetLocalMapFiles(this.mapOutputFiles);
         if (!this._enclosing.IsInterrupted())
         {
             reduce.SetJobFile(this._enclosing.localJobFile.ToString());
             this.localConf.SetUser(reduce.GetUser());
             reduce.LocalizeConfiguration(this.localConf);
             reduce.SetConf(this.localConf);
             try
             {
                 this._enclosing._enclosing.reduce_tasks.GetAndIncrement();
                 this._enclosing._enclosing.myMetrics.LaunchReduce(reduce.GetTaskID());
                 reduce.Run(this.localConf, this._enclosing);
                 this._enclosing._enclosing.myMetrics.CompleteReduce(reduce.GetTaskID());
             }
             finally
             {
                 this._enclosing._enclosing.reduce_tasks.GetAndDecrement();
             }
             LocalJobRunner.Log.Info("Finishing task: " + reduceId);
         }
         else
         {
             throw new Exception();
         }
     }
     catch (Exception t)
     {
         // store this to be rethrown in the initial thread context.
         this.storedException = t;
     }
 }
예제 #3
0
 public override void Run()
 {
     try
     {
         TaskAttemptID mapId = new TaskAttemptID(new TaskID(this.jobId, TaskType.Map, this
                                                            .taskId), 0);
         LocalJobRunner.Log.Info("Starting task: " + mapId);
         this._enclosing.mapIds.AddItem(mapId);
         MapTask map = new MapTask(this._enclosing.systemJobFile.ToString(), mapId, this.taskId
                                   , this.info.GetSplitIndex(), 1);
         map.SetUser(UserGroupInformation.GetCurrentUser().GetShortUserName());
         LocalJobRunner.SetupChildMapredLocalDirs(map, this.localConf);
         MapOutputFile mapOutput = new MROutputFiles();
         mapOutput.SetConf(this.localConf);
         this.mapOutputFiles[mapId] = mapOutput;
         map.SetJobFile(this._enclosing.localJobFile.ToString());
         this.localConf.SetUser(map.GetUser());
         map.LocalizeConfiguration(this.localConf);
         map.SetConf(this.localConf);
         try
         {
             this._enclosing._enclosing.map_tasks.GetAndIncrement();
             this._enclosing._enclosing.myMetrics.LaunchMap(mapId);
             map.Run(this.localConf, this._enclosing);
             this._enclosing._enclosing.myMetrics.CompleteMap(mapId);
         }
         finally
         {
             this._enclosing._enclosing.map_tasks.GetAndDecrement();
         }
         LocalJobRunner.Log.Info("Finishing task: " + mapId);
     }
     catch (Exception e)
     {
         this.storedException = e;
     }
 }