/// <exception cref="System.Exception"/> public virtual Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job Submit(Configuration conf , bool mapSpeculative, bool reduceSpeculative) { string user = conf.Get(MRJobConfig.UserName, UserGroupInformation.GetCurrentUser( ).GetShortUserName()); conf.Set(MRJobConfig.UserName, user); conf.Set(MRJobConfig.MrAmStagingDir, testAbsPath.ToString()); conf.SetBoolean(MRJobConfig.MrAmCreateJhIntermediateBaseDir, true); // TODO: fix the bug where the speculator gets events with // not-fully-constructed objects. For now, disable speculative exec conf.SetBoolean(MRJobConfig.MapSpeculative, mapSpeculative); conf.SetBoolean(MRJobConfig.ReduceSpeculative, reduceSpeculative); Init(conf); Start(); DefaultMetricsSystem.Shutdown(); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = GetContext().GetAllJobs().Values .GetEnumerator().Next(); if (assignedQueue != null) { job.SetQueueName(assignedQueue); } // Write job.xml string jobFile = MRApps.GetJobFile(conf, user, TypeConverter.FromYarn(job.GetID() )); Log.Info("Writing job conf to " + jobFile); new FilePath(jobFile).GetParentFile().Mkdirs(); conf.WriteXml(new FileOutputStream(jobFile)); return(job); }
public static JobStatus[] FromYarnApps(IList <ApplicationReport> applications, Configuration conf) { IList <JobStatus> jobStatuses = new AList <JobStatus>(); foreach (ApplicationReport application in applications) { // each applicationReport has its own jobFile JobID jobId = TypeConverter.FromYarn(application.GetApplicationId()); jobStatuses.AddItem(TypeConverter.FromYarn(application, MRApps.GetJobFile(conf, application .GetUser(), jobId))); } return(Sharpen.Collections.ToArray(jobStatuses, new JobStatus[jobStatuses.Count])); }
/// <exception cref="System.IO.IOException"/> public virtual JobStatus GetJobStatus(JobID oldJobID) { JobId jobId = TypeConverter.ToYarn(oldJobID); GetJobReportRequest request = recordFactory.NewRecordInstance <GetJobReportRequest >(); request.SetJobId(jobId); JobReport report = ((GetJobReportResponse)Invoke("getJobReport", typeof(GetJobReportRequest ), request)).GetJobReport(); JobStatus jobStatus = null; if (report != null) { if (StringUtils.IsEmpty(report.GetJobFile())) { string jobFile = MRApps.GetJobFile(conf, report.GetUser(), oldJobID); report.SetJobFile(jobFile); } string historyTrackingUrl = report.GetTrackingUrl(); string url = StringUtils.IsNotEmpty(historyTrackingUrl) ? historyTrackingUrl : trackingUrl; jobStatus = TypeConverter.FromYarn(report, url); } return(jobStatus); }