/* Verify some expected values based on the history file */ /// <exception cref="System.Exception"/> public virtual void TestCompletedJob() { HistoryFileManager.HistoryFileInfo info = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo >(); Org.Mockito.Mockito.When(info.GetConfFile()).ThenReturn(fullConfPath); //Re-initialize to verify the delayed load. completedJob = new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user", info, jobAclsManager); //Verify tasks loaded based on loadTask parameter. NUnit.Framework.Assert.AreEqual(loadTasks, completedJob.tasksLoaded.Get()); NUnit.Framework.Assert.AreEqual(1, completedJob.GetAMInfos().Count); NUnit.Framework.Assert.AreEqual(10, completedJob.GetCompletedMaps()); NUnit.Framework.Assert.AreEqual(1, completedJob.GetCompletedReduces()); NUnit.Framework.Assert.AreEqual(12, completedJob.GetTasks().Count); //Verify tasks loaded at this point. NUnit.Framework.Assert.AreEqual(true, completedJob.tasksLoaded.Get()); NUnit.Framework.Assert.AreEqual(10, completedJob.GetTasks(TaskType.Map).Count); NUnit.Framework.Assert.AreEqual(2, completedJob.GetTasks(TaskType.Reduce).Count); NUnit.Framework.Assert.AreEqual("user", completedJob.GetUserName()); NUnit.Framework.Assert.AreEqual(JobState.Succeeded, completedJob.GetState()); JobReport jobReport = completedJob.GetReport(); NUnit.Framework.Assert.AreEqual("user", jobReport.GetUser()); NUnit.Framework.Assert.AreEqual(JobState.Succeeded, jobReport.GetJobState()); }
/// <exception cref="System.IO.IOException"/> public virtual JobStatus GetJobStatus(JobID oldJobID) { JobId jobId = TypeConverter.ToYarn(oldJobID); GetJobReportRequest request = recordFactory.NewRecordInstance <GetJobReportRequest >(); request.SetJobId(jobId); JobReport report = ((GetJobReportResponse)Invoke("getJobReport", typeof(GetJobReportRequest ), request)).GetJobReport(); JobStatus jobStatus = null; if (report != null) { if (StringUtils.IsEmpty(report.GetJobFile())) { string jobFile = MRApps.GetJobFile(conf, report.GetUser(), oldJobID); report.SetJobFile(jobFile); } string historyTrackingUrl = report.GetTrackingUrl(); string url = StringUtils.IsNotEmpty(historyTrackingUrl) ? historyTrackingUrl : trackingUrl; jobStatus = TypeConverter.FromYarn(report, url); } return(jobStatus); }
public static JobStatus FromYarn(JobReport jobreport, string trackingUrl) { JobPriority jobPriority = JobPriority.Normal; JobStatus jobStatus = new JobStatus(FromYarn(jobreport.GetJobId()), jobreport.GetSetupProgress (), jobreport.GetMapProgress(), jobreport.GetReduceProgress(), jobreport.GetCleanupProgress (), FromYarn(jobreport.GetJobState()), jobPriority, jobreport.GetUser(), jobreport .GetJobName(), jobreport.GetJobFile(), trackingUrl, jobreport.IsUber()); jobStatus.SetStartTime(jobreport.GetStartTime()); jobStatus.SetFinishTime(jobreport.GetFinishTime()); jobStatus.SetFailureInfo(jobreport.GetDiagnostics()); return(jobStatus); }
/// <exception cref="System.IO.IOException"/> public virtual LogParams GetLogFilePath(JobID oldJobID, TaskAttemptID oldTaskAttemptID ) { JobId jobId = TypeConverter.ToYarn(oldJobID); GetJobReportRequest request = recordFactory.NewRecordInstance <GetJobReportRequest >(); request.SetJobId(jobId); JobReport report = ((GetJobReportResponse)Invoke("getJobReport", typeof(GetJobReportRequest ), request)).GetJobReport(); if (EnumSet.Of(JobState.Succeeded, JobState.Failed, JobState.Killed, JobState.Error ).Contains(report.GetJobState())) { if (oldTaskAttemptID != null) { GetTaskAttemptReportRequest taRequest = recordFactory.NewRecordInstance <GetTaskAttemptReportRequest >(); taRequest.SetTaskAttemptId(TypeConverter.ToYarn(oldTaskAttemptID)); TaskAttemptReport taReport = ((GetTaskAttemptReportResponse)Invoke("getTaskAttemptReport" , typeof(GetTaskAttemptReportRequest), taRequest)).GetTaskAttemptReport(); if (taReport.GetContainerId() == null || taReport.GetNodeManagerHost() == null) { throw new IOException("Unable to get log information for task: " + oldTaskAttemptID ); } return(new LogParams(taReport.GetContainerId().ToString(), taReport.GetContainerId ().GetApplicationAttemptId().GetApplicationId().ToString(), NodeId.NewInstance(taReport .GetNodeManagerHost(), taReport.GetNodeManagerPort()).ToString(), report.GetUser ())); } else { if (report.GetAMInfos() == null || report.GetAMInfos().Count == 0) { throw new IOException("Unable to get log information for job: " + oldJobID); } AMInfo amInfo = report.GetAMInfos()[report.GetAMInfos().Count - 1]; return(new LogParams(amInfo.GetContainerId().ToString(), amInfo.GetAppAttemptId() .GetApplicationId().ToString(), NodeId.NewInstance(amInfo.GetNodeManagerHost(), amInfo.GetNodeManagerPort()).ToString(), report.GetUser())); } } else { throw new IOException("Cannot get log path for a in-progress job"); } }