/* Verify some expected values based on the history file */ /// <exception cref="System.Exception"/> public virtual void TestCompletedJob() { HistoryFileManager.HistoryFileInfo info = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo >(); Org.Mockito.Mockito.When(info.GetConfFile()).ThenReturn(fullConfPath); //Re-initialize to verify the delayed load. completedJob = new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user", info, jobAclsManager); //Verify tasks loaded based on loadTask parameter. NUnit.Framework.Assert.AreEqual(loadTasks, completedJob.tasksLoaded.Get()); NUnit.Framework.Assert.AreEqual(1, completedJob.GetAMInfos().Count); NUnit.Framework.Assert.AreEqual(10, completedJob.GetCompletedMaps()); NUnit.Framework.Assert.AreEqual(1, completedJob.GetCompletedReduces()); NUnit.Framework.Assert.AreEqual(12, completedJob.GetTasks().Count); //Verify tasks loaded at this point. NUnit.Framework.Assert.AreEqual(true, completedJob.tasksLoaded.Get()); NUnit.Framework.Assert.AreEqual(10, completedJob.GetTasks(TaskType.Map).Count); NUnit.Framework.Assert.AreEqual(2, completedJob.GetTasks(TaskType.Reduce).Count); NUnit.Framework.Assert.AreEqual("user", completedJob.GetUserName()); NUnit.Framework.Assert.AreEqual(JobState.Succeeded, completedJob.GetState()); JobReport jobReport = completedJob.GetReport(); NUnit.Framework.Assert.AreEqual("user", jobReport.GetUser()); NUnit.Framework.Assert.AreEqual(JobState.Succeeded, jobReport.GetJobState()); }
/// <exception cref="System.Exception"/> public virtual void WaitForState(Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job, JobState finalState) { int timeoutSecs = 0; JobReport report = job.GetReport(); while (!finalState.Equals(report.GetJobState()) && timeoutSecs++ < 20) { System.Console.Out.WriteLine("Job State is : " + report.GetJobState() + " Waiting for state : " + finalState + " map progress : " + report.GetMapProgress() + " reduce progress : " + report.GetReduceProgress()); report = job.GetReport(); Sharpen.Thread.Sleep(500); } System.Console.Out.WriteLine("Job State is : " + report.GetJobState()); NUnit.Framework.Assert.AreEqual("Job state is not correct (timedout)", finalState , job.GetState()); }
public static JobStatus FromYarn(JobReport jobreport, string trackingUrl) { JobPriority jobPriority = JobPriority.Normal; JobStatus jobStatus = new JobStatus(FromYarn(jobreport.GetJobId()), jobreport.GetSetupProgress (), jobreport.GetMapProgress(), jobreport.GetReduceProgress(), jobreport.GetCleanupProgress (), FromYarn(jobreport.GetJobState()), jobPriority, jobreport.GetUser(), jobreport .GetJobName(), jobreport.GetJobFile(), trackingUrl, jobreport.IsUber()); jobStatus.SetStartTime(jobreport.GetStartTime()); jobStatus.SetFinishTime(jobreport.GetFinishTime()); jobStatus.SetFailureInfo(jobreport.GetDiagnostics()); return(jobStatus); }
/// <summary>Notify a server of the completion of a submitted job.</summary> /// <remarks> /// Notify a server of the completion of a submitted job. The user must have /// configured MRJobConfig.MR_JOB_END_NOTIFICATION_URL /// </remarks> /// <param name="jobReport">JobReport used to read JobId and JobStatus</param> /// <exception cref="System.Exception"/> public virtual void Notify(JobReport jobReport) { // Do we need job-end notification? if (userUrl == null) { Org.Mortbay.Log.Log.Info("Job end notification URL not set, skipping."); return; } //Do string replacements for jobId and jobStatus if (userUrl.Contains(JobId)) { userUrl = userUrl.Replace(JobId, jobReport.GetJobId().ToString()); } if (userUrl.Contains(JobStatus)) { userUrl = userUrl.Replace(JobStatus, jobReport.GetJobState().ToString()); } // Create the URL, ensure sanity try { urlToNotify = new Uri(userUrl); } catch (UriFormatException mue) { Org.Mortbay.Log.Log.Warn("Job end notification couldn't parse " + userUrl, mue); return; } // Send notification bool success = false; while (numTries-- > 0 && !success) { Org.Mortbay.Log.Log.Info("Job end notification attempts left " + numTries); success = NotifyURLOnce(); if (!success) { Sharpen.Thread.Sleep(waitInterval); } } if (!success) { Org.Mortbay.Log.Log.Warn("Job end notification failed to notify : " + urlToNotify ); } else { Org.Mortbay.Log.Log.Info("Job end notification succeeded for " + jobReport.GetJobId ()); } }
/// <exception cref="System.IO.IOException"/> public virtual LogParams GetLogFilePath(JobID oldJobID, TaskAttemptID oldTaskAttemptID ) { JobId jobId = TypeConverter.ToYarn(oldJobID); GetJobReportRequest request = recordFactory.NewRecordInstance <GetJobReportRequest >(); request.SetJobId(jobId); JobReport report = ((GetJobReportResponse)Invoke("getJobReport", typeof(GetJobReportRequest ), request)).GetJobReport(); if (EnumSet.Of(JobState.Succeeded, JobState.Failed, JobState.Killed, JobState.Error ).Contains(report.GetJobState())) { if (oldTaskAttemptID != null) { GetTaskAttemptReportRequest taRequest = recordFactory.NewRecordInstance <GetTaskAttemptReportRequest >(); taRequest.SetTaskAttemptId(TypeConverter.ToYarn(oldTaskAttemptID)); TaskAttemptReport taReport = ((GetTaskAttemptReportResponse)Invoke("getTaskAttemptReport" , typeof(GetTaskAttemptReportRequest), taRequest)).GetTaskAttemptReport(); if (taReport.GetContainerId() == null || taReport.GetNodeManagerHost() == null) { throw new IOException("Unable to get log information for task: " + oldTaskAttemptID ); } return(new LogParams(taReport.GetContainerId().ToString(), taReport.GetContainerId ().GetApplicationAttemptId().GetApplicationId().ToString(), NodeId.NewInstance(taReport .GetNodeManagerHost(), taReport.GetNodeManagerPort()).ToString(), report.GetUser ())); } else { if (report.GetAMInfos() == null || report.GetAMInfos().Count == 0) { throw new IOException("Unable to get log information for job: " + oldJobID); } AMInfo amInfo = report.GetAMInfos()[report.GetAMInfos().Count - 1]; return(new LogParams(amInfo.GetContainerId().ToString(), amInfo.GetAppAttemptId() .GetApplicationId().ToString(), NodeId.NewInstance(amInfo.GetNodeManagerHost(), amInfo.GetNodeManagerPort()).ToString(), report.GetUser())); } } else { throw new IOException("Cannot get log path for a in-progress job"); } }
// pass private void VerifyJobReport(JobReport jr) { NUnit.Framework.Assert.IsNotNull("JobReport is null", jr); IList <AMInfo> amInfos = jr.GetAMInfos(); NUnit.Framework.Assert.AreEqual(1, amInfos.Count); NUnit.Framework.Assert.AreEqual(JobState.Running, jr.GetJobState()); AMInfo amInfo = amInfos[0]; NUnit.Framework.Assert.AreEqual(MRApp.NmHost, amInfo.GetNodeManagerHost()); NUnit.Framework.Assert.AreEqual(MRApp.NmPort, amInfo.GetNodeManagerPort()); NUnit.Framework.Assert.AreEqual(MRApp.NmHttpPort, amInfo.GetNodeManagerHttpPort() ); NUnit.Framework.Assert.AreEqual(1, amInfo.GetAppAttemptId().GetAttemptId()); NUnit.Framework.Assert.AreEqual(1, amInfo.GetContainerId().GetApplicationAttemptId ().GetAttemptId()); NUnit.Framework.Assert.IsTrue(amInfo.GetStartTime() > 0); NUnit.Framework.Assert.AreEqual(false, jr.IsUber()); }
public static JobsInfo GetPartialJobs(ICollection <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job > jobs, long offset, long count, string user, string queue, long sBegin, long sEnd , long fBegin, long fEnd, JobState jobState) { JobsInfo allJobs = new JobsInfo(); if (sBegin == null || sBegin < 0) { sBegin = 0l; } if (sEnd == null) { sEnd = long.MaxValue; } if (fBegin == null || fBegin < 0) { fBegin = 0l; } if (fEnd == null) { fEnd = long.MaxValue; } if (offset == null || offset < 0) { offset = 0l; } if (count == null) { count = long.MaxValue; } if (offset > jobs.Count) { return(allJobs); } long at = 0; long end = offset + count - 1; if (end < 0) { // due to overflow end = long.MaxValue; } foreach (Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job in jobs) { if (at > end) { break; } // can't really validate queue is a valid one since queues could change if (queue != null && !queue.IsEmpty()) { if (!job.GetQueueName().Equals(queue)) { continue; } } if (user != null && !user.IsEmpty()) { if (!job.GetUserName().Equals(user)) { continue; } } JobReport report = job.GetReport(); if (report.GetStartTime() < sBegin || report.GetStartTime() > sEnd) { continue; } if (report.GetFinishTime() < fBegin || report.GetFinishTime() > fEnd) { continue; } if (jobState != null && jobState != report.GetJobState()) { continue; } at++; if ((at - 1) < offset) { continue; } JobInfo jobInfo = new JobInfo(job); allJobs.Add(jobInfo); } return(allJobs); }