/* Verify some expected values based on the history file */
/// <exception cref="System.Exception"/>
public virtual void TestCompletedJob()
{
HistoryFileManager.HistoryFileInfo info = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo
>();
Org.Mockito.Mockito.When(info.GetConfFile()).ThenReturn(fullConfPath);
//Re-initialize to verify the delayed load.
completedJob = new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user",
info, jobAclsManager);
//Verify tasks loaded based on loadTask parameter.
NUnit.Framework.Assert.AreEqual(loadTasks, completedJob.tasksLoaded.Get());
NUnit.Framework.Assert.AreEqual(1, completedJob.GetAMInfos().Count);
NUnit.Framework.Assert.AreEqual(10, completedJob.GetCompletedMaps());
NUnit.Framework.Assert.AreEqual(1, completedJob.GetCompletedReduces());
NUnit.Framework.Assert.AreEqual(12, completedJob.GetTasks().Count);
//Verify tasks loaded at this point.
NUnit.Framework.Assert.AreEqual(true, completedJob.tasksLoaded.Get());
NUnit.Framework.Assert.AreEqual(10, completedJob.GetTasks(TaskType.Map).Count);
NUnit.Framework.Assert.AreEqual(2, completedJob.GetTasks(TaskType.Reduce).Count);
NUnit.Framework.Assert.AreEqual("user", completedJob.GetUserName());
NUnit.Framework.Assert.AreEqual(JobState.Succeeded, completedJob.GetState());
JobReport jobReport = completedJob.GetReport();
NUnit.Framework.Assert.AreEqual("user", jobReport.GetUser());
NUnit.Framework.Assert.AreEqual(JobState.Succeeded, jobReport.GetJobState());
}
/// <exception cref="System.Exception"/>
public virtual void WaitForState(Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job,
JobState finalState)
{
int timeoutSecs = 0;
JobReport report = job.GetReport();
while (!finalState.Equals(report.GetJobState()) && timeoutSecs++ < 20)
{
System.Console.Out.WriteLine("Job State is : " + report.GetJobState() + " Waiting for state : "
+ finalState + " map progress : " + report.GetMapProgress() + " reduce progress : "
+ report.GetReduceProgress());
report = job.GetReport();
Sharpen.Thread.Sleep(500);
}
System.Console.Out.WriteLine("Job State is : " + report.GetJobState());
NUnit.Framework.Assert.AreEqual("Job state is not correct (timedout)", finalState
, job.GetState());
}
public static JobStatus FromYarn(JobReport jobreport, string trackingUrl)
{
JobPriority jobPriority = JobPriority.Normal;
JobStatus jobStatus = new JobStatus(FromYarn(jobreport.GetJobId()), jobreport.GetSetupProgress
(), jobreport.GetMapProgress(), jobreport.GetReduceProgress(), jobreport.GetCleanupProgress
(), FromYarn(jobreport.GetJobState()), jobPriority, jobreport.GetUser(), jobreport
.GetJobName(), jobreport.GetJobFile(), trackingUrl, jobreport.IsUber());
jobStatus.SetStartTime(jobreport.GetStartTime());
jobStatus.SetFinishTime(jobreport.GetFinishTime());
jobStatus.SetFailureInfo(jobreport.GetDiagnostics());
return(jobStatus);
}
/// <summary>Notify a server of the completion of a submitted job.</summary>
/// <remarks>
/// Notify a server of the completion of a submitted job. The user must have
/// configured MRJobConfig.MR_JOB_END_NOTIFICATION_URL
/// </remarks>
/// <param name="jobReport">JobReport used to read JobId and JobStatus</param>
/// <exception cref="System.Exception"/>
public virtual void Notify(JobReport jobReport)
{
// Do we need job-end notification?
if (userUrl == null)
{
Org.Mortbay.Log.Log.Info("Job end notification URL not set, skipping.");
return;
}
//Do string replacements for jobId and jobStatus
if (userUrl.Contains(JobId))
{
userUrl = userUrl.Replace(JobId, jobReport.GetJobId().ToString());
}
if (userUrl.Contains(JobStatus))
{
userUrl = userUrl.Replace(JobStatus, jobReport.GetJobState().ToString());
}
// Create the URL, ensure sanity
try
{
urlToNotify = new Uri(userUrl);
}
catch (UriFormatException mue)
{
Org.Mortbay.Log.Log.Warn("Job end notification couldn't parse " + userUrl, mue);
return;
}
// Send notification
bool success = false;
while (numTries-- > 0 && !success)
{
Org.Mortbay.Log.Log.Info("Job end notification attempts left " + numTries);
success = NotifyURLOnce();
if (!success)
{
Sharpen.Thread.Sleep(waitInterval);
}
}
if (!success)
{
Org.Mortbay.Log.Log.Warn("Job end notification failed to notify : " + urlToNotify
);
}
else
{
Org.Mortbay.Log.Log.Info("Job end notification succeeded for " + jobReport.GetJobId
());
}
}
/// <exception cref="System.IO.IOException"/>
public virtual LogParams GetLogFilePath(JobID oldJobID, TaskAttemptID oldTaskAttemptID
)
{
JobId jobId = TypeConverter.ToYarn(oldJobID);
GetJobReportRequest request = recordFactory.NewRecordInstance <GetJobReportRequest
>();
request.SetJobId(jobId);
JobReport report = ((GetJobReportResponse)Invoke("getJobReport", typeof(GetJobReportRequest
), request)).GetJobReport();
if (EnumSet.Of(JobState.Succeeded, JobState.Failed, JobState.Killed, JobState.Error
).Contains(report.GetJobState()))
{
if (oldTaskAttemptID != null)
{
GetTaskAttemptReportRequest taRequest = recordFactory.NewRecordInstance <GetTaskAttemptReportRequest
>();
taRequest.SetTaskAttemptId(TypeConverter.ToYarn(oldTaskAttemptID));
TaskAttemptReport taReport = ((GetTaskAttemptReportResponse)Invoke("getTaskAttemptReport"
, typeof(GetTaskAttemptReportRequest), taRequest)).GetTaskAttemptReport();
if (taReport.GetContainerId() == null || taReport.GetNodeManagerHost() == null)
{
throw new IOException("Unable to get log information for task: " + oldTaskAttemptID
);
}
return(new LogParams(taReport.GetContainerId().ToString(), taReport.GetContainerId
().GetApplicationAttemptId().GetApplicationId().ToString(), NodeId.NewInstance(taReport
.GetNodeManagerHost(), taReport.GetNodeManagerPort()).ToString(), report.GetUser
()));
}
else
{
if (report.GetAMInfos() == null || report.GetAMInfos().Count == 0)
{
throw new IOException("Unable to get log information for job: " + oldJobID);
}
AMInfo amInfo = report.GetAMInfos()[report.GetAMInfos().Count - 1];
return(new LogParams(amInfo.GetContainerId().ToString(), amInfo.GetAppAttemptId()
.GetApplicationId().ToString(), NodeId.NewInstance(amInfo.GetNodeManagerHost(),
amInfo.GetNodeManagerPort()).ToString(), report.GetUser()));
}
}
else
{
throw new IOException("Cannot get log path for a in-progress job");
}
}
// pass
private void VerifyJobReport(JobReport jr)
{
NUnit.Framework.Assert.IsNotNull("JobReport is null", jr);
IList <AMInfo> amInfos = jr.GetAMInfos();
NUnit.Framework.Assert.AreEqual(1, amInfos.Count);
NUnit.Framework.Assert.AreEqual(JobState.Running, jr.GetJobState());
AMInfo amInfo = amInfos[0];
NUnit.Framework.Assert.AreEqual(MRApp.NmHost, amInfo.GetNodeManagerHost());
NUnit.Framework.Assert.AreEqual(MRApp.NmPort, amInfo.GetNodeManagerPort());
NUnit.Framework.Assert.AreEqual(MRApp.NmHttpPort, amInfo.GetNodeManagerHttpPort()
);
NUnit.Framework.Assert.AreEqual(1, amInfo.GetAppAttemptId().GetAttemptId());
NUnit.Framework.Assert.AreEqual(1, amInfo.GetContainerId().GetApplicationAttemptId
().GetAttemptId());
NUnit.Framework.Assert.IsTrue(amInfo.GetStartTime() > 0);
NUnit.Framework.Assert.AreEqual(false, jr.IsUber());
}
public static JobsInfo GetPartialJobs(ICollection <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
> jobs, long offset, long count, string user, string queue, long sBegin, long sEnd
, long fBegin, long fEnd, JobState jobState)
{
JobsInfo allJobs = new JobsInfo();
if (sBegin == null || sBegin < 0)
{
sBegin = 0l;
}
if (sEnd == null)
{
sEnd = long.MaxValue;
}
if (fBegin == null || fBegin < 0)
{
fBegin = 0l;
}
if (fEnd == null)
{
fEnd = long.MaxValue;
}
if (offset == null || offset < 0)
{
offset = 0l;
}
if (count == null)
{
count = long.MaxValue;
}
if (offset > jobs.Count)
{
return(allJobs);
}
long at = 0;
long end = offset + count - 1;
if (end < 0)
{
// due to overflow
end = long.MaxValue;
}
foreach (Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job in jobs)
{
if (at > end)
{
break;
}
// can't really validate queue is a valid one since queues could change
if (queue != null && !queue.IsEmpty())
{
if (!job.GetQueueName().Equals(queue))
{
continue;
}
}
if (user != null && !user.IsEmpty())
{
if (!job.GetUserName().Equals(user))
{
continue;
}
}
JobReport report = job.GetReport();
if (report.GetStartTime() < sBegin || report.GetStartTime() > sEnd)
{
continue;
}
if (report.GetFinishTime() < fBegin || report.GetFinishTime() > fEnd)
{
continue;
}
if (jobState != null && jobState != report.GetJobState())
{
continue;
}
at++;
if ((at - 1) < offset)
{
continue;
}
JobInfo jobInfo = new JobInfo(job);
allJobs.Add(jobInfo);
}
return(allJobs);
}
