/// <exception cref="System.IO.IOException"/>
private static MockHistoryJobs.JobsPair Split(IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
> mocked)
{
MockHistoryJobs.JobsPair ret = new MockHistoryJobs.JobsPair();
ret.full = Maps.NewHashMap();
ret.partial = Maps.NewHashMap();
foreach (KeyValuePair <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> entry in
mocked)
{
JobId id = entry.Key;
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job j = entry.Value;
MockHistoryJobs.MockCompletedJob mockJob = new MockHistoryJobs.MockCompletedJob(j
);
// use MockCompletedJob to set everything below to make sure
// consistent with what history server would do
ret.full[id] = mockJob;
JobReport report = mockJob.GetReport();
JobIndexInfo info = new JobIndexInfo(report.GetStartTime(), report.GetFinishTime(
), mockJob.GetUserName(), mockJob.GetName(), id, mockJob.GetCompletedMaps(), mockJob
.GetCompletedReduces(), mockJob.GetState().ToString());
info.SetJobStartTime(report.GetStartTime());
info.SetQueueName(mockJob.GetQueueName());
ret.partial[id] = new PartialJob(info, id);
}
return(ret);
}
// acls not being checked since
// we are using mock job instead of CompletedJob
public static void VerifyHsJobGeneric(Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
job, string id, string user, string name, string state, string queue, long startTime
, long finishTime, int mapsTotal, int mapsCompleted, int reducesTotal, int reducesCompleted
)
{
JobReport report = job.GetReport();
WebServicesTestUtils.CheckStringMatch("id", MRApps.ToString(job.GetID()), id);
WebServicesTestUtils.CheckStringMatch("user", job.GetUserName().ToString(), user);
WebServicesTestUtils.CheckStringMatch("name", job.GetName(), name);
WebServicesTestUtils.CheckStringMatch("state", job.GetState().ToString(), state);
WebServicesTestUtils.CheckStringMatch("queue", job.GetQueueName(), queue);
NUnit.Framework.Assert.AreEqual("startTime incorrect", report.GetStartTime(), startTime
);
NUnit.Framework.Assert.AreEqual("finishTime incorrect", report.GetFinishTime(), finishTime
);
NUnit.Framework.Assert.AreEqual("mapsTotal incorrect", job.GetTotalMaps(), mapsTotal
);
NUnit.Framework.Assert.AreEqual("mapsCompleted incorrect", job.GetCompletedMaps()
, mapsCompleted);
NUnit.Framework.Assert.AreEqual("reducesTotal incorrect", job.GetTotalReduces(),
reducesTotal);
NUnit.Framework.Assert.AreEqual("reducesCompleted incorrect", job.GetCompletedReduces
(), reducesCompleted);
}
public virtual void VerifyCompleted()
{
foreach (Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job in GetContext().GetAllJobs
().Values)
{
JobReport jobReport = job.GetReport();
System.Console.Out.WriteLine("Job start time :" + jobReport.GetStartTime());
System.Console.Out.WriteLine("Job finish time :" + jobReport.GetFinishTime());
NUnit.Framework.Assert.IsTrue("Job start time is not less than finish time", jobReport
.GetStartTime() <= jobReport.GetFinishTime());
NUnit.Framework.Assert.IsTrue("Job finish time is in future", jobReport.GetFinishTime
() <= Runtime.CurrentTimeMillis());
foreach (Task task in job.GetTasks().Values)
{
TaskReport taskReport = task.GetReport();
System.Console.Out.WriteLine("Task start time : " + taskReport.GetStartTime());
System.Console.Out.WriteLine("Task finish time : " + taskReport.GetFinishTime());
NUnit.Framework.Assert.IsTrue("Task start time is not less than finish time", taskReport
.GetStartTime() <= taskReport.GetFinishTime());
foreach (TaskAttempt attempt in task.GetAttempts().Values)
{
TaskAttemptReport attemptReport = attempt.GetReport();
NUnit.Framework.Assert.IsTrue("Attempt start time is not less than finish time",
attemptReport.GetStartTime() <= attemptReport.GetFinishTime());
}
}
}
}
private Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job GetJob()
{
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
>();
JobId jobId = new JobIdPBImpl();
ApplicationId appId = ApplicationIdPBImpl.NewInstance(Runtime.CurrentTimeMillis()
, 4);
jobId.SetAppId(appId);
jobId.SetId(1);
Org.Mockito.Mockito.When(job.GetID()).ThenReturn(jobId);
JobReport report = Org.Mockito.Mockito.Mock <JobReport>();
Org.Mockito.Mockito.When(report.GetStartTime()).ThenReturn(100010L);
Org.Mockito.Mockito.When(report.GetFinishTime()).ThenReturn(100015L);
Org.Mockito.Mockito.When(job.GetReport()).ThenReturn(report);
Org.Mockito.Mockito.When(job.GetName()).ThenReturn("JobName");
Org.Mockito.Mockito.When(job.GetUserName()).ThenReturn("UserName");
Org.Mockito.Mockito.When(job.GetQueueName()).ThenReturn("QueueName");
Org.Mockito.Mockito.When(job.GetState()).ThenReturn(JobState.Succeeded);
Org.Mockito.Mockito.When(job.GetTotalMaps()).ThenReturn(3);
Org.Mockito.Mockito.When(job.GetCompletedMaps()).ThenReturn(2);
Org.Mockito.Mockito.When(job.GetTotalReduces()).ThenReturn(2);
Org.Mockito.Mockito.When(job.GetCompletedReduces()).ThenReturn(1);
Org.Mockito.Mockito.When(job.GetCompletedReduces()).ThenReturn(1);
return(job);
}
private void VerifyJobReport(JobReport jobReport, JobId jobId)
{
IList <AMInfo> amInfos = jobReport.GetAMInfos();
NUnit.Framework.Assert.AreEqual(1, amInfos.Count);
AMInfo amInfo = amInfos[0];
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(jobId.GetAppId
(), 1);
ContainerId amContainerId = ContainerId.NewContainerId(appAttemptId, 1);
NUnit.Framework.Assert.AreEqual(appAttemptId, amInfo.GetAppAttemptId());
NUnit.Framework.Assert.AreEqual(amContainerId, amInfo.GetContainerId());
NUnit.Framework.Assert.IsTrue(jobReport.GetSubmitTime() > 0);
NUnit.Framework.Assert.IsTrue(jobReport.GetStartTime() > 0 && jobReport.GetStartTime
() >= jobReport.GetSubmitTime());
NUnit.Framework.Assert.IsTrue(jobReport.GetFinishTime() > 0 && jobReport.GetFinishTime
() >= jobReport.GetStartTime());
}
public JobInfo(Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job)
{
this.id = MRApps.ToString(job.GetID());
JobReport report = job.GetReport();
this.mapsTotal = job.GetTotalMaps();
this.mapsCompleted = job.GetCompletedMaps();
this.reducesTotal = job.GetTotalReduces();
this.reducesCompleted = job.GetCompletedReduces();
this.submitTime = report.GetSubmitTime();
this.startTime = report.GetStartTime();
this.finishTime = report.GetFinishTime();
this.name = job.GetName().ToString();
this.queue = job.GetQueueName();
this.user = job.GetUserName();
this.state = job.GetState().ToString();
this.acls = new AList <ConfEntryInfo>();
if (job is CompletedJob)
{
avgMapTime = 0l;
avgReduceTime = 0l;
avgShuffleTime = 0l;
avgMergeTime = 0l;
failedReduceAttempts = 0;
killedReduceAttempts = 0;
successfulReduceAttempts = 0;
failedMapAttempts = 0;
killedMapAttempts = 0;
successfulMapAttempts = 0;
CountTasksAndAttempts(job);
this.uberized = job.IsUber();
this.diagnostics = string.Empty;
IList <string> diagnostics = job.GetDiagnostics();
if (diagnostics != null && !diagnostics.IsEmpty())
{
StringBuilder b = new StringBuilder();
foreach (string diag in diagnostics)
{
b.Append(diag);
}
this.diagnostics = b.ToString();
}
IDictionary <JobACL, AccessControlList> allacls = job.GetJobACLs();
if (allacls != null)
{
foreach (KeyValuePair <JobACL, AccessControlList> entry in allacls)
{
this.acls.AddItem(new ConfEntryInfo(entry.Key.GetAclName(), entry.Value.GetAclString
()));
}
}
}
}
public JobInfo(Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job, bool hasAccess)
{
// ok for any user to see
// these should only be seen if acls allow
this.id = MRApps.ToString(job.GetID());
JobReport report = job.GetReport();
this.startTime = report.GetStartTime();
this.finishTime = report.GetFinishTime();
this.elapsedTime = Times.Elapsed(this.startTime, this.finishTime);
if (this.elapsedTime == -1)
{
this.elapsedTime = 0;
}
this.name = job.GetName().ToString();
this.user = job.GetUserName();
this.state = job.GetState();
this.mapsTotal = job.GetTotalMaps();
this.mapsCompleted = job.GetCompletedMaps();
this.mapProgress = report.GetMapProgress() * 100;
this.mapProgressPercent = StringHelper.Percent(report.GetMapProgress());
this.reducesTotal = job.GetTotalReduces();
this.reducesCompleted = job.GetCompletedReduces();
this.reduceProgress = report.GetReduceProgress() * 100;
this.reduceProgressPercent = StringHelper.Percent(report.GetReduceProgress());
this.acls = new AList <ConfEntryInfo>();
if (hasAccess)
{
this.diagnostics = string.Empty;
CountTasksAndAttempts(job);
this.uberized = job.IsUber();
IList <string> diagnostics = job.GetDiagnostics();
if (diagnostics != null && !diagnostics.IsEmpty())
{
StringBuilder b = new StringBuilder();
foreach (string diag in diagnostics)
{
b.Append(diag);
}
this.diagnostics = b.ToString();
}
IDictionary <JobACL, AccessControlList> allacls = job.GetJobACLs();
if (allacls != null)
{
foreach (KeyValuePair <JobACL, AccessControlList> entry in allacls)
{
this.acls.AddItem(new ConfEntryInfo(entry.Key.GetAclName(), entry.Value.GetAclString
()));
}
}
}
}
public static JobStatus FromYarn(JobReport jobreport, string trackingUrl)
{
JobPriority jobPriority = JobPriority.Normal;
JobStatus jobStatus = new JobStatus(FromYarn(jobreport.GetJobId()), jobreport.GetSetupProgress
(), jobreport.GetMapProgress(), jobreport.GetReduceProgress(), jobreport.GetCleanupProgress
(), FromYarn(jobreport.GetJobState()), jobPriority, jobreport.GetUser(), jobreport
.GetJobName(), jobreport.GetJobFile(), trackingUrl, jobreport.IsUber());
jobStatus.SetStartTime(jobreport.GetStartTime());
jobStatus.SetFinishTime(jobreport.GetFinishTime());
jobStatus.SetFailureInfo(jobreport.GetDiagnostics());
return(jobStatus);
}
public static JobsInfo GetPartialJobs(ICollection <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
> jobs, long offset, long count, string user, string queue, long sBegin, long sEnd
, long fBegin, long fEnd, JobState jobState)
{
JobsInfo allJobs = new JobsInfo();
if (sBegin == null || sBegin < 0)
{
sBegin = 0l;
}
if (sEnd == null)
{
sEnd = long.MaxValue;
}
if (fBegin == null || fBegin < 0)
{
fBegin = 0l;
}
if (fEnd == null)
{
fEnd = long.MaxValue;
}
if (offset == null || offset < 0)
{
offset = 0l;
}
if (count == null)
{
count = long.MaxValue;
}
if (offset > jobs.Count)
{
return(allJobs);
}
long at = 0;
long end = offset + count - 1;
if (end < 0)
{
// due to overflow
end = long.MaxValue;
}
foreach (Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job in jobs)
{
if (at > end)
{
break;
}
// can't really validate queue is a valid one since queues could change
if (queue != null && !queue.IsEmpty())
{
if (!job.GetQueueName().Equals(queue))
{
continue;
}
}
if (user != null && !user.IsEmpty())
{
if (!job.GetUserName().Equals(user))
{
continue;
}
}
JobReport report = job.GetReport();
if (report.GetStartTime() < sBegin || report.GetStartTime() > sEnd)
{
continue;
}
if (report.GetFinishTime() < fBegin || report.GetFinishTime() > fEnd)
{
continue;
}
if (jobState != null && jobState != report.GetJobState())
{
continue;
}
at++;
if ((at - 1) < offset)
{
continue;
}
JobInfo jobInfo = new JobInfo(job);
allJobs.Add(jobInfo);
}
return(allJobs);
}
