/// <exception cref="System.IO.IOException"/> private static MockHistoryJobs.JobsPair Split(IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job > mocked) { MockHistoryJobs.JobsPair ret = new MockHistoryJobs.JobsPair(); ret.full = Maps.NewHashMap(); ret.partial = Maps.NewHashMap(); foreach (KeyValuePair <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> entry in mocked) { JobId id = entry.Key; Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job j = entry.Value; MockHistoryJobs.MockCompletedJob mockJob = new MockHistoryJobs.MockCompletedJob(j ); // use MockCompletedJob to set everything below to make sure // consistent with what history server would do ret.full[id] = mockJob; JobReport report = mockJob.GetReport(); JobIndexInfo info = new JobIndexInfo(report.GetStartTime(), report.GetFinishTime( ), mockJob.GetUserName(), mockJob.GetName(), id, mockJob.GetCompletedMaps(), mockJob .GetCompletedReduces(), mockJob.GetState().ToString()); info.SetJobStartTime(report.GetStartTime()); info.SetQueueName(mockJob.GetQueueName()); ret.partial[id] = new PartialJob(info, id); } return(ret); }
// acls not being checked since // we are using mock job instead of CompletedJob public static void VerifyHsJobGeneric(Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job, string id, string user, string name, string state, string queue, long startTime , long finishTime, int mapsTotal, int mapsCompleted, int reducesTotal, int reducesCompleted ) { JobReport report = job.GetReport(); WebServicesTestUtils.CheckStringMatch("id", MRApps.ToString(job.GetID()), id); WebServicesTestUtils.CheckStringMatch("user", job.GetUserName().ToString(), user); WebServicesTestUtils.CheckStringMatch("name", job.GetName(), name); WebServicesTestUtils.CheckStringMatch("state", job.GetState().ToString(), state); WebServicesTestUtils.CheckStringMatch("queue", job.GetQueueName(), queue); NUnit.Framework.Assert.AreEqual("startTime incorrect", report.GetStartTime(), startTime ); NUnit.Framework.Assert.AreEqual("finishTime incorrect", report.GetFinishTime(), finishTime ); NUnit.Framework.Assert.AreEqual("mapsTotal incorrect", job.GetTotalMaps(), mapsTotal ); NUnit.Framework.Assert.AreEqual("mapsCompleted incorrect", job.GetCompletedMaps() , mapsCompleted); NUnit.Framework.Assert.AreEqual("reducesTotal incorrect", job.GetTotalReduces(), reducesTotal); NUnit.Framework.Assert.AreEqual("reducesCompleted incorrect", job.GetCompletedReduces (), reducesCompleted); }
public virtual void VerifyCompleted() { foreach (Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job in GetContext().GetAllJobs ().Values) { JobReport jobReport = job.GetReport(); System.Console.Out.WriteLine("Job start time :" + jobReport.GetStartTime()); System.Console.Out.WriteLine("Job finish time :" + jobReport.GetFinishTime()); NUnit.Framework.Assert.IsTrue("Job start time is not less than finish time", jobReport .GetStartTime() <= jobReport.GetFinishTime()); NUnit.Framework.Assert.IsTrue("Job finish time is in future", jobReport.GetFinishTime () <= Runtime.CurrentTimeMillis()); foreach (Task task in job.GetTasks().Values) { TaskReport taskReport = task.GetReport(); System.Console.Out.WriteLine("Task start time : " + taskReport.GetStartTime()); System.Console.Out.WriteLine("Task finish time : " + taskReport.GetFinishTime()); NUnit.Framework.Assert.IsTrue("Task start time is not less than finish time", taskReport .GetStartTime() <= taskReport.GetFinishTime()); foreach (TaskAttempt attempt in task.GetAttempts().Values) { TaskAttemptReport attemptReport = attempt.GetReport(); NUnit.Framework.Assert.IsTrue("Attempt start time is not less than finish time", attemptReport.GetStartTime() <= attemptReport.GetFinishTime()); } } } }
private Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job GetJob() { Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job >(); JobId jobId = new JobIdPBImpl(); ApplicationId appId = ApplicationIdPBImpl.NewInstance(Runtime.CurrentTimeMillis() , 4); jobId.SetAppId(appId); jobId.SetId(1); Org.Mockito.Mockito.When(job.GetID()).ThenReturn(jobId); JobReport report = Org.Mockito.Mockito.Mock <JobReport>(); Org.Mockito.Mockito.When(report.GetStartTime()).ThenReturn(100010L); Org.Mockito.Mockito.When(report.GetFinishTime()).ThenReturn(100015L); Org.Mockito.Mockito.When(job.GetReport()).ThenReturn(report); Org.Mockito.Mockito.When(job.GetName()).ThenReturn("JobName"); Org.Mockito.Mockito.When(job.GetUserName()).ThenReturn("UserName"); Org.Mockito.Mockito.When(job.GetQueueName()).ThenReturn("QueueName"); Org.Mockito.Mockito.When(job.GetState()).ThenReturn(JobState.Succeeded); Org.Mockito.Mockito.When(job.GetTotalMaps()).ThenReturn(3); Org.Mockito.Mockito.When(job.GetCompletedMaps()).ThenReturn(2); Org.Mockito.Mockito.When(job.GetTotalReduces()).ThenReturn(2); Org.Mockito.Mockito.When(job.GetCompletedReduces()).ThenReturn(1); Org.Mockito.Mockito.When(job.GetCompletedReduces()).ThenReturn(1); return(job); }
private void VerifyJobReport(JobReport jobReport, JobId jobId) { IList <AMInfo> amInfos = jobReport.GetAMInfos(); NUnit.Framework.Assert.AreEqual(1, amInfos.Count); AMInfo amInfo = amInfos[0]; ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(jobId.GetAppId (), 1); ContainerId amContainerId = ContainerId.NewContainerId(appAttemptId, 1); NUnit.Framework.Assert.AreEqual(appAttemptId, amInfo.GetAppAttemptId()); NUnit.Framework.Assert.AreEqual(amContainerId, amInfo.GetContainerId()); NUnit.Framework.Assert.IsTrue(jobReport.GetSubmitTime() > 0); NUnit.Framework.Assert.IsTrue(jobReport.GetStartTime() > 0 && jobReport.GetStartTime () >= jobReport.GetSubmitTime()); NUnit.Framework.Assert.IsTrue(jobReport.GetFinishTime() > 0 && jobReport.GetFinishTime () >= jobReport.GetStartTime()); }
public JobInfo(Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job) { this.id = MRApps.ToString(job.GetID()); JobReport report = job.GetReport(); this.mapsTotal = job.GetTotalMaps(); this.mapsCompleted = job.GetCompletedMaps(); this.reducesTotal = job.GetTotalReduces(); this.reducesCompleted = job.GetCompletedReduces(); this.submitTime = report.GetSubmitTime(); this.startTime = report.GetStartTime(); this.finishTime = report.GetFinishTime(); this.name = job.GetName().ToString(); this.queue = job.GetQueueName(); this.user = job.GetUserName(); this.state = job.GetState().ToString(); this.acls = new AList <ConfEntryInfo>(); if (job is CompletedJob) { avgMapTime = 0l; avgReduceTime = 0l; avgShuffleTime = 0l; avgMergeTime = 0l; failedReduceAttempts = 0; killedReduceAttempts = 0; successfulReduceAttempts = 0; failedMapAttempts = 0; killedMapAttempts = 0; successfulMapAttempts = 0; CountTasksAndAttempts(job); this.uberized = job.IsUber(); this.diagnostics = string.Empty; IList <string> diagnostics = job.GetDiagnostics(); if (diagnostics != null && !diagnostics.IsEmpty()) { StringBuilder b = new StringBuilder(); foreach (string diag in diagnostics) { b.Append(diag); } this.diagnostics = b.ToString(); } IDictionary <JobACL, AccessControlList> allacls = job.GetJobACLs(); if (allacls != null) { foreach (KeyValuePair <JobACL, AccessControlList> entry in allacls) { this.acls.AddItem(new ConfEntryInfo(entry.Key.GetAclName(), entry.Value.GetAclString ())); } } } }
public JobInfo(Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job, bool hasAccess) { // ok for any user to see // these should only be seen if acls allow this.id = MRApps.ToString(job.GetID()); JobReport report = job.GetReport(); this.startTime = report.GetStartTime(); this.finishTime = report.GetFinishTime(); this.elapsedTime = Times.Elapsed(this.startTime, this.finishTime); if (this.elapsedTime == -1) { this.elapsedTime = 0; } this.name = job.GetName().ToString(); this.user = job.GetUserName(); this.state = job.GetState(); this.mapsTotal = job.GetTotalMaps(); this.mapsCompleted = job.GetCompletedMaps(); this.mapProgress = report.GetMapProgress() * 100; this.mapProgressPercent = StringHelper.Percent(report.GetMapProgress()); this.reducesTotal = job.GetTotalReduces(); this.reducesCompleted = job.GetCompletedReduces(); this.reduceProgress = report.GetReduceProgress() * 100; this.reduceProgressPercent = StringHelper.Percent(report.GetReduceProgress()); this.acls = new AList <ConfEntryInfo>(); if (hasAccess) { this.diagnostics = string.Empty; CountTasksAndAttempts(job); this.uberized = job.IsUber(); IList <string> diagnostics = job.GetDiagnostics(); if (diagnostics != null && !diagnostics.IsEmpty()) { StringBuilder b = new StringBuilder(); foreach (string diag in diagnostics) { b.Append(diag); } this.diagnostics = b.ToString(); } IDictionary <JobACL, AccessControlList> allacls = job.GetJobACLs(); if (allacls != null) { foreach (KeyValuePair <JobACL, AccessControlList> entry in allacls) { this.acls.AddItem(new ConfEntryInfo(entry.Key.GetAclName(), entry.Value.GetAclString ())); } } } }
public static JobStatus FromYarn(JobReport jobreport, string trackingUrl) { JobPriority jobPriority = JobPriority.Normal; JobStatus jobStatus = new JobStatus(FromYarn(jobreport.GetJobId()), jobreport.GetSetupProgress (), jobreport.GetMapProgress(), jobreport.GetReduceProgress(), jobreport.GetCleanupProgress (), FromYarn(jobreport.GetJobState()), jobPriority, jobreport.GetUser(), jobreport .GetJobName(), jobreport.GetJobFile(), trackingUrl, jobreport.IsUber()); jobStatus.SetStartTime(jobreport.GetStartTime()); jobStatus.SetFinishTime(jobreport.GetFinishTime()); jobStatus.SetFailureInfo(jobreport.GetDiagnostics()); return(jobStatus); }
public static JobsInfo GetPartialJobs(ICollection <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job > jobs, long offset, long count, string user, string queue, long sBegin, long sEnd , long fBegin, long fEnd, JobState jobState) { JobsInfo allJobs = new JobsInfo(); if (sBegin == null || sBegin < 0) { sBegin = 0l; } if (sEnd == null) { sEnd = long.MaxValue; } if (fBegin == null || fBegin < 0) { fBegin = 0l; } if (fEnd == null) { fEnd = long.MaxValue; } if (offset == null || offset < 0) { offset = 0l; } if (count == null) { count = long.MaxValue; } if (offset > jobs.Count) { return(allJobs); } long at = 0; long end = offset + count - 1; if (end < 0) { // due to overflow end = long.MaxValue; } foreach (Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job in jobs) { if (at > end) { break; } // can't really validate queue is a valid one since queues could change if (queue != null && !queue.IsEmpty()) { if (!job.GetQueueName().Equals(queue)) { continue; } } if (user != null && !user.IsEmpty()) { if (!job.GetUserName().Equals(user)) { continue; } } JobReport report = job.GetReport(); if (report.GetStartTime() < sBegin || report.GetStartTime() > sEnd) { continue; } if (report.GetFinishTime() < fBegin || report.GetFinishTime() > fEnd) { continue; } if (jobState != null && jobState != report.GetJobState()) { continue; } at++; if ((at - 1) < offset) { continue; } JobInfo jobInfo = new JobInfo(job); allJobs.Add(jobInfo); } return(allJobs); }