public JobInfo(Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job) { this.id = MRApps.ToString(job.GetID()); JobReport report = job.GetReport(); this.mapsTotal = job.GetTotalMaps(); this.mapsCompleted = job.GetCompletedMaps(); this.reducesTotal = job.GetTotalReduces(); this.reducesCompleted = job.GetCompletedReduces(); this.submitTime = report.GetSubmitTime(); this.startTime = report.GetStartTime(); this.finishTime = report.GetFinishTime(); this.name = job.GetName().ToString(); this.queue = job.GetQueueName(); this.user = job.GetUserName(); this.state = job.GetState().ToString(); this.acls = new AList <ConfEntryInfo>(); if (job is CompletedJob) { avgMapTime = 0l; avgReduceTime = 0l; avgShuffleTime = 0l; avgMergeTime = 0l; failedReduceAttempts = 0; killedReduceAttempts = 0; successfulReduceAttempts = 0; failedMapAttempts = 0; killedMapAttempts = 0; successfulMapAttempts = 0; CountTasksAndAttempts(job); this.uberized = job.IsUber(); this.diagnostics = string.Empty; IList <string> diagnostics = job.GetDiagnostics(); if (diagnostics != null && !diagnostics.IsEmpty()) { StringBuilder b = new StringBuilder(); foreach (string diag in diagnostics) { b.Append(diag); } this.diagnostics = b.ToString(); } IDictionary <JobACL, AccessControlList> allacls = job.GetJobACLs(); if (allacls != null) { foreach (KeyValuePair <JobACL, AccessControlList> entry in allacls) { this.acls.AddItem(new ConfEntryInfo(entry.Key.GetAclName(), entry.Value.GetAclString ())); } } } }
public JobInfo(Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job, bool hasAccess) { // ok for any user to see // these should only be seen if acls allow this.id = MRApps.ToString(job.GetID()); JobReport report = job.GetReport(); this.startTime = report.GetStartTime(); this.finishTime = report.GetFinishTime(); this.elapsedTime = Times.Elapsed(this.startTime, this.finishTime); if (this.elapsedTime == -1) { this.elapsedTime = 0; } this.name = job.GetName().ToString(); this.user = job.GetUserName(); this.state = job.GetState(); this.mapsTotal = job.GetTotalMaps(); this.mapsCompleted = job.GetCompletedMaps(); this.mapProgress = report.GetMapProgress() * 100; this.mapProgressPercent = StringHelper.Percent(report.GetMapProgress()); this.reducesTotal = job.GetTotalReduces(); this.reducesCompleted = job.GetCompletedReduces(); this.reduceProgress = report.GetReduceProgress() * 100; this.reduceProgressPercent = StringHelper.Percent(report.GetReduceProgress()); this.acls = new AList <ConfEntryInfo>(); if (hasAccess) { this.diagnostics = string.Empty; CountTasksAndAttempts(job); this.uberized = job.IsUber(); IList <string> diagnostics = job.GetDiagnostics(); if (diagnostics != null && !diagnostics.IsEmpty()) { StringBuilder b = new StringBuilder(); foreach (string diag in diagnostics) { b.Append(diag); } this.diagnostics = b.ToString(); } IDictionary <JobACL, AccessControlList> allacls = job.GetJobACLs(); if (allacls != null) { foreach (KeyValuePair <JobACL, AccessControlList> entry in allacls) { this.acls.AddItem(new ConfEntryInfo(entry.Key.GetAclName(), entry.Value.GetAclString ())); } } } }
//Empty public override IList <string> GetDiagnostics() { return(job.GetDiagnostics()); }
public virtual IList <string> GetDiagnostics() { return(mockJob.GetDiagnostics()); }
protected internal virtual void DoUnregistration() { FinalApplicationStatus finishState = FinalApplicationStatus.Undefined; JobImpl jobImpl = (JobImpl)job; if (jobImpl.GetInternalState() == JobStateInternal.Succeeded) { finishState = FinalApplicationStatus.Succeeded; } else { if (jobImpl.GetInternalState() == JobStateInternal.Killed || (jobImpl.GetInternalState () == JobStateInternal.Running && isSignalled)) { finishState = FinalApplicationStatus.Killed; } else { if (jobImpl.GetInternalState() == JobStateInternal.Failed || jobImpl.GetInternalState () == JobStateInternal.Error) { finishState = FinalApplicationStatus.Failed; } } } StringBuilder sb = new StringBuilder(); foreach (string s in job.GetDiagnostics()) { sb.Append(s).Append("\n"); } Log.Info("Setting job diagnostics to " + sb.ToString()); string historyUrl = MRWebAppUtil.GetApplicationWebURLOnJHSWithScheme(GetConfig(), context.GetApplicationID()); Log.Info("History url is " + historyUrl); FinishApplicationMasterRequest request = FinishApplicationMasterRequest.NewInstance (finishState, sb.ToString(), historyUrl); try { while (true) { FinishApplicationMasterResponse response = scheduler.FinishApplicationMaster(request ); if (response.GetIsUnregistered()) { // When excepting ClientService, other services are already stopped, // it is safe to let clients know the final states. ClientService // should wait for some time so clients have enough time to know the // final states. MRAppMaster.RunningAppContext raContext = (MRAppMaster.RunningAppContext)context; raContext.MarkSuccessfulUnregistration(); break; } Log.Info("Waiting for application to be successfully unregistered."); Sharpen.Thread.Sleep(rmPollInterval); } } catch (ApplicationMasterNotRegisteredException) { // RM might have restarted or failed over and so lost the fact that AM had // registered before. Register(); DoUnregistration(); } }
/// <exception cref="System.Exception"/> public virtual void TestDiagnosticsForKilledJob() { Log.Info("STARTING testDiagnosticsForKilledJob"); try { Configuration conf = new Configuration(); conf.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey, typeof(TestJobHistoryParsing.MyResolver), typeof(DNSToSwitchMapping)); RackResolver.Init(conf); MRApp app = new TestJobHistoryParsing.MRAppWithHistoryWithJobKilled(2, 1, true, this .GetType().FullName, true); app.Submit(conf); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values .GetEnumerator().Next(); JobId jobId = job.GetID(); app.WaitForState(job, JobState.Killed); // make sure all events are flushed app.WaitForState(Service.STATE.Stopped); JobHistory jobHistory = new JobHistory(); jobHistory.Init(conf); HistoryFileManager.HistoryFileInfo fileInfo = jobHistory.GetJobFileInfo(jobId); JobHistoryParser parser; JobHistoryParser.JobInfo jobInfo; lock (fileInfo) { Path historyFilePath = fileInfo.GetHistoryFile(); FSDataInputStream @in = null; FileContext fc = null; try { fc = FileContext.GetFileContext(conf); @in = fc.Open(fc.MakeQualified(historyFilePath)); } catch (IOException ioe) { Log.Info("Can not open history file: " + historyFilePath, ioe); throw (new Exception("Can not open History File")); } parser = new JobHistoryParser(@in); jobInfo = parser.Parse(); } Exception parseException = parser.GetParseException(); NUnit.Framework.Assert.IsNull("Caught an expected exception " + parseException, parseException ); IList <string> originalDiagnostics = job.GetDiagnostics(); string historyError = jobInfo.GetErrorInfo(); NUnit.Framework.Assert.IsTrue("No original diagnostics for a failed job", originalDiagnostics != null && !originalDiagnostics.IsEmpty()); NUnit.Framework.Assert.IsNotNull("No history error info for a failed job ", historyError ); foreach (string diagString in originalDiagnostics) { NUnit.Framework.Assert.IsTrue(historyError.Contains(diagString)); } NUnit.Framework.Assert.IsTrue("No killed message in diagnostics", historyError.Contains (JobImpl.JobKilledDiag)); } finally { Log.Info("FINISHED testDiagnosticsForKilledJob"); } }