public AppInfo(ApplicationReport app) { // JAXB needs this appId = app.GetApplicationId().ToString(); if (app.GetCurrentApplicationAttemptId() != null) { currentAppAttemptId = app.GetCurrentApplicationAttemptId().ToString(); } user = app.GetUser(); queue = app.GetQueue(); name = app.GetName(); type = app.GetApplicationType(); host = app.GetHost(); rpcPort = app.GetRpcPort(); appState = app.GetYarnApplicationState(); diagnosticsInfo = app.GetDiagnostics(); trackingUrl = app.GetTrackingUrl(); originalTrackingUrl = app.GetOriginalTrackingUrl(); submittedTime = app.GetStartTime(); startedTime = app.GetStartTime(); finishedTime = app.GetFinishTime(); elapsedTime = Times.Elapsed(startedTime, finishedTime); finalAppStatus = app.GetFinalApplicationStatus(); progress = app.GetProgress() * 100; // in percent if (app.GetApplicationTags() != null && !app.GetApplicationTags().IsEmpty()) { this.applicationTags = StringHelper.CsvJoiner.Join(app.GetApplicationTags()); } }
/// <exception cref="System.Exception"/> public virtual void TestJobSubmissionFailure() { Org.Mockito.Mockito.When(resourceMgrDelegate.SubmitApplication(Matchers.Any <ApplicationSubmissionContext >())).ThenReturn(appId); ApplicationReport report = Org.Mockito.Mockito.Mock <ApplicationReport>(); Org.Mockito.Mockito.When(report.GetApplicationId()).ThenReturn(appId); Org.Mockito.Mockito.When(report.GetDiagnostics()).ThenReturn(failString); Org.Mockito.Mockito.When(report.GetYarnApplicationState()).ThenReturn(YarnApplicationState .Failed); Org.Mockito.Mockito.When(resourceMgrDelegate.GetApplicationReport(appId)).ThenReturn (report); Credentials credentials = new Credentials(); FilePath jobxml = new FilePath(testWorkDir, "job.xml"); OutputStream @out = new FileOutputStream(jobxml); conf.WriteXml(@out); @out.Close(); try { yarnRunner.SubmitJob(jobId, testWorkDir.GetAbsolutePath().ToString(), credentials ); } catch (IOException io) { Log.Info("Logging exception:", io); NUnit.Framework.Assert.IsTrue(io.GetLocalizedMessage().Contains(failString)); } }
public static JobStatus FromYarn(ApplicationReport application, string jobFile) { string trackingUrl = application.GetTrackingUrl(); trackingUrl = trackingUrl == null ? string.Empty : trackingUrl; JobStatus jobStatus = new JobStatus(TypeConverter.FromYarn(application.GetApplicationId ()), 0.0f, 0.0f, 0.0f, 0.0f, TypeConverter.FromYarn(application.GetYarnApplicationState (), application.GetFinalApplicationStatus()), JobPriority.Normal, application.GetUser (), application.GetName(), application.GetQueue(), jobFile, trackingUrl, false); jobStatus.SetSchedulingInfo(trackingUrl); // Set AM tracking url jobStatus.SetStartTime(application.GetStartTime()); jobStatus.SetFinishTime(application.GetFinishTime()); jobStatus.SetFailureInfo(application.GetDiagnostics()); ApplicationResourceUsageReport resourceUsageReport = application.GetApplicationResourceUsageReport (); if (resourceUsageReport != null) { jobStatus.SetNeededMem(resourceUsageReport.GetNeededResources().GetMemory()); jobStatus.SetNumReservedSlots(resourceUsageReport.GetNumReservedContainers()); jobStatus.SetNumUsedSlots(resourceUsageReport.GetNumUsedContainers()); jobStatus.SetReservedMem(resourceUsageReport.GetReservedResources().GetMemory()); jobStatus.SetUsedMem(resourceUsageReport.GetUsedResources().GetMemory()); } return(jobStatus); }
private void VerifyEnemyAppReport(ApplicationReport appReport) { NUnit.Framework.Assert.AreEqual("Enemy should not see app host!", Unavailable, appReport .GetHost()); NUnit.Framework.Assert.AreEqual("Enemy should not see app rpc port!", -1, appReport .GetRpcPort()); NUnit.Framework.Assert.AreEqual("Enemy should not see app client token!", null, appReport .GetClientToAMToken()); NUnit.Framework.Assert.AreEqual("Enemy should not see app diagnostics!", Unavailable , appReport.GetDiagnostics()); NUnit.Framework.Assert.AreEqual("Enemy should not see app tracking url!", Unavailable , appReport.GetTrackingUrl()); NUnit.Framework.Assert.AreEqual("Enemy should not see app original tracking url!" , Unavailable, appReport.GetOriginalTrackingUrl()); ApplicationResourceUsageReport usageReport = appReport.GetApplicationResourceUsageReport (); NUnit.Framework.Assert.AreEqual("Enemy should not see app used containers", -1, usageReport .GetNumUsedContainers()); NUnit.Framework.Assert.AreEqual("Enemy should not see app reserved containers", - 1, usageReport.GetNumReservedContainers()); NUnit.Framework.Assert.AreEqual("Enemy should not see app used resources", -1, usageReport .GetUsedResources().GetMemory()); NUnit.Framework.Assert.AreEqual("Enemy should not see app reserved resources", -1 , usageReport.GetReservedResources().GetMemory()); NUnit.Framework.Assert.AreEqual("Enemy should not see app needed resources", -1, usageReport.GetNeededResources().GetMemory()); }
/// <summary>Monitor the submitted application for completion.</summary> /// <remarks> /// Monitor the submitted application for completion. /// Kill application if time expires. /// </remarks> /// <param name="appId">Application Id of application to be monitored</param> /// <returns>true if application completed successfully</returns> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> private bool MonitorApplication(ApplicationId appId) { while (true) { // Check app status every 1 second. try { Sharpen.Thread.Sleep(1000); } catch (Exception) { Log.Debug("Thread sleep in monitoring loop interrupted"); } // Get application report for the appId we are interested in ApplicationReport report = yarnClient.GetApplicationReport(appId); Log.Info("Got application report from ASM for" + ", appId=" + appId.GetId() + ", clientToAMToken=" + report.GetClientToAMToken() + ", appDiagnostics=" + report.GetDiagnostics() + ", appMasterHost=" + report.GetHost() + ", appQueue=" + report.GetQueue() + ", appMasterRpcPort=" + report.GetRpcPort() + ", appStartTime=" + report.GetStartTime() + ", yarnAppState=" + report.GetYarnApplicationState().ToString() + ", distributedFinalState=" + report .GetFinalApplicationStatus().ToString() + ", appTrackingUrl=" + report.GetTrackingUrl () + ", appUser="******"Application has completed successfully. Breaking monitoring loop"); return(true); } else { Log.Info("Application did finished unsuccessfully." + " YarnState=" + state.ToString () + ", DSFinalStatus=" + dsStatus.ToString() + ". Breaking monitoring loop"); return(false); } } else { if (YarnApplicationState.Killed == state || YarnApplicationState.Failed == state) { Log.Info("Application did not finish." + " YarnState=" + state.ToString() + ", DSFinalStatus=" + dsStatus.ToString() + ". Breaking monitoring loop"); return(false); } } if (Runtime.CurrentTimeMillis() > (clientStartTime + clientTimeout)) { Log.Info("Reached client specified timeout for application. Killing application"); ForceKillApplication(appId); return(false); } } }
/// <exception cref="System.IO.IOException"/> public virtual GetJobReportResponse GetJobReport(GetJobReportRequest request) { JobReport jobReport = recordFactory.NewRecordInstance <JobReport>(); jobReport.SetJobId(request.GetJobId()); jobReport.SetJobState(jobState); jobReport.SetUser(applicationReport.GetUser()); jobReport.SetStartTime(applicationReport.GetStartTime()); jobReport.SetDiagnostics(applicationReport.GetDiagnostics()); jobReport.SetJobName(applicationReport.GetName()); jobReport.SetTrackingUrl(applicationReport.GetTrackingUrl()); jobReport.SetFinishTime(applicationReport.GetFinishTime()); GetJobReportResponse resp = recordFactory.NewRecordInstance <GetJobReportResponse> (); resp.SetJobReport(jobReport); return(resp); }
/// <summary>Monitor the submitted application for completion.</summary> /// <remarks> /// Monitor the submitted application for completion. Kill application if time /// expires. /// </remarks> /// <param name="appId">Application Id of application to be monitored</param> /// <returns>true if application completed successfully</returns> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> private ApplicationReport MonitorApplication(ApplicationId appId, ICollection <YarnApplicationState > finalState) { long foundAMCompletedTime = 0; StringBuilder expectedFinalState = new StringBuilder(); bool first = true; foreach (YarnApplicationState state in finalState) { if (first) { first = false; expectedFinalState.Append(state.ToString()); } else { expectedFinalState.Append("," + state.ToString()); } } while (true) { // Check app status every 1 second. try { Sharpen.Thread.Sleep(1000); } catch (Exception) { Log.Debug("Thread sleep in monitoring loop interrupted"); } // Get application report for the appId we are interested in ApplicationReport report = rmClient.GetApplicationReport(appId); Log.Info("Got application report from ASM for" + ", appId=" + appId.GetId() + ", appAttemptId=" + report.GetCurrentApplicationAttemptId() + ", clientToAMToken=" + report.GetClientToAMToken () + ", appDiagnostics=" + report.GetDiagnostics() + ", appMasterHost=" + report .GetHost() + ", appQueue=" + report.GetQueue() + ", appMasterRpcPort=" + report. GetRpcPort() + ", appStartTime=" + report.GetStartTime() + ", yarnAppState=" + report .GetYarnApplicationState().ToString() + ", distributedFinalState=" + report.GetFinalApplicationStatus ().ToString() + ", appTrackingUrl=" + report.GetTrackingUrl() + ", appUser="******"Waited " + AmStateWaitTimeoutMs / 1000 + " seconds after process completed for AppReport" + " to reach desired final state. Not waiting anymore." + "CurrentState = " + state_1 + ", ExpectedStates = " + expectedFinalState.ToString()); throw new RuntimeException("Failed to receive final expected state" + " in ApplicationReport" + ", CurrentState=" + state_1 + ", ExpectedStates=" + expectedFinalState.ToString ()); } } } } }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> public override ApplicationId SubmitApplication(ApplicationSubmissionContext appContext ) { ApplicationId applicationId = appContext.GetApplicationId(); if (applicationId == null) { throw new ApplicationIdNotProvidedException("ApplicationId is not provided in ApplicationSubmissionContext" ); } SubmitApplicationRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord < SubmitApplicationRequest>(); request.SetApplicationSubmissionContext(appContext); // Automatically add the timeline DT into the CLC // Only when the security and the timeline service are both enabled if (IsSecurityEnabled() && timelineServiceEnabled) { AddTimelineDelegationToken(appContext.GetAMContainerSpec()); } //TODO: YARN-1763:Handle RM failovers during the submitApplication call. rmClient.SubmitApplication(request); int pollCount = 0; long startTime = Runtime.CurrentTimeMillis(); EnumSet <YarnApplicationState> waitingStates = EnumSet.Of(YarnApplicationState.New , YarnApplicationState.NewSaving, YarnApplicationState.Submitted); EnumSet <YarnApplicationState> failToSubmitStates = EnumSet.Of(YarnApplicationState .Failed, YarnApplicationState.Killed); while (true) { try { ApplicationReport appReport = GetApplicationReport(applicationId); YarnApplicationState state = appReport.GetYarnApplicationState(); if (!waitingStates.Contains(state)) { if (failToSubmitStates.Contains(state)) { throw new YarnException("Failed to submit " + applicationId + " to YARN : " + appReport .GetDiagnostics()); } Log.Info("Submitted application " + applicationId); break; } long elapsedMillis = Runtime.CurrentTimeMillis() - startTime; if (EnforceAsyncAPITimeout() && elapsedMillis >= asyncApiPollTimeoutMillis) { throw new YarnException("Timed out while waiting for application " + applicationId + " to be submitted successfully"); } // Notify the client through the log every 10 poll, in case the client // is blocked here too long. if (++pollCount % 10 == 0) { Log.Info("Application submission is not finished, " + "submitted application " + applicationId + " is still in " + state); } try { Sharpen.Thread.Sleep(submitPollIntervalMillis); } catch (Exception) { Log.Error("Interrupted while waiting for application " + applicationId + " to be successfully submitted." ); } } catch (ApplicationNotFoundException) { // FailOver or RM restart happens before RMStateStore saves // ApplicationState Log.Info("Re-submit application " + applicationId + "with the " + "same ApplicationSubmissionContext" ); rmClient.SubmitApplication(request); } } return(applicationId); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public override JobStatus SubmitJob(JobID jobId, string jobSubmitDir, Credentials ts) { AddHistoryToken(ts); // Construct necessary information to start the MR AM ApplicationSubmissionContext appContext = CreateApplicationSubmissionContext(conf , jobSubmitDir, ts); // Submit to ResourceManager try { ApplicationId applicationId = resMgrDelegate.SubmitApplication(appContext); ApplicationReport appMaster = resMgrDelegate.GetApplicationReport(applicationId); string diagnostics = (appMaster == null ? "application report is null" : appMaster .GetDiagnostics()); if (appMaster == null || appMaster.GetYarnApplicationState() == YarnApplicationState .Failed || appMaster.GetYarnApplicationState() == YarnApplicationState.Killed) { throw new IOException("Failed to run job : " + diagnostics); } return(clientCache.GetClient(jobId).GetJobStatus(jobId)); } catch (YarnException e) { throw new IOException(e); } }
/// <summary>Prints the application report for an application id.</summary> /// <param name="applicationId"/> /// <returns>exitCode</returns> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> private int PrintApplicationReport(string applicationId) { ApplicationReport appReport = null; try { appReport = client.GetApplicationReport(ConverterUtils.ToApplicationId(applicationId )); } catch (ApplicationNotFoundException) { sysout.WriteLine("Application with id '" + applicationId + "' doesn't exist in RM or Timeline Server." ); return(-1); } // Use PrintWriter.println, which uses correct platform line ending. ByteArrayOutputStream baos = new ByteArrayOutputStream(); PrintWriter appReportStr = new PrintWriter(new OutputStreamWriter(baos, Sharpen.Extensions.GetEncoding ("UTF-8"))); if (appReport != null) { appReportStr.WriteLine("Application Report : "); appReportStr.Write("\tApplication-Id : "); appReportStr.WriteLine(appReport.GetApplicationId()); appReportStr.Write("\tApplication-Name : "); appReportStr.WriteLine(appReport.GetName()); appReportStr.Write("\tApplication-Type : "); appReportStr.WriteLine(appReport.GetApplicationType()); appReportStr.Write("\tUser : "******"\tQueue : "); appReportStr.WriteLine(appReport.GetQueue()); appReportStr.Write("\tStart-Time : "); appReportStr.WriteLine(appReport.GetStartTime()); appReportStr.Write("\tFinish-Time : "); appReportStr.WriteLine(appReport.GetFinishTime()); appReportStr.Write("\tProgress : "); DecimalFormat formatter = new DecimalFormat("###.##%"); string progress = formatter.Format(appReport.GetProgress()); appReportStr.WriteLine(progress); appReportStr.Write("\tState : "); appReportStr.WriteLine(appReport.GetYarnApplicationState()); appReportStr.Write("\tFinal-State : "); appReportStr.WriteLine(appReport.GetFinalApplicationStatus()); appReportStr.Write("\tTracking-URL : "); appReportStr.WriteLine(appReport.GetOriginalTrackingUrl()); appReportStr.Write("\tRPC Port : "); appReportStr.WriteLine(appReport.GetRpcPort()); appReportStr.Write("\tAM Host : "); appReportStr.WriteLine(appReport.GetHost()); appReportStr.Write("\tAggregate Resource Allocation : "); ApplicationResourceUsageReport usageReport = appReport.GetApplicationResourceUsageReport (); if (usageReport != null) { //completed app report in the timeline server doesn't have usage report appReportStr.Write(usageReport.GetMemorySeconds() + " MB-seconds, "); appReportStr.WriteLine(usageReport.GetVcoreSeconds() + " vcore-seconds"); } else { appReportStr.WriteLine("N/A"); } appReportStr.Write("\tDiagnostics : "); appReportStr.Write(appReport.GetDiagnostics()); } else { appReportStr.Write("Application with id '" + applicationId + "' doesn't exist in RM." ); appReportStr.Close(); sysout.WriteLine(baos.ToString("UTF-8")); return(-1); } appReportStr.Close(); sysout.WriteLine(baos.ToString("UTF-8")); return(0); }