public AppInfo(ApplicationReport app) { // JAXB needs this appId = app.GetApplicationId().ToString(); if (app.GetCurrentApplicationAttemptId() != null) { currentAppAttemptId = app.GetCurrentApplicationAttemptId().ToString(); } user = app.GetUser(); queue = app.GetQueue(); name = app.GetName(); type = app.GetApplicationType(); host = app.GetHost(); rpcPort = app.GetRpcPort(); appState = app.GetYarnApplicationState(); diagnosticsInfo = app.GetDiagnostics(); trackingUrl = app.GetTrackingUrl(); originalTrackingUrl = app.GetOriginalTrackingUrl(); submittedTime = app.GetStartTime(); startedTime = app.GetStartTime(); finishedTime = app.GetFinishTime(); elapsedTime = Times.Elapsed(startedTime, finishedTime); finalAppStatus = app.GetFinalApplicationStatus(); progress = app.GetProgress() * 100; // in percent if (app.GetApplicationTags() != null && !app.GetApplicationTags().IsEmpty()) { this.applicationTags = StringHelper.CsvJoiner.Join(app.GetApplicationTags()); } }
public virtual void TestApplicationReport() { long timestamp = Runtime.CurrentTimeMillis(); ApplicationReport appReport1 = CreateApplicationReport(1, 1, timestamp); ApplicationReport appReport2 = CreateApplicationReport(1, 1, timestamp); ApplicationReport appReport3 = CreateApplicationReport(1, 1, timestamp); NUnit.Framework.Assert.AreEqual(appReport1, appReport2); NUnit.Framework.Assert.AreEqual(appReport2, appReport3); appReport1.SetApplicationId(null); NUnit.Framework.Assert.IsNull(appReport1.GetApplicationId()); NUnit.Framework.Assert.AreNotSame(appReport1, appReport2); appReport2.SetCurrentApplicationAttemptId(null); NUnit.Framework.Assert.IsNull(appReport2.GetCurrentApplicationAttemptId()); NUnit.Framework.Assert.AreNotSame(appReport2, appReport3); NUnit.Framework.Assert.IsNull(appReport1.GetAMRMToken()); }
public virtual void TestApplicationReport() { ApplicationId appId = null; appId = ApplicationId.NewInstance(0, 1); WriteApplicationStartData(appId); WriteApplicationFinishData(appId); ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, 1); WriteApplicationAttemptStartData(appAttemptId); WriteApplicationAttemptFinishData(appAttemptId); ApplicationReport appReport = applicationHistoryManagerImpl.GetApplication(appId); NUnit.Framework.Assert.IsNotNull(appReport); NUnit.Framework.Assert.AreEqual(appId, appReport.GetApplicationId()); NUnit.Framework.Assert.AreEqual(appAttemptId, appReport.GetCurrentApplicationAttemptId ()); NUnit.Framework.Assert.AreEqual(appAttemptId.ToString(), appReport.GetHost()); NUnit.Framework.Assert.AreEqual("test type", appReport.GetApplicationType().ToString ()); NUnit.Framework.Assert.AreEqual("test queue", appReport.GetQueue().ToString()); }
public virtual void Setup() { // start minicluster conf = new YarnConfiguration(); yarnCluster = new MiniYARNCluster(typeof(TestAMRMClient).FullName, nodeCount, 1, 1); yarnCluster.Init(conf); yarnCluster.Start(); NUnit.Framework.Assert.IsNotNull(yarnCluster); NUnit.Framework.Assert.AreEqual(Service.STATE.Started, yarnCluster.GetServiceState ()); // start rm client yarnClient = (YarnClientImpl)YarnClient.CreateYarnClient(); yarnClient.Init(conf); yarnClient.Start(); NUnit.Framework.Assert.IsNotNull(yarnClient); NUnit.Framework.Assert.AreEqual(Service.STATE.Started, yarnClient.GetServiceState ()); // get node info nodeReports = yarnClient.GetNodeReports(NodeState.Running); // submit new app ApplicationSubmissionContext appContext = yarnClient.CreateApplication().GetApplicationSubmissionContext (); ApplicationId appId = appContext.GetApplicationId(); // set the application name appContext.SetApplicationName("Test"); // Set the priority for the application master Priority pri = Priority.NewInstance(0); appContext.SetPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.SetQueue("default"); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <ContainerLaunchContext>(); appContext.SetAMContainerSpec(amContainer); // unmanaged AM appContext.SetUnmanagedAM(true); // Create the request to send to the applications manager SubmitApplicationRequest appRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <SubmitApplicationRequest>(); appRequest.SetApplicationSubmissionContext(appContext); // Submit the application to the applications manager yarnClient.SubmitApplication(appContext); // wait for app to start int iterationsLeft = 30; RMAppAttempt appAttempt = null; while (iterationsLeft > 0) { ApplicationReport appReport = yarnClient.GetApplicationReport(appId); if (appReport.GetYarnApplicationState() == YarnApplicationState.Accepted) { attemptId = appReport.GetCurrentApplicationAttemptId(); appAttempt = yarnCluster.GetResourceManager().GetRMContext().GetRMApps()[attemptId .GetApplicationId()].GetCurrentAppAttempt(); while (true) { if (appAttempt.GetAppAttemptState() == RMAppAttemptState.Launched) { break; } } break; } Sleep(1000); --iterationsLeft; } if (iterationsLeft == 0) { NUnit.Framework.Assert.Fail("Application hasn't bee started"); } // Just dig into the ResourceManager and get the AMRMToken just for the sake // of testing. UserGroupInformation.SetLoginUser(UserGroupInformation.CreateRemoteUser(UserGroupInformation .GetCurrentUser().GetUserName())); UserGroupInformation.GetCurrentUser().AddToken(appAttempt.GetAMRMToken()); //creating an instance NMTokenCase nmTokenCache = new NMTokenCache(); // start am rm client rmClient = (AMRMClientImpl <AMRMClient.ContainerRequest>)AMRMClient.CreateAMRMClient <AMRMClient.ContainerRequest>(); //setting an instance NMTokenCase rmClient.SetNMTokenCache(nmTokenCache); rmClient.Init(conf); rmClient.Start(); NUnit.Framework.Assert.IsNotNull(rmClient); NUnit.Framework.Assert.AreEqual(Service.STATE.Started, rmClient.GetServiceState() ); // start am nm client nmClient = (NMClientImpl)NMClient.CreateNMClient(); //propagating the AMRMClient NMTokenCache instance nmClient.SetNMTokenCache(rmClient.GetNMTokenCache()); nmClient.Init(conf); nmClient.Start(); NUnit.Framework.Assert.IsNotNull(nmClient); NUnit.Framework.Assert.AreEqual(Service.STATE.Started, nmClient.GetServiceState() ); }
/// <summary>Monitor the submitted application for completion.</summary> /// <remarks> /// Monitor the submitted application for completion. Kill application if time /// expires. /// </remarks> /// <param name="appId">Application Id of application to be monitored</param> /// <returns>true if application completed successfully</returns> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> private ApplicationReport MonitorApplication(ApplicationId appId, ICollection <YarnApplicationState > finalState) { long foundAMCompletedTime = 0; StringBuilder expectedFinalState = new StringBuilder(); bool first = true; foreach (YarnApplicationState state in finalState) { if (first) { first = false; expectedFinalState.Append(state.ToString()); } else { expectedFinalState.Append("," + state.ToString()); } } while (true) { // Check app status every 1 second. try { Sharpen.Thread.Sleep(1000); } catch (Exception) { Log.Debug("Thread sleep in monitoring loop interrupted"); } // Get application report for the appId we are interested in ApplicationReport report = rmClient.GetApplicationReport(appId); Log.Info("Got application report from ASM for" + ", appId=" + appId.GetId() + ", appAttemptId=" + report.GetCurrentApplicationAttemptId() + ", clientToAMToken=" + report.GetClientToAMToken () + ", appDiagnostics=" + report.GetDiagnostics() + ", appMasterHost=" + report .GetHost() + ", appQueue=" + report.GetQueue() + ", appMasterRpcPort=" + report. GetRpcPort() + ", appStartTime=" + report.GetStartTime() + ", yarnAppState=" + report .GetYarnApplicationState().ToString() + ", distributedFinalState=" + report.GetFinalApplicationStatus ().ToString() + ", appTrackingUrl=" + report.GetTrackingUrl() + ", appUser="******"Waited " + AmStateWaitTimeoutMs / 1000 + " seconds after process completed for AppReport" + " to reach desired final state. Not waiting anymore." + "CurrentState = " + state_1 + ", ExpectedStates = " + expectedFinalState.ToString()); throw new RuntimeException("Failed to receive final expected state" + " in ApplicationReport" + ", CurrentState=" + state_1 + ", ExpectedStates=" + expectedFinalState.ToString ()); } } } } }
private void CreateAppReports() { ApplicationId applicationId = ApplicationId.NewInstance(1234, 5); ApplicationReport newApplicationReport = ApplicationReport.NewInstance(applicationId , ApplicationAttemptId.NewInstance(applicationId, 1), "user", "queue", "appname" , "host", 124, null, YarnApplicationState.Running, "diagnostics", "url", 0, 0, FinalApplicationStatus .Succeeded, null, "N/A", 0.53789f, "YARN", null); IList <ApplicationReport> applicationReports = new AList <ApplicationReport>(); applicationReports.AddItem(newApplicationReport); IList <ApplicationAttemptReport> appAttempts = new AList <ApplicationAttemptReport> (); ApplicationAttemptReport attempt = ApplicationAttemptReport.NewInstance(ApplicationAttemptId .NewInstance(applicationId, 1), "host", 124, "url", "oUrl", "diagnostics", YarnApplicationAttemptState .Finished, ContainerId.NewContainerId(newApplicationReport.GetCurrentApplicationAttemptId (), 1)); appAttempts.AddItem(attempt); ApplicationAttemptReport attempt1 = ApplicationAttemptReport.NewInstance(ApplicationAttemptId .NewInstance(applicationId, 2), "host", 124, "url", "oUrl", "diagnostics", YarnApplicationAttemptState .Finished, ContainerId.NewContainerId(newApplicationReport.GetCurrentApplicationAttemptId (), 2)); appAttempts.AddItem(attempt1); attempts[applicationId] = appAttempts; IList <ContainerReport> containerReports = new AList <ContainerReport>(); ContainerReport container = ContainerReport.NewInstance(ContainerId.NewContainerId (attempt.GetApplicationAttemptId(), 1), null, NodeId.NewInstance("host", 1234), Priority.Undefined, 1234, 5678, "diagnosticInfo", "logURL", 0, ContainerState.Complete , "http://" + NodeId.NewInstance("host", 2345).ToString()); containerReports.AddItem(container); ContainerReport container1 = ContainerReport.NewInstance(ContainerId.NewContainerId (attempt.GetApplicationAttemptId(), 2), null, NodeId.NewInstance("host", 1234), Priority.Undefined, 1234, 5678, "diagnosticInfo", "logURL", 0, ContainerState.Complete , "http://" + NodeId.NewInstance("host", 2345).ToString()); containerReports.AddItem(container1); containers[attempt.GetApplicationAttemptId()] = containerReports; ApplicationId applicationId2 = ApplicationId.NewInstance(1234, 6); ApplicationReport newApplicationReport2 = ApplicationReport.NewInstance(applicationId2 , ApplicationAttemptId.NewInstance(applicationId2, 2), "user2", "queue2", "appname2" , "host2", 125, null, YarnApplicationState.Finished, "diagnostics2", "url2", 2, 2, FinalApplicationStatus.Succeeded, null, "N/A", 0.63789f, "NON-YARN", null); applicationReports.AddItem(newApplicationReport2); ApplicationId applicationId3 = ApplicationId.NewInstance(1234, 7); ApplicationReport newApplicationReport3 = ApplicationReport.NewInstance(applicationId3 , ApplicationAttemptId.NewInstance(applicationId3, 3), "user3", "queue3", "appname3" , "host3", 126, null, YarnApplicationState.Running, "diagnostics3", "url3", 3, 3 , FinalApplicationStatus.Succeeded, null, "N/A", 0.73789f, "MAPREDUCE", null); applicationReports.AddItem(newApplicationReport3); ApplicationId applicationId4 = ApplicationId.NewInstance(1234, 8); ApplicationReport newApplicationReport4 = ApplicationReport.NewInstance(applicationId4 , ApplicationAttemptId.NewInstance(applicationId4, 4), "user4", "queue4", "appname4" , "host4", 127, null, YarnApplicationState.Failed, "diagnostics4", "url4", 4, 4, FinalApplicationStatus.Succeeded, null, "N/A", 0.83789f, "NON-MAPREDUCE", null); applicationReports.AddItem(newApplicationReport4); reports = applicationReports; }