/// <summary>Monitor the submitted application for completion.</summary> /// <remarks> /// Monitor the submitted application for completion. /// Kill application if time expires. /// </remarks> /// <param name="appId">Application Id of application to be monitored</param> /// <returns>true if application completed successfully</returns> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> private bool MonitorApplication(ApplicationId appId) { while (true) { // Check app status every 1 second. try { Sharpen.Thread.Sleep(1000); } catch (Exception) { Log.Debug("Thread sleep in monitoring loop interrupted"); } // Get application report for the appId we are interested in ApplicationReport report = yarnClient.GetApplicationReport(appId); Log.Info("Got application report from ASM for" + ", appId=" + appId.GetId() + ", clientToAMToken=" + report.GetClientToAMToken() + ", appDiagnostics=" + report.GetDiagnostics() + ", appMasterHost=" + report.GetHost() + ", appQueue=" + report.GetQueue() + ", appMasterRpcPort=" + report.GetRpcPort() + ", appStartTime=" + report.GetStartTime() + ", yarnAppState=" + report.GetYarnApplicationState().ToString() + ", distributedFinalState=" + report .GetFinalApplicationStatus().ToString() + ", appTrackingUrl=" + report.GetTrackingUrl () + ", appUser="******"Application has completed successfully. Breaking monitoring loop"); return(true); } else { Log.Info("Application did finished unsuccessfully." + " YarnState=" + state.ToString () + ", DSFinalStatus=" + dsStatus.ToString() + ". Breaking monitoring loop"); return(false); } } else { if (YarnApplicationState.Killed == state || YarnApplicationState.Failed == state) { Log.Info("Application did not finish." + " YarnState=" + state.ToString() + ", DSFinalStatus=" + dsStatus.ToString() + ". Breaking monitoring loop"); return(false); } } if (Runtime.CurrentTimeMillis() > (clientStartTime + clientTimeout)) { Log.Info("Reached client specified timeout for application. Killing application"); ForceKillApplication(appId); return(false); } } }
private string ClairfyAppFinalStatus(FinalApplicationStatus status) { if (status == FinalApplicationStatus.Undefined) { return("Application has not completed yet."); } return(status.ToString()); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> public virtual bool Run() { Log.Info("Starting Client"); // Connect to ResourceManager rmClient.Start(); try { // Create launch context for app master Log.Info("Setting up application submission context for ASM"); ApplicationSubmissionContext appContext = rmClient.CreateApplication().GetApplicationSubmissionContext (); ApplicationId appId = appContext.GetApplicationId(); // set the application name appContext.SetApplicationName(appName); // Set the priority for the application master Priority pri = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <Priority>(); pri.SetPriority(amPriority); appContext.SetPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.SetQueue(amQueue); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <ContainerLaunchContext>(); appContext.SetAMContainerSpec(amContainer); // unmanaged AM appContext.SetUnmanagedAM(true); Log.Info("Setting unmanaged AM"); // Submit the application to the applications manager Log.Info("Submitting application to ASM"); rmClient.SubmitApplication(appContext); ApplicationReport appReport = MonitorApplication(appId, EnumSet.Of(YarnApplicationState .Accepted, YarnApplicationState.Killed, YarnApplicationState.Failed, YarnApplicationState .Finished)); if (appReport.GetYarnApplicationState() == YarnApplicationState.Accepted) { // Monitor the application attempt to wait for launch state ApplicationAttemptReport attemptReport = MonitorCurrentAppAttempt(appId, YarnApplicationAttemptState .Launched); ApplicationAttemptId attemptId = attemptReport.GetApplicationAttemptId(); Log.Info("Launching AM with application attempt id " + attemptId); // launch AM LaunchAM(attemptId); // Monitor the application for end state appReport = MonitorApplication(appId, EnumSet.Of(YarnApplicationState.Killed, YarnApplicationState .Failed, YarnApplicationState.Finished)); } YarnApplicationState appState = appReport.GetYarnApplicationState(); FinalApplicationStatus appStatus = appReport.GetFinalApplicationStatus(); Log.Info("App ended with state: " + appReport.GetYarnApplicationState() + " and status: " + appStatus); bool success; if (YarnApplicationState.Finished == appState && FinalApplicationStatus.Succeeded == appStatus) { Log.Info("Application has completed successfully."); success = true; } else { Log.Info("Application did finished unsuccessfully." + " YarnState=" + appState.ToString () + ", FinalStatus=" + appStatus.ToString()); success = false; } return(success); } finally { rmClient.Stop(); } }
/* * FinalApplicationStatus */ public static YarnProtos.FinalApplicationStatusProto ConvertToProtoFormat(FinalApplicationStatus e) { return(YarnProtos.FinalApplicationStatusProto.ValueOf(FinalApplicationStatusPrefix + e.ToString())); }