public RMAppAttemptUnregistrationEvent(ApplicationAttemptId appAttemptId, string trackingUrl, FinalApplicationStatus finalStatus, string diagnostics) : base(appAttemptId, RMAppAttemptEventType.Unregistered, diagnostics) { this.finalTrackingUrl = trackingUrl; this.finalStatus = finalStatus; }
public static ApplicationReport NewApplicationReport(ApplicationId applicationId, ApplicationAttemptId applicationAttemptId, string user, string queue, string name , string host, int rpcPort, Org.Apache.Hadoop.Yarn.Api.Records.Token clientToAMToken , YarnApplicationState state, string diagnostics, string url, long startTime, long finishTime, FinalApplicationStatus finalStatus, ApplicationResourceUsageReport appResources, string origTrackingUrl, float progress, string appType, Org.Apache.Hadoop.Yarn.Api.Records.Token amRmToken, ICollection <string> tags) { ApplicationReport report = recordFactory.NewRecordInstance <ApplicationReport>(); report.SetApplicationId(applicationId); report.SetCurrentApplicationAttemptId(applicationAttemptId); report.SetUser(user); report.SetQueue(queue); report.SetName(name); report.SetHost(host); report.SetRpcPort(rpcPort); report.SetClientToAMToken(clientToAMToken); report.SetYarnApplicationState(state); report.SetDiagnostics(diagnostics); report.SetTrackingUrl(url); report.SetStartTime(startTime); report.SetFinishTime(finishTime); report.SetFinalApplicationStatus(finalStatus); report.SetApplicationResourceUsageReport(appResources); report.SetOriginalTrackingUrl(origTrackingUrl); report.SetProgress(progress); report.SetApplicationType(appType); report.SetAMRMToken(amRmToken); report.SetApplicationTags(tags); return(report); }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> public override void UnregisterApplicationMaster(FinalApplicationStatus appStatus , string appMessage, string appTrackingUrl) { Preconditions.CheckArgument(appStatus != null, "AppStatus should not be null."); FinishApplicationMasterRequest request = FinishApplicationMasterRequest.NewInstance (appStatus, appMessage, appTrackingUrl); try { while (true) { FinishApplicationMasterResponse response = rmClient.FinishApplicationMaster(request ); if (response.GetIsUnregistered()) { break; } Log.Info("Waiting for application to be successfully unregistered."); Sharpen.Thread.Sleep(100); } } catch (Exception) { Log.Info("Interrupted while waiting for application" + " to be removed from RMStateStore" ); } catch (ApplicationMasterNotRegisteredException) { Log.Warn("ApplicationMaster is out of sync with ResourceManager," + " hence resyncing." ); // re register with RM RegisterApplicationMaster(); UnregisterApplicationMaster(appStatus, appMessage, appTrackingUrl); } }
public AppInfo(ApplicationReport app) { // JAXB needs this appId = app.GetApplicationId().ToString(); if (app.GetCurrentApplicationAttemptId() != null) { currentAppAttemptId = app.GetCurrentApplicationAttemptId().ToString(); } user = app.GetUser(); queue = app.GetQueue(); name = app.GetName(); type = app.GetApplicationType(); host = app.GetHost(); rpcPort = app.GetRpcPort(); appState = app.GetYarnApplicationState(); diagnosticsInfo = app.GetDiagnostics(); trackingUrl = app.GetTrackingUrl(); originalTrackingUrl = app.GetOriginalTrackingUrl(); submittedTime = app.GetStartTime(); startedTime = app.GetStartTime(); finishedTime = app.GetFinishTime(); elapsedTime = Times.Elapsed(startedTime, finishedTime); finalAppStatus = app.GetFinalApplicationStatus(); progress = app.GetProgress() * 100; // in percent if (app.GetApplicationTags() != null && !app.GetApplicationTags().IsEmpty()) { this.applicationTags = StringHelper.CsvJoiner.Join(app.GetApplicationTags()); } }
private string ClairfyAppFinalStatus(FinalApplicationStatus status) { if (status == FinalApplicationStatus.Undefined) { return("Application has not completed yet."); } return(status.ToString()); }
/// <summary>Unregister the application master.</summary> /// <remarks>Unregister the application master. This must be called in the end.</remarks> /// <param name="appStatus">Success/Failure status of the master</param> /// <param name="appMessage">Diagnostics message on failure</param> /// <param name="appTrackingUrl">New URL to get master info</param> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> public override void UnregisterApplicationMaster(FinalApplicationStatus appStatus , string appMessage, string appTrackingUrl) { lock (unregisterHeartbeatLock) { keepRunning = false; client.UnregisterApplicationMaster(appStatus, appMessage, appTrackingUrl); } }
public override void SetFinalApplicationStatus(FinalApplicationStatus finalState) { MaybeInitBuilder(); if (finalState == null) { builder.ClearFinalApplicationStatus(); return; } builder.SetFinalApplicationStatus(ConvertToProtoFormat(finalState)); }
/// <summary>Monitor the submitted application for completion.</summary> /// <remarks> /// Monitor the submitted application for completion. /// Kill application if time expires. /// </remarks> /// <param name="appId">Application Id of application to be monitored</param> /// <returns>true if application completed successfully</returns> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> private bool MonitorApplication(ApplicationId appId) { while (true) { // Check app status every 1 second. try { Sharpen.Thread.Sleep(1000); } catch (Exception) { Log.Debug("Thread sleep in monitoring loop interrupted"); } // Get application report for the appId we are interested in ApplicationReport report = yarnClient.GetApplicationReport(appId); Log.Info("Got application report from ASM for" + ", appId=" + appId.GetId() + ", clientToAMToken=" + report.GetClientToAMToken() + ", appDiagnostics=" + report.GetDiagnostics() + ", appMasterHost=" + report.GetHost() + ", appQueue=" + report.GetQueue() + ", appMasterRpcPort=" + report.GetRpcPort() + ", appStartTime=" + report.GetStartTime() + ", yarnAppState=" + report.GetYarnApplicationState().ToString() + ", distributedFinalState=" + report .GetFinalApplicationStatus().ToString() + ", appTrackingUrl=" + report.GetTrackingUrl () + ", appUser="******"Application has completed successfully. Breaking monitoring loop"); return(true); } else { Log.Info("Application did finished unsuccessfully." + " YarnState=" + state.ToString () + ", DSFinalStatus=" + dsStatus.ToString() + ". Breaking monitoring loop"); return(false); } } else { if (YarnApplicationState.Killed == state || YarnApplicationState.Failed == state) { Log.Info("Application did not finish." + " YarnState=" + state.ToString() + ", DSFinalStatus=" + dsStatus.ToString() + ". Breaking monitoring loop"); return(false); } } if (Runtime.CurrentTimeMillis() > (clientStartTime + clientTimeout)) { Log.Info("Reached client specified timeout for application. Killing application"); ForceKillApplication(appId); return(false); } } }
public static FinishApplicationMasterRequest NewInstance(FinalApplicationStatus finalAppStatus , string diagnostics, string url) { FinishApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <FinishApplicationMasterRequest>(); request.SetFinalApplicationStatus(finalAppStatus); request.SetDiagnostics(diagnostics); request.SetTrackingUrl(url); return(request); }
public ApplicationFinishedEvent(ApplicationId appId, string diagnosticsInfo, FinalApplicationStatus appStatus, YarnApplicationState state, ApplicationAttemptId latestAppAttemptId, long finishedTime, RMAppMetrics appMetrics) : base(SystemMetricsEventType.AppFinished, finishedTime) { this.appId = appId; this.diagnosticsInfo = diagnosticsInfo; this.appStatus = appStatus; this.latestAppAttemptId = latestAppAttemptId; this.state = state; this.appMetrics = appMetrics; }
public AppAttemptFinishedEvent(ApplicationAttemptId appAttemptId, string trackingUrl , string originalTrackingUrl, string diagnosticsInfo, FinalApplicationStatus appStatus , YarnApplicationAttemptState state, long finishedTime) : base(SystemMetricsEventType.AppAttemptFinished, finishedTime) { this.appAttemptId = appAttemptId; // This is the tracking URL after the application attempt is finished this.trackingUrl = trackingUrl; this.originalTrackingUrl = originalTrackingUrl; this.diagnosticsInfo = diagnosticsInfo; this.appStatus = appStatus; this.state = state; }
public static JobStatus.State FromYarn(YarnApplicationState yarnApplicationState, FinalApplicationStatus finalApplicationStatus) { switch (yarnApplicationState) { case YarnApplicationState.New: case YarnApplicationState.NewSaving: case YarnApplicationState.Submitted: case YarnApplicationState.Accepted: { return(JobStatus.State.Prep); } case YarnApplicationState.Running: { return(JobStatus.State.Running); } case YarnApplicationState.Finished: { if (finalApplicationStatus == FinalApplicationStatus.Succeeded) { return(JobStatus.State.Succeeded); } else { if (finalApplicationStatus == FinalApplicationStatus.Killed) { return(JobStatus.State.Killed); } } goto case YarnApplicationState.Failed; } case YarnApplicationState.Failed: { return(JobStatus.State.Failed); } case YarnApplicationState.Killed: { return(JobStatus.State.Killed); } } throw new YarnRuntimeException("Unrecognized application state: " + yarnApplicationState ); }
private ApplicationReport GetApplicationReport(YarnApplicationState yarnApplicationState , FinalApplicationStatus finalApplicationStatus) { ApplicationReport appReport = Org.Mockito.Mockito.Mock <ApplicationReport>(); ApplicationResourceUsageReport appResources = Org.Mockito.Mockito.Mock <ApplicationResourceUsageReport >(); Org.Mockito.Mockito.When(appReport.GetApplicationId()).ThenReturn(ApplicationId.NewInstance (0, 0)); Org.Mockito.Mockito.When(appResources.GetNeededResources()).ThenReturn(Org.Apache.Hadoop.Yarn.Util.Records .NewRecord <Resource>()); Org.Mockito.Mockito.When(appResources.GetReservedResources()).ThenReturn(Org.Apache.Hadoop.Yarn.Util.Records .NewRecord <Resource>()); Org.Mockito.Mockito.When(appResources.GetUsedResources()).ThenReturn(Org.Apache.Hadoop.Yarn.Util.Records .NewRecord <Resource>()); Org.Mockito.Mockito.When(appReport.GetApplicationResourceUsageReport()).ThenReturn (appResources); Org.Mockito.Mockito.When(appReport.GetYarnApplicationState()).ThenReturn(yarnApplicationState ); Org.Mockito.Mockito.When(appReport.GetFinalApplicationStatus()).ThenReturn(finalApplicationStatus ); return(appReport); }
private static ApplicationHistoryManagerOnTimelineStore.ApplicationReportExt ConvertToApplicationReport (TimelineEntity entity, ApplicationHistoryManagerOnTimelineStore.ApplicationReportField field) { string user = null; string queue = null; string name = null; string type = null; long createdTime = 0; long finishedTime = 0; ApplicationAttemptId latestApplicationAttemptId = null; string diagnosticsInfo = null; FinalApplicationStatus finalStatus = FinalApplicationStatus.Undefined; YarnApplicationState state = null; ApplicationResourceUsageReport appResources = null; IDictionary <ApplicationAccessType, string> appViewACLs = new Dictionary <ApplicationAccessType , string>(); IDictionary <string, object> entityInfo = entity.GetOtherInfo(); if (entityInfo != null) { if (entityInfo.Contains(ApplicationMetricsConstants.UserEntityInfo)) { user = entityInfo[ApplicationMetricsConstants.UserEntityInfo].ToString(); } if (entityInfo.Contains(ApplicationMetricsConstants.AppViewAclsEntityInfo)) { string appViewACLsStr = entityInfo[ApplicationMetricsConstants.AppViewAclsEntityInfo ].ToString(); if (appViewACLsStr.Length > 0) { appViewACLs[ApplicationAccessType.ViewApp] = appViewACLsStr; } } if (field == ApplicationHistoryManagerOnTimelineStore.ApplicationReportField.UserAndAcls) { return(new ApplicationHistoryManagerOnTimelineStore.ApplicationReportExt(ApplicationReport .NewInstance(ConverterUtils.ToApplicationId(entity.GetEntityId()), latestApplicationAttemptId , user, queue, name, null, -1, null, state, diagnosticsInfo, null, createdTime, finishedTime, finalStatus, null, null, 1.0F, type, null), appViewACLs)); } if (entityInfo.Contains(ApplicationMetricsConstants.QueueEntityInfo)) { queue = entityInfo[ApplicationMetricsConstants.QueueEntityInfo].ToString(); } if (entityInfo.Contains(ApplicationMetricsConstants.NameEntityInfo)) { name = entityInfo[ApplicationMetricsConstants.NameEntityInfo].ToString(); } if (entityInfo.Contains(ApplicationMetricsConstants.TypeEntityInfo)) { type = entityInfo[ApplicationMetricsConstants.TypeEntityInfo].ToString(); } if (entityInfo.Contains(ApplicationMetricsConstants.AppCpuMetrics)) { long vcoreSeconds = long.Parse(entityInfo[ApplicationMetricsConstants.AppCpuMetrics ].ToString()); long memorySeconds = long.Parse(entityInfo[ApplicationMetricsConstants.AppMemMetrics ].ToString()); appResources = ApplicationResourceUsageReport.NewInstance(0, 0, null, null, null, memorySeconds, vcoreSeconds); } } IList <TimelineEvent> events = entity.GetEvents(); if (events != null) { foreach (TimelineEvent @event in events) { if (@event.GetEventType().Equals(ApplicationMetricsConstants.CreatedEventType)) { createdTime = @event.GetTimestamp(); } else { if (@event.GetEventType().Equals(ApplicationMetricsConstants.FinishedEventType)) { finishedTime = @event.GetTimestamp(); IDictionary <string, object> eventInfo = @event.GetEventInfo(); if (eventInfo == null) { continue; } if (eventInfo.Contains(ApplicationMetricsConstants.LatestAppAttemptEventInfo)) { latestApplicationAttemptId = ConverterUtils.ToApplicationAttemptId(eventInfo[ApplicationMetricsConstants .LatestAppAttemptEventInfo].ToString()); } if (eventInfo.Contains(ApplicationMetricsConstants.DiagnosticsInfoEventInfo)) { diagnosticsInfo = eventInfo[ApplicationMetricsConstants.DiagnosticsInfoEventInfo] .ToString(); } if (eventInfo.Contains(ApplicationMetricsConstants.FinalStatusEventInfo)) { finalStatus = FinalApplicationStatus.ValueOf(eventInfo[ApplicationMetricsConstants .FinalStatusEventInfo].ToString()); } if (eventInfo.Contains(ApplicationMetricsConstants.StateEventInfo)) { state = YarnApplicationState.ValueOf(eventInfo[ApplicationMetricsConstants.StateEventInfo ].ToString()); } } } } } return(new ApplicationHistoryManagerOnTimelineStore.ApplicationReportExt(ApplicationReport .NewInstance(ConverterUtils.ToApplicationId(entity.GetEntityId()), latestApplicationAttemptId , user, queue, name, null, -1, null, state, diagnosticsInfo, null, createdTime, finishedTime, finalStatus, appResources, null, 1.0F, type, null), appViewACLs)); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> public virtual bool Run() { Log.Info("Starting Client"); // Connect to ResourceManager rmClient.Start(); try { // Create launch context for app master Log.Info("Setting up application submission context for ASM"); ApplicationSubmissionContext appContext = rmClient.CreateApplication().GetApplicationSubmissionContext (); ApplicationId appId = appContext.GetApplicationId(); // set the application name appContext.SetApplicationName(appName); // Set the priority for the application master Priority pri = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <Priority>(); pri.SetPriority(amPriority); appContext.SetPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.SetQueue(amQueue); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <ContainerLaunchContext>(); appContext.SetAMContainerSpec(amContainer); // unmanaged AM appContext.SetUnmanagedAM(true); Log.Info("Setting unmanaged AM"); // Submit the application to the applications manager Log.Info("Submitting application to ASM"); rmClient.SubmitApplication(appContext); ApplicationReport appReport = MonitorApplication(appId, EnumSet.Of(YarnApplicationState .Accepted, YarnApplicationState.Killed, YarnApplicationState.Failed, YarnApplicationState .Finished)); if (appReport.GetYarnApplicationState() == YarnApplicationState.Accepted) { // Monitor the application attempt to wait for launch state ApplicationAttemptReport attemptReport = MonitorCurrentAppAttempt(appId, YarnApplicationAttemptState .Launched); ApplicationAttemptId attemptId = attemptReport.GetApplicationAttemptId(); Log.Info("Launching AM with application attempt id " + attemptId); // launch AM LaunchAM(attemptId); // Monitor the application for end state appReport = MonitorApplication(appId, EnumSet.Of(YarnApplicationState.Killed, YarnApplicationState .Failed, YarnApplicationState.Finished)); } YarnApplicationState appState = appReport.GetYarnApplicationState(); FinalApplicationStatus appStatus = appReport.GetFinalApplicationStatus(); Log.Info("App ended with state: " + appReport.GetYarnApplicationState() + " and status: " + appStatus); bool success; if (YarnApplicationState.Finished == appState && FinalApplicationStatus.Succeeded == appStatus) { Log.Info("Application has completed successfully."); success = true; } else { Log.Info("Application did finished unsuccessfully." + " YarnState=" + appState.ToString () + ", FinalStatus=" + appStatus.ToString()); success = false; } return(success); } finally { rmClient.Stop(); } }
public static ApplicationFinishData NewInstance(ApplicationId applicationId, long finishTime, string diagnosticsInfo, FinalApplicationStatus finalApplicationStatus , YarnApplicationState yarnApplicationState) { ApplicationFinishData appFD = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <ApplicationFinishData >(); appFD.SetApplicationId(applicationId); appFD.SetFinishTime(finishTime); appFD.SetDiagnosticsInfo(diagnosticsInfo); appFD.SetFinalApplicationStatus(finalApplicationStatus); appFD.SetYarnApplicationState(yarnApplicationState); return(appFD); }
public static ApplicationAttemptFinishData NewInstance(ApplicationAttemptId appAttemptId , string diagnosticsInfo, string trackingURL, FinalApplicationStatus finalApplicationStatus , YarnApplicationAttemptState yarnApplicationAttemptState) { ApplicationAttemptFinishData appAttemptFD = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <ApplicationAttemptFinishData>(); appAttemptFD.SetApplicationAttemptId(appAttemptId); appAttemptFD.SetDiagnosticsInfo(diagnosticsInfo); appAttemptFD.SetTrackingURL(trackingURL); appAttemptFD.SetFinalApplicationStatus(finalApplicationStatus); appAttemptFD.SetYarnApplicationAttemptState(yarnApplicationAttemptState); return(appAttemptFD); }
public abstract void SetFinalApplicationStatus(FinalApplicationStatus finalApplicationStatus );
public virtual void SetFinalApplicationStatus(FinalApplicationStatus finalApplicationStatus ) { this.finalApplicationStatus = finalApplicationStatus; }
public static ApplicationAttemptHistoryData NewInstance(ApplicationAttemptId appAttemptId , string host, int rpcPort, ContainerId masterContainerId, string diagnosticsInfo , string trackingURL, FinalApplicationStatus finalApplicationStatus, YarnApplicationAttemptState yarnApplicationAttemptState) { ApplicationAttemptHistoryData appAttemptHD = new ApplicationAttemptHistoryData(); appAttemptHD.SetApplicationAttemptId(appAttemptId); appAttemptHD.SetHost(host); appAttemptHD.SetRPCPort(rpcPort); appAttemptHD.SetMasterContainerId(masterContainerId); appAttemptHD.SetDiagnosticsInfo(diagnosticsInfo); appAttemptHD.SetTrackingURL(trackingURL); appAttemptHD.SetFinalApplicationStatus(finalApplicationStatus); appAttemptHD.SetYarnApplicationAttemptState(yarnApplicationAttemptState); return(appAttemptHD); }
public AppInfo(ResourceManager rm, RMApp app, bool hasAccess, string schemePrefix ) { // these are ok for any user to see // these are only allowed if acls allow // preemption info fields // JAXB needs this this.schemePrefix = schemePrefix; if (app != null) { string trackingUrl = app.GetTrackingUrl(); this.state = app.CreateApplicationState(); this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.IsEmpty() || YarnApplicationState .New == this.state || YarnApplicationState.NewSaving == this.state || YarnApplicationState .Submitted == this.state || YarnApplicationState.Accepted == this.state; this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app.GetFinishTime( ) == 0 ? "ApplicationMaster" : "History"); if (!trackingUrlIsNotReady) { this.trackingUrl = WebAppUtils.GetURLWithScheme(schemePrefix, trackingUrl); this.trackingUrlPretty = this.trackingUrl; } else { this.trackingUrlPretty = "UNASSIGNED"; } this.applicationId = app.GetApplicationId(); this.applicationType = app.GetApplicationType(); this.appIdNum = app.GetApplicationId().GetId().ToString(); this.id = app.GetApplicationId().ToString(); this.user = app.GetUser().ToString(); this.name = app.GetName().ToString(); this.queue = app.GetQueue().ToString(); this.progress = app.GetProgress() * 100; this.diagnostics = app.GetDiagnostics().ToString(); if (diagnostics == null || diagnostics.IsEmpty()) { this.diagnostics = string.Empty; } if (app.GetApplicationTags() != null && !app.GetApplicationTags().IsEmpty()) { this.applicationTags = Joiner.On(',').Join(app.GetApplicationTags()); } this.finalStatus = app.GetFinalApplicationStatus(); this.clusterId = ResourceManager.GetClusterTimeStamp(); if (hasAccess) { this.startedTime = app.GetStartTime(); this.finishedTime = app.GetFinishTime(); this.elapsedTime = Times.Elapsed(app.GetStartTime(), app.GetFinishTime()); RMAppAttempt attempt = app.GetCurrentAppAttempt(); if (attempt != null) { Container masterContainer = attempt.GetMasterContainer(); if (masterContainer != null) { this.amContainerLogsExist = true; this.amContainerLogs = WebAppUtils.GetRunningLogURL(schemePrefix + masterContainer .GetNodeHttpAddress(), ConverterUtils.ToString(masterContainer.GetId()), app.GetUser ()); this.amHostHttpAddress = masterContainer.GetNodeHttpAddress(); } ApplicationResourceUsageReport resourceReport = attempt.GetApplicationResourceUsageReport (); if (resourceReport != null) { Resource usedResources = resourceReport.GetUsedResources(); allocatedMB = usedResources.GetMemory(); allocatedVCores = usedResources.GetVirtualCores(); runningContainers = resourceReport.GetNumUsedContainers(); } resourceRequests = ((AbstractYarnScheduler)rm.GetRMContext().GetScheduler()).GetPendingResourceRequestsForAttempt (attempt.GetAppAttemptId()); } } // copy preemption info fields RMAppMetrics appMetrics = app.GetRMAppMetrics(); numAMContainerPreempted = appMetrics.GetNumAMContainersPreempted(); preemptedResourceMB = appMetrics.GetResourcePreempted().GetMemory(); numNonAMContainerPreempted = appMetrics.GetNumNonAMContainersPreempted(); preemptedResourceVCores = appMetrics.GetResourcePreempted().GetVirtualCores(); memorySeconds = appMetrics.GetMemorySeconds(); vcoreSeconds = appMetrics.GetVcoreSeconds(); } }
private YarnProtos.FinalApplicationStatusProto ConvertToProtoFormat(FinalApplicationStatus s) { return(ProtoUtils.ConvertToProtoFormat(s)); }
/* * FinalApplicationStatus */ public static YarnProtos.FinalApplicationStatusProto ConvertToProtoFormat(FinalApplicationStatus e) { return(YarnProtos.FinalApplicationStatusProto.ValueOf(FinalApplicationStatusPrefix + e.ToString())); }
public static FinalApplicationStatus ConvertFromProtoFormat(YarnProtos.FinalApplicationStatusProto e) { return(FinalApplicationStatus.ValueOf(e.ToString().Replace(FinalApplicationStatusPrefix , string.Empty))); }
public static ApplicationHistoryData NewInstance(ApplicationId applicationId, string applicationName, string applicationType, string queue, string user, long submitTime , long startTime, long finishTime, string diagnosticsInfo, FinalApplicationStatus finalApplicationStatus, YarnApplicationState yarnApplicationState) { ApplicationHistoryData appHD = new ApplicationHistoryData(); appHD.SetApplicationId(applicationId); appHD.SetApplicationName(applicationName); appHD.SetApplicationType(applicationType); appHD.SetQueue(queue); appHD.SetUser(user); appHD.SetSubmitTime(submitTime); appHD.SetStartTime(startTime); appHD.SetFinishTime(finishTime); appHD.SetDiagnosticsInfo(diagnosticsInfo); appHD.SetFinalApplicationStatus(finalApplicationStatus); appHD.SetYarnApplicationState(yarnApplicationState); return(appHD); }
/// <summary>Unregister the application master.</summary> /// <remarks>Unregister the application master. This must be called in the end.</remarks> /// <param name="appStatus">Success/Failure status of the master</param> /// <param name="appMessage">Diagnostics message on failure</param> /// <param name="appTrackingUrl">New URL to get master info</param> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> public abstract void UnregisterApplicationMaster(FinalApplicationStatus appStatus , string appMessage, string appTrackingUrl);
public virtual AppsInfo GetApps(HttpServletRequest req, HttpServletResponse res, string stateQuery, ICollection <string> statesQuery, string finalStatusQuery, string userQuery, string queueQuery, string count, string startedBegin, string startedEnd , string finishBegin, string finishEnd, ICollection <string> applicationTypes) { UserGroupInformation callerUGI = GetUser(req); bool checkStart = false; bool checkEnd = false; bool checkAppTypes = false; bool checkAppStates = false; long countNum = long.MaxValue; // set values suitable in case both of begin/end not specified long sBegin = 0; long sEnd = long.MaxValue; long fBegin = 0; long fEnd = long.MaxValue; if (count != null && !count.IsEmpty()) { countNum = long.Parse(count); if (countNum <= 0) { throw new BadRequestException("limit value must be greater then 0"); } } if (startedBegin != null && !startedBegin.IsEmpty()) { checkStart = true; sBegin = long.Parse(startedBegin); if (sBegin < 0) { throw new BadRequestException("startedTimeBegin must be greater than 0"); } } if (startedEnd != null && !startedEnd.IsEmpty()) { checkStart = true; sEnd = long.Parse(startedEnd); if (sEnd < 0) { throw new BadRequestException("startedTimeEnd must be greater than 0"); } } if (sBegin > sEnd) { throw new BadRequestException("startedTimeEnd must be greater than startTimeBegin" ); } if (finishBegin != null && !finishBegin.IsEmpty()) { checkEnd = true; fBegin = long.Parse(finishBegin); if (fBegin < 0) { throw new BadRequestException("finishTimeBegin must be greater than 0"); } } if (finishEnd != null && !finishEnd.IsEmpty()) { checkEnd = true; fEnd = long.Parse(finishEnd); if (fEnd < 0) { throw new BadRequestException("finishTimeEnd must be greater than 0"); } } if (fBegin > fEnd) { throw new BadRequestException("finishTimeEnd must be greater than finishTimeBegin" ); } ICollection <string> appTypes = ParseQueries(applicationTypes, false); if (!appTypes.IsEmpty()) { checkAppTypes = true; } // stateQuery is deprecated. if (stateQuery != null && !stateQuery.IsEmpty()) { statesQuery.AddItem(stateQuery); } ICollection <string> appStates = ParseQueries(statesQuery, true); if (!appStates.IsEmpty()) { checkAppStates = true; } AppsInfo allApps = new AppsInfo(); ICollection <ApplicationReport> appReports = null; GetApplicationsRequest request = GetApplicationsRequest.NewInstance(); request.SetLimit(countNum); try { if (callerUGI == null) { // TODO: the request should take the params like what RMWebServices does // in YARN-1819. appReports = appBaseProt.GetApplications(request).GetApplicationList(); } else { appReports = callerUGI.DoAs(new _PrivilegedExceptionAction_161(this, request)); } } catch (Exception e) { RewrapAndThrowException(e); } foreach (ApplicationReport appReport in appReports) { if (checkAppStates && !appStates.Contains(StringUtils.ToLowerCase(appReport.GetYarnApplicationState ().ToString()))) { continue; } if (finalStatusQuery != null && !finalStatusQuery.IsEmpty()) { FinalApplicationStatus.ValueOf(finalStatusQuery); if (!Sharpen.Runtime.EqualsIgnoreCase(appReport.GetFinalApplicationStatus().ToString (), finalStatusQuery)) { continue; } } if (userQuery != null && !userQuery.IsEmpty()) { if (!appReport.GetUser().Equals(userQuery)) { continue; } } if (queueQuery != null && !queueQuery.IsEmpty()) { if (!appReport.GetQueue().Equals(queueQuery)) { continue; } } if (checkAppTypes && !appTypes.Contains(StringUtils.ToLowerCase(appReport.GetApplicationType ().Trim()))) { continue; } if (checkStart && (appReport.GetStartTime() < sBegin || appReport.GetStartTime() > sEnd)) { continue; } if (checkEnd && (appReport.GetFinishTime() < fBegin || appReport.GetFinishTime() > fEnd)) { continue; } AppInfo app = new AppInfo(appReport); allApps.Add(app); } return(allApps); }
protected internal virtual void DoUnregistration() { FinalApplicationStatus finishState = FinalApplicationStatus.Undefined; JobImpl jobImpl = (JobImpl)job; if (jobImpl.GetInternalState() == JobStateInternal.Succeeded) { finishState = FinalApplicationStatus.Succeeded; } else { if (jobImpl.GetInternalState() == JobStateInternal.Killed || (jobImpl.GetInternalState () == JobStateInternal.Running && isSignalled)) { finishState = FinalApplicationStatus.Killed; } else { if (jobImpl.GetInternalState() == JobStateInternal.Failed || jobImpl.GetInternalState () == JobStateInternal.Error) { finishState = FinalApplicationStatus.Failed; } } } StringBuilder sb = new StringBuilder(); foreach (string s in job.GetDiagnostics()) { sb.Append(s).Append("\n"); } Log.Info("Setting job diagnostics to " + sb.ToString()); string historyUrl = MRWebAppUtil.GetApplicationWebURLOnJHSWithScheme(GetConfig(), context.GetApplicationID()); Log.Info("History url is " + historyUrl); FinishApplicationMasterRequest request = FinishApplicationMasterRequest.NewInstance (finishState, sb.ToString(), historyUrl); try { while (true) { FinishApplicationMasterResponse response = scheduler.FinishApplicationMaster(request ); if (response.GetIsUnregistered()) { // When excepting ClientService, other services are already stopped, // it is safe to let clients know the final states. ClientService // should wait for some time so clients have enough time to know the // final states. MRAppMaster.RunningAppContext raContext = (MRAppMaster.RunningAppContext)context; raContext.MarkSuccessfulUnregistration(); break; } Log.Info("Waiting for application to be successfully unregistered."); Sharpen.Thread.Sleep(rmPollInterval); } } catch (ApplicationMasterNotRegisteredException) { // RM might have restarted or failed over and so lost the fact that AM had // registered before. Register(); DoUnregistration(); } }
/* * Contains the state data that needs to be persisted for an ApplicationAttempt */ public static ApplicationAttemptStateData NewInstance(ApplicationAttemptId attemptId , Container container, Credentials attemptTokens, long startTime, RMAppAttemptState finalState, string finalTrackingUrl, string diagnostics, FinalApplicationStatus amUnregisteredFinalStatus, int exitStatus, long finishTime, long memorySeconds, long vcoreSeconds) { ApplicationAttemptStateData attemptStateData = Org.Apache.Hadoop.Yarn.Util.Records .NewRecord <ApplicationAttemptStateData>(); attemptStateData.SetAttemptId(attemptId); attemptStateData.SetMasterContainer(container); attemptStateData.SetAppAttemptTokens(attemptTokens); attemptStateData.SetState(finalState); attemptStateData.SetFinalTrackingUrl(finalTrackingUrl); attemptStateData.SetDiagnostics(diagnostics == null ? string.Empty : diagnostics); attemptStateData.SetStartTime(startTime); attemptStateData.SetFinalApplicationStatus(amUnregisteredFinalStatus); attemptStateData.SetAMContainerExitStatus(exitStatus); attemptStateData.SetFinishTime(finishTime); attemptStateData.SetMemorySeconds(memorySeconds); attemptStateData.SetVcoreSeconds(vcoreSeconds); return(attemptStateData); }