public RMAppAttemptUnregistrationEvent(ApplicationAttemptId appAttemptId, string
trackingUrl, FinalApplicationStatus finalStatus, string diagnostics)
: base(appAttemptId, RMAppAttemptEventType.Unregistered, diagnostics)
{
this.finalTrackingUrl = trackingUrl;
this.finalStatus = finalStatus;
}
public static ApplicationReport NewApplicationReport(ApplicationId applicationId,
ApplicationAttemptId applicationAttemptId, string user, string queue, string name
, string host, int rpcPort, Org.Apache.Hadoop.Yarn.Api.Records.Token clientToAMToken
, YarnApplicationState state, string diagnostics, string url, long startTime, long
finishTime, FinalApplicationStatus finalStatus, ApplicationResourceUsageReport
appResources, string origTrackingUrl, float progress, string appType, Org.Apache.Hadoop.Yarn.Api.Records.Token
amRmToken, ICollection <string> tags)
{
ApplicationReport report = recordFactory.NewRecordInstance <ApplicationReport>();
report.SetApplicationId(applicationId);
report.SetCurrentApplicationAttemptId(applicationAttemptId);
report.SetUser(user);
report.SetQueue(queue);
report.SetName(name);
report.SetHost(host);
report.SetRpcPort(rpcPort);
report.SetClientToAMToken(clientToAMToken);
report.SetYarnApplicationState(state);
report.SetDiagnostics(diagnostics);
report.SetTrackingUrl(url);
report.SetStartTime(startTime);
report.SetFinishTime(finishTime);
report.SetFinalApplicationStatus(finalStatus);
report.SetApplicationResourceUsageReport(appResources);
report.SetOriginalTrackingUrl(origTrackingUrl);
report.SetProgress(progress);
report.SetApplicationType(appType);
report.SetAMRMToken(amRmToken);
report.SetApplicationTags(tags);
return(report);
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public override void UnregisterApplicationMaster(FinalApplicationStatus appStatus
, string appMessage, string appTrackingUrl)
{
Preconditions.CheckArgument(appStatus != null, "AppStatus should not be null.");
FinishApplicationMasterRequest request = FinishApplicationMasterRequest.NewInstance
(appStatus, appMessage, appTrackingUrl);
try
{
while (true)
{
FinishApplicationMasterResponse response = rmClient.FinishApplicationMaster(request
);
if (response.GetIsUnregistered())
{
break;
}
Log.Info("Waiting for application to be successfully unregistered.");
Sharpen.Thread.Sleep(100);
}
}
catch (Exception)
{
Log.Info("Interrupted while waiting for application" + " to be removed from RMStateStore"
);
}
catch (ApplicationMasterNotRegisteredException)
{
Log.Warn("ApplicationMaster is out of sync with ResourceManager," + " hence resyncing."
);
// re register with RM
RegisterApplicationMaster();
UnregisterApplicationMaster(appStatus, appMessage, appTrackingUrl);
}
}
public AppInfo(ApplicationReport app)
{
// JAXB needs this
appId = app.GetApplicationId().ToString();
if (app.GetCurrentApplicationAttemptId() != null)
{
currentAppAttemptId = app.GetCurrentApplicationAttemptId().ToString();
}
user = app.GetUser();
queue = app.GetQueue();
name = app.GetName();
type = app.GetApplicationType();
host = app.GetHost();
rpcPort = app.GetRpcPort();
appState = app.GetYarnApplicationState();
diagnosticsInfo = app.GetDiagnostics();
trackingUrl = app.GetTrackingUrl();
originalTrackingUrl = app.GetOriginalTrackingUrl();
submittedTime = app.GetStartTime();
startedTime = app.GetStartTime();
finishedTime = app.GetFinishTime();
elapsedTime = Times.Elapsed(startedTime, finishedTime);
finalAppStatus = app.GetFinalApplicationStatus();
progress = app.GetProgress() * 100;
// in percent
if (app.GetApplicationTags() != null && !app.GetApplicationTags().IsEmpty())
{
this.applicationTags = StringHelper.CsvJoiner.Join(app.GetApplicationTags());
}
}
private string ClairfyAppFinalStatus(FinalApplicationStatus status)
{
if (status == FinalApplicationStatus.Undefined)
{
return("Application has not completed yet.");
}
return(status.ToString());
}
/// <summary>Unregister the application master.</summary>
/// <remarks>Unregister the application master. This must be called in the end.</remarks>
/// <param name="appStatus">Success/Failure status of the master</param>
/// <param name="appMessage">Diagnostics message on failure</param>
/// <param name="appTrackingUrl">New URL to get master info</param>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public override void UnregisterApplicationMaster(FinalApplicationStatus appStatus
, string appMessage, string appTrackingUrl)
{
lock (unregisterHeartbeatLock)
{
keepRunning = false;
client.UnregisterApplicationMaster(appStatus, appMessage, appTrackingUrl);
}
}
public override void SetFinalApplicationStatus(FinalApplicationStatus finalState)
{
MaybeInitBuilder();
if (finalState == null)
{
builder.ClearFinalApplicationStatus();
return;
}
builder.SetFinalApplicationStatus(ConvertToProtoFormat(finalState));
}
/// <summary>Monitor the submitted application for completion.</summary>
/// <remarks>
/// Monitor the submitted application for completion.
/// Kill application if time expires.
/// </remarks>
/// <param name="appId">Application Id of application to be monitored</param>
/// <returns>true if application completed successfully</returns>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
private bool MonitorApplication(ApplicationId appId)
{
while (true)
{
// Check app status every 1 second.
try
{
Sharpen.Thread.Sleep(1000);
}
catch (Exception)
{
Log.Debug("Thread sleep in monitoring loop interrupted");
}
// Get application report for the appId we are interested in
ApplicationReport report = yarnClient.GetApplicationReport(appId);
Log.Info("Got application report from ASM for" + ", appId=" + appId.GetId() + ", clientToAMToken="
+ report.GetClientToAMToken() + ", appDiagnostics=" + report.GetDiagnostics() +
", appMasterHost=" + report.GetHost() + ", appQueue=" + report.GetQueue() + ", appMasterRpcPort="
+ report.GetRpcPort() + ", appStartTime=" + report.GetStartTime() + ", yarnAppState="
+ report.GetYarnApplicationState().ToString() + ", distributedFinalState=" + report
.GetFinalApplicationStatus().ToString() + ", appTrackingUrl=" + report.GetTrackingUrl
() + ", appUser="******"Application has completed successfully. Breaking monitoring loop");
return(true);
}
else
{
Log.Info("Application did finished unsuccessfully." + " YarnState=" + state.ToString
() + ", DSFinalStatus=" + dsStatus.ToString() + ". Breaking monitoring loop");
return(false);
}
}
else
{
if (YarnApplicationState.Killed == state || YarnApplicationState.Failed == state)
{
Log.Info("Application did not finish." + " YarnState=" + state.ToString() + ", DSFinalStatus="
+ dsStatus.ToString() + ". Breaking monitoring loop");
return(false);
}
}
if (Runtime.CurrentTimeMillis() > (clientStartTime + clientTimeout))
{
Log.Info("Reached client specified timeout for application. Killing application");
ForceKillApplication(appId);
return(false);
}
}
}
public static FinishApplicationMasterRequest NewInstance(FinalApplicationStatus finalAppStatus
, string diagnostics, string url)
{
FinishApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<FinishApplicationMasterRequest>();
request.SetFinalApplicationStatus(finalAppStatus);
request.SetDiagnostics(diagnostics);
request.SetTrackingUrl(url);
return(request);
}
public ApplicationFinishedEvent(ApplicationId appId, string diagnosticsInfo, FinalApplicationStatus
appStatus, YarnApplicationState state, ApplicationAttemptId latestAppAttemptId,
long finishedTime, RMAppMetrics appMetrics)
: base(SystemMetricsEventType.AppFinished, finishedTime)
{
this.appId = appId;
this.diagnosticsInfo = diagnosticsInfo;
this.appStatus = appStatus;
this.latestAppAttemptId = latestAppAttemptId;
this.state = state;
this.appMetrics = appMetrics;
}
public AppAttemptFinishedEvent(ApplicationAttemptId appAttemptId, string trackingUrl
, string originalTrackingUrl, string diagnosticsInfo, FinalApplicationStatus appStatus
, YarnApplicationAttemptState state, long finishedTime)
: base(SystemMetricsEventType.AppAttemptFinished, finishedTime)
{
this.appAttemptId = appAttemptId;
// This is the tracking URL after the application attempt is finished
this.trackingUrl = trackingUrl;
this.originalTrackingUrl = originalTrackingUrl;
this.diagnosticsInfo = diagnosticsInfo;
this.appStatus = appStatus;
this.state = state;
}
public static JobStatus.State FromYarn(YarnApplicationState yarnApplicationState,
FinalApplicationStatus finalApplicationStatus)
{
switch (yarnApplicationState)
{
case YarnApplicationState.New:
case YarnApplicationState.NewSaving:
case YarnApplicationState.Submitted:
case YarnApplicationState.Accepted:
{
return(JobStatus.State.Prep);
}
case YarnApplicationState.Running:
{
return(JobStatus.State.Running);
}
case YarnApplicationState.Finished:
{
if (finalApplicationStatus == FinalApplicationStatus.Succeeded)
{
return(JobStatus.State.Succeeded);
}
else
{
if (finalApplicationStatus == FinalApplicationStatus.Killed)
{
return(JobStatus.State.Killed);
}
}
goto case YarnApplicationState.Failed;
}
case YarnApplicationState.Failed:
{
return(JobStatus.State.Failed);
}
case YarnApplicationState.Killed:
{
return(JobStatus.State.Killed);
}
}
throw new YarnRuntimeException("Unrecognized application state: " + yarnApplicationState
);
}
private ApplicationReport GetApplicationReport(YarnApplicationState yarnApplicationState
, FinalApplicationStatus finalApplicationStatus)
{
ApplicationReport appReport = Org.Mockito.Mockito.Mock <ApplicationReport>();
ApplicationResourceUsageReport appResources = Org.Mockito.Mockito.Mock <ApplicationResourceUsageReport
>();
Org.Mockito.Mockito.When(appReport.GetApplicationId()).ThenReturn(ApplicationId.NewInstance
(0, 0));
Org.Mockito.Mockito.When(appResources.GetNeededResources()).ThenReturn(Org.Apache.Hadoop.Yarn.Util.Records
.NewRecord <Resource>());
Org.Mockito.Mockito.When(appResources.GetReservedResources()).ThenReturn(Org.Apache.Hadoop.Yarn.Util.Records
.NewRecord <Resource>());
Org.Mockito.Mockito.When(appResources.GetUsedResources()).ThenReturn(Org.Apache.Hadoop.Yarn.Util.Records
.NewRecord <Resource>());
Org.Mockito.Mockito.When(appReport.GetApplicationResourceUsageReport()).ThenReturn
(appResources);
Org.Mockito.Mockito.When(appReport.GetYarnApplicationState()).ThenReturn(yarnApplicationState
);
Org.Mockito.Mockito.When(appReport.GetFinalApplicationStatus()).ThenReturn(finalApplicationStatus
);
return(appReport);
}
private static ApplicationHistoryManagerOnTimelineStore.ApplicationReportExt ConvertToApplicationReport
(TimelineEntity entity, ApplicationHistoryManagerOnTimelineStore.ApplicationReportField
field)
{
string user = null;
string queue = null;
string name = null;
string type = null;
long createdTime = 0;
long finishedTime = 0;
ApplicationAttemptId latestApplicationAttemptId = null;
string diagnosticsInfo = null;
FinalApplicationStatus finalStatus = FinalApplicationStatus.Undefined;
YarnApplicationState state = null;
ApplicationResourceUsageReport appResources = null;
IDictionary <ApplicationAccessType, string> appViewACLs = new Dictionary <ApplicationAccessType
, string>();
IDictionary <string, object> entityInfo = entity.GetOtherInfo();
if (entityInfo != null)
{
if (entityInfo.Contains(ApplicationMetricsConstants.UserEntityInfo))
{
user = entityInfo[ApplicationMetricsConstants.UserEntityInfo].ToString();
}
if (entityInfo.Contains(ApplicationMetricsConstants.AppViewAclsEntityInfo))
{
string appViewACLsStr = entityInfo[ApplicationMetricsConstants.AppViewAclsEntityInfo
].ToString();
if (appViewACLsStr.Length > 0)
{
appViewACLs[ApplicationAccessType.ViewApp] = appViewACLsStr;
}
}
if (field == ApplicationHistoryManagerOnTimelineStore.ApplicationReportField.UserAndAcls)
{
return(new ApplicationHistoryManagerOnTimelineStore.ApplicationReportExt(ApplicationReport
.NewInstance(ConverterUtils.ToApplicationId(entity.GetEntityId()), latestApplicationAttemptId
, user, queue, name, null, -1, null, state, diagnosticsInfo, null, createdTime,
finishedTime, finalStatus, null, null, 1.0F, type, null), appViewACLs));
}
if (entityInfo.Contains(ApplicationMetricsConstants.QueueEntityInfo))
{
queue = entityInfo[ApplicationMetricsConstants.QueueEntityInfo].ToString();
}
if (entityInfo.Contains(ApplicationMetricsConstants.NameEntityInfo))
{
name = entityInfo[ApplicationMetricsConstants.NameEntityInfo].ToString();
}
if (entityInfo.Contains(ApplicationMetricsConstants.TypeEntityInfo))
{
type = entityInfo[ApplicationMetricsConstants.TypeEntityInfo].ToString();
}
if (entityInfo.Contains(ApplicationMetricsConstants.AppCpuMetrics))
{
long vcoreSeconds = long.Parse(entityInfo[ApplicationMetricsConstants.AppCpuMetrics
].ToString());
long memorySeconds = long.Parse(entityInfo[ApplicationMetricsConstants.AppMemMetrics
].ToString());
appResources = ApplicationResourceUsageReport.NewInstance(0, 0, null, null, null,
memorySeconds, vcoreSeconds);
}
}
IList <TimelineEvent> events = entity.GetEvents();
if (events != null)
{
foreach (TimelineEvent @event in events)
{
if (@event.GetEventType().Equals(ApplicationMetricsConstants.CreatedEventType))
{
createdTime = @event.GetTimestamp();
}
else
{
if (@event.GetEventType().Equals(ApplicationMetricsConstants.FinishedEventType))
{
finishedTime = @event.GetTimestamp();
IDictionary <string, object> eventInfo = @event.GetEventInfo();
if (eventInfo == null)
{
continue;
}
if (eventInfo.Contains(ApplicationMetricsConstants.LatestAppAttemptEventInfo))
{
latestApplicationAttemptId = ConverterUtils.ToApplicationAttemptId(eventInfo[ApplicationMetricsConstants
.LatestAppAttemptEventInfo].ToString());
}
if (eventInfo.Contains(ApplicationMetricsConstants.DiagnosticsInfoEventInfo))
{
diagnosticsInfo = eventInfo[ApplicationMetricsConstants.DiagnosticsInfoEventInfo]
.ToString();
}
if (eventInfo.Contains(ApplicationMetricsConstants.FinalStatusEventInfo))
{
finalStatus = FinalApplicationStatus.ValueOf(eventInfo[ApplicationMetricsConstants
.FinalStatusEventInfo].ToString());
}
if (eventInfo.Contains(ApplicationMetricsConstants.StateEventInfo))
{
state = YarnApplicationState.ValueOf(eventInfo[ApplicationMetricsConstants.StateEventInfo
].ToString());
}
}
}
}
}
return(new ApplicationHistoryManagerOnTimelineStore.ApplicationReportExt(ApplicationReport
.NewInstance(ConverterUtils.ToApplicationId(entity.GetEntityId()), latestApplicationAttemptId
, user, queue, name, null, -1, null, state, diagnosticsInfo, null, createdTime,
finishedTime, finalStatus, appResources, null, 1.0F, type, null), appViewACLs));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public virtual bool Run()
{
Log.Info("Starting Client");
// Connect to ResourceManager
rmClient.Start();
try
{
// Create launch context for app master
Log.Info("Setting up application submission context for ASM");
ApplicationSubmissionContext appContext = rmClient.CreateApplication().GetApplicationSubmissionContext
();
ApplicationId appId = appContext.GetApplicationId();
// set the application name
appContext.SetApplicationName(appName);
// Set the priority for the application master
Priority pri = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <Priority>();
pri.SetPriority(amPriority);
appContext.SetPriority(pri);
// Set the queue to which this application is to be submitted in the RM
appContext.SetQueue(amQueue);
// Set up the container launch context for the application master
ContainerLaunchContext amContainer = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<ContainerLaunchContext>();
appContext.SetAMContainerSpec(amContainer);
// unmanaged AM
appContext.SetUnmanagedAM(true);
Log.Info("Setting unmanaged AM");
// Submit the application to the applications manager
Log.Info("Submitting application to ASM");
rmClient.SubmitApplication(appContext);
ApplicationReport appReport = MonitorApplication(appId, EnumSet.Of(YarnApplicationState
.Accepted, YarnApplicationState.Killed, YarnApplicationState.Failed, YarnApplicationState
.Finished));
if (appReport.GetYarnApplicationState() == YarnApplicationState.Accepted)
{
// Monitor the application attempt to wait for launch state
ApplicationAttemptReport attemptReport = MonitorCurrentAppAttempt(appId, YarnApplicationAttemptState
.Launched);
ApplicationAttemptId attemptId = attemptReport.GetApplicationAttemptId();
Log.Info("Launching AM with application attempt id " + attemptId);
// launch AM
LaunchAM(attemptId);
// Monitor the application for end state
appReport = MonitorApplication(appId, EnumSet.Of(YarnApplicationState.Killed, YarnApplicationState
.Failed, YarnApplicationState.Finished));
}
YarnApplicationState appState = appReport.GetYarnApplicationState();
FinalApplicationStatus appStatus = appReport.GetFinalApplicationStatus();
Log.Info("App ended with state: " + appReport.GetYarnApplicationState() + " and status: "
+ appStatus);
bool success;
if (YarnApplicationState.Finished == appState && FinalApplicationStatus.Succeeded
== appStatus)
{
Log.Info("Application has completed successfully.");
success = true;
}
else
{
Log.Info("Application did finished unsuccessfully." + " YarnState=" + appState.ToString
() + ", FinalStatus=" + appStatus.ToString());
success = false;
}
return(success);
}
finally
{
rmClient.Stop();
}
}
public static ApplicationFinishData NewInstance(ApplicationId applicationId, long
finishTime, string diagnosticsInfo, FinalApplicationStatus finalApplicationStatus
, YarnApplicationState yarnApplicationState)
{
ApplicationFinishData appFD = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <ApplicationFinishData
>();
appFD.SetApplicationId(applicationId);
appFD.SetFinishTime(finishTime);
appFD.SetDiagnosticsInfo(diagnosticsInfo);
appFD.SetFinalApplicationStatus(finalApplicationStatus);
appFD.SetYarnApplicationState(yarnApplicationState);
return(appFD);
}
public static ApplicationAttemptFinishData NewInstance(ApplicationAttemptId appAttemptId
, string diagnosticsInfo, string trackingURL, FinalApplicationStatus finalApplicationStatus
, YarnApplicationAttemptState yarnApplicationAttemptState)
{
ApplicationAttemptFinishData appAttemptFD = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<ApplicationAttemptFinishData>();
appAttemptFD.SetApplicationAttemptId(appAttemptId);
appAttemptFD.SetDiagnosticsInfo(diagnosticsInfo);
appAttemptFD.SetTrackingURL(trackingURL);
appAttemptFD.SetFinalApplicationStatus(finalApplicationStatus);
appAttemptFD.SetYarnApplicationAttemptState(yarnApplicationAttemptState);
return(appAttemptFD);
}
public abstract void SetFinalApplicationStatus(FinalApplicationStatus finalApplicationStatus
);
public virtual void SetFinalApplicationStatus(FinalApplicationStatus finalApplicationStatus
)
{
this.finalApplicationStatus = finalApplicationStatus;
}
public static ApplicationAttemptHistoryData NewInstance(ApplicationAttemptId appAttemptId
, string host, int rpcPort, ContainerId masterContainerId, string diagnosticsInfo
, string trackingURL, FinalApplicationStatus finalApplicationStatus, YarnApplicationAttemptState
yarnApplicationAttemptState)
{
ApplicationAttemptHistoryData appAttemptHD = new ApplicationAttemptHistoryData();
appAttemptHD.SetApplicationAttemptId(appAttemptId);
appAttemptHD.SetHost(host);
appAttemptHD.SetRPCPort(rpcPort);
appAttemptHD.SetMasterContainerId(masterContainerId);
appAttemptHD.SetDiagnosticsInfo(diagnosticsInfo);
appAttemptHD.SetTrackingURL(trackingURL);
appAttemptHD.SetFinalApplicationStatus(finalApplicationStatus);
appAttemptHD.SetYarnApplicationAttemptState(yarnApplicationAttemptState);
return(appAttemptHD);
}
public AppInfo(ResourceManager rm, RMApp app, bool hasAccess, string schemePrefix
)
{
// these are ok for any user to see
// these are only allowed if acls allow
// preemption info fields
// JAXB needs this
this.schemePrefix = schemePrefix;
if (app != null)
{
string trackingUrl = app.GetTrackingUrl();
this.state = app.CreateApplicationState();
this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.IsEmpty() || YarnApplicationState
.New == this.state || YarnApplicationState.NewSaving == this.state || YarnApplicationState
.Submitted == this.state || YarnApplicationState.Accepted == this.state;
this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app.GetFinishTime(
) == 0 ? "ApplicationMaster" : "History");
if (!trackingUrlIsNotReady)
{
this.trackingUrl = WebAppUtils.GetURLWithScheme(schemePrefix, trackingUrl);
this.trackingUrlPretty = this.trackingUrl;
}
else
{
this.trackingUrlPretty = "UNASSIGNED";
}
this.applicationId = app.GetApplicationId();
this.applicationType = app.GetApplicationType();
this.appIdNum = app.GetApplicationId().GetId().ToString();
this.id = app.GetApplicationId().ToString();
this.user = app.GetUser().ToString();
this.name = app.GetName().ToString();
this.queue = app.GetQueue().ToString();
this.progress = app.GetProgress() * 100;
this.diagnostics = app.GetDiagnostics().ToString();
if (diagnostics == null || diagnostics.IsEmpty())
{
this.diagnostics = string.Empty;
}
if (app.GetApplicationTags() != null && !app.GetApplicationTags().IsEmpty())
{
this.applicationTags = Joiner.On(',').Join(app.GetApplicationTags());
}
this.finalStatus = app.GetFinalApplicationStatus();
this.clusterId = ResourceManager.GetClusterTimeStamp();
if (hasAccess)
{
this.startedTime = app.GetStartTime();
this.finishedTime = app.GetFinishTime();
this.elapsedTime = Times.Elapsed(app.GetStartTime(), app.GetFinishTime());
RMAppAttempt attempt = app.GetCurrentAppAttempt();
if (attempt != null)
{
Container masterContainer = attempt.GetMasterContainer();
if (masterContainer != null)
{
this.amContainerLogsExist = true;
this.amContainerLogs = WebAppUtils.GetRunningLogURL(schemePrefix + masterContainer
.GetNodeHttpAddress(), ConverterUtils.ToString(masterContainer.GetId()), app.GetUser
());
this.amHostHttpAddress = masterContainer.GetNodeHttpAddress();
}
ApplicationResourceUsageReport resourceReport = attempt.GetApplicationResourceUsageReport
();
if (resourceReport != null)
{
Resource usedResources = resourceReport.GetUsedResources();
allocatedMB = usedResources.GetMemory();
allocatedVCores = usedResources.GetVirtualCores();
runningContainers = resourceReport.GetNumUsedContainers();
}
resourceRequests = ((AbstractYarnScheduler)rm.GetRMContext().GetScheduler()).GetPendingResourceRequestsForAttempt
(attempt.GetAppAttemptId());
}
}
// copy preemption info fields
RMAppMetrics appMetrics = app.GetRMAppMetrics();
numAMContainerPreempted = appMetrics.GetNumAMContainersPreempted();
preemptedResourceMB = appMetrics.GetResourcePreempted().GetMemory();
numNonAMContainerPreempted = appMetrics.GetNumNonAMContainersPreempted();
preemptedResourceVCores = appMetrics.GetResourcePreempted().GetVirtualCores();
memorySeconds = appMetrics.GetMemorySeconds();
vcoreSeconds = appMetrics.GetVcoreSeconds();
}
}
private YarnProtos.FinalApplicationStatusProto ConvertToProtoFormat(FinalApplicationStatus
s)
{
return(ProtoUtils.ConvertToProtoFormat(s));
}
/*
* FinalApplicationStatus
*/
public static YarnProtos.FinalApplicationStatusProto ConvertToProtoFormat(FinalApplicationStatus
e)
{
return(YarnProtos.FinalApplicationStatusProto.ValueOf(FinalApplicationStatusPrefix
+ e.ToString()));
}
public static FinalApplicationStatus ConvertFromProtoFormat(YarnProtos.FinalApplicationStatusProto
e)
{
return(FinalApplicationStatus.ValueOf(e.ToString().Replace(FinalApplicationStatusPrefix
, string.Empty)));
}
public static ApplicationHistoryData NewInstance(ApplicationId applicationId, string
applicationName, string applicationType, string queue, string user, long submitTime
, long startTime, long finishTime, string diagnosticsInfo, FinalApplicationStatus
finalApplicationStatus, YarnApplicationState yarnApplicationState)
{
ApplicationHistoryData appHD = new ApplicationHistoryData();
appHD.SetApplicationId(applicationId);
appHD.SetApplicationName(applicationName);
appHD.SetApplicationType(applicationType);
appHD.SetQueue(queue);
appHD.SetUser(user);
appHD.SetSubmitTime(submitTime);
appHD.SetStartTime(startTime);
appHD.SetFinishTime(finishTime);
appHD.SetDiagnosticsInfo(diagnosticsInfo);
appHD.SetFinalApplicationStatus(finalApplicationStatus);
appHD.SetYarnApplicationState(yarnApplicationState);
return(appHD);
}
/// <summary>Unregister the application master.</summary>
/// <remarks>Unregister the application master. This must be called in the end.</remarks>
/// <param name="appStatus">Success/Failure status of the master</param>
/// <param name="appMessage">Diagnostics message on failure</param>
/// <param name="appTrackingUrl">New URL to get master info</param>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public abstract void UnregisterApplicationMaster(FinalApplicationStatus appStatus
, string appMessage, string appTrackingUrl);
public virtual AppsInfo GetApps(HttpServletRequest req, HttpServletResponse res,
string stateQuery, ICollection <string> statesQuery, string finalStatusQuery, string
userQuery, string queueQuery, string count, string startedBegin, string startedEnd
, string finishBegin, string finishEnd, ICollection <string> applicationTypes)
{
UserGroupInformation callerUGI = GetUser(req);
bool checkStart = false;
bool checkEnd = false;
bool checkAppTypes = false;
bool checkAppStates = false;
long countNum = long.MaxValue;
// set values suitable in case both of begin/end not specified
long sBegin = 0;
long sEnd = long.MaxValue;
long fBegin = 0;
long fEnd = long.MaxValue;
if (count != null && !count.IsEmpty())
{
countNum = long.Parse(count);
if (countNum <= 0)
{
throw new BadRequestException("limit value must be greater then 0");
}
}
if (startedBegin != null && !startedBegin.IsEmpty())
{
checkStart = true;
sBegin = long.Parse(startedBegin);
if (sBegin < 0)
{
throw new BadRequestException("startedTimeBegin must be greater than 0");
}
}
if (startedEnd != null && !startedEnd.IsEmpty())
{
checkStart = true;
sEnd = long.Parse(startedEnd);
if (sEnd < 0)
{
throw new BadRequestException("startedTimeEnd must be greater than 0");
}
}
if (sBegin > sEnd)
{
throw new BadRequestException("startedTimeEnd must be greater than startTimeBegin"
);
}
if (finishBegin != null && !finishBegin.IsEmpty())
{
checkEnd = true;
fBegin = long.Parse(finishBegin);
if (fBegin < 0)
{
throw new BadRequestException("finishTimeBegin must be greater than 0");
}
}
if (finishEnd != null && !finishEnd.IsEmpty())
{
checkEnd = true;
fEnd = long.Parse(finishEnd);
if (fEnd < 0)
{
throw new BadRequestException("finishTimeEnd must be greater than 0");
}
}
if (fBegin > fEnd)
{
throw new BadRequestException("finishTimeEnd must be greater than finishTimeBegin"
);
}
ICollection <string> appTypes = ParseQueries(applicationTypes, false);
if (!appTypes.IsEmpty())
{
checkAppTypes = true;
}
// stateQuery is deprecated.
if (stateQuery != null && !stateQuery.IsEmpty())
{
statesQuery.AddItem(stateQuery);
}
ICollection <string> appStates = ParseQueries(statesQuery, true);
if (!appStates.IsEmpty())
{
checkAppStates = true;
}
AppsInfo allApps = new AppsInfo();
ICollection <ApplicationReport> appReports = null;
GetApplicationsRequest request = GetApplicationsRequest.NewInstance();
request.SetLimit(countNum);
try
{
if (callerUGI == null)
{
// TODO: the request should take the params like what RMWebServices does
// in YARN-1819.
appReports = appBaseProt.GetApplications(request).GetApplicationList();
}
else
{
appReports = callerUGI.DoAs(new _PrivilegedExceptionAction_161(this, request));
}
}
catch (Exception e)
{
RewrapAndThrowException(e);
}
foreach (ApplicationReport appReport in appReports)
{
if (checkAppStates && !appStates.Contains(StringUtils.ToLowerCase(appReport.GetYarnApplicationState
().ToString())))
{
continue;
}
if (finalStatusQuery != null && !finalStatusQuery.IsEmpty())
{
FinalApplicationStatus.ValueOf(finalStatusQuery);
if (!Sharpen.Runtime.EqualsIgnoreCase(appReport.GetFinalApplicationStatus().ToString
(), finalStatusQuery))
{
continue;
}
}
if (userQuery != null && !userQuery.IsEmpty())
{
if (!appReport.GetUser().Equals(userQuery))
{
continue;
}
}
if (queueQuery != null && !queueQuery.IsEmpty())
{
if (!appReport.GetQueue().Equals(queueQuery))
{
continue;
}
}
if (checkAppTypes && !appTypes.Contains(StringUtils.ToLowerCase(appReport.GetApplicationType
().Trim())))
{
continue;
}
if (checkStart && (appReport.GetStartTime() < sBegin || appReport.GetStartTime()
> sEnd))
{
continue;
}
if (checkEnd && (appReport.GetFinishTime() < fBegin || appReport.GetFinishTime()
> fEnd))
{
continue;
}
AppInfo app = new AppInfo(appReport);
allApps.Add(app);
}
return(allApps);
}
protected internal virtual void DoUnregistration()
{
FinalApplicationStatus finishState = FinalApplicationStatus.Undefined;
JobImpl jobImpl = (JobImpl)job;
if (jobImpl.GetInternalState() == JobStateInternal.Succeeded)
{
finishState = FinalApplicationStatus.Succeeded;
}
else
{
if (jobImpl.GetInternalState() == JobStateInternal.Killed || (jobImpl.GetInternalState
() == JobStateInternal.Running && isSignalled))
{
finishState = FinalApplicationStatus.Killed;
}
else
{
if (jobImpl.GetInternalState() == JobStateInternal.Failed || jobImpl.GetInternalState
() == JobStateInternal.Error)
{
finishState = FinalApplicationStatus.Failed;
}
}
}
StringBuilder sb = new StringBuilder();
foreach (string s in job.GetDiagnostics())
{
sb.Append(s).Append("\n");
}
Log.Info("Setting job diagnostics to " + sb.ToString());
string historyUrl = MRWebAppUtil.GetApplicationWebURLOnJHSWithScheme(GetConfig(),
context.GetApplicationID());
Log.Info("History url is " + historyUrl);
FinishApplicationMasterRequest request = FinishApplicationMasterRequest.NewInstance
(finishState, sb.ToString(), historyUrl);
try
{
while (true)
{
FinishApplicationMasterResponse response = scheduler.FinishApplicationMaster(request
);
if (response.GetIsUnregistered())
{
// When excepting ClientService, other services are already stopped,
// it is safe to let clients know the final states. ClientService
// should wait for some time so clients have enough time to know the
// final states.
MRAppMaster.RunningAppContext raContext = (MRAppMaster.RunningAppContext)context;
raContext.MarkSuccessfulUnregistration();
break;
}
Log.Info("Waiting for application to be successfully unregistered.");
Sharpen.Thread.Sleep(rmPollInterval);
}
}
catch (ApplicationMasterNotRegisteredException)
{
// RM might have restarted or failed over and so lost the fact that AM had
// registered before.
Register();
DoUnregistration();
}
}
/*
* Contains the state data that needs to be persisted for an ApplicationAttempt
*/
public static ApplicationAttemptStateData NewInstance(ApplicationAttemptId attemptId
, Container container, Credentials attemptTokens, long startTime, RMAppAttemptState
finalState, string finalTrackingUrl, string diagnostics, FinalApplicationStatus
amUnregisteredFinalStatus, int exitStatus, long finishTime, long memorySeconds,
long vcoreSeconds)
{
ApplicationAttemptStateData attemptStateData = Org.Apache.Hadoop.Yarn.Util.Records
.NewRecord <ApplicationAttemptStateData>();
attemptStateData.SetAttemptId(attemptId);
attemptStateData.SetMasterContainer(container);
attemptStateData.SetAppAttemptTokens(attemptTokens);
attemptStateData.SetState(finalState);
attemptStateData.SetFinalTrackingUrl(finalTrackingUrl);
attemptStateData.SetDiagnostics(diagnostics == null ? string.Empty : diagnostics);
attemptStateData.SetStartTime(startTime);
attemptStateData.SetFinalApplicationStatus(amUnregisteredFinalStatus);
attemptStateData.SetAMContainerExitStatus(exitStatus);
attemptStateData.SetFinishTime(finishTime);
attemptStateData.SetMemorySeconds(memorySeconds);
attemptStateData.SetVcoreSeconds(vcoreSeconds);
return(attemptStateData);
}
