public static ApplicationReport NewInstance(ApplicationId applicationId, ApplicationAttemptId
applicationAttemptId, string user, string queue, string name, string host, int
rpcPort, Token clientToAMToken, YarnApplicationState state, string diagnostics,
string url, long startTime, long finishTime, FinalApplicationStatus finalStatus,
ApplicationResourceUsageReport appResources, string origTrackingUrl, float progress
, string applicationType, Token amRmToken)
{
ApplicationReport report = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <ApplicationReport
>();
report.SetApplicationId(applicationId);
report.SetCurrentApplicationAttemptId(applicationAttemptId);
report.SetUser(user);
report.SetQueue(queue);
report.SetName(name);
report.SetHost(host);
report.SetRpcPort(rpcPort);
report.SetClientToAMToken(clientToAMToken);
report.SetYarnApplicationState(state);
report.SetDiagnostics(diagnostics);
report.SetTrackingUrl(url);
report.SetStartTime(startTime);
report.SetFinishTime(finishTime);
report.SetFinalApplicationStatus(finalStatus);
report.SetApplicationResourceUsageReport(appResources);
report.SetOriginalTrackingUrl(origTrackingUrl);
report.SetProgress(progress);
report.SetApplicationType(applicationType);
report.SetAMRMToken(amRmToken);
return(report);
}
public virtual void TestEnums()
{
foreach (YarnApplicationState applicationState in YarnApplicationState.Values())
{
TypeConverter.FromYarn(applicationState, FinalApplicationStatus.Failed);
}
// ad hoc test of NEW_SAVING, which is newly added
NUnit.Framework.Assert.AreEqual(JobStatus.State.Prep, TypeConverter.FromYarn(YarnApplicationState
.NewSaving, FinalApplicationStatus.Failed));
foreach (TaskType taskType in TaskType.Values())
{
TypeConverter.FromYarn(taskType);
}
foreach (JobState jobState in JobState.Values())
{
TypeConverter.FromYarn(jobState);
}
foreach (QueueState queueState in QueueState.Values())
{
TypeConverter.FromYarn(queueState);
}
foreach (TaskState taskState in TaskState.Values())
{
TypeConverter.FromYarn(taskState);
}
}
public virtual void TestFromYarn()
{
int appStartTime = 612354;
int appFinishTime = 612355;
YarnApplicationState state = YarnApplicationState.Running;
ApplicationId applicationId = ApplicationId.NewInstance(0, 0);
ApplicationReport applicationReport = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<ApplicationReport>();
applicationReport.SetApplicationId(applicationId);
applicationReport.SetYarnApplicationState(state);
applicationReport.SetStartTime(appStartTime);
applicationReport.SetFinishTime(appFinishTime);
applicationReport.SetUser("TestTypeConverter-user");
ApplicationResourceUsageReport appUsageRpt = Org.Apache.Hadoop.Yarn.Util.Records.
NewRecord <ApplicationResourceUsageReport>();
Resource r = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <Resource>();
r.SetMemory(2048);
appUsageRpt.SetNeededResources(r);
appUsageRpt.SetNumReservedContainers(1);
appUsageRpt.SetNumUsedContainers(3);
appUsageRpt.SetReservedResources(r);
appUsageRpt.SetUsedResources(r);
applicationReport.SetApplicationResourceUsageReport(appUsageRpt);
JobStatus jobStatus = TypeConverter.FromYarn(applicationReport, "dummy-jobfile");
NUnit.Framework.Assert.AreEqual(appStartTime, jobStatus.GetStartTime());
NUnit.Framework.Assert.AreEqual(appFinishTime, jobStatus.GetFinishTime());
NUnit.Framework.Assert.AreEqual(state.ToString(), jobStatus.GetState().ToString()
);
}
private static void ValidateStates(string stateQuery, ICollection <string> statesQuery
)
{
// stateQuery is deprecated.
if (stateQuery != null && !stateQuery.IsEmpty())
{
statesQuery.AddItem(stateQuery);
}
ICollection <string> appStates = ParseQueries(statesQuery, true);
foreach (string appState in appStates)
{
switch (YarnApplicationState.ValueOf(StringUtils.ToUpperCase(appState)))
{
case YarnApplicationState.Finished:
case YarnApplicationState.Failed:
case YarnApplicationState.Killed:
{
continue;
}
default:
{
throw new BadRequestException("Invalid application-state " + appState + " specified. It should be a final state"
);
}
}
}
}
public AppInfo(ApplicationReport app)
{
// JAXB needs this
appId = app.GetApplicationId().ToString();
if (app.GetCurrentApplicationAttemptId() != null)
{
currentAppAttemptId = app.GetCurrentApplicationAttemptId().ToString();
}
user = app.GetUser();
queue = app.GetQueue();
name = app.GetName();
type = app.GetApplicationType();
host = app.GetHost();
rpcPort = app.GetRpcPort();
appState = app.GetYarnApplicationState();
diagnosticsInfo = app.GetDiagnostics();
trackingUrl = app.GetTrackingUrl();
originalTrackingUrl = app.GetOriginalTrackingUrl();
submittedTime = app.GetStartTime();
startedTime = app.GetStartTime();
finishedTime = app.GetFinishTime();
elapsedTime = Times.Elapsed(startedTime, finishedTime);
finalAppStatus = app.GetFinalApplicationStatus();
progress = app.GetProgress() * 100;
// in percent
if (app.GetApplicationTags() != null && !app.GetApplicationTags().IsEmpty())
{
this.applicationTags = StringHelper.CsvJoiner.Join(app.GetApplicationTags());
}
}
/// <exception cref="System.Exception"/>
private void VerifySubmitApp(MockRM rm, RMApp app, ApplicationId expectedAppId)
{
int maxWaittingTimes = 20;
int count = 0;
while (true)
{
YarnApplicationState state = rm.GetApplicationReport(app.GetApplicationId()).GetYarnApplicationState
();
if (!state.Equals(YarnApplicationState.New) && !state.Equals(YarnApplicationState
.NewSaving))
{
break;
}
if (count > maxWaittingTimes)
{
break;
}
Sharpen.Thread.Sleep(200);
count++;
}
// Verify submittion is successful
YarnApplicationState state_1 = rm.GetApplicationReport(app.GetApplicationId()).GetYarnApplicationState
();
NUnit.Framework.Assert.IsTrue(state_1 == YarnApplicationState.Accepted || state_1
== YarnApplicationState.Submitted);
NUnit.Framework.Assert.AreEqual(expectedAppId, app.GetApplicationId());
}
public StatisticsItemInfo(YarnApplicationState state, string type, long count)
{
// JAXB needs this
this.state = state;
this.type = type;
this.count = count;
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected internal virtual void FetchData()
{
reqAppStates = EnumSet.NoneOf <YarnApplicationState>();
string reqStateString = $(YarnWebParams.AppState);
if (reqStateString != null && !reqStateString.IsEmpty())
{
string[] appStateStrings = reqStateString.Split(",");
foreach (string stateString in appStateStrings)
{
reqAppStates.AddItem(YarnApplicationState.ValueOf(stateString.Trim()));
}
}
callerUGI = GetCallerUGI();
GetApplicationsRequest request = GetApplicationsRequest.NewInstance(reqAppStates);
string appsNumStr = $(YarnWebParams.AppsNum);
if (appsNumStr != null && !appsNumStr.IsEmpty())
{
long appsNum = long.Parse(appsNumStr);
request.SetLimit(appsNum);
}
if (callerUGI == null)
{
appReports = appBaseProt.GetApplications(request).GetApplicationList();
}
else
{
appReports = callerUGI.DoAs(new _PrivilegedExceptionAction_87(this, request));
}
}
public override void SetYarnApplicationState(YarnApplicationState state)
{
MaybeInitBuilder();
if (state == null)
{
builder.ClearYarnApplicationState();
return;
}
builder.SetYarnApplicationState(ConvertToProtoFormat(state));
}
/// <summary>Monitor the submitted application for completion.</summary>
/// <remarks>
/// Monitor the submitted application for completion.
/// Kill application if time expires.
/// </remarks>
/// <param name="appId">Application Id of application to be monitored</param>
/// <returns>true if application completed successfully</returns>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
private bool MonitorApplication(ApplicationId appId)
{
while (true)
{
// Check app status every 1 second.
try
{
Sharpen.Thread.Sleep(1000);
}
catch (Exception)
{
Log.Debug("Thread sleep in monitoring loop interrupted");
}
// Get application report for the appId we are interested in
ApplicationReport report = yarnClient.GetApplicationReport(appId);
Log.Info("Got application report from ASM for" + ", appId=" + appId.GetId() + ", clientToAMToken="
+ report.GetClientToAMToken() + ", appDiagnostics=" + report.GetDiagnostics() +
", appMasterHost=" + report.GetHost() + ", appQueue=" + report.GetQueue() + ", appMasterRpcPort="
+ report.GetRpcPort() + ", appStartTime=" + report.GetStartTime() + ", yarnAppState="
+ report.GetYarnApplicationState().ToString() + ", distributedFinalState=" + report
.GetFinalApplicationStatus().ToString() + ", appTrackingUrl=" + report.GetTrackingUrl
() + ", appUser="******"Application has completed successfully. Breaking monitoring loop");
return(true);
}
else
{
Log.Info("Application did finished unsuccessfully." + " YarnState=" + state.ToString
() + ", DSFinalStatus=" + dsStatus.ToString() + ". Breaking monitoring loop");
return(false);
}
}
else
{
if (YarnApplicationState.Killed == state || YarnApplicationState.Failed == state)
{
Log.Info("Application did not finish." + " YarnState=" + state.ToString() + ", DSFinalStatus="
+ dsStatus.ToString() + ". Breaking monitoring loop");
return(false);
}
}
if (Runtime.CurrentTimeMillis() > (clientStartTime + clientTimeout))
{
Log.Info("Reached client specified timeout for application. Killing application");
ForceKillApplication(appId);
return(false);
}
}
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
private YarnClient CreateMockYarnClient(YarnApplicationState appState)
{
YarnClient mockClient = Org.Mockito.Mockito.Mock <YarnClient>();
ApplicationReport mockAppReport = Org.Mockito.Mockito.Mock <ApplicationReport>();
Org.Mockito.Mockito.DoReturn(appState).When(mockAppReport).GetYarnApplicationState
();
Org.Mockito.Mockito.DoReturn(mockAppReport).When(mockClient).GetApplicationReport
(Matchers.Any <ApplicationId>());
return(mockClient);
}
public ApplicationFinishedEvent(ApplicationId appId, string diagnosticsInfo, FinalApplicationStatus
appStatus, YarnApplicationState state, ApplicationAttemptId latestAppAttemptId,
long finishedTime, RMAppMetrics appMetrics)
: base(SystemMetricsEventType.AppFinished, finishedTime)
{
this.appId = appId;
this.diagnosticsInfo = diagnosticsInfo;
this.appStatus = appStatus;
this.latestAppAttemptId = latestAppAttemptId;
this.state = state;
this.appMetrics = appMetrics;
}
public static ApplicationFinishData NewInstance(ApplicationId applicationId, long
finishTime, string diagnosticsInfo, FinalApplicationStatus finalApplicationStatus
, YarnApplicationState yarnApplicationState)
{
ApplicationFinishData appFD = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <ApplicationFinishData
>();
appFD.SetApplicationId(applicationId);
appFD.SetFinishTime(finishTime);
appFD.SetDiagnosticsInfo(diagnosticsInfo);
appFD.SetFinalApplicationStatus(finalApplicationStatus);
appFD.SetYarnApplicationState(yarnApplicationState);
return(appFD);
}
private string GetAllValidApplicationStates()
{
StringBuilder sb = new StringBuilder();
sb.Append("The valid application state can be" + " one of the following: ");
sb.Append(AllstatesOption + ",");
foreach (YarnApplicationState appState in YarnApplicationState.Values())
{
sb.Append(appState + ",");
}
string output = sb.ToString();
return(Sharpen.Runtime.Substring(output, 0, output.Length - 1));
}
public static JobStatus.State FromYarn(YarnApplicationState yarnApplicationState,
FinalApplicationStatus finalApplicationStatus)
{
switch (yarnApplicationState)
{
case YarnApplicationState.New:
case YarnApplicationState.NewSaving:
case YarnApplicationState.Submitted:
case YarnApplicationState.Accepted:
{
return(JobStatus.State.Prep);
}
case YarnApplicationState.Running:
{
return(JobStatus.State.Running);
}
case YarnApplicationState.Finished:
{
if (finalApplicationStatus == FinalApplicationStatus.Succeeded)
{
return(JobStatus.State.Succeeded);
}
else
{
if (finalApplicationStatus == FinalApplicationStatus.Killed)
{
return(JobStatus.State.Killed);
}
}
goto case YarnApplicationState.Failed;
}
case YarnApplicationState.Failed:
{
return(JobStatus.State.Failed);
}
case YarnApplicationState.Killed:
{
return(JobStatus.State.Killed);
}
}
throw new YarnRuntimeException("Unrecognized application state: " + yarnApplicationState
);
}
protected override void Render(HtmlBlock.Block html)
{
Hamlet.UL <Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > mainList = html
.Div("#nav").H3("Cluster").Ul().Li().A(Url("cluster"), "About").().Li().A(Url("nodes"
), "Nodes").().Li().A(Url("nodelabels"), "Node Labels").();
Hamlet.UL <Hamlet.LI <Hamlet.UL <Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet
> > > > subAppsList = mainList.Li().A(Url("apps"), "Applications").Ul();
subAppsList.Li().();
foreach (YarnApplicationState state in YarnApplicationState.Values())
{
subAppsList.Li().A(Url("apps", state.ToString()), state.ToString()).();
}
subAppsList.().();
mainList.Li().A(Url("scheduler"), "Scheduler").().().H3("Tools").Ul().Li().A("/conf"
, "Configuration").().Li().A("/logs", "Local logs").().Li().A("/stacks", "Server stacks"
).().Li().A("/jmx?qry=Hadoop:*", "Server metrics").().().();
}
public static ApplicationHistoryData NewInstance(ApplicationId applicationId, string
applicationName, string applicationType, string queue, string user, long submitTime
, long startTime, long finishTime, string diagnosticsInfo, FinalApplicationStatus
finalApplicationStatus, YarnApplicationState yarnApplicationState)
{
ApplicationHistoryData appHD = new ApplicationHistoryData();
appHD.SetApplicationId(applicationId);
appHD.SetApplicationName(applicationName);
appHD.SetApplicationType(applicationType);
appHD.SetQueue(queue);
appHD.SetUser(user);
appHD.SetSubmitTime(submitTime);
appHD.SetStartTime(startTime);
appHD.SetFinishTime(finishTime);
appHD.SetDiagnosticsInfo(diagnosticsInfo);
appHD.SetFinalApplicationStatus(finalApplicationStatus);
appHD.SetYarnApplicationState(yarnApplicationState);
return(appHD);
}
public override void SetApplicationStates(ICollection <string> applicationStates)
{
EnumSet <YarnApplicationState> appStates = null;
foreach (YarnApplicationState state in YarnApplicationState.Values())
{
if (applicationStates.Contains(StringUtils.ToLowerCase(state.ToString())))
{
if (appStates == null)
{
appStates = EnumSet.Of(state);
}
else
{
appStates.AddItem(state);
}
}
}
SetApplicationStates(appStates);
}
protected internal static ICollection <string> ParseQueries(ICollection <string> queries
, bool isState)
{
ICollection <string> @params = new HashSet <string>();
if (!queries.IsEmpty())
{
foreach (string query in queries)
{
if (query != null && !query.Trim().IsEmpty())
{
string[] paramStrs = query.Split(",");
foreach (string paramStr in paramStrs)
{
if (paramStr != null && !paramStr.Trim().IsEmpty())
{
if (isState)
{
try
{
// enum string is in the uppercase
YarnApplicationState.ValueOf(StringUtils.ToUpperCase(paramStr.Trim()));
}
catch (RuntimeException)
{
YarnApplicationState[] stateArray = YarnApplicationState.Values();
string allAppStates = Arrays.ToString(stateArray);
throw new BadRequestException("Invalid application-state " + paramStr.Trim() + " specified. It should be one of "
+ allAppStates);
}
}
@params.AddItem(StringUtils.ToLowerCase(paramStr.Trim()));
}
}
}
}
}
return(@params);
}
/// <summary>
/// Lists the applications matching the given application Types And application
/// States present in the Resource Manager
/// </summary>
/// <param name="appTypes"/>
/// <param name="appStates"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
private void ListApplications(ICollection <string> appTypes, EnumSet <YarnApplicationState
> appStates)
{
PrintWriter writer = new PrintWriter(new OutputStreamWriter(sysout, Sharpen.Extensions.GetEncoding
("UTF-8")));
if (allAppStates)
{
foreach (YarnApplicationState appState in YarnApplicationState.Values())
{
appStates.AddItem(appState);
}
}
else
{
if (appStates.IsEmpty())
{
appStates.AddItem(YarnApplicationState.Running);
appStates.AddItem(YarnApplicationState.Accepted);
appStates.AddItem(YarnApplicationState.Submitted);
}
}
IList <ApplicationReport> appsReport = client.GetApplications(appTypes, appStates);
writer.WriteLine("Total number of applications (application-types: " + appTypes +
" and states: " + appStates + ")" + ":" + appsReport.Count);
writer.Printf(ApplicationsPattern, "Application-Id", "Application-Name", "Application-Type"
, "User", "Queue", "State", "Final-State", "Progress", "Tracking-URL");
foreach (ApplicationReport appReport in appsReport)
{
DecimalFormat formatter = new DecimalFormat("###.##%");
string progress = formatter.Format(appReport.GetProgress());
writer.Printf(ApplicationsPattern, appReport.GetApplicationId(), appReport.GetName
(), appReport.GetApplicationType(), appReport.GetUser(), appReport.GetQueue(), appReport
.GetYarnApplicationState(), appReport.GetFinalApplicationStatus(), progress, appReport
.GetOriginalTrackingUrl());
}
writer.Flush();
}
private ApplicationReport GetApplicationReport(YarnApplicationState yarnApplicationState
, FinalApplicationStatus finalApplicationStatus)
{
ApplicationReport appReport = Org.Mockito.Mockito.Mock <ApplicationReport>();
ApplicationResourceUsageReport appResources = Org.Mockito.Mockito.Mock <ApplicationResourceUsageReport
>();
Org.Mockito.Mockito.When(appReport.GetApplicationId()).ThenReturn(ApplicationId.NewInstance
(0, 0));
Org.Mockito.Mockito.When(appResources.GetNeededResources()).ThenReturn(Org.Apache.Hadoop.Yarn.Util.Records
.NewRecord <Resource>());
Org.Mockito.Mockito.When(appResources.GetReservedResources()).ThenReturn(Org.Apache.Hadoop.Yarn.Util.Records
.NewRecord <Resource>());
Org.Mockito.Mockito.When(appResources.GetUsedResources()).ThenReturn(Org.Apache.Hadoop.Yarn.Util.Records
.NewRecord <Resource>());
Org.Mockito.Mockito.When(appReport.GetApplicationResourceUsageReport()).ThenReturn
(appResources);
Org.Mockito.Mockito.When(appReport.GetYarnApplicationState()).ThenReturn(yarnApplicationState
);
Org.Mockito.Mockito.When(appReport.GetFinalApplicationStatus()).ThenReturn(finalApplicationStatus
);
return(appReport);
}
/// <exception cref="System.IO.IOException"/>
private static bool IsApplicationTerminated(ApplicationId appId, ApplicationClientProtocol
rmClient)
{
ApplicationReport appReport = null;
try
{
appReport = rmClient.GetApplicationReport(GetApplicationReportRequest.NewInstance
(appId)).GetApplicationReport();
}
catch (ApplicationNotFoundException)
{
return(true);
}
catch (YarnException e)
{
throw new IOException(e);
}
YarnApplicationState currentState = appReport.GetYarnApplicationState();
return(currentState == YarnApplicationState.Failed || currentState == YarnApplicationState
.Killed || currentState == YarnApplicationState.Finished);
}
private string ClarifyAppState(YarnApplicationState state)
{
string ret = state.ToString();
switch (state)
{
case YarnApplicationState.New:
{
return(ret + ": waiting for application to be initialized");
}
case YarnApplicationState.NewSaving:
{
return(ret + ": waiting for application to be persisted in state-store.");
}
case YarnApplicationState.Submitted:
{
return(ret + ": waiting for application to be accepted by scheduler.");
}
case YarnApplicationState.Accepted:
{
return(ret + ": waiting for AM container to be allocated, launched and" + " register with RM.");
}
case YarnApplicationState.Running:
{
return(ret + ": AM has registered with RM and started running.");
}
default:
{
return(ret);
}
}
}
public static RMApp NewApplication(int i)
{
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(NewAppID(i),
0);
Container masterContainer = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <Container
>();
ContainerId containerId = ContainerId.NewContainerId(appAttemptId, 0);
masterContainer.SetId(containerId);
masterContainer.SetNodeHttpAddress("node:port");
string user = NewUserName();
string name = NewAppName();
string queue = NewQueue();
long start = 123456 + i * 1000;
long finish = 234567 + i * 1000;
string type = YarnConfiguration.DefaultApplicationType;
YarnApplicationState[] allStates = YarnApplicationState.Values();
YarnApplicationState state = allStates[i % allStates.Length];
int maxAppAttempts = i % 1000;
return(new _ApplicationBase_211(appAttemptId, user, name, type, queue, start, finish
, state, maxAppAttempts));
}
public virtual void SetYarnApplicationState(YarnApplicationState yarnApplicationState
)
{
this.yarnApplicationState = yarnApplicationState;
}
/// <summary>Monitor the submitted application for completion.</summary>
/// <remarks>
/// Monitor the submitted application for completion. Kill application if time
/// expires.
/// </remarks>
/// <param name="appId">Application Id of application to be monitored</param>
/// <returns>true if application completed successfully</returns>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
private ApplicationReport MonitorApplication(ApplicationId appId, ICollection <YarnApplicationState
> finalState)
{
long foundAMCompletedTime = 0;
StringBuilder expectedFinalState = new StringBuilder();
bool first = true;
foreach (YarnApplicationState state in finalState)
{
if (first)
{
first = false;
expectedFinalState.Append(state.ToString());
}
else
{
expectedFinalState.Append("," + state.ToString());
}
}
while (true)
{
// Check app status every 1 second.
try
{
Sharpen.Thread.Sleep(1000);
}
catch (Exception)
{
Log.Debug("Thread sleep in monitoring loop interrupted");
}
// Get application report for the appId we are interested in
ApplicationReport report = rmClient.GetApplicationReport(appId);
Log.Info("Got application report from ASM for" + ", appId=" + appId.GetId() + ", appAttemptId="
+ report.GetCurrentApplicationAttemptId() + ", clientToAMToken=" + report.GetClientToAMToken
() + ", appDiagnostics=" + report.GetDiagnostics() + ", appMasterHost=" + report
.GetHost() + ", appQueue=" + report.GetQueue() + ", appMasterRpcPort=" + report.
GetRpcPort() + ", appStartTime=" + report.GetStartTime() + ", yarnAppState=" + report
.GetYarnApplicationState().ToString() + ", distributedFinalState=" + report.GetFinalApplicationStatus
().ToString() + ", appTrackingUrl=" + report.GetTrackingUrl() + ", appUser="******"Waited " + AmStateWaitTimeoutMs / 1000 + " seconds after process completed for AppReport"
+ " to reach desired final state. Not waiting anymore." + "CurrentState = " + state_1
+ ", ExpectedStates = " + expectedFinalState.ToString());
throw new RuntimeException("Failed to receive final expected state" + " in ApplicationReport"
+ ", CurrentState=" + state_1 + ", ExpectedStates=" + expectedFinalState.ToString
());
}
}
}
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public virtual bool Run()
{
Log.Info("Starting Client");
// Connect to ResourceManager
rmClient.Start();
try
{
// Create launch context for app master
Log.Info("Setting up application submission context for ASM");
ApplicationSubmissionContext appContext = rmClient.CreateApplication().GetApplicationSubmissionContext
();
ApplicationId appId = appContext.GetApplicationId();
// set the application name
appContext.SetApplicationName(appName);
// Set the priority for the application master
Priority pri = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <Priority>();
pri.SetPriority(amPriority);
appContext.SetPriority(pri);
// Set the queue to which this application is to be submitted in the RM
appContext.SetQueue(amQueue);
// Set up the container launch context for the application master
ContainerLaunchContext amContainer = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<ContainerLaunchContext>();
appContext.SetAMContainerSpec(amContainer);
// unmanaged AM
appContext.SetUnmanagedAM(true);
Log.Info("Setting unmanaged AM");
// Submit the application to the applications manager
Log.Info("Submitting application to ASM");
rmClient.SubmitApplication(appContext);
ApplicationReport appReport = MonitorApplication(appId, EnumSet.Of(YarnApplicationState
.Accepted, YarnApplicationState.Killed, YarnApplicationState.Failed, YarnApplicationState
.Finished));
if (appReport.GetYarnApplicationState() == YarnApplicationState.Accepted)
{
// Monitor the application attempt to wait for launch state
ApplicationAttemptReport attemptReport = MonitorCurrentAppAttempt(appId, YarnApplicationAttemptState
.Launched);
ApplicationAttemptId attemptId = attemptReport.GetApplicationAttemptId();
Log.Info("Launching AM with application attempt id " + attemptId);
// launch AM
LaunchAM(attemptId);
// Monitor the application for end state
appReport = MonitorApplication(appId, EnumSet.Of(YarnApplicationState.Killed, YarnApplicationState
.Failed, YarnApplicationState.Finished));
}
YarnApplicationState appState = appReport.GetYarnApplicationState();
FinalApplicationStatus appStatus = appReport.GetFinalApplicationStatus();
Log.Info("App ended with state: " + appReport.GetYarnApplicationState() + " and status: "
+ appStatus);
bool success;
if (YarnApplicationState.Finished == appState && FinalApplicationStatus.Succeeded
== appStatus)
{
Log.Info("Application has completed successfully.");
success = true;
}
else
{
Log.Info("Application did finished unsuccessfully." + " YarnState=" + appState.ToString
() + ", FinalStatus=" + appStatus.ToString());
success = false;
}
return(success);
}
finally
{
rmClient.Stop();
}
}
private YarnProtos.YarnApplicationStateProto ConvertToProtoFormat(YarnApplicationState
s)
{
return(ProtoUtils.ConvertToProtoFormat(s));
}
public AppInfo(ResourceManager rm, RMApp app, bool hasAccess, string schemePrefix
)
{
// these are ok for any user to see
// these are only allowed if acls allow
// preemption info fields
// JAXB needs this
this.schemePrefix = schemePrefix;
if (app != null)
{
string trackingUrl = app.GetTrackingUrl();
this.state = app.CreateApplicationState();
this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.IsEmpty() || YarnApplicationState
.New == this.state || YarnApplicationState.NewSaving == this.state || YarnApplicationState
.Submitted == this.state || YarnApplicationState.Accepted == this.state;
this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app.GetFinishTime(
) == 0 ? "ApplicationMaster" : "History");
if (!trackingUrlIsNotReady)
{
this.trackingUrl = WebAppUtils.GetURLWithScheme(schemePrefix, trackingUrl);
this.trackingUrlPretty = this.trackingUrl;
}
else
{
this.trackingUrlPretty = "UNASSIGNED";
}
this.applicationId = app.GetApplicationId();
this.applicationType = app.GetApplicationType();
this.appIdNum = app.GetApplicationId().GetId().ToString();
this.id = app.GetApplicationId().ToString();
this.user = app.GetUser().ToString();
this.name = app.GetName().ToString();
this.queue = app.GetQueue().ToString();
this.progress = app.GetProgress() * 100;
this.diagnostics = app.GetDiagnostics().ToString();
if (diagnostics == null || diagnostics.IsEmpty())
{
this.diagnostics = string.Empty;
}
if (app.GetApplicationTags() != null && !app.GetApplicationTags().IsEmpty())
{
this.applicationTags = Joiner.On(',').Join(app.GetApplicationTags());
}
this.finalStatus = app.GetFinalApplicationStatus();
this.clusterId = ResourceManager.GetClusterTimeStamp();
if (hasAccess)
{
this.startedTime = app.GetStartTime();
this.finishedTime = app.GetFinishTime();
this.elapsedTime = Times.Elapsed(app.GetStartTime(), app.GetFinishTime());
RMAppAttempt attempt = app.GetCurrentAppAttempt();
if (attempt != null)
{
Container masterContainer = attempt.GetMasterContainer();
if (masterContainer != null)
{
this.amContainerLogsExist = true;
this.amContainerLogs = WebAppUtils.GetRunningLogURL(schemePrefix + masterContainer
.GetNodeHttpAddress(), ConverterUtils.ToString(masterContainer.GetId()), app.GetUser
());
this.amHostHttpAddress = masterContainer.GetNodeHttpAddress();
}
ApplicationResourceUsageReport resourceReport = attempt.GetApplicationResourceUsageReport
();
if (resourceReport != null)
{
Resource usedResources = resourceReport.GetUsedResources();
allocatedMB = usedResources.GetMemory();
allocatedVCores = usedResources.GetVirtualCores();
runningContainers = resourceReport.GetNumUsedContainers();
}
resourceRequests = ((AbstractYarnScheduler)rm.GetRMContext().GetScheduler()).GetPendingResourceRequestsForAttempt
(attempt.GetAppAttemptId());
}
}
// copy preemption info fields
RMAppMetrics appMetrics = app.GetRMAppMetrics();
numAMContainerPreempted = appMetrics.GetNumAMContainersPreempted();
preemptedResourceMB = appMetrics.GetResourcePreempted().GetMemory();
numNonAMContainerPreempted = appMetrics.GetNumNonAMContainersPreempted();
preemptedResourceVCores = appMetrics.GetResourcePreempted().GetVirtualCores();
memorySeconds = appMetrics.GetMemorySeconds();
vcoreSeconds = appMetrics.GetVcoreSeconds();
}
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public override ApplicationId SubmitApplication(ApplicationSubmissionContext appContext
)
{
ApplicationId applicationId = appContext.GetApplicationId();
if (applicationId == null)
{
throw new ApplicationIdNotProvidedException("ApplicationId is not provided in ApplicationSubmissionContext"
);
}
SubmitApplicationRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <
SubmitApplicationRequest>();
request.SetApplicationSubmissionContext(appContext);
// Automatically add the timeline DT into the CLC
// Only when the security and the timeline service are both enabled
if (IsSecurityEnabled() && timelineServiceEnabled)
{
AddTimelineDelegationToken(appContext.GetAMContainerSpec());
}
//TODO: YARN-1763:Handle RM failovers during the submitApplication call.
rmClient.SubmitApplication(request);
int pollCount = 0;
long startTime = Runtime.CurrentTimeMillis();
EnumSet <YarnApplicationState> waitingStates = EnumSet.Of(YarnApplicationState.New
, YarnApplicationState.NewSaving, YarnApplicationState.Submitted);
EnumSet <YarnApplicationState> failToSubmitStates = EnumSet.Of(YarnApplicationState
.Failed, YarnApplicationState.Killed);
while (true)
{
try
{
ApplicationReport appReport = GetApplicationReport(applicationId);
YarnApplicationState state = appReport.GetYarnApplicationState();
if (!waitingStates.Contains(state))
{
if (failToSubmitStates.Contains(state))
{
throw new YarnException("Failed to submit " + applicationId + " to YARN : " + appReport
.GetDiagnostics());
}
Log.Info("Submitted application " + applicationId);
break;
}
long elapsedMillis = Runtime.CurrentTimeMillis() - startTime;
if (EnforceAsyncAPITimeout() && elapsedMillis >= asyncApiPollTimeoutMillis)
{
throw new YarnException("Timed out while waiting for application " + applicationId
+ " to be submitted successfully");
}
// Notify the client through the log every 10 poll, in case the client
// is blocked here too long.
if (++pollCount % 10 == 0)
{
Log.Info("Application submission is not finished, " + "submitted application " +
applicationId + " is still in " + state);
}
try
{
Sharpen.Thread.Sleep(submitPollIntervalMillis);
}
catch (Exception)
{
Log.Error("Interrupted while waiting for application " + applicationId + " to be successfully submitted."
);
}
}
catch (ApplicationNotFoundException)
{
// FailOver or RM restart happens before RMStateStore saves
// ApplicationState
Log.Info("Re-submit application " + applicationId + "with the " + "same ApplicationSubmissionContext"
);
rmClient.SubmitApplication(request);
}
}
return(applicationId);
}
