/// <summary>Kills the application with the application id as appId</summary>
/// <param name="applicationId"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
private void KillApplication(string applicationId)
{
ApplicationId appId = ConverterUtils.ToApplicationId(applicationId);
ApplicationReport appReport = null;
try
{
appReport = client.GetApplicationReport(appId);
}
catch (ApplicationNotFoundException e)
{
sysout.WriteLine("Application with id '" + applicationId + "' doesn't exist in RM."
);
throw;
}
if (appReport.GetYarnApplicationState() == YarnApplicationState.Finished || appReport
.GetYarnApplicationState() == YarnApplicationState.Killed || appReport.GetYarnApplicationState
() == YarnApplicationState.Failed)
{
sysout.WriteLine("Application " + applicationId + " has already finished ");
}
else
{
sysout.WriteLine("Killing application " + applicationId);
client.KillApplication(appId);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public override JobStatus SubmitJob(JobID jobId, string jobSubmitDir, Credentials
ts)
{
AddHistoryToken(ts);
// Construct necessary information to start the MR AM
ApplicationSubmissionContext appContext = CreateApplicationSubmissionContext(conf
, jobSubmitDir, ts);
// Submit to ResourceManager
try
{
ApplicationId applicationId = resMgrDelegate.SubmitApplication(appContext);
ApplicationReport appMaster = resMgrDelegate.GetApplicationReport(applicationId);
string diagnostics = (appMaster == null ? "application report is null" : appMaster
.GetDiagnostics());
if (appMaster == null || appMaster.GetYarnApplicationState() == YarnApplicationState
.Failed || appMaster.GetYarnApplicationState() == YarnApplicationState.Killed)
{
throw new IOException("Failed to run job : " + diagnostics);
}
return(clientCache.GetClient(jobId).GetJobStatus(jobId));
}
catch (YarnException e)
{
throw new IOException(e);
}
}
/// <summary>Monitor the submitted application for completion.</summary>
/// <remarks>
/// Monitor the submitted application for completion.
/// Kill application if time expires.
/// </remarks>
/// <param name="appId">Application Id of application to be monitored</param>
/// <returns>true if application completed successfully</returns>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
private bool MonitorApplication(ApplicationId appId)
{
while (true)
{
// Check app status every 1 second.
try
{
Sharpen.Thread.Sleep(1000);
}
catch (Exception)
{
Log.Debug("Thread sleep in monitoring loop interrupted");
}
// Get application report for the appId we are interested in
ApplicationReport report = yarnClient.GetApplicationReport(appId);
Log.Info("Got application report from ASM for" + ", appId=" + appId.GetId() + ", clientToAMToken="
+ report.GetClientToAMToken() + ", appDiagnostics=" + report.GetDiagnostics() +
", appMasterHost=" + report.GetHost() + ", appQueue=" + report.GetQueue() + ", appMasterRpcPort="
+ report.GetRpcPort() + ", appStartTime=" + report.GetStartTime() + ", yarnAppState="
+ report.GetYarnApplicationState().ToString() + ", distributedFinalState=" + report
.GetFinalApplicationStatus().ToString() + ", appTrackingUrl=" + report.GetTrackingUrl
() + ", appUser="******"Application has completed successfully. Breaking monitoring loop");
return(true);
}
else
{
Log.Info("Application did finished unsuccessfully." + " YarnState=" + state.ToString
() + ", DSFinalStatus=" + dsStatus.ToString() + ". Breaking monitoring loop");
return(false);
}
}
else
{
if (YarnApplicationState.Killed == state || YarnApplicationState.Failed == state)
{
Log.Info("Application did not finish." + " YarnState=" + state.ToString() + ", DSFinalStatus="
+ dsStatus.ToString() + ". Breaking monitoring loop");
return(false);
}
}
if (Runtime.CurrentTimeMillis() > (clientStartTime + clientTimeout))
{
Log.Info("Reached client specified timeout for application. Killing application");
ForceKillApplication(appId);
return(false);
}
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
private int VerifyApplicationState(ApplicationId appId)
{
YarnClient yarnClient = CreateYarnClient();
try
{
ApplicationReport appReport = yarnClient.GetApplicationReport(appId);
switch (appReport.GetYarnApplicationState())
{
case YarnApplicationState.New:
case YarnApplicationState.NewSaving:
case YarnApplicationState.Submitted:
{
return(-1);
}
case YarnApplicationState.Accepted:
case YarnApplicationState.Running:
case YarnApplicationState.Failed:
case YarnApplicationState.Finished:
case YarnApplicationState.Killed:
default:
{
break;
}
}
}
finally
{
yarnClient.Close();
}
return(0);
}
/// <exception cref="System.Exception"/>
public virtual void TestJobSubmissionFailure()
{
Org.Mockito.Mockito.When(resourceMgrDelegate.SubmitApplication(Matchers.Any <ApplicationSubmissionContext
>())).ThenReturn(appId);
ApplicationReport report = Org.Mockito.Mockito.Mock <ApplicationReport>();
Org.Mockito.Mockito.When(report.GetApplicationId()).ThenReturn(appId);
Org.Mockito.Mockito.When(report.GetDiagnostics()).ThenReturn(failString);
Org.Mockito.Mockito.When(report.GetYarnApplicationState()).ThenReturn(YarnApplicationState
.Failed);
Org.Mockito.Mockito.When(resourceMgrDelegate.GetApplicationReport(appId)).ThenReturn
(report);
Credentials credentials = new Credentials();
FilePath jobxml = new FilePath(testWorkDir, "job.xml");
OutputStream @out = new FileOutputStream(jobxml);
conf.WriteXml(@out);
@out.Close();
try
{
yarnRunner.SubmitJob(jobId, testWorkDir.GetAbsolutePath().ToString(), credentials
);
}
catch (IOException io)
{
Log.Info("Logging exception:", io);
NUnit.Framework.Assert.IsTrue(io.GetLocalizedMessage().Contains(failString));
}
}
public AppInfo(ApplicationReport app)
{
// JAXB needs this
appId = app.GetApplicationId().ToString();
if (app.GetCurrentApplicationAttemptId() != null)
{
currentAppAttemptId = app.GetCurrentApplicationAttemptId().ToString();
}
user = app.GetUser();
queue = app.GetQueue();
name = app.GetName();
type = app.GetApplicationType();
host = app.GetHost();
rpcPort = app.GetRpcPort();
appState = app.GetYarnApplicationState();
diagnosticsInfo = app.GetDiagnostics();
trackingUrl = app.GetTrackingUrl();
originalTrackingUrl = app.GetOriginalTrackingUrl();
submittedTime = app.GetStartTime();
startedTime = app.GetStartTime();
finishedTime = app.GetFinishTime();
elapsedTime = Times.Elapsed(startedTime, finishedTime);
finalAppStatus = app.GetFinalApplicationStatus();
progress = app.GetProgress() * 100;
// in percent
if (app.GetApplicationTags() != null && !app.GetApplicationTags().IsEmpty())
{
this.applicationTags = StringHelper.CsvJoiner.Join(app.GetApplicationTags());
}
}
/// <summary>Moves the application with the given ID to the given queue.</summary>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
private void MoveApplicationAcrossQueues(string applicationId, string queue)
{
ApplicationId appId = ConverterUtils.ToApplicationId(applicationId);
ApplicationReport appReport = client.GetApplicationReport(appId);
if (appReport.GetYarnApplicationState() == YarnApplicationState.Finished || appReport
.GetYarnApplicationState() == YarnApplicationState.Killed || appReport.GetYarnApplicationState
() == YarnApplicationState.Failed)
{
sysout.WriteLine("Application " + applicationId + " has already finished ");
}
else
{
sysout.WriteLine("Moving application " + applicationId + " to queue " + queue);
client.MoveApplicationAcrossQueues(appId, queue);
sysout.WriteLine("Successfully completed move.");
}
}
/// <exception cref="System.IO.IOException"/>
private void KillUnFinishedApplication(ApplicationId appId)
{
ApplicationReport application = null;
try
{
application = resMgrDelegate.GetApplicationReport(appId);
}
catch (YarnException e)
{
throw new IOException(e);
}
if (application.GetYarnApplicationState() == YarnApplicationState.Finished || application
.GetYarnApplicationState() == YarnApplicationState.Failed || application.GetYarnApplicationState
() == YarnApplicationState.Killed)
{
return;
}
KillApplication(appId);
}
private static GetApplicationReportResponse CreateApplicationReportWithFinishedApplication
()
{
ApplicationReport report = Org.Mockito.Mockito.Mock <ApplicationReport>();
Org.Mockito.Mockito.When(report.GetYarnApplicationState()).ThenReturn(YarnApplicationState
.Finished);
GetApplicationReportResponse response = Org.Mockito.Mockito.Mock <GetApplicationReportResponse
>();
Org.Mockito.Mockito.When(response.GetApplicationReport()).ThenReturn(report);
return(response);
}
public virtual void TestHandleRMHAafterSubmitApplicationCallWithSavedApplicationState
()
{
// Test scenario 1 when RM failover happens
// after SubmitApplication Call:
// RMStateStore already saved the ApplicationState when failover happens
StartRMs();
// Submit Application
// After submission, the applicationState will be saved in RMStateStore.
RMApp app0 = rm1.SubmitApp(200);
// Do the failover
ExplicitFailover();
// Since the applicationState has already been saved in RMStateStore
// before failover happens, the current active rm can load the previous
// applicationState.
ApplicationReport appReport = rm2.GetApplicationReport(app0.GetApplicationId());
// verify previous submission is successful.
NUnit.Framework.Assert.IsTrue(appReport.GetYarnApplicationState() == YarnApplicationState
.Accepted || appReport.GetYarnApplicationState() == YarnApplicationState.Submitted
);
}
public virtual void TestGetApplicationReportIdempotent()
{
// start two RMs, and transit rm1 to active, rm2 to standby
StartRMs();
// Submit Application
// After submission, the applicationState will be saved in RMStateStore.
RMApp app = rm1.SubmitApp(200);
ApplicationReport appReport1 = rm1.GetApplicationReport(app.GetApplicationId());
NUnit.Framework.Assert.IsTrue(appReport1.GetYarnApplicationState() == YarnApplicationState
.Accepted || appReport1.GetYarnApplicationState() == YarnApplicationState.Submitted
);
// call getApplicationReport again
ApplicationReport appReport2 = rm1.GetApplicationReport(app.GetApplicationId());
NUnit.Framework.Assert.AreEqual(appReport1.GetApplicationId(), appReport2.GetApplicationId
());
NUnit.Framework.Assert.AreEqual(appReport1.GetYarnApplicationState(), appReport2.
GetYarnApplicationState());
// Do the failover
ExplicitFailover();
// call getApplicationReport
ApplicationReport appReport3 = rm2.GetApplicationReport(app.GetApplicationId());
NUnit.Framework.Assert.AreEqual(appReport1.GetApplicationId(), appReport3.GetApplicationId
());
NUnit.Framework.Assert.AreEqual(appReport1.GetYarnApplicationState(), appReport3.
GetYarnApplicationState());
// call getApplicationReport again
ApplicationReport appReport4 = rm2.GetApplicationReport(app.GetApplicationId());
NUnit.Framework.Assert.AreEqual(appReport3.GetApplicationId(), appReport4.GetApplicationId
());
NUnit.Framework.Assert.AreEqual(appReport3.GetYarnApplicationState(), appReport4.
GetYarnApplicationState());
}
private ApplicationReport GetApplicationReport(YarnApplicationState yarnApplicationState
, FinalApplicationStatus finalApplicationStatus)
{
ApplicationReport appReport = Org.Mockito.Mockito.Mock <ApplicationReport>();
ApplicationResourceUsageReport appResources = Org.Mockito.Mockito.Mock <ApplicationResourceUsageReport
>();
Org.Mockito.Mockito.When(appReport.GetApplicationId()).ThenReturn(ApplicationId.NewInstance
(0, 0));
Org.Mockito.Mockito.When(appResources.GetNeededResources()).ThenReturn(Org.Apache.Hadoop.Yarn.Util.Records
.NewRecord <Resource>());
Org.Mockito.Mockito.When(appResources.GetReservedResources()).ThenReturn(Org.Apache.Hadoop.Yarn.Util.Records
.NewRecord <Resource>());
Org.Mockito.Mockito.When(appResources.GetUsedResources()).ThenReturn(Org.Apache.Hadoop.Yarn.Util.Records
.NewRecord <Resource>());
Org.Mockito.Mockito.When(appReport.GetApplicationResourceUsageReport()).ThenReturn
(appResources);
Org.Mockito.Mockito.When(appReport.GetYarnApplicationState()).ThenReturn(yarnApplicationState
);
Org.Mockito.Mockito.When(appReport.GetFinalApplicationStatus()).ThenReturn(finalApplicationStatus
);
return(appReport);
}
/// <exception cref="System.IO.IOException"/>
private static bool IsApplicationTerminated(ApplicationId appId, ApplicationClientProtocol
rmClient)
{
ApplicationReport appReport = null;
try
{
appReport = rmClient.GetApplicationReport(GetApplicationReportRequest.NewInstance
(appId)).GetApplicationReport();
}
catch (ApplicationNotFoundException)
{
return(true);
}
catch (YarnException e)
{
throw new IOException(e);
}
YarnApplicationState currentState = appReport.GetYarnApplicationState();
return(currentState == YarnApplicationState.Failed || currentState == YarnApplicationState
.Killed || currentState == YarnApplicationState.Finished);
}
public override bool IsApplicationActive(ApplicationId id)
{
ApplicationReport report = null;
try
{
report = client.GetApplicationReport(id);
}
catch (ApplicationNotFoundException)
{
// the app does not exist
return(false);
}
catch (IOException e)
{
throw new YarnException(e);
}
if (report == null)
{
// the app does not exist
return(false);
}
return(ActiveStates.Contains(report.GetYarnApplicationState()));
}
/// <exception cref="System.IO.IOException"/>
private MRClientProtocol GetProxy()
{
if (realProxy != null)
{
return(realProxy);
}
// Possibly allow nulls through the PB tunnel, otherwise deal with an exception
// and redirect to the history server.
ApplicationReport application = null;
try
{
application = rm.GetApplicationReport(appId);
}
catch (ApplicationNotFoundException)
{
application = null;
}
catch (YarnException e2)
{
throw new IOException(e2);
}
if (application != null)
{
trackingUrl = application.GetTrackingUrl();
}
IPEndPoint serviceAddr = null;
while (application == null || YarnApplicationState.Running == application.GetYarnApplicationState
())
{
if (application == null)
{
Log.Info("Could not get Job info from RM for job " + jobId + ". Redirecting to job history server."
);
return(CheckAndGetHSProxy(null, JobState.New));
}
try
{
if (application.GetHost() == null || string.Empty.Equals(application.GetHost()))
{
Log.Debug("AM not assigned to Job. Waiting to get the AM ...");
Sharpen.Thread.Sleep(2000);
Log.Debug("Application state is " + application.GetYarnApplicationState());
application = rm.GetApplicationReport(appId);
continue;
}
else
{
if (Unavailable.Equals(application.GetHost()))
{
if (!amAclDisabledStatusLogged)
{
Log.Info("Job " + jobId + " is running, but the host is unknown." + " Verify user has VIEW_JOB access."
);
amAclDisabledStatusLogged = true;
}
return(GetNotRunningJob(application, JobState.Running));
}
}
if (!conf.GetBoolean(MRJobConfig.JobAmAccessDisabled, false))
{
UserGroupInformation newUgi = UserGroupInformation.CreateRemoteUser(UserGroupInformation
.GetCurrentUser().GetUserName());
serviceAddr = NetUtils.CreateSocketAddrForHost(application.GetHost(), application
.GetRpcPort());
if (UserGroupInformation.IsSecurityEnabled())
{
Token clientToAMToken = application.GetClientToAMToken();
Org.Apache.Hadoop.Security.Token.Token <ClientToAMTokenIdentifier> token = ConverterUtils
.ConvertFromYarn(clientToAMToken, serviceAddr);
newUgi.AddToken(token);
}
Log.Debug("Connecting to " + serviceAddr);
IPEndPoint finalServiceAddr = serviceAddr;
realProxy = newUgi.DoAs(new _PrivilegedExceptionAction_202(this, finalServiceAddr
));
}
else
{
if (!amAclDisabledStatusLogged)
{
Log.Info("Network ACL closed to AM for job " + jobId + ". Not going to try to reach the AM."
);
amAclDisabledStatusLogged = true;
}
return(GetNotRunningJob(null, JobState.Running));
}
return(realProxy);
}
catch (IOException)
{
//possibly the AM has crashed
//there may be some time before AM is restarted
//keep retrying by getting the address from RM
Log.Info("Could not connect to " + serviceAddr + ". Waiting for getting the latest AM address..."
);
try
{
Sharpen.Thread.Sleep(2000);
}
catch (Exception e1)
{
Log.Warn("getProxy() call interruped", e1);
throw new YarnRuntimeException(e1);
}
try
{
application = rm.GetApplicationReport(appId);
}
catch (YarnException e1)
{
throw new IOException(e1);
}
if (application == null)
{
Log.Info("Could not get Job info from RM for job " + jobId + ". Redirecting to job history server."
);
return(CheckAndGetHSProxy(null, JobState.Running));
}
}
catch (Exception e)
{
Log.Warn("getProxy() call interruped", e);
throw new YarnRuntimeException(e);
}
catch (YarnException e)
{
throw new IOException(e);
}
}
string user = application.GetUser();
if (user == null)
{
throw new IOException("User is not set in the application report");
}
if (application.GetYarnApplicationState() == YarnApplicationState.New || application
.GetYarnApplicationState() == YarnApplicationState.NewSaving || application.GetYarnApplicationState
() == YarnApplicationState.Submitted || application.GetYarnApplicationState() ==
YarnApplicationState.Accepted)
{
realProxy = null;
return(GetNotRunningJob(application, JobState.New));
}
if (application.GetYarnApplicationState() == YarnApplicationState.Failed)
{
realProxy = null;
return(GetNotRunningJob(application, JobState.Failed));
}
if (application.GetYarnApplicationState() == YarnApplicationState.Killed)
{
realProxy = null;
return(GetNotRunningJob(application, JobState.Killed));
}
//History server can serve a job only if application
//succeeded.
if (application.GetYarnApplicationState() == YarnApplicationState.Finished)
{
Log.Info("Application state is completed. FinalApplicationStatus=" + application.
GetFinalApplicationStatus().ToString() + ". Redirecting to job history server");
realProxy = CheckAndGetHSProxy(application, JobState.Succeeded);
}
return(realProxy);
}
public virtual void Setup()
{
// start minicluster
conf = new YarnConfiguration();
yarnCluster = new MiniYARNCluster(typeof(TestAMRMClient).FullName, nodeCount, 1,
1);
yarnCluster.Init(conf);
yarnCluster.Start();
NUnit.Framework.Assert.IsNotNull(yarnCluster);
NUnit.Framework.Assert.AreEqual(Service.STATE.Started, yarnCluster.GetServiceState
());
// start rm client
yarnClient = (YarnClientImpl)YarnClient.CreateYarnClient();
yarnClient.Init(conf);
yarnClient.Start();
NUnit.Framework.Assert.IsNotNull(yarnClient);
NUnit.Framework.Assert.AreEqual(Service.STATE.Started, yarnClient.GetServiceState
());
// get node info
nodeReports = yarnClient.GetNodeReports(NodeState.Running);
// submit new app
ApplicationSubmissionContext appContext = yarnClient.CreateApplication().GetApplicationSubmissionContext
();
ApplicationId appId = appContext.GetApplicationId();
// set the application name
appContext.SetApplicationName("Test");
// Set the priority for the application master
Priority pri = Priority.NewInstance(0);
appContext.SetPriority(pri);
// Set the queue to which this application is to be submitted in the RM
appContext.SetQueue("default");
// Set up the container launch context for the application master
ContainerLaunchContext amContainer = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<ContainerLaunchContext>();
appContext.SetAMContainerSpec(amContainer);
// unmanaged AM
appContext.SetUnmanagedAM(true);
// Create the request to send to the applications manager
SubmitApplicationRequest appRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<SubmitApplicationRequest>();
appRequest.SetApplicationSubmissionContext(appContext);
// Submit the application to the applications manager
yarnClient.SubmitApplication(appContext);
// wait for app to start
int iterationsLeft = 30;
RMAppAttempt appAttempt = null;
while (iterationsLeft > 0)
{
ApplicationReport appReport = yarnClient.GetApplicationReport(appId);
if (appReport.GetYarnApplicationState() == YarnApplicationState.Accepted)
{
attemptId = appReport.GetCurrentApplicationAttemptId();
appAttempt = yarnCluster.GetResourceManager().GetRMContext().GetRMApps()[attemptId
.GetApplicationId()].GetCurrentAppAttempt();
while (true)
{
if (appAttempt.GetAppAttemptState() == RMAppAttemptState.Launched)
{
break;
}
}
break;
}
Sleep(1000);
--iterationsLeft;
}
if (iterationsLeft == 0)
{
NUnit.Framework.Assert.Fail("Application hasn't bee started");
}
// Just dig into the ResourceManager and get the AMRMToken just for the sake
// of testing.
UserGroupInformation.SetLoginUser(UserGroupInformation.CreateRemoteUser(UserGroupInformation
.GetCurrentUser().GetUserName()));
UserGroupInformation.GetCurrentUser().AddToken(appAttempt.GetAMRMToken());
//creating an instance NMTokenCase
nmTokenCache = new NMTokenCache();
// start am rm client
rmClient = (AMRMClientImpl <AMRMClient.ContainerRequest>)AMRMClient.CreateAMRMClient
<AMRMClient.ContainerRequest>();
//setting an instance NMTokenCase
rmClient.SetNMTokenCache(nmTokenCache);
rmClient.Init(conf);
rmClient.Start();
NUnit.Framework.Assert.IsNotNull(rmClient);
NUnit.Framework.Assert.AreEqual(Service.STATE.Started, rmClient.GetServiceState()
);
// start am nm client
nmClient = (NMClientImpl)NMClient.CreateNMClient();
//propagating the AMRMClient NMTokenCache instance
nmClient.SetNMTokenCache(rmClient.GetNMTokenCache());
nmClient.Init(conf);
nmClient.Start();
NUnit.Framework.Assert.IsNotNull(nmClient);
NUnit.Framework.Assert.AreEqual(Service.STATE.Started, nmClient.GetServiceState()
);
}
/// <exception cref="System.Exception"/>
public virtual void TestDSShell(bool haveDomain)
{
string[] args = new string[] { "--jar", AppmasterJar, "--num_containers", "2", "--shell_command"
, Shell.Windows ? "dir" : "ls", "--master_memory", "512", "--master_vcores", "2"
, "--container_memory", "128", "--container_vcores", "1" };
if (haveDomain)
{
string[] domainArgs = new string[] { "--domain", "TEST_DOMAIN", "--view_acls", "reader_user reader_group"
, "--modify_acls", "writer_user writer_group", "--create" };
IList <string> argsList = new AList <string>(Arrays.AsList(args));
Sharpen.Collections.AddAll(argsList, Arrays.AsList(domainArgs));
args = Sharpen.Collections.ToArray(argsList, new string[argsList.Count]);
}
Log.Info("Initializing DS Client");
Client client = new Client(new Configuration(yarnCluster.GetConfig()));
bool initSuccess = client.Init(args);
NUnit.Framework.Assert.IsTrue(initSuccess);
Log.Info("Running DS Client");
AtomicBoolean result = new AtomicBoolean(false);
Sharpen.Thread t = new _Thread_194(result, client);
t.Start();
YarnClient yarnClient = YarnClient.CreateYarnClient();
yarnClient.Init(new Configuration(yarnCluster.GetConfig()));
yarnClient.Start();
string hostName = NetUtils.GetHostname();
bool verified = false;
string errorMessage = string.Empty;
while (!verified)
{
IList <ApplicationReport> apps = yarnClient.GetApplications();
if (apps.Count == 0)
{
Sharpen.Thread.Sleep(10);
continue;
}
ApplicationReport appReport = apps[0];
if (appReport.GetHost().Equals("N/A"))
{
Sharpen.Thread.Sleep(10);
continue;
}
errorMessage = "Expected host name to start with '" + hostName + "', was '" + appReport
.GetHost() + "'. Expected rpc port to be '-1', was '" + appReport.GetRpcPort() +
"'.";
if (CheckHostname(appReport.GetHost()) && appReport.GetRpcPort() == -1)
{
verified = true;
}
if (appReport.GetYarnApplicationState() == YarnApplicationState.Finished)
{
break;
}
}
NUnit.Framework.Assert.IsTrue(errorMessage, verified);
t.Join();
Log.Info("Client run completed. Result=" + result);
NUnit.Framework.Assert.IsTrue(result.Get());
TimelineDomain domain = null;
if (haveDomain)
{
domain = yarnCluster.GetApplicationHistoryServer().GetTimelineStore().GetDomain("TEST_DOMAIN"
);
NUnit.Framework.Assert.IsNotNull(domain);
NUnit.Framework.Assert.AreEqual("reader_user reader_group", domain.GetReaders());
NUnit.Framework.Assert.AreEqual("writer_user writer_group", domain.GetWriters());
}
TimelineEntities entitiesAttempts = yarnCluster.GetApplicationHistoryServer().GetTimelineStore
().GetEntities(ApplicationMaster.DSEntity.DsAppAttempt.ToString(), null, null, null
, null, null, null, null, null, null);
NUnit.Framework.Assert.IsNotNull(entitiesAttempts);
NUnit.Framework.Assert.AreEqual(1, entitiesAttempts.GetEntities().Count);
NUnit.Framework.Assert.AreEqual(2, entitiesAttempts.GetEntities()[0].GetEvents().
Count);
NUnit.Framework.Assert.AreEqual(entitiesAttempts.GetEntities()[0].GetEntityType()
.ToString(), ApplicationMaster.DSEntity.DsAppAttempt.ToString());
if (haveDomain)
{
NUnit.Framework.Assert.AreEqual(domain.GetId(), entitiesAttempts.GetEntities()[0]
.GetDomainId());
}
else
{
NUnit.Framework.Assert.AreEqual("DEFAULT", entitiesAttempts.GetEntities()[0].GetDomainId
());
}
TimelineEntities entities = yarnCluster.GetApplicationHistoryServer().GetTimelineStore
().GetEntities(ApplicationMaster.DSEntity.DsContainer.ToString(), null, null, null
, null, null, null, null, null, null);
NUnit.Framework.Assert.IsNotNull(entities);
NUnit.Framework.Assert.AreEqual(2, entities.GetEntities().Count);
NUnit.Framework.Assert.AreEqual(entities.GetEntities()[0].GetEntityType().ToString
(), ApplicationMaster.DSEntity.DsContainer.ToString());
if (haveDomain)
{
NUnit.Framework.Assert.AreEqual(domain.GetId(), entities.GetEntities()[0].GetDomainId
());
}
else
{
NUnit.Framework.Assert.AreEqual("DEFAULT", entities.GetEntities()[0].GetDomainId(
));
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public virtual bool Run()
{
Log.Info("Starting Client");
// Connect to ResourceManager
rmClient.Start();
try
{
// Create launch context for app master
Log.Info("Setting up application submission context for ASM");
ApplicationSubmissionContext appContext = rmClient.CreateApplication().GetApplicationSubmissionContext
();
ApplicationId appId = appContext.GetApplicationId();
// set the application name
appContext.SetApplicationName(appName);
// Set the priority for the application master
Priority pri = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <Priority>();
pri.SetPriority(amPriority);
appContext.SetPriority(pri);
// Set the queue to which this application is to be submitted in the RM
appContext.SetQueue(amQueue);
// Set up the container launch context for the application master
ContainerLaunchContext amContainer = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<ContainerLaunchContext>();
appContext.SetAMContainerSpec(amContainer);
// unmanaged AM
appContext.SetUnmanagedAM(true);
Log.Info("Setting unmanaged AM");
// Submit the application to the applications manager
Log.Info("Submitting application to ASM");
rmClient.SubmitApplication(appContext);
ApplicationReport appReport = MonitorApplication(appId, EnumSet.Of(YarnApplicationState
.Accepted, YarnApplicationState.Killed, YarnApplicationState.Failed, YarnApplicationState
.Finished));
if (appReport.GetYarnApplicationState() == YarnApplicationState.Accepted)
{
// Monitor the application attempt to wait for launch state
ApplicationAttemptReport attemptReport = MonitorCurrentAppAttempt(appId, YarnApplicationAttemptState
.Launched);
ApplicationAttemptId attemptId = attemptReport.GetApplicationAttemptId();
Log.Info("Launching AM with application attempt id " + attemptId);
// launch AM
LaunchAM(attemptId);
// Monitor the application for end state
appReport = MonitorApplication(appId, EnumSet.Of(YarnApplicationState.Killed, YarnApplicationState
.Failed, YarnApplicationState.Finished));
}
YarnApplicationState appState = appReport.GetYarnApplicationState();
FinalApplicationStatus appStatus = appReport.GetFinalApplicationStatus();
Log.Info("App ended with state: " + appReport.GetYarnApplicationState() + " and status: "
+ appStatus);
bool success;
if (YarnApplicationState.Finished == appState && FinalApplicationStatus.Succeeded
== appStatus)
{
Log.Info("Application has completed successfully.");
success = true;
}
else
{
Log.Info("Application did finished unsuccessfully." + " YarnState=" + appState.ToString
() + ", FinalStatus=" + appStatus.ToString());
success = false;
}
return(success);
}
finally
{
rmClient.Stop();
}
}
/// <summary>Monitor the submitted application for completion.</summary>
/// <remarks>
/// Monitor the submitted application for completion. Kill application if time
/// expires.
/// </remarks>
/// <param name="appId">Application Id of application to be monitored</param>
/// <returns>true if application completed successfully</returns>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
private ApplicationReport MonitorApplication(ApplicationId appId, ICollection <YarnApplicationState
> finalState)
{
long foundAMCompletedTime = 0;
StringBuilder expectedFinalState = new StringBuilder();
bool first = true;
foreach (YarnApplicationState state in finalState)
{
if (first)
{
first = false;
expectedFinalState.Append(state.ToString());
}
else
{
expectedFinalState.Append("," + state.ToString());
}
}
while (true)
{
// Check app status every 1 second.
try
{
Sharpen.Thread.Sleep(1000);
}
catch (Exception)
{
Log.Debug("Thread sleep in monitoring loop interrupted");
}
// Get application report for the appId we are interested in
ApplicationReport report = rmClient.GetApplicationReport(appId);
Log.Info("Got application report from ASM for" + ", appId=" + appId.GetId() + ", appAttemptId="
+ report.GetCurrentApplicationAttemptId() + ", clientToAMToken=" + report.GetClientToAMToken
() + ", appDiagnostics=" + report.GetDiagnostics() + ", appMasterHost=" + report
.GetHost() + ", appQueue=" + report.GetQueue() + ", appMasterRpcPort=" + report.
GetRpcPort() + ", appStartTime=" + report.GetStartTime() + ", yarnAppState=" + report
.GetYarnApplicationState().ToString() + ", distributedFinalState=" + report.GetFinalApplicationStatus
().ToString() + ", appTrackingUrl=" + report.GetTrackingUrl() + ", appUser="******"Waited " + AmStateWaitTimeoutMs / 1000 + " seconds after process completed for AppReport"
+ " to reach desired final state. Not waiting anymore." + "CurrentState = " + state_1
+ ", ExpectedStates = " + expectedFinalState.ToString());
throw new RuntimeException("Failed to receive final expected state" + " in ApplicationReport"
+ ", CurrentState=" + state_1 + ", ExpectedStates=" + expectedFinalState.ToString
());
}
}
}
}
}
/// <summary>Prints the application report for an application id.</summary>
/// <param name="applicationId"/>
/// <returns>exitCode</returns>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
private int PrintApplicationReport(string applicationId)
{
ApplicationReport appReport = null;
try
{
appReport = client.GetApplicationReport(ConverterUtils.ToApplicationId(applicationId
));
}
catch (ApplicationNotFoundException)
{
sysout.WriteLine("Application with id '" + applicationId + "' doesn't exist in RM or Timeline Server."
);
return(-1);
}
// Use PrintWriter.println, which uses correct platform line ending.
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter appReportStr = new PrintWriter(new OutputStreamWriter(baos, Sharpen.Extensions.GetEncoding
("UTF-8")));
if (appReport != null)
{
appReportStr.WriteLine("Application Report : ");
appReportStr.Write("\tApplication-Id : ");
appReportStr.WriteLine(appReport.GetApplicationId());
appReportStr.Write("\tApplication-Name : ");
appReportStr.WriteLine(appReport.GetName());
appReportStr.Write("\tApplication-Type : ");
appReportStr.WriteLine(appReport.GetApplicationType());
appReportStr.Write("\tUser : "******"\tQueue : ");
appReportStr.WriteLine(appReport.GetQueue());
appReportStr.Write("\tStart-Time : ");
appReportStr.WriteLine(appReport.GetStartTime());
appReportStr.Write("\tFinish-Time : ");
appReportStr.WriteLine(appReport.GetFinishTime());
appReportStr.Write("\tProgress : ");
DecimalFormat formatter = new DecimalFormat("###.##%");
string progress = formatter.Format(appReport.GetProgress());
appReportStr.WriteLine(progress);
appReportStr.Write("\tState : ");
appReportStr.WriteLine(appReport.GetYarnApplicationState());
appReportStr.Write("\tFinal-State : ");
appReportStr.WriteLine(appReport.GetFinalApplicationStatus());
appReportStr.Write("\tTracking-URL : ");
appReportStr.WriteLine(appReport.GetOriginalTrackingUrl());
appReportStr.Write("\tRPC Port : ");
appReportStr.WriteLine(appReport.GetRpcPort());
appReportStr.Write("\tAM Host : ");
appReportStr.WriteLine(appReport.GetHost());
appReportStr.Write("\tAggregate Resource Allocation : ");
ApplicationResourceUsageReport usageReport = appReport.GetApplicationResourceUsageReport
();
if (usageReport != null)
{
//completed app report in the timeline server doesn't have usage report
appReportStr.Write(usageReport.GetMemorySeconds() + " MB-seconds, ");
appReportStr.WriteLine(usageReport.GetVcoreSeconds() + " vcore-seconds");
}
else
{
appReportStr.WriteLine("N/A");
}
appReportStr.Write("\tDiagnostics : ");
appReportStr.Write(appReport.GetDiagnostics());
}
else
{
appReportStr.Write("Application with id '" + applicationId + "' doesn't exist in RM."
);
appReportStr.Close();
sysout.WriteLine(baos.ToString("UTF-8"));
return(-1);
}
appReportStr.Close();
sysout.WriteLine(baos.ToString("UTF-8"));
return(0);
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public override ApplicationId SubmitApplication(ApplicationSubmissionContext appContext
)
{
ApplicationId applicationId = appContext.GetApplicationId();
if (applicationId == null)
{
throw new ApplicationIdNotProvidedException("ApplicationId is not provided in ApplicationSubmissionContext"
);
}
SubmitApplicationRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <
SubmitApplicationRequest>();
request.SetApplicationSubmissionContext(appContext);
// Automatically add the timeline DT into the CLC
// Only when the security and the timeline service are both enabled
if (IsSecurityEnabled() && timelineServiceEnabled)
{
AddTimelineDelegationToken(appContext.GetAMContainerSpec());
}
//TODO: YARN-1763:Handle RM failovers during the submitApplication call.
rmClient.SubmitApplication(request);
int pollCount = 0;
long startTime = Runtime.CurrentTimeMillis();
EnumSet <YarnApplicationState> waitingStates = EnumSet.Of(YarnApplicationState.New
, YarnApplicationState.NewSaving, YarnApplicationState.Submitted);
EnumSet <YarnApplicationState> failToSubmitStates = EnumSet.Of(YarnApplicationState
.Failed, YarnApplicationState.Killed);
while (true)
{
try
{
ApplicationReport appReport = GetApplicationReport(applicationId);
YarnApplicationState state = appReport.GetYarnApplicationState();
if (!waitingStates.Contains(state))
{
if (failToSubmitStates.Contains(state))
{
throw new YarnException("Failed to submit " + applicationId + " to YARN : " + appReport
.GetDiagnostics());
}
Log.Info("Submitted application " + applicationId);
break;
}
long elapsedMillis = Runtime.CurrentTimeMillis() - startTime;
if (EnforceAsyncAPITimeout() && elapsedMillis >= asyncApiPollTimeoutMillis)
{
throw new YarnException("Timed out while waiting for application " + applicationId
+ " to be submitted successfully");
}
// Notify the client through the log every 10 poll, in case the client
// is blocked here too long.
if (++pollCount % 10 == 0)
{
Log.Info("Application submission is not finished, " + "submitted application " +
applicationId + " is still in " + state);
}
try
{
Sharpen.Thread.Sleep(submitPollIntervalMillis);
}
catch (Exception)
{
Log.Error("Interrupted while waiting for application " + applicationId + " to be successfully submitted."
);
}
}
catch (ApplicationNotFoundException)
{
// FailOver or RM restart happens before RMStateStore saves
// ApplicationState
Log.Info("Re-submit application " + applicationId + "with the " + "same ApplicationSubmissionContext"
);
rmClient.SubmitApplication(request);
}
}
return(applicationId);
}
public virtual void TestFromYarnApplicationReport()
{
ApplicationId mockAppId = Org.Mockito.Mockito.Mock <ApplicationId>();
Org.Mockito.Mockito.When(mockAppId.GetClusterTimestamp()).ThenReturn(12345L);
Org.Mockito.Mockito.When(mockAppId.GetId()).ThenReturn(6789);
ApplicationReport mockReport = Org.Mockito.Mockito.Mock <ApplicationReport>();
Org.Mockito.Mockito.When(mockReport.GetTrackingUrl()).ThenReturn("dummy-tracking-url"
);
Org.Mockito.Mockito.When(mockReport.GetApplicationId()).ThenReturn(mockAppId);
Org.Mockito.Mockito.When(mockReport.GetYarnApplicationState()).ThenReturn(YarnApplicationState
.Killed);
Org.Mockito.Mockito.When(mockReport.GetUser()).ThenReturn("dummy-user");
Org.Mockito.Mockito.When(mockReport.GetQueue()).ThenReturn("dummy-queue");
string jobFile = "dummy-path/job.xml";
try
{
JobStatus status = TypeConverter.FromYarn(mockReport, jobFile);
}
catch (ArgumentNullException)
{
NUnit.Framework.Assert.Fail("Type converstion from YARN fails for jobs without "
+ "ApplicationUsageReport");
}
ApplicationResourceUsageReport appUsageRpt = Org.Apache.Hadoop.Yarn.Util.Records.
NewRecord <ApplicationResourceUsageReport>();
Resource r = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <Resource>();
r.SetMemory(2048);
appUsageRpt.SetNeededResources(r);
appUsageRpt.SetNumReservedContainers(1);
appUsageRpt.SetNumUsedContainers(3);
appUsageRpt.SetReservedResources(r);
appUsageRpt.SetUsedResources(r);
Org.Mockito.Mockito.When(mockReport.GetApplicationResourceUsageReport()).ThenReturn
(appUsageRpt);
JobStatus status_1 = TypeConverter.FromYarn(mockReport, jobFile);
NUnit.Framework.Assert.IsNotNull("fromYarn returned null status", status_1);
NUnit.Framework.Assert.AreEqual("jobFile set incorrectly", "dummy-path/job.xml",
status_1.GetJobFile());
NUnit.Framework.Assert.AreEqual("queue set incorrectly", "dummy-queue", status_1.
GetQueue());
NUnit.Framework.Assert.AreEqual("trackingUrl set incorrectly", "dummy-tracking-url"
, status_1.GetTrackingUrl());
NUnit.Framework.Assert.AreEqual("user set incorrectly", "dummy-user", status_1.GetUsername
());
NUnit.Framework.Assert.AreEqual("schedulingInfo set incorrectly", "dummy-tracking-url"
, status_1.GetSchedulingInfo());
NUnit.Framework.Assert.AreEqual("jobId set incorrectly", 6789, status_1.GetJobID(
).GetId());
NUnit.Framework.Assert.AreEqual("state set incorrectly", JobStatus.State.Killed,
status_1.GetState());
NUnit.Framework.Assert.AreEqual("needed mem info set incorrectly", 2048, status_1
.GetNeededMem());
NUnit.Framework.Assert.AreEqual("num rsvd slots info set incorrectly", 1, status_1
.GetNumReservedSlots());
NUnit.Framework.Assert.AreEqual("num used slots info set incorrectly", 3, status_1
.GetNumUsedSlots());
NUnit.Framework.Assert.AreEqual("rsvd mem info set incorrectly", 2048, status_1.GetReservedMem
());
NUnit.Framework.Assert.AreEqual("used mem info set incorrectly", 2048, status_1.GetUsedMem
());
}
public static JobStatus FromYarn(ApplicationReport application, string jobFile)
{
string trackingUrl = application.GetTrackingUrl();
trackingUrl = trackingUrl == null ? string.Empty : trackingUrl;
JobStatus jobStatus = new JobStatus(TypeConverter.FromYarn(application.GetApplicationId
()), 0.0f, 0.0f, 0.0f, 0.0f, TypeConverter.FromYarn(application.GetYarnApplicationState
(), application.GetFinalApplicationStatus()), JobPriority.Normal, application.GetUser
(), application.GetName(), application.GetQueue(), jobFile, trackingUrl, false);
jobStatus.SetSchedulingInfo(trackingUrl);
// Set AM tracking url
jobStatus.SetStartTime(application.GetStartTime());
jobStatus.SetFinishTime(application.GetFinishTime());
jobStatus.SetFailureInfo(application.GetDiagnostics());
ApplicationResourceUsageReport resourceUsageReport = application.GetApplicationResourceUsageReport
();
if (resourceUsageReport != null)
{
jobStatus.SetNeededMem(resourceUsageReport.GetNeededResources().GetMemory());
jobStatus.SetNumReservedSlots(resourceUsageReport.GetNumReservedContainers());
jobStatus.SetNumUsedSlots(resourceUsageReport.GetNumUsedContainers());
jobStatus.SetReservedMem(resourceUsageReport.GetReservedResources().GetMemory());
jobStatus.SetUsedMem(resourceUsageReport.GetUsedResources().GetMemory());
}
return(jobStatus);
}
/// <exception cref="System.IO.IOException"/>
protected override void DoGet(HttpServletRequest req, HttpServletResponse resp)
{
try
{
string userApprovedParamS = req.GetParameter(ProxyUriUtils.ProxyApprovalParam);
bool userWasWarned = false;
bool userApproved = Sharpen.Extensions.ValueOf(userApprovedParamS);
bool securityEnabled = IsSecurityEnabled();
string remoteUser = req.GetRemoteUser();
string pathInfo = req.GetPathInfo();
string[] parts = pathInfo.Split("/", 3);
if (parts.Length < 2)
{
Log.Warn("{} gave an invalid proxy path {}", remoteUser, pathInfo);
NotFound(resp, "Your path appears to be formatted incorrectly.");
return;
}
//parts[0] is empty because path info always starts with a /
string appId = parts[1];
string rest = parts.Length > 2 ? parts[2] : string.Empty;
ApplicationId id = Apps.ToAppID(appId);
if (id == null)
{
Log.Warn("{} attempting to access {} that is invalid", remoteUser, appId);
NotFound(resp, appId + " appears to be formatted incorrectly.");
return;
}
if (securityEnabled)
{
string cookieName = GetCheckCookieName(id);
Cookie[] cookies = req.GetCookies();
if (cookies != null)
{
foreach (Cookie c in cookies)
{
if (cookieName.Equals(c.GetName()))
{
userWasWarned = true;
userApproved = userApproved || Sharpen.Extensions.ValueOf(c.GetValue());
break;
}
}
}
}
bool checkUser = securityEnabled && (!userWasWarned || !userApproved);
AppReportFetcher.FetchedAppReport fetchedAppReport = null;
ApplicationReport applicationReport = null;
try
{
fetchedAppReport = GetApplicationReport(id);
if (fetchedAppReport != null)
{
if (fetchedAppReport.GetAppReportSource() != AppReportFetcher.AppReportSource.Rm &&
fetchedAppReport.GetAppReportSource() != AppReportFetcher.AppReportSource.Ahs)
{
throw new NotSupportedException("Application report not " + "fetched from RM or history server."
);
}
applicationReport = fetchedAppReport.GetApplicationReport();
}
}
catch (ApplicationNotFoundException)
{
applicationReport = null;
}
if (applicationReport == null)
{
Log.Warn("{} attempting to access {} that was not found", remoteUser, id);
URI toFetch = ProxyUriUtils.GetUriFromTrackingPlugins(id, this.trackingUriPlugins
);
if (toFetch != null)
{
ProxyUtils.SendRedirect(req, resp, toFetch.ToString());
return;
}
NotFound(resp, "Application " + appId + " could not be found " + "in RM or history server"
);
return;
}
string original = applicationReport.GetOriginalTrackingUrl();
URI trackingUri;
if (original == null || original.Equals("N/A") || original.Equals(string.Empty))
{
if (fetchedAppReport.GetAppReportSource() == AppReportFetcher.AppReportSource.Rm)
{
// fallback to ResourceManager's app page if no tracking URI provided
// and Application Report was fetched from RM
Log.Debug("Original tracking url is '{}'. Redirecting to RM app page", original ==
null ? "NULL" : original);
ProxyUtils.SendRedirect(req, resp, StringHelper.Pjoin(rmAppPageUrlBase, id.ToString
()));
}
else
{
if (fetchedAppReport.GetAppReportSource() == AppReportFetcher.AppReportSource.Ahs)
{
// fallback to Application History Server app page if the application
// report was fetched from AHS
Log.Debug("Original tracking url is '{}'. Redirecting to AHS app page", original
== null ? "NULL" : original);
ProxyUtils.SendRedirect(req, resp, StringHelper.Pjoin(ahsAppPageUrlBase, id.ToString
()));
}
}
return;
}
else
{
if (ProxyUriUtils.GetSchemeFromUrl(original).IsEmpty())
{
trackingUri = ProxyUriUtils.GetUriFromAMUrl(WebAppUtils.GetHttpSchemePrefix(conf)
, original);
}
else
{
trackingUri = new URI(original);
}
}
string runningUser = applicationReport.GetUser();
if (checkUser && !runningUser.Equals(remoteUser))
{
Log.Info("Asking {} if they want to connect to the " + "app master GUI of {} owned by {}"
, remoteUser, appId, runningUser);
WarnUserPage(resp, ProxyUriUtils.GetPathAndQuery(id, rest, req.GetQueryString(),
true), runningUser, id);
return;
}
// Append the user-provided path and query parameter to the original
// tracking url.
IList <NameValuePair> queryPairs = URLEncodedUtils.Parse(req.GetQueryString(), null
);
UriBuilder builder = UriBuilder.FromUri(trackingUri);
foreach (NameValuePair pair in queryPairs)
{
builder.QueryParam(pair.GetName(), pair.GetValue());
}
URI toFetch_1 = builder.Path(rest).Build();
Log.Info("{} is accessing unchecked {}" + " which is the app master GUI of {} owned by {}"
, remoteUser, toFetch_1, appId, runningUser);
switch (applicationReport.GetYarnApplicationState())
{
case YarnApplicationState.Killed:
case YarnApplicationState.Finished:
case YarnApplicationState.Failed:
{
ProxyUtils.SendRedirect(req, resp, toFetch_1.ToString());
return;
}
default:
{
break;
}
}
// fall out of the switch
Cookie c_1 = null;
if (userWasWarned && userApproved)
{
c_1 = MakeCheckCookie(id, true);
}
ProxyLink(req, resp, toFetch_1, c_1, GetProxyHost());
}
catch (Exception e)
{
throw new IOException(e);
}
}