/// <summary>Monitor the submitted application for completion.</summary>
/// <remarks>
/// Monitor the submitted application for completion.
/// Kill application if time expires.
/// </remarks>
/// <param name="appId">Application Id of application to be monitored</param>
/// <returns>true if application completed successfully</returns>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
private bool MonitorApplication(ApplicationId appId)
{
while (true)
{
// Check app status every 1 second.
try
{
Sharpen.Thread.Sleep(1000);
}
catch (Exception)
{
Log.Debug("Thread sleep in monitoring loop interrupted");
}
// Get application report for the appId we are interested in
ApplicationReport report = yarnClient.GetApplicationReport(appId);
Log.Info("Got application report from ASM for" + ", appId=" + appId.GetId() + ", clientToAMToken="
+ report.GetClientToAMToken() + ", appDiagnostics=" + report.GetDiagnostics() +
", appMasterHost=" + report.GetHost() + ", appQueue=" + report.GetQueue() + ", appMasterRpcPort="
+ report.GetRpcPort() + ", appStartTime=" + report.GetStartTime() + ", yarnAppState="
+ report.GetYarnApplicationState().ToString() + ", distributedFinalState=" + report
.GetFinalApplicationStatus().ToString() + ", appTrackingUrl=" + report.GetTrackingUrl
() + ", appUser="******"Application has completed successfully. Breaking monitoring loop");
return(true);
}
else
{
Log.Info("Application did finished unsuccessfully." + " YarnState=" + state.ToString
() + ", DSFinalStatus=" + dsStatus.ToString() + ". Breaking monitoring loop");
return(false);
}
}
else
{
if (YarnApplicationState.Killed == state || YarnApplicationState.Failed == state)
{
Log.Info("Application did not finish." + " YarnState=" + state.ToString() + ", DSFinalStatus="
+ dsStatus.ToString() + ". Breaking monitoring loop");
return(false);
}
}
if (Runtime.CurrentTimeMillis() > (clientStartTime + clientTimeout))
{
Log.Info("Reached client specified timeout for application. Killing application");
ForceKillApplication(appId);
return(false);
}
}
}
/// <summary>Monitor the submitted application for completion.</summary>
/// <remarks>
/// Monitor the submitted application for completion. Kill application if time
/// expires.
/// </remarks>
/// <param name="appId">Application Id of application to be monitored</param>
/// <returns>true if application completed successfully</returns>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
private ApplicationReport MonitorApplication(ApplicationId appId, ICollection <YarnApplicationState
> finalState)
{
long foundAMCompletedTime = 0;
StringBuilder expectedFinalState = new StringBuilder();
bool first = true;
foreach (YarnApplicationState state in finalState)
{
if (first)
{
first = false;
expectedFinalState.Append(state.ToString());
}
else
{
expectedFinalState.Append("," + state.ToString());
}
}
while (true)
{
// Check app status every 1 second.
try
{
Sharpen.Thread.Sleep(1000);
}
catch (Exception)
{
Log.Debug("Thread sleep in monitoring loop interrupted");
}
// Get application report for the appId we are interested in
ApplicationReport report = rmClient.GetApplicationReport(appId);
Log.Info("Got application report from ASM for" + ", appId=" + appId.GetId() + ", appAttemptId="
+ report.GetCurrentApplicationAttemptId() + ", clientToAMToken=" + report.GetClientToAMToken
() + ", appDiagnostics=" + report.GetDiagnostics() + ", appMasterHost=" + report
.GetHost() + ", appQueue=" + report.GetQueue() + ", appMasterRpcPort=" + report.
GetRpcPort() + ", appStartTime=" + report.GetStartTime() + ", yarnAppState=" + report
.GetYarnApplicationState().ToString() + ", distributedFinalState=" + report.GetFinalApplicationStatus
().ToString() + ", appTrackingUrl=" + report.GetTrackingUrl() + ", appUser="******"Waited " + AmStateWaitTimeoutMs / 1000 + " seconds after process completed for AppReport"
+ " to reach desired final state. Not waiting anymore." + "CurrentState = " + state_1
+ ", ExpectedStates = " + expectedFinalState.ToString());
throw new RuntimeException("Failed to receive final expected state" + " in ApplicationReport"
+ ", CurrentState=" + state_1 + ", ExpectedStates=" + expectedFinalState.ToString
());
}
}
}
}
}
/// <exception cref="System.IO.IOException"/>
private MRClientProtocol GetProxy()
{
if (realProxy != null)
{
return(realProxy);
}
// Possibly allow nulls through the PB tunnel, otherwise deal with an exception
// and redirect to the history server.
ApplicationReport application = null;
try
{
application = rm.GetApplicationReport(appId);
}
catch (ApplicationNotFoundException)
{
application = null;
}
catch (YarnException e2)
{
throw new IOException(e2);
}
if (application != null)
{
trackingUrl = application.GetTrackingUrl();
}
IPEndPoint serviceAddr = null;
while (application == null || YarnApplicationState.Running == application.GetYarnApplicationState
())
{
if (application == null)
{
Log.Info("Could not get Job info from RM for job " + jobId + ". Redirecting to job history server."
);
return(CheckAndGetHSProxy(null, JobState.New));
}
try
{
if (application.GetHost() == null || string.Empty.Equals(application.GetHost()))
{
Log.Debug("AM not assigned to Job. Waiting to get the AM ...");
Sharpen.Thread.Sleep(2000);
Log.Debug("Application state is " + application.GetYarnApplicationState());
application = rm.GetApplicationReport(appId);
continue;
}
else
{
if (Unavailable.Equals(application.GetHost()))
{
if (!amAclDisabledStatusLogged)
{
Log.Info("Job " + jobId + " is running, but the host is unknown." + " Verify user has VIEW_JOB access."
);
amAclDisabledStatusLogged = true;
}
return(GetNotRunningJob(application, JobState.Running));
}
}
if (!conf.GetBoolean(MRJobConfig.JobAmAccessDisabled, false))
{
UserGroupInformation newUgi = UserGroupInformation.CreateRemoteUser(UserGroupInformation
.GetCurrentUser().GetUserName());
serviceAddr = NetUtils.CreateSocketAddrForHost(application.GetHost(), application
.GetRpcPort());
if (UserGroupInformation.IsSecurityEnabled())
{
Token clientToAMToken = application.GetClientToAMToken();
Org.Apache.Hadoop.Security.Token.Token <ClientToAMTokenIdentifier> token = ConverterUtils
.ConvertFromYarn(clientToAMToken, serviceAddr);
newUgi.AddToken(token);
}
Log.Debug("Connecting to " + serviceAddr);
IPEndPoint finalServiceAddr = serviceAddr;
realProxy = newUgi.DoAs(new _PrivilegedExceptionAction_202(this, finalServiceAddr
));
}
else
{
if (!amAclDisabledStatusLogged)
{
Log.Info("Network ACL closed to AM for job " + jobId + ". Not going to try to reach the AM."
);
amAclDisabledStatusLogged = true;
}
return(GetNotRunningJob(null, JobState.Running));
}
return(realProxy);
}
catch (IOException)
{
//possibly the AM has crashed
//there may be some time before AM is restarted
//keep retrying by getting the address from RM
Log.Info("Could not connect to " + serviceAddr + ". Waiting for getting the latest AM address..."
);
try
{
Sharpen.Thread.Sleep(2000);
}
catch (Exception e1)
{
Log.Warn("getProxy() call interruped", e1);
throw new YarnRuntimeException(e1);
}
try
{
application = rm.GetApplicationReport(appId);
}
catch (YarnException e1)
{
throw new IOException(e1);
}
if (application == null)
{
Log.Info("Could not get Job info from RM for job " + jobId + ". Redirecting to job history server."
);
return(CheckAndGetHSProxy(null, JobState.Running));
}
}
catch (Exception e)
{
Log.Warn("getProxy() call interruped", e);
throw new YarnRuntimeException(e);
}
catch (YarnException e)
{
throw new IOException(e);
}
}
string user = application.GetUser();
if (user == null)
{
throw new IOException("User is not set in the application report");
}
if (application.GetYarnApplicationState() == YarnApplicationState.New || application
.GetYarnApplicationState() == YarnApplicationState.NewSaving || application.GetYarnApplicationState
() == YarnApplicationState.Submitted || application.GetYarnApplicationState() ==
YarnApplicationState.Accepted)
{
realProxy = null;
return(GetNotRunningJob(application, JobState.New));
}
if (application.GetYarnApplicationState() == YarnApplicationState.Failed)
{
realProxy = null;
return(GetNotRunningJob(application, JobState.Failed));
}
if (application.GetYarnApplicationState() == YarnApplicationState.Killed)
{
realProxy = null;
return(GetNotRunningJob(application, JobState.Killed));
}
//History server can serve a job only if application
//succeeded.
if (application.GetYarnApplicationState() == YarnApplicationState.Finished)
{
Log.Info("Application state is completed. FinalApplicationStatus=" + application.
GetFinalApplicationStatus().ToString() + ". Redirecting to job history server");
realProxy = CheckAndGetHSProxy(application, JobState.Succeeded);
}
return(realProxy);
}
/// <exception cref="System.Exception"/>
public virtual void TestClientTokenRace()
{
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
ContainerManagementProtocol containerManager = Org.Mockito.Mockito.Mock <ContainerManagementProtocol
>();
StartContainersResponse mockResponse = Org.Mockito.Mockito.Mock <StartContainersResponse
>();
Org.Mockito.Mockito.When(containerManager.StartContainers((StartContainersRequest
)Matchers.Any())).ThenReturn(mockResponse);
DrainDispatcher dispatcher = new DrainDispatcher();
MockRM rm = new _MockRMWithCustomAMLauncher_433(dispatcher, conf, containerManager
);
rm.Start();
// Submit an app
RMApp app = rm.SubmitApp(1024);
// Set up a node.
MockNM nm1 = rm.RegisterNode("localhost:1234", 3072);
nm1.NodeHeartbeat(true);
dispatcher.Await();
nm1.NodeHeartbeat(true);
dispatcher.Await();
ApplicationAttemptId appAttempt = app.GetCurrentAppAttempt().GetAppAttemptId();
MockAM mockAM = new MockAM(rm.GetRMContext(), rm.GetApplicationMasterService(), app
.GetCurrentAppAttempt().GetAppAttemptId());
UserGroupInformation appUgi = UserGroupInformation.CreateRemoteUser(appAttempt.ToString
());
RegisterApplicationMasterResponse response = appUgi.DoAs(new _PrivilegedAction_469
(mockAM));
// Get the app-report.
GetApplicationReportRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<GetApplicationReportRequest>();
request.SetApplicationId(app.GetApplicationId());
GetApplicationReportResponse reportResponse = rm.GetClientRMService().GetApplicationReport
(request);
ApplicationReport appReport = reportResponse.GetApplicationReport();
Org.Apache.Hadoop.Yarn.Api.Records.Token originalClientToAMToken = appReport.GetClientToAMToken
();
// ClientToAMToken master key should have been received on register
// application master response.
ByteBuffer clientMasterKey = response.GetClientToAMTokenMasterKey();
NUnit.Framework.Assert.IsNotNull(clientMasterKey);
NUnit.Framework.Assert.IsTrue(((byte[])clientMasterKey.Array()).Length > 0);
// Start the AM with the correct shared-secret.
ApplicationAttemptId appAttemptId = app.GetAppAttempts().Keys.GetEnumerator().Next
();
NUnit.Framework.Assert.IsNotNull(appAttemptId);
TestClientToAMTokens.CustomAM am = new TestClientToAMTokens.CustomAM(appAttemptId
, null);
am.Init(conf);
am.Start();
// Now the real test!
// Set up clients to be able to pick up correct tokens.
SecurityUtil.SetSecurityInfoProviders(new TestClientToAMTokens.CustomSecurityInfo
());
Org.Apache.Hadoop.Security.Token.Token <ClientToAMTokenIdentifier> token = ConverterUtils
.ConvertFromYarn(originalClientToAMToken, am.address);
// Schedule the key to be set after a significant delay
Timer timer = new Timer();
TimerTask timerTask = new _TimerTask_516(am, clientMasterKey);
timer.Schedule(timerTask, 250);
// connect should pause waiting for the master key to arrive
VerifyValidToken(conf, am, token);
am.Stop();
rm.Stop();
}
public virtual void TestClientToAMTokens()
{
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
ContainerManagementProtocol containerManager = Org.Mockito.Mockito.Mock <ContainerManagementProtocol
>();
StartContainersResponse mockResponse = Org.Mockito.Mockito.Mock <StartContainersResponse
>();
Org.Mockito.Mockito.When(containerManager.StartContainers((StartContainersRequest
)Matchers.Any())).ThenReturn(mockResponse);
DrainDispatcher dispatcher = new DrainDispatcher();
MockRM rm = new _MockRMWithCustomAMLauncher_192(dispatcher, conf, containerManager
);
rm.Start();
// Submit an app
RMApp app = rm.SubmitApp(1024);
// Set up a node.
MockNM nm1 = rm.RegisterNode("localhost:1234", 3072);
nm1.NodeHeartbeat(true);
dispatcher.Await();
nm1.NodeHeartbeat(true);
dispatcher.Await();
ApplicationAttemptId appAttempt = app.GetCurrentAppAttempt().GetAppAttemptId();
MockAM mockAM = new MockAM(rm.GetRMContext(), rm.GetApplicationMasterService(), app
.GetCurrentAppAttempt().GetAppAttemptId());
UserGroupInformation appUgi = UserGroupInformation.CreateRemoteUser(appAttempt.ToString
());
RegisterApplicationMasterResponse response = appUgi.DoAs(new _PrivilegedAction_229
(mockAM));
// Get the app-report.
GetApplicationReportRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<GetApplicationReportRequest>();
request.SetApplicationId(app.GetApplicationId());
GetApplicationReportResponse reportResponse = rm.GetClientRMService().GetApplicationReport
(request);
ApplicationReport appReport = reportResponse.GetApplicationReport();
Org.Apache.Hadoop.Yarn.Api.Records.Token originalClientToAMToken = appReport.GetClientToAMToken
();
// ClientToAMToken master key should have been received on register
// application master response.
NUnit.Framework.Assert.IsNotNull(response.GetClientToAMTokenMasterKey());
NUnit.Framework.Assert.IsTrue(((byte[])response.GetClientToAMTokenMasterKey().Array
()).Length > 0);
// Start the AM with the correct shared-secret.
ApplicationAttemptId appAttemptId = app.GetAppAttempts().Keys.GetEnumerator().Next
();
NUnit.Framework.Assert.IsNotNull(appAttemptId);
TestClientToAMTokens.CustomAM am = new TestClientToAMTokens.CustomAM(appAttemptId
, ((byte[])response.GetClientToAMTokenMasterKey().Array()));
am.Init(conf);
am.Start();
// Now the real test!
// Set up clients to be able to pick up correct tokens.
SecurityUtil.SetSecurityInfoProviders(new TestClientToAMTokens.CustomSecurityInfo
());
// Verify denial for unauthenticated user
try
{
TestClientToAMTokens.CustomProtocol client = (TestClientToAMTokens.CustomProtocol
)RPC.GetProxy <TestClientToAMTokens.CustomProtocol>(1L, am.address, conf);
client.Ping();
NUnit.Framework.Assert.Fail("Access by unauthenticated user should fail!!");
}
catch (Exception)
{
NUnit.Framework.Assert.IsFalse(am.pinged);
}
Org.Apache.Hadoop.Security.Token.Token <ClientToAMTokenIdentifier> token = ConverterUtils
.ConvertFromYarn(originalClientToAMToken, am.address);
// Verify denial for a malicious user with tampered ID
VerifyTokenWithTamperedID(conf, am, token);
// Verify denial for a malicious user with tampered user-name
VerifyTokenWithTamperedUserName(conf, am, token);
// Now for an authenticated user
VerifyValidToken(conf, am, token);
// Verify for a new version token
VerifyNewVersionToken(conf, am, token, rm);
am.Stop();
rm.Stop();
}
private void VerifyEnemyAppReport(ApplicationReport appReport)
{
NUnit.Framework.Assert.AreEqual("Enemy should not see app host!", Unavailable, appReport
.GetHost());
NUnit.Framework.Assert.AreEqual("Enemy should not see app rpc port!", -1, appReport
.GetRpcPort());
NUnit.Framework.Assert.AreEqual("Enemy should not see app client token!", null, appReport
.GetClientToAMToken());
NUnit.Framework.Assert.AreEqual("Enemy should not see app diagnostics!", Unavailable
, appReport.GetDiagnostics());
NUnit.Framework.Assert.AreEqual("Enemy should not see app tracking url!", Unavailable
, appReport.GetTrackingUrl());
NUnit.Framework.Assert.AreEqual("Enemy should not see app original tracking url!"
, Unavailable, appReport.GetOriginalTrackingUrl());
ApplicationResourceUsageReport usageReport = appReport.GetApplicationResourceUsageReport
();
NUnit.Framework.Assert.AreEqual("Enemy should not see app used containers", -1, usageReport
.GetNumUsedContainers());
NUnit.Framework.Assert.AreEqual("Enemy should not see app reserved containers", -
1, usageReport.GetNumReservedContainers());
NUnit.Framework.Assert.AreEqual("Enemy should not see app used resources", -1, usageReport
.GetUsedResources().GetMemory());
NUnit.Framework.Assert.AreEqual("Enemy should not see app reserved resources", -1
, usageReport.GetReservedResources().GetMemory());
NUnit.Framework.Assert.AreEqual("Enemy should not see app needed resources", -1,
usageReport.GetNeededResources().GetMemory());
}