/// <exception cref="System.Exception"/>
private static ApplicationClientProtocol CreateMockRMClient(IList <ApplicationId>
finishedApplicaitons, IList <ApplicationId> runningApplications)
{
ApplicationClientProtocol mockProtocol = Org.Mockito.Mockito.Mock <ApplicationClientProtocol
>();
if (finishedApplicaitons != null && !finishedApplicaitons.IsEmpty())
{
foreach (ApplicationId appId in finishedApplicaitons)
{
GetApplicationReportRequest request = GetApplicationReportRequest.NewInstance(appId
);
GetApplicationReportResponse response = CreateApplicationReportWithFinishedApplication
();
Org.Mockito.Mockito.When(mockProtocol.GetApplicationReport(request)).ThenReturn(response
);
}
}
if (runningApplications != null && !runningApplications.IsEmpty())
{
foreach (ApplicationId appId in runningApplications)
{
GetApplicationReportRequest request = GetApplicationReportRequest.NewInstance(appId
);
GetApplicationReportResponse response = CreateApplicationReportWithRunningApplication
();
Org.Mockito.Mockito.When(mockProtocol.GetApplicationReport(request)).ThenReturn(response
);
}
}
return(mockProtocol);
}
public virtual void TestApplicationNotFound()
{
ApplicationId appId = null;
appId = ApplicationId.NewInstance(0, MaxApps + 1);
GetApplicationReportRequest request = GetApplicationReportRequest.NewInstance(appId
);
try
{
GetApplicationReportResponse response = clientService.GetApplicationReport(request
);
NUnit.Framework.Assert.Fail("Exception should have been thrown before we reach here."
);
}
catch (ApplicationNotFoundException e)
{
//This exception is expected.
NUnit.Framework.Assert.IsTrue(e.Message.Contains("doesn't exist in the timeline store"
));
}
catch (Exception)
{
NUnit.Framework.Assert.Fail("Undesired exception caught");
}
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public override ApplicationReport GetApplicationReport(ApplicationId appId)
{
GetApplicationReportRequest request = GetApplicationReportRequest.NewInstance(appId
);
GetApplicationReportResponse response = ahsClient.GetApplicationReport(request);
return(response.GetApplicationReport());
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual ApplicationReport GetApplicationReport(ApplicationId appId)
{
ApplicationClientProtocol client = GetClientRMService();
GetApplicationReportResponse response = client.GetApplicationReport(GetApplicationReportRequest
.NewInstance(appId));
return(response.GetApplicationReport());
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public override GetApplicationReportResponse GetApplicationReport(GetApplicationReportRequest
request)
{
this._enclosing.ResetStartFailoverFlag(true);
// make sure failover has been triggered
NUnit.Framework.Assert.IsTrue(this._enclosing.WaittingForFailOver());
// create a fake application report
ApplicationReport report = this._enclosing.CreateFakeAppReport();
GetApplicationReportResponse response = GetApplicationReportResponse.NewInstance(
report);
return(response);
}
private static GetApplicationReportResponse CreateApplicationReportWithFinishedApplication
()
{
ApplicationReport report = Org.Mockito.Mockito.Mock <ApplicationReport>();
Org.Mockito.Mockito.When(report.GetYarnApplicationState()).ThenReturn(YarnApplicationState
.Finished);
GetApplicationReportResponse response = Org.Mockito.Mockito.Mock <GetApplicationReportResponse
>();
Org.Mockito.Mockito.When(response.GetApplicationReport()).ThenReturn(report);
return(response);
}
/// <exception cref="System.Exception"/>
private void VerifyGetClientAMToken(string submitter, string queueAdmin, string queueName
, bool setupACLs)
{
ApplicationId applicationId = SubmitAppAndGetAppId(submitter, queueName, setupACLs
);
GetApplicationReportRequest appReportRequest = GetApplicationReportRequest.NewInstance
(applicationId);
ApplicationClientProtocol submitterClient = GetRMClientForUser(submitter);
ApplicationClientProtocol adMinUserClient = GetRMClientForUser(queueAdmin);
GetApplicationReportResponse submitterGetReport = submitterClient.GetApplicationReport
(appReportRequest);
GetApplicationReportResponse adMinUserGetReport = adMinUserClient.GetApplicationReport
(appReportRequest);
NUnit.Framework.Assert.AreEqual(submitterGetReport.GetApplicationReport().GetClientToAMToken
(), adMinUserGetReport.GetApplicationReport().GetClientToAMToken());
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual GetApplicationReportResponse GetApplicationReport(GetApplicationReportRequest
request)
{
ApplicationId applicationId = request.GetApplicationId();
try
{
GetApplicationReportResponse response = GetApplicationReportResponse.NewInstance(
history.GetApplication(applicationId));
return(response);
}
catch (IOException e)
{
Log.Error(e.Message, e);
throw;
}
}
/// <exception cref="System.IO.IOException"/>
protected internal override ApplicationHistoryProtocol GetAHSProxy(Configuration
conf)
{
GetApplicationReportResponse resp = Org.Mockito.Mockito.Mock <GetApplicationReportResponse
>();
historyManager = Org.Mockito.Mockito.Mock <ApplicationHistoryProtocol>();
try
{
Org.Mockito.Mockito.When(historyManager.GetApplicationReport(Org.Mockito.Mockito.
Any <GetApplicationReportRequest>())).ThenReturn(resp);
}
catch (YarnException e)
{
// TODO Auto-generated catch block
Sharpen.Runtime.PrintStackTrace(e);
}
return(historyManager);
}
/// <exception cref="Com.Google.Protobuf.ServiceException"/>
public virtual YarnServiceProtos.GetApplicationReportResponseProto GetApplicationReport
(RpcController arg0, YarnServiceProtos.GetApplicationReportRequestProto proto)
{
GetApplicationReportRequestPBImpl request = new GetApplicationReportRequestPBImpl
(proto);
try
{
GetApplicationReportResponse response = real.GetApplicationReport(request);
return(((GetApplicationReportResponsePBImpl)response).GetProto());
}
catch (YarnException e)
{
throw new ServiceException(e);
}
catch (IOException e)
{
throw new ServiceException(e);
}
}
/// <exception cref="System.IO.IOException"/>
public virtual GetApplicationReportResponse GetApplicationReport(GetApplicationReportRequest
request)
{
ApplicationId applicationId = request.GetApplicationId();
ApplicationReport application = TestClientRedirect.recordFactory.NewRecordInstance
<ApplicationReport>();
application.SetApplicationId(applicationId);
application.SetFinalApplicationStatus(FinalApplicationStatus.Undefined);
if (this._enclosing.amRunning)
{
application.SetYarnApplicationState(YarnApplicationState.Running);
}
else
{
if (this._enclosing.amRestarting)
{
application.SetYarnApplicationState(YarnApplicationState.Submitted);
}
else
{
application.SetYarnApplicationState(YarnApplicationState.Finished);
application.SetFinalApplicationStatus(FinalApplicationStatus.Succeeded);
}
}
string[] split = TestClientRedirect.Amhostaddress.Split(":");
application.SetHost(split[0]);
application.SetRpcPort(System.Convert.ToInt32(split[1]));
application.SetUser("TestClientRedirect-user");
application.SetName("N/A");
application.SetQueue("N/A");
application.SetStartTime(0);
application.SetFinishTime(0);
application.SetTrackingUrl("N/A");
application.SetDiagnostics("N/A");
GetApplicationReportResponse response = TestClientRedirect.recordFactory.NewRecordInstance
<GetApplicationReportResponse>();
response.SetApplicationReport(application);
return(response);
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public override ApplicationReport GetApplicationReport(ApplicationId appId)
{
GetApplicationReportResponse response = null;
try
{
GetApplicationReportRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<GetApplicationReportRequest>();
request.SetApplicationId(appId);
response = rmClient.GetApplicationReport(request);
}
catch (ApplicationNotFoundException e)
{
if (!historyServiceEnabled)
{
// Just throw it as usual if historyService is not enabled.
throw;
}
return(historyClient.GetApplicationReport(appId));
}
return(response.GetApplicationReport());
}
public virtual void TestApplicationReport()
{
ApplicationId appId = null;
appId = ApplicationId.NewInstance(0, 1);
GetApplicationReportRequest request = GetApplicationReportRequest.NewInstance(appId
);
GetApplicationReportResponse response = clientService.GetApplicationReport(request
);
ApplicationReport appReport = response.GetApplicationReport();
NUnit.Framework.Assert.IsNotNull(appReport);
NUnit.Framework.Assert.AreEqual(123, appReport.GetApplicationResourceUsageReport(
).GetMemorySeconds());
NUnit.Framework.Assert.AreEqual(345, appReport.GetApplicationResourceUsageReport(
).GetVcoreSeconds());
NUnit.Framework.Assert.AreEqual("application_0_0001", appReport.GetApplicationId(
).ToString());
NUnit.Framework.Assert.AreEqual("test app type", appReport.GetApplicationType().ToString
());
NUnit.Framework.Assert.AreEqual("test queue", appReport.GetQueue().ToString());
}
/// <exception cref="System.Exception"/>
public virtual void TestClientTokenRace()
{
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
ContainerManagementProtocol containerManager = Org.Mockito.Mockito.Mock <ContainerManagementProtocol
>();
StartContainersResponse mockResponse = Org.Mockito.Mockito.Mock <StartContainersResponse
>();
Org.Mockito.Mockito.When(containerManager.StartContainers((StartContainersRequest
)Matchers.Any())).ThenReturn(mockResponse);
DrainDispatcher dispatcher = new DrainDispatcher();
MockRM rm = new _MockRMWithCustomAMLauncher_433(dispatcher, conf, containerManager
);
rm.Start();
// Submit an app
RMApp app = rm.SubmitApp(1024);
// Set up a node.
MockNM nm1 = rm.RegisterNode("localhost:1234", 3072);
nm1.NodeHeartbeat(true);
dispatcher.Await();
nm1.NodeHeartbeat(true);
dispatcher.Await();
ApplicationAttemptId appAttempt = app.GetCurrentAppAttempt().GetAppAttemptId();
MockAM mockAM = new MockAM(rm.GetRMContext(), rm.GetApplicationMasterService(), app
.GetCurrentAppAttempt().GetAppAttemptId());
UserGroupInformation appUgi = UserGroupInformation.CreateRemoteUser(appAttempt.ToString
());
RegisterApplicationMasterResponse response = appUgi.DoAs(new _PrivilegedAction_469
(mockAM));
// Get the app-report.
GetApplicationReportRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<GetApplicationReportRequest>();
request.SetApplicationId(app.GetApplicationId());
GetApplicationReportResponse reportResponse = rm.GetClientRMService().GetApplicationReport
(request);
ApplicationReport appReport = reportResponse.GetApplicationReport();
Org.Apache.Hadoop.Yarn.Api.Records.Token originalClientToAMToken = appReport.GetClientToAMToken
();
// ClientToAMToken master key should have been received on register
// application master response.
ByteBuffer clientMasterKey = response.GetClientToAMTokenMasterKey();
NUnit.Framework.Assert.IsNotNull(clientMasterKey);
NUnit.Framework.Assert.IsTrue(((byte[])clientMasterKey.Array()).Length > 0);
// Start the AM with the correct shared-secret.
ApplicationAttemptId appAttemptId = app.GetAppAttempts().Keys.GetEnumerator().Next
();
NUnit.Framework.Assert.IsNotNull(appAttemptId);
TestClientToAMTokens.CustomAM am = new TestClientToAMTokens.CustomAM(appAttemptId
, null);
am.Init(conf);
am.Start();
// Now the real test!
// Set up clients to be able to pick up correct tokens.
SecurityUtil.SetSecurityInfoProviders(new TestClientToAMTokens.CustomSecurityInfo
());
Org.Apache.Hadoop.Security.Token.Token <ClientToAMTokenIdentifier> token = ConverterUtils
.ConvertFromYarn(originalClientToAMToken, am.address);
// Schedule the key to be set after a significant delay
Timer timer = new Timer();
TimerTask timerTask = new _TimerTask_516(am, clientMasterKey);
timer.Schedule(timerTask, 250);
// connect should pause waiting for the master key to arrive
VerifyValidToken(conf, am, token);
am.Stop();
rm.Stop();
}
public virtual void TestClientToAMTokens()
{
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
ContainerManagementProtocol containerManager = Org.Mockito.Mockito.Mock <ContainerManagementProtocol
>();
StartContainersResponse mockResponse = Org.Mockito.Mockito.Mock <StartContainersResponse
>();
Org.Mockito.Mockito.When(containerManager.StartContainers((StartContainersRequest
)Matchers.Any())).ThenReturn(mockResponse);
DrainDispatcher dispatcher = new DrainDispatcher();
MockRM rm = new _MockRMWithCustomAMLauncher_192(dispatcher, conf, containerManager
);
rm.Start();
// Submit an app
RMApp app = rm.SubmitApp(1024);
// Set up a node.
MockNM nm1 = rm.RegisterNode("localhost:1234", 3072);
nm1.NodeHeartbeat(true);
dispatcher.Await();
nm1.NodeHeartbeat(true);
dispatcher.Await();
ApplicationAttemptId appAttempt = app.GetCurrentAppAttempt().GetAppAttemptId();
MockAM mockAM = new MockAM(rm.GetRMContext(), rm.GetApplicationMasterService(), app
.GetCurrentAppAttempt().GetAppAttemptId());
UserGroupInformation appUgi = UserGroupInformation.CreateRemoteUser(appAttempt.ToString
());
RegisterApplicationMasterResponse response = appUgi.DoAs(new _PrivilegedAction_229
(mockAM));
// Get the app-report.
GetApplicationReportRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<GetApplicationReportRequest>();
request.SetApplicationId(app.GetApplicationId());
GetApplicationReportResponse reportResponse = rm.GetClientRMService().GetApplicationReport
(request);
ApplicationReport appReport = reportResponse.GetApplicationReport();
Org.Apache.Hadoop.Yarn.Api.Records.Token originalClientToAMToken = appReport.GetClientToAMToken
();
// ClientToAMToken master key should have been received on register
// application master response.
NUnit.Framework.Assert.IsNotNull(response.GetClientToAMTokenMasterKey());
NUnit.Framework.Assert.IsTrue(((byte[])response.GetClientToAMTokenMasterKey().Array
()).Length > 0);
// Start the AM with the correct shared-secret.
ApplicationAttemptId appAttemptId = app.GetAppAttempts().Keys.GetEnumerator().Next
();
NUnit.Framework.Assert.IsNotNull(appAttemptId);
TestClientToAMTokens.CustomAM am = new TestClientToAMTokens.CustomAM(appAttemptId
, ((byte[])response.GetClientToAMTokenMasterKey().Array()));
am.Init(conf);
am.Start();
// Now the real test!
// Set up clients to be able to pick up correct tokens.
SecurityUtil.SetSecurityInfoProviders(new TestClientToAMTokens.CustomSecurityInfo
());
// Verify denial for unauthenticated user
try
{
TestClientToAMTokens.CustomProtocol client = (TestClientToAMTokens.CustomProtocol
)RPC.GetProxy <TestClientToAMTokens.CustomProtocol>(1L, am.address, conf);
client.Ping();
NUnit.Framework.Assert.Fail("Access by unauthenticated user should fail!!");
}
catch (Exception)
{
NUnit.Framework.Assert.IsFalse(am.pinged);
}
Org.Apache.Hadoop.Security.Token.Token <ClientToAMTokenIdentifier> token = ConverterUtils
.ConvertFromYarn(originalClientToAMToken, am.address);
// Verify denial for a malicious user with tampered ID
VerifyTokenWithTamperedID(conf, am, token);
// Verify denial for a malicious user with tampered user-name
VerifyTokenWithTamperedUserName(conf, am, token);
// Now for an authenticated user
VerifyValidToken(conf, am, token);
// Verify for a new version token
VerifyNewVersionToken(conf, am, token, rm);
am.Stop();
rm.Stop();
}
