public virtual void TestUnknownCall()
{
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.IpcRpcImpl, typeof(HadoopYarnProtoRPC).FullName);
YarnRPC rpc = YarnRPC.Create(conf);
string bindAddr = "localhost:0";
IPEndPoint addr = NetUtils.CreateSocketAddr(bindAddr);
Server server = rpc.GetServer(typeof(ContainerManagementProtocol), new TestRPC.DummyContainerManager
(this), addr, conf, null, 1);
server.Start();
// Any unrelated protocol would do
ApplicationClientProtocol proxy = (ApplicationClientProtocol)rpc.GetProxy(typeof(
ApplicationClientProtocol), NetUtils.GetConnectAddress(server), conf);
try
{
proxy.GetNewApplication(Records.NewRecord <GetNewApplicationRequest>());
NUnit.Framework.Assert.Fail("Excepted RPC call to fail with unknown method.");
}
catch (YarnException e)
{
NUnit.Framework.Assert.IsTrue(e.Message.Matches("Unknown method getNewApplication called on.*"
+ "org.apache.hadoop.yarn.proto.ApplicationClientProtocol" + "\\$ApplicationClientProtocolService\\$BlockingInterface protocol."
));
}
catch (Exception e)
{
Sharpen.Runtime.PrintStackTrace(e);
}
}
/// <exception cref="System.Exception"/>
private ApplicationId SubmitAppAndGetAppId(string submitter, string queueName, bool
setupACLs)
{
GetNewApplicationRequest newAppRequest = GetNewApplicationRequest.NewInstance();
ApplicationClientProtocol submitterClient = GetRMClientForUser(submitter);
ApplicationId applicationId = submitterClient.GetNewApplication(newAppRequest).GetApplicationId
();
Resource resource = BuilderUtils.NewResource(1024, 1);
IDictionary <ApplicationAccessType, string> acls = CreateACLs(submitter, setupACLs
);
ContainerLaunchContext amContainerSpec = ContainerLaunchContext.NewInstance(null,
null, null, null, null, acls);
ApplicationSubmissionContext appSubmissionContext = ApplicationSubmissionContext.
NewInstance(applicationId, "applicationName", queueName, null, amContainerSpec,
false, true, 1, resource, "applicationType");
appSubmissionContext.SetApplicationId(applicationId);
appSubmissionContext.SetQueue(queueName);
SubmitApplicationRequest submitRequest = SubmitApplicationRequest.NewInstance(appSubmissionContext
);
submitterClient.SubmitApplication(submitRequest);
resourceManager.WaitForState(applicationId, RMAppState.Accepted);
return(applicationId);
}
// get new application id
/// <exception cref="System.Exception"/>
public virtual GetNewApplicationResponse GetNewAppId()
{
ApplicationClientProtocol client = GetClientRMService();
return(client.GetNewApplication(Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <GetNewApplicationRequest
>()));
}
/// <exception cref="System.Exception"/>
private ApplicationId SubmitAppAndGetAppId(AccessControlList viewACL, AccessControlList
modifyACL)
{
SubmitApplicationRequest submitRequest = recordFactory.NewRecordInstance <SubmitApplicationRequest
>();
ApplicationSubmissionContext context = recordFactory.NewRecordInstance <ApplicationSubmissionContext
>();
ApplicationId applicationId = rmClient.GetNewApplication(recordFactory.NewRecordInstance
<GetNewApplicationRequest>()).GetApplicationId();
context.SetApplicationId(applicationId);
IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
, string>();
acls[ApplicationAccessType.ViewApp] = viewACL.GetAclString();
acls[ApplicationAccessType.ModifyApp] = modifyACL.GetAclString();
ContainerLaunchContext amContainer = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
Resource resource = BuilderUtils.NewResource(1024, 1);
context.SetResource(resource);
amContainer.SetApplicationACLs(acls);
context.SetAMContainerSpec(amContainer);
submitRequest.SetApplicationSubmissionContext(context);
rmClient.SubmitApplication(submitRequest);
resourceManager.WaitForState(applicationId, RMAppState.Accepted);
return(applicationId);
}
/// <exception cref="Com.Google.Protobuf.ServiceException"/>
public virtual YarnServiceProtos.GetNewApplicationResponseProto GetNewApplication
(RpcController arg0, YarnServiceProtos.GetNewApplicationRequestProto proto)
{
GetNewApplicationRequestPBImpl request = new GetNewApplicationRequestPBImpl(proto
);
try
{
GetNewApplicationResponse response = real.GetNewApplication(request);
return(((GetNewApplicationResponsePBImpl)response).GetProto());
}
catch (YarnException e)
{
throw new ServiceException(e);
}
catch (IOException e)
{
throw new ServiceException(e);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestResourceMgrDelegate()
{
/* we not want a mock of resource mgr delegate */
ApplicationClientProtocol clientRMProtocol = Org.Mockito.Mockito.Mock <ApplicationClientProtocol
>();
ResourceMgrDelegate delegate_ = new _ResourceMgrDelegate_256(clientRMProtocol, conf
);
/* make sure kill calls finish application master */
Org.Mockito.Mockito.When(clientRMProtocol.ForceKillApplication(Matchers.Any <KillApplicationRequest
>())).ThenReturn(KillApplicationResponse.NewInstance(true));
delegate_.KillApplication(appId);
Org.Mockito.Mockito.Verify(clientRMProtocol).ForceKillApplication(Matchers.Any <KillApplicationRequest
>());
/* make sure getalljobs calls get all applications */
Org.Mockito.Mockito.When(clientRMProtocol.GetApplications(Matchers.Any <GetApplicationsRequest
>())).ThenReturn(recordFactory.NewRecordInstance <GetApplicationsResponse>());
delegate_.GetAllJobs();
Org.Mockito.Mockito.Verify(clientRMProtocol).GetApplications(Matchers.Any <GetApplicationsRequest
>());
/* make sure getapplication report is called */
Org.Mockito.Mockito.When(clientRMProtocol.GetApplicationReport(Matchers.Any <GetApplicationReportRequest
>())).ThenReturn(recordFactory.NewRecordInstance <GetApplicationReportResponse>()
);
delegate_.GetApplicationReport(appId);
Org.Mockito.Mockito.Verify(clientRMProtocol).GetApplicationReport(Matchers.Any <GetApplicationReportRequest
>());
/* make sure metrics is called */
GetClusterMetricsResponse clusterMetricsResponse = recordFactory.NewRecordInstance
<GetClusterMetricsResponse>();
clusterMetricsResponse.SetClusterMetrics(recordFactory.NewRecordInstance <YarnClusterMetrics
>());
Org.Mockito.Mockito.When(clientRMProtocol.GetClusterMetrics(Matchers.Any <GetClusterMetricsRequest
>())).ThenReturn(clusterMetricsResponse);
delegate_.GetClusterMetrics();
Org.Mockito.Mockito.Verify(clientRMProtocol).GetClusterMetrics(Matchers.Any <GetClusterMetricsRequest
>());
Org.Mockito.Mockito.When(clientRMProtocol.GetClusterNodes(Matchers.Any <GetClusterNodesRequest
>())).ThenReturn(recordFactory.NewRecordInstance <GetClusterNodesResponse>());
delegate_.GetActiveTrackers();
Org.Mockito.Mockito.Verify(clientRMProtocol).GetClusterNodes(Matchers.Any <GetClusterNodesRequest
>());
GetNewApplicationResponse newAppResponse = recordFactory.NewRecordInstance <GetNewApplicationResponse
>();
newAppResponse.SetApplicationId(appId);
Org.Mockito.Mockito.When(clientRMProtocol.GetNewApplication(Matchers.Any <GetNewApplicationRequest
>())).ThenReturn(newAppResponse);
delegate_.GetNewJobID();
Org.Mockito.Mockito.Verify(clientRMProtocol).GetNewApplication(Matchers.Any <GetNewApplicationRequest
>());
GetQueueInfoResponse queueInfoResponse = recordFactory.NewRecordInstance <GetQueueInfoResponse
>();
queueInfoResponse.SetQueueInfo(recordFactory.NewRecordInstance <QueueInfo>());
Org.Mockito.Mockito.When(clientRMProtocol.GetQueueInfo(Matchers.Any <GetQueueInfoRequest
>())).ThenReturn(queueInfoResponse);
delegate_.GetQueues();
Org.Mockito.Mockito.Verify(clientRMProtocol).GetQueueInfo(Matchers.Any <GetQueueInfoRequest
>());
GetQueueUserAclsInfoResponse aclResponse = recordFactory.NewRecordInstance <GetQueueUserAclsInfoResponse
>();
Org.Mockito.Mockito.When(clientRMProtocol.GetQueueUserAcls(Matchers.Any <GetQueueUserAclsInfoRequest
>())).ThenReturn(aclResponse);
delegate_.GetQueueAclsForCurrentUser();
Org.Mockito.Mockito.Verify(clientRMProtocol).GetQueueUserAcls(Matchers.Any <GetQueueUserAclsInfoRequest
>());
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
private GetNewApplicationResponse GetNewApplication()
{
GetNewApplicationRequest request = Records.NewRecord <GetNewApplicationRequest>();
return(rmClient.GetNewApplication(request));
}
public virtual void TestDelegationToken()
{
YarnConfiguration conf = new YarnConfiguration();
conf.Set(YarnConfiguration.RmPrincipal, "testuser/[email protected]");
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
ResourceScheduler scheduler = CreateMockScheduler(conf);
long initialInterval = 10000l;
long maxLifetime = 20000l;
long renewInterval = 10000l;
RMDelegationTokenSecretManager rmDtSecretManager = CreateRMDelegationTokenSecretManager
(initialInterval, maxLifetime, renewInterval);
rmDtSecretManager.StartThreads();
Log.Info("Creating DelegationTokenSecretManager with initialInterval: " + initialInterval
+ ", maxLifetime: " + maxLifetime + ", renewInterval: " + renewInterval);
ClientRMService clientRMService = new TestClientRMTokens.ClientRMServiceForTest(this
, conf, scheduler, rmDtSecretManager);
clientRMService.Init(conf);
clientRMService.Start();
ApplicationClientProtocol clientRMWithDT = null;
try
{
// Create a user for the renewr and fake the authentication-method
UserGroupInformation loggedInUser = UserGroupInformation.CreateRemoteUser("*****@*****.**"
);
NUnit.Framework.Assert.AreEqual("testrenewer", loggedInUser.GetShortUserName());
// Default realm is APACHE.ORG
loggedInUser.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Kerberos
);
Token token = GetDelegationToken(loggedInUser, clientRMService, loggedInUser.GetShortUserName
());
long tokenFetchTime = Runtime.CurrentTimeMillis();
Log.Info("Got delegation token at: " + tokenFetchTime);
// Now try talking to RMService using the delegation token
clientRMWithDT = GetClientRMProtocolWithDT(token, clientRMService.GetBindAddress(
), "loginuser1", conf);
GetNewApplicationRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <
GetNewApplicationRequest>();
try
{
clientRMWithDT.GetNewApplication(request);
}
catch (IOException e)
{
NUnit.Framework.Assert.Fail("Unexpected exception" + e);
}
catch (YarnException e)
{
NUnit.Framework.Assert.Fail("Unexpected exception" + e);
}
// Renew after 50% of token age.
while (Runtime.CurrentTimeMillis() < tokenFetchTime + initialInterval / 2)
{
Sharpen.Thread.Sleep(500l);
}
long nextExpTime = RenewDelegationToken(loggedInUser, clientRMService, token);
long renewalTime = Runtime.CurrentTimeMillis();
Log.Info("Renewed token at: " + renewalTime + ", NextExpiryTime: " + nextExpTime);
// Wait for first expiry, but before renewed expiry.
while (Runtime.CurrentTimeMillis() > tokenFetchTime + initialInterval && Runtime.
CurrentTimeMillis() < nextExpTime)
{
Sharpen.Thread.Sleep(500l);
}
Sharpen.Thread.Sleep(50l);
// Valid token because of renewal.
try
{
clientRMWithDT.GetNewApplication(request);
}
catch (IOException e)
{
NUnit.Framework.Assert.Fail("Unexpected exception" + e);
}
catch (YarnException e)
{
NUnit.Framework.Assert.Fail("Unexpected exception" + e);
}
// Wait for expiry.
while (Runtime.CurrentTimeMillis() < renewalTime + renewInterval)
{
Sharpen.Thread.Sleep(500l);
}
Sharpen.Thread.Sleep(50l);
Log.Info("At time: " + Runtime.CurrentTimeMillis() + ", token should be invalid");
// Token should have expired.
try
{
clientRMWithDT.GetNewApplication(request);
NUnit.Framework.Assert.Fail("Should not have succeeded with an expired token");
}
catch (Exception e)
{
NUnit.Framework.Assert.AreEqual(typeof(SecretManager.InvalidToken).FullName, e.GetType
().FullName);
NUnit.Framework.Assert.IsTrue(e.Message.Contains("is expired"));
}
// Test cancellation
// Stop the existing proxy, start another.
if (clientRMWithDT != null)
{
RPC.StopProxy(clientRMWithDT);
clientRMWithDT = null;
}
token = GetDelegationToken(loggedInUser, clientRMService, loggedInUser.GetShortUserName
());
tokenFetchTime = Runtime.CurrentTimeMillis();
Log.Info("Got delegation token at: " + tokenFetchTime);
// Now try talking to RMService using the delegation token
clientRMWithDT = GetClientRMProtocolWithDT(token, clientRMService.GetBindAddress(
), "loginuser2", conf);
request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <GetNewApplicationRequest>
();
try
{
clientRMWithDT.GetNewApplication(request);
}
catch (IOException e)
{
NUnit.Framework.Assert.Fail("Unexpected exception" + e);
}
catch (YarnException e)
{
NUnit.Framework.Assert.Fail("Unexpected exception" + e);
}
CancelDelegationToken(loggedInUser, clientRMService, token);
if (clientRMWithDT != null)
{
RPC.StopProxy(clientRMWithDT);
clientRMWithDT = null;
}
// Creating a new connection.
clientRMWithDT = GetClientRMProtocolWithDT(token, clientRMService.GetBindAddress(
), "loginuser2", conf);
Log.Info("Cancelled delegation token at: " + Runtime.CurrentTimeMillis());
// Verify cancellation worked.
try
{
clientRMWithDT.GetNewApplication(request);
NUnit.Framework.Assert.Fail("Should not have succeeded with a cancelled delegation token"
);
}
catch (IOException)
{
}
catch (YarnException)
{
}
// Test new version token
// Stop the existing proxy, start another.
if (clientRMWithDT != null)
{
RPC.StopProxy(clientRMWithDT);
clientRMWithDT = null;
}
token = GetDelegationToken(loggedInUser, clientRMService, loggedInUser.GetShortUserName
());
byte[] tokenIdentifierContent = ((byte[])token.GetIdentifier().Array());
RMDelegationTokenIdentifier tokenIdentifier = new RMDelegationTokenIdentifier();
DataInputBuffer dib = new DataInputBuffer();
dib.Reset(tokenIdentifierContent, tokenIdentifierContent.Length);
tokenIdentifier.ReadFields(dib);
// Construct new version RMDelegationTokenIdentifier with additional field
RMDelegationTokenIdentifierForTest newVersionTokenIdentifier = new RMDelegationTokenIdentifierForTest
(tokenIdentifier, "message");
Org.Apache.Hadoop.Security.Token.Token <RMDelegationTokenIdentifier> newRMDTtoken =
new Org.Apache.Hadoop.Security.Token.Token <RMDelegationTokenIdentifier>(newVersionTokenIdentifier
, rmDtSecretManager);
Org.Apache.Hadoop.Yarn.Api.Records.Token newToken = BuilderUtils.NewDelegationToken
(newRMDTtoken.GetIdentifier(), newRMDTtoken.GetKind().ToString(), newRMDTtoken.GetPassword
(), newRMDTtoken.GetService().ToString());
// Now try talking to RMService using the new version delegation token
clientRMWithDT = GetClientRMProtocolWithDT(newToken, clientRMService.GetBindAddress
(), "loginuser3", conf);
request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <GetNewApplicationRequest>
();
try
{
clientRMWithDT.GetNewApplication(request);
}
catch (IOException e)
{
NUnit.Framework.Assert.Fail("Unexpected exception" + e);
}
catch (YarnException e)
{
NUnit.Framework.Assert.Fail("Unexpected exception" + e);
}
}
finally
{
rmDtSecretManager.StopThreads();
// TODO PRECOMMIT Close proxies.
if (clientRMWithDT != null)
{
RPC.StopProxy(clientRMWithDT);
}
}
}
/// <exception cref="System.Exception"/>
public virtual RMApp SubmitApp(int masterMemory, string name, string user, IDictionary
<ApplicationAccessType, string> acls, bool unmanaged, string queue, int maxAppAttempts
, Credentials ts, string appType, bool waitForAccepted, bool keepContainers, bool
isAppIdProvided, ApplicationId applicationId, long attemptFailuresValidityInterval
, LogAggregationContext logAggregationContext, bool cancelTokensWhenComplete)
{
ApplicationId appId = isAppIdProvided ? applicationId : null;
ApplicationClientProtocol client = GetClientRMService();
if (!isAppIdProvided)
{
GetNewApplicationResponse resp = client.GetNewApplication(Org.Apache.Hadoop.Yarn.Util.Records
.NewRecord <GetNewApplicationRequest>());
appId = resp.GetApplicationId();
}
SubmitApplicationRequest req = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <SubmitApplicationRequest
>();
ApplicationSubmissionContext sub = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <
ApplicationSubmissionContext>();
sub.SetKeepContainersAcrossApplicationAttempts(keepContainers);
sub.SetApplicationId(appId);
sub.SetApplicationName(name);
sub.SetMaxAppAttempts(maxAppAttempts);
if (unmanaged)
{
sub.SetUnmanagedAM(true);
}
if (queue != null)
{
sub.SetQueue(queue);
}
sub.SetApplicationType(appType);
ContainerLaunchContext clc = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <ContainerLaunchContext
>();
Resource capability = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <Resource>();
capability.SetMemory(masterMemory);
sub.SetResource(capability);
clc.SetApplicationACLs(acls);
if (ts != null && UserGroupInformation.IsSecurityEnabled())
{
DataOutputBuffer dob = new DataOutputBuffer();
ts.WriteTokenStorageToStream(dob);
ByteBuffer securityTokens = ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength());
clc.SetTokens(securityTokens);
}
sub.SetAMContainerSpec(clc);
sub.SetAttemptFailuresValidityInterval(attemptFailuresValidityInterval);
if (logAggregationContext != null)
{
sub.SetLogAggregationContext(logAggregationContext);
}
sub.SetCancelTokensWhenComplete(cancelTokensWhenComplete);
req.SetApplicationSubmissionContext(sub);
UserGroupInformation fakeUser = UserGroupInformation.CreateUserForTesting(user, new
string[] { "someGroup" });
PrivilegedAction <SubmitApplicationResponse> action = new _PrivilegedAction_415().
SetClientReq(client, req);
fakeUser.DoAs(action);
// make sure app is immediately available after submit
if (waitForAccepted)
{
WaitForState(appId, RMAppState.Accepted);
}
RMApp rmApp = GetRMContext().GetRMApps()[appId];
// unmanaged AM won't go to RMAppAttemptState.SCHEDULED.
if (waitForAccepted && !unmanaged)
{
WaitForState(rmApp.GetCurrentAppAttempt().GetAppAttemptId(), RMAppAttemptState.Scheduled
);
}
return(rmApp);
}