// Test the DT mast key in the state-store when the mast key is being rolled.
/// <exception cref="System.Exception"/>
public virtual void TestRMDTMasterKeyStateOnRollingMasterKey()
{
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.Init(conf);
RMStateStore.RMState rmState = memStore.GetState();
IDictionary <RMDelegationTokenIdentifier, long> rmDTState = rmState.GetRMDTSecretManagerState
().GetTokenState();
ICollection <DelegationKey> rmDTMasterKeyState = rmState.GetRMDTSecretManagerState
().GetMasterKeyState();
MockRM rm1 = new TestRMDelegationTokens.MyMockRM(this, conf, memStore);
rm1.Start();
// on rm start, two master keys are created.
// One is created at RMDTSecretMgr.startThreads.updateCurrentKey();
// the other is created on the first run of
// tokenRemoverThread.rollMasterKey()
RMDelegationTokenSecretManager dtSecretManager = rm1.GetRMContext().GetRMDelegationTokenSecretManager
();
// assert all master keys are saved
NUnit.Framework.Assert.AreEqual(dtSecretManager.GetAllMasterKeys(), rmDTMasterKeyState
);
ICollection <DelegationKey> expiringKeys = new HashSet <DelegationKey>();
Sharpen.Collections.AddAll(expiringKeys, dtSecretManager.GetAllMasterKeys());
// request to generate a RMDelegationToken
GetDelegationTokenRequest request = Org.Mockito.Mockito.Mock <GetDelegationTokenRequest
>();
Org.Mockito.Mockito.When(request.GetRenewer()).ThenReturn("renewer1");
GetDelegationTokenResponse response = rm1.GetClientRMService().GetDelegationToken
(request);
Org.Apache.Hadoop.Yarn.Api.Records.Token delegationToken = response.GetRMDelegationToken
();
Org.Apache.Hadoop.Security.Token.Token <RMDelegationTokenIdentifier> token1 = ConverterUtils
.ConvertFromYarn(delegationToken, (Text)null);
RMDelegationTokenIdentifier dtId1 = token1.DecodeIdentifier();
// For all keys that still remain in memory, we should have them stored
// in state-store also.
while (((TestRMDelegationTokens.TestRMDelegationTokenSecretManager)dtSecretManager
).numUpdatedKeys.Get() < 3)
{
((TestRMDelegationTokens.TestRMDelegationTokenSecretManager)dtSecretManager).CheckCurrentKeyInStateStore
(rmDTMasterKeyState);
Sharpen.Thread.Sleep(100);
}
// wait for token to expire and remove from state-store
// rollMasterKey is called every 1 second.
int count = 0;
while (rmDTState.Contains(dtId1) && count < 100)
{
Sharpen.Thread.Sleep(100);
count++;
}
rm1.Stop();
}
/// <exception cref="System.Exception"/>
public virtual RegisterApplicationMasterResponse RegisterAppAttempt(bool wait)
{
if (wait)
{
WaitForState(RMAppAttemptState.Launched);
}
responseId = 0;
RegisterApplicationMasterRequest req = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<RegisterApplicationMasterRequest>();
req.SetHost(string.Empty);
req.SetRpcPort(1);
req.SetTrackingUrl(string.Empty);
if (ugi == null)
{
ugi = UserGroupInformation.CreateRemoteUser(attemptId.ToString());
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> token = context.GetRMApps
()[attemptId.GetApplicationId()].GetRMAppAttempt(attemptId).GetAMRMToken();
ugi.AddTokenIdentifier(token.DecodeIdentifier());
}
try
{
return(ugi.DoAs(new _PrivilegedExceptionAction_117(this, req)));
}
catch (UndeclaredThrowableException e)
{
throw (Exception)e.InnerException;
}
}
/// <exception cref="System.IO.IOException"/>
public static ContainerTokenIdentifier NewContainerTokenIdentifier(Token containerToken
)
{
Org.Apache.Hadoop.Security.Token.Token <ContainerTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token
<ContainerTokenIdentifier>(((byte[])containerToken.GetIdentifier().Array()), ((byte
[])containerToken.GetPassword().Array()), new Text(containerToken.GetKind()), new
Text(containerToken.GetService()));
return(token.DecodeIdentifier());
}
/// <exception cref="System.Exception"/>
public virtual AllocateResponse Allocate(AllocateRequest allocateRequest)
{
UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser(attemptId.ToString
());
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> token = context.GetRMApps
()[attemptId.GetApplicationId()].GetRMAppAttempt(attemptId).GetAMRMToken();
ugi.AddTokenIdentifier(token.DecodeIdentifier());
lastResponse = DoAllocateAs(ugi, allocateRequest);
return(lastResponse);
}
/// <exception cref="System.Exception"/>
private AllocateResponse Allocate(ApplicationAttemptId attemptId, AllocateRequest
req)
{
UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser(attemptId.ToString
());
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> token = rm.GetRMContext
().GetRMApps()[attemptId.GetApplicationId()].GetRMAppAttempt(attemptId).GetAMRMToken
();
ugi.AddTokenIdentifier(token.DecodeIdentifier());
return(ugi.DoAs(new _PrivilegedExceptionAction_67(this, req)));
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestDecodeIdentifier()
{
TestDelegationToken.TestDelegationTokenSecretManager secretManager = new TestDelegationToken.TestDelegationTokenSecretManager
(0, 0, 0, 0);
secretManager.StartThreads();
TestDelegationToken.TestDelegationTokenIdentifier id = new TestDelegationToken.TestDelegationTokenIdentifier
(new Text("owner"), new Text("renewer"), new Text("realUser"));
Org.Apache.Hadoop.Security.Token.Token <TestDelegationToken.TestDelegationTokenIdentifier
> token = new Org.Apache.Hadoop.Security.Token.Token <TestDelegationToken.TestDelegationTokenIdentifier
>(id, secretManager);
TokenIdentifier idCopy = token.DecodeIdentifier();
NUnit.Framework.Assert.AreNotSame(id, idCopy);
Assert.Equal(id, idCopy);
}
/// <exception cref="System.Exception"/>
public Void Call()
{
string token = string.Empty;
string owner = string.Empty;
string renewer = "renewer";
string body = "{\"renewer\":\"" + renewer + "\"}";
Uri url = new Uri("http://localhost:8088/ws/v1/cluster/delegation-token?doAs=client2"
);
HttpURLConnection conn = (HttpURLConnection)url.OpenConnection();
Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Webapp.TestRMWebServicesDelegationTokenAuthentication
.SetupConn(conn, "POST", MediaType.ApplicationJson, body);
InputStream response = conn.GetInputStream();
NUnit.Framework.Assert.AreEqual(ClientResponse.Status.Ok.GetStatusCode(), conn.GetResponseCode
());
BufferedReader reader = null;
try
{
reader = new BufferedReader(new InputStreamReader(response, "UTF8"));
for (string line; (line = reader.ReadLine()) != null;)
{
JSONObject obj = new JSONObject(line);
if (obj.Has("token"))
{
token = obj.GetString("token");
}
if (obj.Has("owner"))
{
owner = obj.GetString("owner");
}
}
}
finally
{
IOUtils.CloseQuietly(reader);
IOUtils.CloseQuietly(response);
}
NUnit.Framework.Assert.AreEqual("client2", owner);
Org.Apache.Hadoop.Security.Token.Token <RMDelegationTokenIdentifier> realToken = new
Org.Apache.Hadoop.Security.Token.Token <RMDelegationTokenIdentifier>();
realToken.DecodeFromUrlString(token);
NUnit.Framework.Assert.AreEqual("client2", realToken.DecodeIdentifier().GetOwner(
).ToString());
return(null);
}
/// <exception cref="System.Exception"/>
public virtual void UnregisterAppAttempt(FinishApplicationMasterRequest req, bool
waitForStateRunning)
{
if (waitForStateRunning)
{
WaitForState(RMAppAttemptState.Running);
}
if (ugi == null)
{
ugi = UserGroupInformation.CreateRemoteUser(attemptId.ToString());
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> token = context.GetRMApps
()[attemptId.GetApplicationId()].GetRMAppAttempt(attemptId).GetAMRMToken();
ugi.AddTokenIdentifier(token.DecodeIdentifier());
}
try
{
ugi.DoAs(new _PrivilegedExceptionAction_276(this, req));
}
catch (UndeclaredThrowableException e)
{
throw (Exception)e.InnerException;
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void TestEditsForCancelOnTokenExpire()
{
long renewInterval = 2000;
Configuration conf = new Configuration();
conf.SetBoolean(DFSConfigKeys.DfsNamenodeDelegationTokenAlwaysUseKey, true);
conf.SetLong(DfsNamenodeDelegationTokenRenewIntervalKey, renewInterval);
conf.SetLong(DfsNamenodeDelegationTokenMaxLifetimeKey, renewInterval * 2);
Text renewer = new Text(UserGroupInformation.GetCurrentUser().GetUserName());
FSImage fsImage = Org.Mockito.Mockito.Mock <FSImage>();
FSEditLog log = Org.Mockito.Mockito.Mock <FSEditLog>();
Org.Mockito.Mockito.DoReturn(log).When(fsImage).GetEditLog();
FSNamesystem fsn = new FSNamesystem(conf, fsImage);
DelegationTokenSecretManager dtsm = fsn.GetDelegationTokenSecretManager();
try
{
dtsm.StartThreads();
// get two tokens
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token1 = fsn.GetDelegationToken
(renewer);
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token2 = fsn.GetDelegationToken
(renewer);
DelegationTokenIdentifier ident1 = token1.DecodeIdentifier();
DelegationTokenIdentifier ident2 = token2.DecodeIdentifier();
// verify we got the tokens
Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(1)).LogGetDelegationToken
(Eq(ident1), AnyLong());
Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(1)).LogGetDelegationToken
(Eq(ident2), AnyLong());
// this is a little tricky because DTSM doesn't let us set scan interval
// so need to periodically sleep, then stop/start threads to force scan
// renew first token 1/2 to expire
Sharpen.Thread.Sleep(renewInterval / 2);
fsn.RenewDelegationToken(token2);
Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(1)).LogRenewDelegationToken
(Eq(ident2), AnyLong());
// force scan and give it a little time to complete
dtsm.StopThreads();
dtsm.StartThreads();
Sharpen.Thread.Sleep(250);
// no token has expired yet
Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(0)).LogCancelDelegationToken
(Eq(ident1));
Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(0)).LogCancelDelegationToken
(Eq(ident2));
// sleep past expiration of 1st non-renewed token
Sharpen.Thread.Sleep(renewInterval / 2);
dtsm.StopThreads();
dtsm.StartThreads();
Sharpen.Thread.Sleep(250);
// non-renewed token should have implicitly been cancelled
Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(1)).LogCancelDelegationToken
(Eq(ident1));
Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(0)).LogCancelDelegationToken
(Eq(ident2));
// sleep past expiration of 2nd renewed token
Sharpen.Thread.Sleep(renewInterval / 2);
dtsm.StopThreads();
dtsm.StartThreads();
Sharpen.Thread.Sleep(250);
// both tokens should have been implicitly cancelled by now
Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(1)).LogCancelDelegationToken
(Eq(ident1));
Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(1)).LogCancelDelegationToken
(Eq(ident2));
}
finally
{
dtsm.StopThreads();
}
}
public virtual void TestDelegationTokenOperations()
{
TimelineClient httpUserClient = KerberosTestUtils.DoAs(HttpUser + "/localhost", new
_Callable_221(this));
UserGroupInformation httpUser = KerberosTestUtils.DoAs(HttpUser + "/localhost", new
_Callable_228());
// Let HTTP user to get the delegation for itself
Org.Apache.Hadoop.Security.Token.Token <TimelineDelegationTokenIdentifier> token =
httpUserClient.GetDelegationToken(httpUser.GetShortUserName());
NUnit.Framework.Assert.IsNotNull(token);
TimelineDelegationTokenIdentifier tDT = token.DecodeIdentifier();
NUnit.Framework.Assert.IsNotNull(tDT);
NUnit.Framework.Assert.AreEqual(new Text(HttpUser), tDT.GetOwner());
// Renew token
NUnit.Framework.Assert.IsFalse(token.GetService().ToString().IsEmpty());
// Renew the token from the token service address
long renewTime1 = httpUserClient.RenewDelegationToken(token);
Sharpen.Thread.Sleep(100);
token.SetService(new Text());
NUnit.Framework.Assert.IsTrue(token.GetService().ToString().IsEmpty());
// If the token service address is not avaiable, it still can be renewed
// from the configured address
long renewTime2 = httpUserClient.RenewDelegationToken(token);
NUnit.Framework.Assert.IsTrue(renewTime1 < renewTime2);
// Cancel token
NUnit.Framework.Assert.IsTrue(token.GetService().ToString().IsEmpty());
// If the token service address is not avaiable, it still can be canceled
// from the configured address
httpUserClient.CancelDelegationToken(token);
// Renew should not be successful because the token is canceled
try
{
httpUserClient.RenewDelegationToken(token);
NUnit.Framework.Assert.Fail();
}
catch (Exception e)
{
NUnit.Framework.Assert.IsTrue(e.Message.Contains("Renewal request for unknown token"
));
}
// Let HTTP user to get the delegation token for FOO user
UserGroupInformation fooUgi = UserGroupInformation.CreateProxyUser(FooUser, httpUser
);
TimelineClient fooUserClient = fooUgi.DoAs(new _PrivilegedExceptionAction_272(this
));
token = fooUserClient.GetDelegationToken(httpUser.GetShortUserName());
NUnit.Framework.Assert.IsNotNull(token);
tDT = token.DecodeIdentifier();
NUnit.Framework.Assert.IsNotNull(tDT);
NUnit.Framework.Assert.AreEqual(new Text(FooUser), tDT.GetOwner());
NUnit.Framework.Assert.AreEqual(new Text(HttpUser), tDT.GetRealUser());
// Renew token as the renewer
Org.Apache.Hadoop.Security.Token.Token <TimelineDelegationTokenIdentifier> tokenToRenew
= token;
renewTime1 = httpUserClient.RenewDelegationToken(tokenToRenew);
renewTime2 = httpUserClient.RenewDelegationToken(tokenToRenew);
NUnit.Framework.Assert.IsTrue(renewTime1 < renewTime2);
// Cancel token
NUnit.Framework.Assert.IsFalse(tokenToRenew.GetService().ToString().IsEmpty());
// Cancel the token from the token service address
fooUserClient.CancelDelegationToken(tokenToRenew);
// Renew should not be successful because the token is canceled
try
{
httpUserClient.RenewDelegationToken(tokenToRenew);
NUnit.Framework.Assert.Fail();
}
catch (Exception e)
{
NUnit.Framework.Assert.IsTrue(e.Message.Contains("Renewal request for unknown token"
));
}
// Let HTTP user to get the delegation token for BAR user
UserGroupInformation barUgi = UserGroupInformation.CreateProxyUser(BarUser, httpUser
);
TimelineClient barUserClient = barUgi.DoAs(new _PrivilegedExceptionAction_309(this
));
try
{
barUserClient.GetDelegationToken(httpUser.GetShortUserName());
NUnit.Framework.Assert.Fail();
}
catch (Exception e)
{
NUnit.Framework.Assert.IsTrue(e.InnerException is AuthorizationException || e.InnerException
is AuthenticationException);
}
}
