/// <exception cref="System.Exception"/>
public virtual void TestGetCreds <T>()
where T : TokenIdentifier
{
// from Mockito mocks
UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser("someone");
Text service = new Text("service");
Org.Apache.Hadoop.Security.Token.Token <T> t1 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token
>();
Org.Mockito.Mockito.When(t1.GetService()).ThenReturn(service);
Org.Apache.Hadoop.Security.Token.Token <T> t2 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token
>();
Org.Mockito.Mockito.When(t2.GetService()).ThenReturn(new Text("service2"));
Org.Apache.Hadoop.Security.Token.Token <T> t3 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token
>();
Org.Mockito.Mockito.When(t3.GetService()).ThenReturn(service);
// add token to ugi
ugi.AddToken(t1);
ugi.AddToken(t2);
CheckTokens(ugi, t1, t2);
Credentials creds = ugi.GetCredentials();
creds.AddToken(t3.GetService(), t3);
NUnit.Framework.Assert.AreSame(t3, creds.GetToken(service));
// check that ugi wasn't modified
CheckTokens(ugi, t1, t2);
}
/// <exception cref="System.Exception"/>
public virtual void TestAddToken <T>()
where T : TokenIdentifier
{
// from Mockito mocks
UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser("someone");
Org.Apache.Hadoop.Security.Token.Token <T> t1 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token
>();
Org.Apache.Hadoop.Security.Token.Token <T> t2 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token
>();
Org.Apache.Hadoop.Security.Token.Token <T> t3 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token
>();
// add token to ugi
ugi.AddToken(t1);
CheckTokens(ugi, t1);
// replace token t1 with t2 - with same key (null)
ugi.AddToken(t2);
CheckTokens(ugi, t2);
// change t1 service and add token
Org.Mockito.Mockito.When(t1.GetService()).ThenReturn(new Text("t1"));
ugi.AddToken(t1);
CheckTokens(ugi, t1, t2);
// overwrite t1 token with t3 - same key (!null)
Org.Mockito.Mockito.When(t3.GetService()).ThenReturn(new Text("t1"));
ugi.AddToken(t3);
CheckTokens(ugi, t2, t3);
// just try to re-add with new name
Org.Mockito.Mockito.When(t1.GetService()).ThenReturn(new Text("t1.1"));
ugi.AddToken(t1);
CheckTokens(ugi, t1, t2, t3);
// just try to re-add with new name again
ugi.AddToken(t1);
CheckTokens(ugi, t1, t2, t3);
}
/// <exception cref="System.Exception"/>
public virtual void TestAddCreds <T>()
where T : TokenIdentifier
{
// from Mockito mocks
UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser("someone");
Text service = new Text("service");
Org.Apache.Hadoop.Security.Token.Token <T> t1 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token
>();
Org.Mockito.Mockito.When(t1.GetService()).ThenReturn(service);
Org.Apache.Hadoop.Security.Token.Token <T> t2 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token
>();
Org.Mockito.Mockito.When(t2.GetService()).ThenReturn(new Text("service2"));
byte[] secret = new byte[] { };
Text secretKey = new Text("sshhh");
// fill credentials
Credentials creds = new Credentials();
creds.AddToken(t1.GetService(), t1);
creds.AddToken(t2.GetService(), t2);
creds.AddSecretKey(secretKey, secret);
// add creds to ugi, and check ugi
ugi.AddCredentials(creds);
CheckTokens(ugi, t1, t2);
NUnit.Framework.Assert.AreSame(secret, ugi.GetCredentials().GetSecretKey(secretKey
));
}
private bool CheckService <_T0>(Text service, Org.Apache.Hadoop.Security.Token.Token
<_T0> token)
where _T0 : TokenIdentifier
{
if (service == null || token.GetService() == null)
{
return(false);
}
return(token.GetService().ToString().Contains(service.ToString()));
}
/// <exception cref="System.Exception"/>
public Void Run()
{
// make sure it is not the same as the login user because we use the
// same UGI object for every instantiation of the login user and you
// won't run into the race condition otherwise
Assert.AssertNotEquals(UserGroupInformation.GetLoginUser(), UserGroupInformation.
GetCurrentUser());
TestUserGroupInformation.GetTokenThread thread = new TestUserGroupInformation.GetTokenThread
();
try
{
thread.Start();
for (int i = 0; i < 100; i++)
{
Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> t = Org.Mockito.Mockito.Mock
<Org.Apache.Hadoop.Security.Token.Token>();
Org.Mockito.Mockito.When(t.GetService()).ThenReturn(new Text("t" + i));
UserGroupInformation.GetCurrentUser().AddToken(t);
NUnit.Framework.Assert.IsNull("ConcurrentModificationException encountered", thread
.cme);
}
}
catch (ConcurrentModificationException cme)
{
Runtime.PrintStackTrace(cme);
NUnit.Framework.Assert.Fail("ConcurrentModificationException encountered");
}
finally
{
thread.runThread = false;
thread.Join(5 * 1000);
}
return(null);
}
public virtual void TestGetTokensForNamenodes()
{
Path TestRootDir = new Path(Runtime.GetProperty("test.build.data", "test/build/data"
));
// ick, but need fq path minus file:/
string binaryTokenFile = FileSystem.GetLocal(conf).MakeQualified(new Path(TestRootDir
, "tokenFile")).ToUri().GetPath();
FileSystemTestHelper.MockFileSystem fs1 = CreateFileSystemForServiceName("service1"
);
Credentials creds = new Credentials();
Org.Apache.Hadoop.Security.Token.Token <object> token1 = fs1.GetDelegationToken(renewer
);
creds.AddToken(token1.GetService(), token1);
// wait to set, else the obtain tokens call above will fail with FNF
conf.Set(MRJobConfig.MapreduceJobCredentialsBinary, binaryTokenFile);
creds.WriteTokenStorageFile(new Path(binaryTokenFile), conf);
TokenCache.ObtainTokensForNamenodesInternal(fs1, creds, conf);
string fs_addr = fs1.GetCanonicalServiceName();
Org.Apache.Hadoop.Security.Token.Token <object> nnt = TokenCache.GetDelegationToken
(creds, fs_addr);
NUnit.Framework.Assert.IsNotNull("Token for nn is null", nnt);
}
/// <exception cref="System.IO.IOException"/>
private void SetupTokens(ContainerLaunchContext container, ContainerId containerID
)
{
IDictionary <string, string> environment = container.GetEnvironment();
environment[ApplicationConstants.ApplicationWebProxyBaseEnv] = application.GetWebProxyBase
();
// Set AppSubmitTime and MaxAppAttempts to be consumable by the AM.
ApplicationId applicationId = application.GetAppAttemptId().GetApplicationId();
environment[ApplicationConstants.AppSubmitTimeEnv] = rmContext.GetRMApps()[applicationId
].GetSubmitTime().ToString();
environment[ApplicationConstants.MaxAppAttemptsEnv] = rmContext.GetRMApps()[applicationId
].GetMaxAppAttempts().ToString();
Credentials credentials = new Credentials();
DataInputByteBuffer dibb = new DataInputByteBuffer();
if (container.GetTokens() != null)
{
// TODO: Don't do this kind of checks everywhere.
dibb.Reset(container.GetTokens());
credentials.ReadTokenStorageStream(dibb);
}
// Add AMRMToken
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> amrmToken = CreateAndSetAMRMToken
();
if (amrmToken != null)
{
credentials.AddToken(amrmToken.GetService(), amrmToken);
}
DataOutputBuffer dob = new DataOutputBuffer();
credentials.WriteTokenStorageToStream(dob);
container.SetTokens(ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength()));
}
/// <exception cref="System.IO.IOException"/>
private static ApplicationClientProtocol GetRmClient <_T0>(Org.Apache.Hadoop.Security.Token.Token
<_T0> token, Configuration conf)
where _T0 : TokenIdentifier
{
string[] services = token.GetService().ToString().Split(",");
foreach (string service in services)
{
IPEndPoint addr = NetUtils.CreateSocketAddr(service);
if (localSecretManager != null)
{
// return null if it's our token
if (localServiceAddress.Address.IsAnyLocalAddress())
{
if (NetUtils.IsLocalAddress(addr.Address) && addr.Port == localServiceAddress.Port)
{
return(null);
}
}
else
{
if (addr.Equals(localServiceAddress))
{
return(null);
}
}
}
}
return(ClientRMProxy.CreateRMProxy <ApplicationClientProtocol>(conf));
}
/// <exception cref="System.IO.IOException"/>
private void VerifyServiceInToken(ServletContext context, HttpServletRequest request
, string expected)
{
UserGroupInformation ugi = JspHelper.GetUGI(context, request, conf);
Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> tokenInUgi = ugi.GetTokens
().GetEnumerator().Next();
NUnit.Framework.Assert.AreEqual(expected, tokenInUgi.GetService().ToString());
}
/// <returns>a string representation of the token</returns>
/// <exception cref="System.IO.IOException"/>
public static string StringifyToken <_T0>(Org.Apache.Hadoop.Security.Token.Token <_T0
> token)
where _T0 : TokenIdentifier
{
Org.Apache.Hadoop.Hdfs.Security.Token.Delegation.DelegationTokenIdentifier ident =
new Org.Apache.Hadoop.Hdfs.Security.Token.Delegation.DelegationTokenIdentifier();
ByteArrayInputStream buf = new ByteArrayInputStream(token.GetIdentifier());
DataInputStream @in = new DataInputStream(buf);
ident.ReadFields(@in);
if (token.GetService().GetLength() > 0)
{
return(ident + " on " + token.GetService());
}
else
{
return(ident.ToString());
}
}
/// <exception cref="System.Exception"/>
private void TestBinaryCredentials(bool hasScheme)
{
Path TestRootDir = new Path(Runtime.GetProperty("test.build.data", "test/build/data"
));
// ick, but need fq path minus file:/
string binaryTokenFile = hasScheme ? FileSystem.GetLocal(conf).MakeQualified(new
Path(TestRootDir, "tokenFile")).ToString() : FileSystem.GetLocal(conf).MakeQualified
(new Path(TestRootDir, "tokenFile")).ToUri().GetPath();
FileSystemTestHelper.MockFileSystem fs1 = CreateFileSystemForServiceName("service1"
);
FileSystemTestHelper.MockFileSystem fs2 = CreateFileSystemForServiceName("service2"
);
FileSystemTestHelper.MockFileSystem fs3 = CreateFileSystemForServiceName("service3"
);
// get the tokens for fs1 & fs2 and write out to binary creds file
Credentials creds = new Credentials();
Org.Apache.Hadoop.Security.Token.Token <object> token1 = fs1.GetDelegationToken(renewer
);
Org.Apache.Hadoop.Security.Token.Token <object> token2 = fs2.GetDelegationToken(renewer
);
creds.AddToken(token1.GetService(), token1);
creds.AddToken(token2.GetService(), token2);
// wait to set, else the obtain tokens call above will fail with FNF
conf.Set(MRJobConfig.MapreduceJobCredentialsBinary, binaryTokenFile);
creds.WriteTokenStorageFile(new Path(binaryTokenFile), conf);
// re-init creds and add a newer token for fs1
creds = new Credentials();
Org.Apache.Hadoop.Security.Token.Token <object> newerToken1 = fs1.GetDelegationToken
(renewer);
NUnit.Framework.Assert.AreNotSame(newerToken1, token1);
creds.AddToken(newerToken1.GetService(), newerToken1);
CheckToken(creds, newerToken1);
// get token for fs1, see that fs2's token was loaded
TokenCache.ObtainTokensForNamenodesInternal(fs1, creds, conf);
CheckToken(creds, newerToken1, token2);
// get token for fs2, nothing should change since already present
TokenCache.ObtainTokensForNamenodesInternal(fs2, creds, conf);
CheckToken(creds, newerToken1, token2);
// get token for fs3, should only add token for fs3
TokenCache.ObtainTokensForNamenodesInternal(fs3, creds, conf);
Org.Apache.Hadoop.Security.Token.Token <object> token3 = creds.GetToken(new Text(fs3
.GetCanonicalServiceName()));
NUnit.Framework.Assert.IsTrue(token3 != null);
CheckToken(creds, newerToken1, token2, token3);
// be paranoid, check one last time that nothing changes
TokenCache.ObtainTokensForNamenodesInternal(fs1, creds, conf);
TokenCache.ObtainTokensForNamenodesInternal(fs2, creds, conf);
TokenCache.ObtainTokensForNamenodesInternal(fs3, creds, conf);
CheckToken(creds, newerToken1, token2, token3);
}
/// <summary>Parse the file system URI out of the provided token.</summary>
public static URI GetServiceUriFromToken <_T0>(string scheme, Org.Apache.Hadoop.Security.Token.Token
<_T0> token)
where _T0 : TokenIdentifier
{
string tokStr = token.GetService().ToString();
string prefix = BuildTokenServicePrefixForLogicalUri(scheme);
if (tokStr.StartsWith(prefix))
{
tokStr = tokStr.ReplaceFirst(prefix, string.Empty);
}
return(URI.Create(scheme + "://" + tokStr));
}
internal void InitDelegationToken(UserGroupInformation ugi)
{
lock (this)
{
Org.Apache.Hadoop.Security.Token.Token <object> token = SelectDelegationToken(ugi);
if (token != null)
{
Log.Debug("Found existing DT for " + token.GetService());
fs.SetDelegationToken(token);
hasInitedToken = true;
}
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
private void VerifyNewVersionToken(Configuration conf, TestClientToAMTokens.CustomAM
am, Org.Apache.Hadoop.Security.Token.Token <ClientToAMTokenIdentifier> token, MockRM
rm)
{
UserGroupInformation ugi;
ugi = UserGroupInformation.CreateRemoteUser("me");
Org.Apache.Hadoop.Security.Token.Token <ClientToAMTokenIdentifier> newToken = new
Org.Apache.Hadoop.Security.Token.Token <ClientToAMTokenIdentifier>(new ClientToAMTokenIdentifierForTest
(token.DecodeIdentifier(), "message"), am.GetClientToAMTokenSecretManager());
newToken.SetService(token.GetService());
ugi.AddToken(newToken);
ugi.DoAs(new _PrivilegedExceptionAction_386(am, conf));
}
/// <exception cref="System.Exception"/>
public virtual void TestUGITokens <T>()
where T : TokenIdentifier
{
// from Mockito mocks
UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting("TheDoctor",
new string[] { "TheTARDIS" });
Org.Apache.Hadoop.Security.Token.Token <T> t1 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token
>();
Org.Mockito.Mockito.When(t1.GetService()).ThenReturn(new Text("t1"));
Org.Apache.Hadoop.Security.Token.Token <T> t2 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token
>();
Org.Mockito.Mockito.When(t2.GetService()).ThenReturn(new Text("t2"));
Credentials creds = new Credentials();
byte[] secretKey = new byte[] { };
Text secretName = new Text("shhh");
creds.AddSecretKey(secretName, secretKey);
ugi.AddToken(t1);
ugi.AddToken(t2);
ugi.AddCredentials(creds);
ICollection <Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> > z = ugi.GetTokens
();
Assert.True(z.Contains(t1));
Assert.True(z.Contains(t2));
Assert.Equal(2, z.Count);
Credentials ugiCreds = ugi.GetCredentials();
NUnit.Framework.Assert.AreSame(secretKey, ugiCreds.GetSecretKey(secretName));
Assert.Equal(1, ugiCreds.NumberOfSecretKeys());
try
{
z.Remove(t1);
NUnit.Framework.Assert.Fail("Shouldn't be able to modify token collection from UGI"
);
}
catch (NotSupportedException)
{
}
// Can't modify tokens
// ensure that the tokens are passed through doAs
ICollection <Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> > otherSet = ugi
.DoAs(new _PrivilegedExceptionAction_612());
Assert.True(otherSet.Contains(t1));
Assert.True(otherSet.Contains(t2));
}
/// <exception cref="System.Exception"/>
public virtual void TestHdfsGetCanonicalServiceName()
{
Configuration conf = dfs.GetConf();
URI haUri = HATestUtil.GetLogicalUri(cluster);
AbstractFileSystem afs = AbstractFileSystem.CreateFileSystem(haUri, conf);
string haService = HAUtil.BuildTokenServiceForLogicalUri(haUri, HdfsConstants.HdfsUriScheme
).ToString();
NUnit.Framework.Assert.AreEqual(haService, afs.GetCanonicalServiceName());
Org.Apache.Hadoop.Security.Token.Token <object> token = afs.GetDelegationTokens(UserGroupInformation
.GetCurrentUser().GetShortUserName())[0];
NUnit.Framework.Assert.AreEqual(haService, token.GetService().ToString());
// make sure the logical uri is handled correctly
token.Renew(conf);
token.Cancel(conf);
}
/// <summary>
/// HDFS-3062: DistributedFileSystem.getCanonicalServiceName() throws an
/// exception if the URI is a logical URI.
/// </summary>
/// <remarks>
/// HDFS-3062: DistributedFileSystem.getCanonicalServiceName() throws an
/// exception if the URI is a logical URI. This bug fails the combination of
/// ha + mapred + security.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestDFSGetCanonicalServiceName()
{
URI hAUri = HATestUtil.GetLogicalUri(cluster);
string haService = HAUtil.BuildTokenServiceForLogicalUri(hAUri, HdfsConstants.HdfsUriScheme
).ToString();
NUnit.Framework.Assert.AreEqual(haService, dfs.GetCanonicalServiceName());
string renewer = UserGroupInformation.GetCurrentUser().GetShortUserName();
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = GetDelegationToken
(dfs, renewer);
NUnit.Framework.Assert.AreEqual(haService, token.GetService().ToString());
// make sure the logical uri is handled correctly
token.Renew(dfs.GetConf());
token.Cancel(dfs.GetConf());
}
// check:
// 1) buildTokenService honors use_ip setting
// 2) setTokenService & getService works
// 3) getTokenServiceAddr decodes to the identical socket addr
private void VerifyTokenService(IPEndPoint addr, string host, string ip, int port
, bool useIp)
{
//LOG.info("address:"+addr+" host:"+host+" ip:"+ip+" port:"+port);
SecurityUtil.SetTokenServiceUseIp(useIp);
string serviceHost = useIp ? ip : StringUtils.ToLowerCase(host);
Org.Apache.Hadoop.Security.Token.Token <object> token = new Org.Apache.Hadoop.Security.Token.Token
<TokenIdentifier>();
Text service = new Text(serviceHost + ":" + port);
Assert.Equal(service, SecurityUtil.BuildTokenService(addr));
SecurityUtil.SetTokenService(token, addr);
Assert.Equal(service, token.GetService());
IPEndPoint serviceAddr = SecurityUtil.GetTokenServiceAddr(token);
NUnit.Framework.Assert.IsNotNull(serviceAddr);
VerifyValues(serviceAddr, serviceHost, ip, port);
}
/// <exception cref="System.Exception"/>
public virtual void TestAddNamedToken <T>()
where T : TokenIdentifier
{
// from Mockito mocks
UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser("someone");
Org.Apache.Hadoop.Security.Token.Token <T> t1 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token
>();
Text service1 = new Text("t1");
Text service2 = new Text("t2");
Org.Mockito.Mockito.When(t1.GetService()).ThenReturn(service1);
// add token
ugi.AddToken(service1, t1);
NUnit.Framework.Assert.AreSame(t1, ugi.GetCredentials().GetToken(service1));
// add token with another name
ugi.AddToken(service2, t1);
NUnit.Framework.Assert.AreSame(t1, ugi.GetCredentials().GetToken(service1));
NUnit.Framework.Assert.AreSame(t1, ugi.GetCredentials().GetToken(service2));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public override void Cancel <_T0>(Org.Apache.Hadoop.Security.Token.Token <_T0> token
, Configuration conf)
{
Org.Apache.Hadoop.Yarn.Api.Records.Token dToken = Org.Apache.Hadoop.Yarn.Api.Records.Token
.NewInstance(token.GetIdentifier(), token.GetKind().ToString(), token.GetPassword
(), token.GetService().ToString());
MRClientProtocol histProxy = InstantiateHistoryProxy(conf, SecurityUtil.GetTokenServiceAddr
(token));
try
{
CancelDelegationTokenRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<CancelDelegationTokenRequest>();
request.SetDelegationToken(dToken);
histProxy.CancelDelegationToken(request);
}
finally
{
StopHistoryProxy(histProxy);
}
}
/// <exception cref="System.IO.IOException"/>
internal void EnsureTokenInitialized()
{
lock (this)
{
// we haven't inited yet, or we used to have a token but it expired
if (!hasInitedToken || (action != null && !action.IsValid()))
{
//since we don't already have a token, go get one
Org.Apache.Hadoop.Security.Token.Token <object> token = fs.GetDelegationToken(null
);
// security might be disabled
if (token != null)
{
fs.SetDelegationToken(token);
AddRenewAction(fs);
Log.Debug("Created new DT for " + token.GetService());
}
hasInitedToken = true;
}
}
}
public virtual void TestAMRMTokenUpdate()
{
Configuration conf = new Configuration();
ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(ApplicationId.NewInstance
(1, 1), 1);
AMRMTokenIdentifier oldTokenId = new AMRMTokenIdentifier(attemptId, 1);
AMRMTokenIdentifier newTokenId = new AMRMTokenIdentifier(attemptId, 2);
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> oldToken = new Org.Apache.Hadoop.Security.Token.Token
<AMRMTokenIdentifier>(oldTokenId.GetBytes(), Sharpen.Runtime.GetBytesForString("oldpassword"
), oldTokenId.GetKind(), new Text());
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> newToken = new Org.Apache.Hadoop.Security.Token.Token
<AMRMTokenIdentifier>(newTokenId.GetBytes(), Sharpen.Runtime.GetBytesForString("newpassword"
), newTokenId.GetKind(), new Text());
TestLocalContainerAllocator.MockScheduler scheduler = new TestLocalContainerAllocator.MockScheduler
();
scheduler.amToken = newToken;
LocalContainerAllocator lca = new TestLocalContainerAllocator.StubbedLocalContainerAllocator
(scheduler);
lca.Init(conf);
lca.Start();
UserGroupInformation testUgi = UserGroupInformation.CreateUserForTesting("someuser"
, new string[0]);
testUgi.AddToken(oldToken);
testUgi.DoAs(new _PrivilegedExceptionAction_144(lca));
lca.Close();
// verify there is only one AMRM token in the UGI and it matches the
// updated token from the RM
int tokenCount = 0;
Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> ugiToken = null;
foreach (Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> token in testUgi
.GetTokens())
{
if (AMRMTokenIdentifier.KindName.Equals(token.GetKind()))
{
ugiToken = token;
++tokenCount;
}
}
NUnit.Framework.Assert.AreEqual("too many AMRM tokens", 1, tokenCount);
Assert.AssertArrayEquals("token identifier not updated", newToken.GetIdentifier()
, ugiToken.GetIdentifier());
Assert.AssertArrayEquals("token password not updated", newToken.GetPassword(), ugiToken
.GetPassword());
NUnit.Framework.Assert.AreEqual("AMRM token service not updated", new Text(ClientRMProxy
.GetAMRMTokenService(conf)), ugiToken.GetService());
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual AllocateResponse Allocate(AllocateRequest request)
{
NUnit.Framework.Assert.AreEqual("response ID mismatch", responseId, request.GetResponseId
());
++responseId;
Org.Apache.Hadoop.Yarn.Api.Records.Token yarnToken = null;
if (amToken != null)
{
yarnToken = Org.Apache.Hadoop.Yarn.Api.Records.Token.NewInstance(amToken.GetIdentifier
(), amToken.GetKind().ToString(), amToken.GetPassword(), amToken.GetService().ToString
());
}
return(AllocateResponse.NewInstance(responseId, Collections.EmptyList <ContainerStatus
>(), Collections.EmptyList <Container>(), Collections.EmptyList <NodeReport>(), Resources
.None(), null, 1, null, Collections.EmptyList <NMToken>(), yarnToken, Collections
.EmptyList <ContainerResourceIncrease>(), Collections.EmptyList <ContainerResourceDecrease
>()));
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual AllocateResponse Allocate(AllocateRequest request)
{
AMRMTokenIdentifier amrmTokenIdentifier = AuthorizeRequest();
ApplicationAttemptId appAttemptId = amrmTokenIdentifier.GetApplicationAttemptId();
ApplicationId applicationId = appAttemptId.GetApplicationId();
this.amLivelinessMonitor.ReceivedPing(appAttemptId);
/* check if its in cache */
ApplicationMasterService.AllocateResponseLock Lock = responseMap[appAttemptId];
if (Lock == null)
{
string message = "Application attempt " + appAttemptId + " doesn't exist in ApplicationMasterService cache.";
Log.Error(message);
throw new ApplicationAttemptNotFoundException(message);
}
lock (Lock)
{
AllocateResponse lastResponse = Lock.GetAllocateResponse();
if (!HasApplicationMasterRegistered(appAttemptId))
{
string message = "AM is not registered for known application attempt: " + appAttemptId
+ " or RM had restarted after AM registered . AM should re-register.";
Log.Info(message);
RMAuditLogger.LogFailure(this.rmContext.GetRMApps()[appAttemptId.GetApplicationId
()].GetUser(), RMAuditLogger.AuditConstants.AmAllocate, string.Empty, "ApplicationMasterService"
, message, applicationId, appAttemptId);
throw new ApplicationMasterNotRegisteredException(message);
}
if ((request.GetResponseId() + 1) == lastResponse.GetResponseId())
{
/* old heartbeat */
return(lastResponse);
}
else
{
if (request.GetResponseId() + 1 < lastResponse.GetResponseId())
{
string message = "Invalid responseId in AllocateRequest from application attempt: "
+ appAttemptId + ", expect responseId to be " + (lastResponse.GetResponseId() +
1);
throw new InvalidApplicationMasterRequestException(message);
}
}
//filter illegal progress values
float filteredProgress = request.GetProgress();
if (float.IsNaN(filteredProgress) || filteredProgress == float.NegativeInfinity ||
filteredProgress < 0)
{
request.SetProgress(0);
}
else
{
if (filteredProgress > 1 || filteredProgress == float.PositiveInfinity)
{
request.SetProgress(1);
}
}
// Send the status update to the appAttempt.
this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMAppAttemptStatusupdateEvent
(appAttemptId, request.GetProgress()));
IList <ResourceRequest> ask = request.GetAskList();
IList <ContainerId> release = request.GetReleaseList();
ResourceBlacklistRequest blacklistRequest = request.GetResourceBlacklistRequest();
IList <string> blacklistAdditions = (blacklistRequest != null) ? blacklistRequest.
GetBlacklistAdditions() : Sharpen.Collections.EmptyList;
IList <string> blacklistRemovals = (blacklistRequest != null) ? blacklistRequest.GetBlacklistRemovals
() : Sharpen.Collections.EmptyList;
RMApp app = this.rmContext.GetRMApps()[applicationId];
// set label expression for Resource Requests if resourceName=ANY
ApplicationSubmissionContext asc = app.GetApplicationSubmissionContext();
foreach (ResourceRequest req in ask)
{
if (null == req.GetNodeLabelExpression() && ResourceRequest.Any.Equals(req.GetResourceName
()))
{
req.SetNodeLabelExpression(asc.GetNodeLabelExpression());
}
}
// sanity check
try
{
RMServerUtils.NormalizeAndValidateRequests(ask, rScheduler.GetMaximumResourceCapability
(), app.GetQueue(), rScheduler, rmContext);
}
catch (InvalidResourceRequestException e)
{
Log.Warn("Invalid resource ask by application " + appAttemptId, e);
throw;
}
try
{
RMServerUtils.ValidateBlacklistRequest(blacklistRequest);
}
catch (InvalidResourceBlacklistRequestException e)
{
Log.Warn("Invalid blacklist request by application " + appAttemptId, e);
throw;
}
// In the case of work-preserving AM restart, it's possible for the
// AM to release containers from the earlier attempt.
if (!app.GetApplicationSubmissionContext().GetKeepContainersAcrossApplicationAttempts
())
{
try
{
RMServerUtils.ValidateContainerReleaseRequest(release, appAttemptId);
}
catch (InvalidContainerReleaseException e)
{
Log.Warn("Invalid container release by application " + appAttemptId, e);
throw;
}
}
// Send new requests to appAttempt.
Allocation allocation = this.rScheduler.Allocate(appAttemptId, ask, release, blacklistAdditions
, blacklistRemovals);
if (!blacklistAdditions.IsEmpty() || !blacklistRemovals.IsEmpty())
{
Log.Info("blacklist are updated in Scheduler." + "blacklistAdditions: " + blacklistAdditions
+ ", " + "blacklistRemovals: " + blacklistRemovals);
}
RMAppAttempt appAttempt = app.GetRMAppAttempt(appAttemptId);
AllocateResponse allocateResponse = recordFactory.NewRecordInstance <AllocateResponse
>();
if (!allocation.GetContainers().IsEmpty())
{
allocateResponse.SetNMTokens(allocation.GetNMTokens());
}
// update the response with the deltas of node status changes
IList <RMNode> updatedNodes = new AList <RMNode>();
if (app.PullRMNodeUpdates(updatedNodes) > 0)
{
IList <NodeReport> updatedNodeReports = new AList <NodeReport>();
foreach (RMNode rmNode in updatedNodes)
{
SchedulerNodeReport schedulerNodeReport = rScheduler.GetNodeReport(rmNode.GetNodeID
());
Resource used = BuilderUtils.NewResource(0, 0);
int numContainers = 0;
if (schedulerNodeReport != null)
{
used = schedulerNodeReport.GetUsedResource();
numContainers = schedulerNodeReport.GetNumContainers();
}
NodeId nodeId = rmNode.GetNodeID();
NodeReport report = BuilderUtils.NewNodeReport(nodeId, rmNode.GetState(), rmNode.
GetHttpAddress(), rmNode.GetRackName(), used, rmNode.GetTotalCapability(), numContainers
, rmNode.GetHealthReport(), rmNode.GetLastHealthReportTime(), rmNode.GetNodeLabels
());
updatedNodeReports.AddItem(report);
}
allocateResponse.SetUpdatedNodes(updatedNodeReports);
}
allocateResponse.SetAllocatedContainers(allocation.GetContainers());
allocateResponse.SetCompletedContainersStatuses(appAttempt.PullJustFinishedContainers
());
allocateResponse.SetResponseId(lastResponse.GetResponseId() + 1);
allocateResponse.SetAvailableResources(allocation.GetResourceLimit());
allocateResponse.SetNumClusterNodes(this.rScheduler.GetNumClusterNodes());
// add preemption to the allocateResponse message (if any)
allocateResponse.SetPreemptionMessage(GeneratePreemptionMessage(allocation));
// update AMRMToken if the token is rolled-up
MasterKeyData nextMasterKey = this.rmContext.GetAMRMTokenSecretManager().GetNextMasterKeyData
();
if (nextMasterKey != null && nextMasterKey.GetMasterKey().GetKeyId() != amrmTokenIdentifier
.GetKeyId())
{
RMAppAttemptImpl appAttemptImpl = (RMAppAttemptImpl)appAttempt;
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> amrmToken = appAttempt
.GetAMRMToken();
if (nextMasterKey.GetMasterKey().GetKeyId() != appAttemptImpl.GetAMRMTokenKeyId())
{
Log.Info("The AMRMToken has been rolled-over. Send new AMRMToken back" + " to application: "
+ applicationId);
amrmToken = rmContext.GetAMRMTokenSecretManager().CreateAndGetAMRMToken(appAttemptId
);
appAttemptImpl.SetAMRMToken(amrmToken);
}
allocateResponse.SetAMRMToken(Org.Apache.Hadoop.Yarn.Api.Records.Token.NewInstance
(amrmToken.GetIdentifier(), amrmToken.GetKind().ToString(), amrmToken.GetPassword
(), amrmToken.GetService().ToString()));
}
/*
* As we are updating the response inside the lock object so we don't
* need to worry about unregister call occurring in between (which
* removes the lock object).
*/
Lock.SetAllocateResponse(allocateResponse);
return(allocateResponse);
}
}
public virtual void TestDelegationToken()
{
YarnConfiguration conf = new YarnConfiguration();
conf.Set(YarnConfiguration.RmPrincipal, "testuser/[email protected]");
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
ResourceScheduler scheduler = CreateMockScheduler(conf);
long initialInterval = 10000l;
long maxLifetime = 20000l;
long renewInterval = 10000l;
RMDelegationTokenSecretManager rmDtSecretManager = CreateRMDelegationTokenSecretManager
(initialInterval, maxLifetime, renewInterval);
rmDtSecretManager.StartThreads();
Log.Info("Creating DelegationTokenSecretManager with initialInterval: " + initialInterval
+ ", maxLifetime: " + maxLifetime + ", renewInterval: " + renewInterval);
ClientRMService clientRMService = new TestClientRMTokens.ClientRMServiceForTest(this
, conf, scheduler, rmDtSecretManager);
clientRMService.Init(conf);
clientRMService.Start();
ApplicationClientProtocol clientRMWithDT = null;
try
{
// Create a user for the renewr and fake the authentication-method
UserGroupInformation loggedInUser = UserGroupInformation.CreateRemoteUser("*****@*****.**"
);
NUnit.Framework.Assert.AreEqual("testrenewer", loggedInUser.GetShortUserName());
// Default realm is APACHE.ORG
loggedInUser.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Kerberos
);
Token token = GetDelegationToken(loggedInUser, clientRMService, loggedInUser.GetShortUserName
());
long tokenFetchTime = Runtime.CurrentTimeMillis();
Log.Info("Got delegation token at: " + tokenFetchTime);
// Now try talking to RMService using the delegation token
clientRMWithDT = GetClientRMProtocolWithDT(token, clientRMService.GetBindAddress(
), "loginuser1", conf);
GetNewApplicationRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <
GetNewApplicationRequest>();
try
{
clientRMWithDT.GetNewApplication(request);
}
catch (IOException e)
{
NUnit.Framework.Assert.Fail("Unexpected exception" + e);
}
catch (YarnException e)
{
NUnit.Framework.Assert.Fail("Unexpected exception" + e);
}
// Renew after 50% of token age.
while (Runtime.CurrentTimeMillis() < tokenFetchTime + initialInterval / 2)
{
Sharpen.Thread.Sleep(500l);
}
long nextExpTime = RenewDelegationToken(loggedInUser, clientRMService, token);
long renewalTime = Runtime.CurrentTimeMillis();
Log.Info("Renewed token at: " + renewalTime + ", NextExpiryTime: " + nextExpTime);
// Wait for first expiry, but before renewed expiry.
while (Runtime.CurrentTimeMillis() > tokenFetchTime + initialInterval && Runtime.
CurrentTimeMillis() < nextExpTime)
{
Sharpen.Thread.Sleep(500l);
}
Sharpen.Thread.Sleep(50l);
// Valid token because of renewal.
try
{
clientRMWithDT.GetNewApplication(request);
}
catch (IOException e)
{
NUnit.Framework.Assert.Fail("Unexpected exception" + e);
}
catch (YarnException e)
{
NUnit.Framework.Assert.Fail("Unexpected exception" + e);
}
// Wait for expiry.
while (Runtime.CurrentTimeMillis() < renewalTime + renewInterval)
{
Sharpen.Thread.Sleep(500l);
}
Sharpen.Thread.Sleep(50l);
Log.Info("At time: " + Runtime.CurrentTimeMillis() + ", token should be invalid");
// Token should have expired.
try
{
clientRMWithDT.GetNewApplication(request);
NUnit.Framework.Assert.Fail("Should not have succeeded with an expired token");
}
catch (Exception e)
{
NUnit.Framework.Assert.AreEqual(typeof(SecretManager.InvalidToken).FullName, e.GetType
().FullName);
NUnit.Framework.Assert.IsTrue(e.Message.Contains("is expired"));
}
// Test cancellation
// Stop the existing proxy, start another.
if (clientRMWithDT != null)
{
RPC.StopProxy(clientRMWithDT);
clientRMWithDT = null;
}
token = GetDelegationToken(loggedInUser, clientRMService, loggedInUser.GetShortUserName
());
tokenFetchTime = Runtime.CurrentTimeMillis();
Log.Info("Got delegation token at: " + tokenFetchTime);
// Now try talking to RMService using the delegation token
clientRMWithDT = GetClientRMProtocolWithDT(token, clientRMService.GetBindAddress(
), "loginuser2", conf);
request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <GetNewApplicationRequest>
();
try
{
clientRMWithDT.GetNewApplication(request);
}
catch (IOException e)
{
NUnit.Framework.Assert.Fail("Unexpected exception" + e);
}
catch (YarnException e)
{
NUnit.Framework.Assert.Fail("Unexpected exception" + e);
}
CancelDelegationToken(loggedInUser, clientRMService, token);
if (clientRMWithDT != null)
{
RPC.StopProxy(clientRMWithDT);
clientRMWithDT = null;
}
// Creating a new connection.
clientRMWithDT = GetClientRMProtocolWithDT(token, clientRMService.GetBindAddress(
), "loginuser2", conf);
Log.Info("Cancelled delegation token at: " + Runtime.CurrentTimeMillis());
// Verify cancellation worked.
try
{
clientRMWithDT.GetNewApplication(request);
NUnit.Framework.Assert.Fail("Should not have succeeded with a cancelled delegation token"
);
}
catch (IOException)
{
}
catch (YarnException)
{
}
// Test new version token
// Stop the existing proxy, start another.
if (clientRMWithDT != null)
{
RPC.StopProxy(clientRMWithDT);
clientRMWithDT = null;
}
token = GetDelegationToken(loggedInUser, clientRMService, loggedInUser.GetShortUserName
());
byte[] tokenIdentifierContent = ((byte[])token.GetIdentifier().Array());
RMDelegationTokenIdentifier tokenIdentifier = new RMDelegationTokenIdentifier();
DataInputBuffer dib = new DataInputBuffer();
dib.Reset(tokenIdentifierContent, tokenIdentifierContent.Length);
tokenIdentifier.ReadFields(dib);
// Construct new version RMDelegationTokenIdentifier with additional field
RMDelegationTokenIdentifierForTest newVersionTokenIdentifier = new RMDelegationTokenIdentifierForTest
(tokenIdentifier, "message");
Org.Apache.Hadoop.Security.Token.Token <RMDelegationTokenIdentifier> newRMDTtoken =
new Org.Apache.Hadoop.Security.Token.Token <RMDelegationTokenIdentifier>(newVersionTokenIdentifier
, rmDtSecretManager);
Org.Apache.Hadoop.Yarn.Api.Records.Token newToken = BuilderUtils.NewDelegationToken
(newRMDTtoken.GetIdentifier(), newRMDTtoken.GetKind().ToString(), newRMDTtoken.GetPassword
(), newRMDTtoken.GetService().ToString());
// Now try talking to RMService using the new version delegation token
clientRMWithDT = GetClientRMProtocolWithDT(newToken, clientRMService.GetBindAddress
(), "loginuser3", conf);
request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <GetNewApplicationRequest>
();
try
{
clientRMWithDT.GetNewApplication(request);
}
catch (IOException e)
{
NUnit.Framework.Assert.Fail("Unexpected exception" + e);
}
catch (YarnException e)
{
NUnit.Framework.Assert.Fail("Unexpected exception" + e);
}
}
finally
{
rmDtSecretManager.StopThreads();
// TODO PRECOMMIT Close proxies.
if (clientRMWithDT != null)
{
RPC.StopProxy(clientRMWithDT);
}
}
}
private void VerifyTamperedToken(Configuration conf, TestClientToAMTokens.CustomAM
am, Org.Apache.Hadoop.Security.Token.Token <ClientToAMTokenIdentifier> token, UserGroupInformation
ugi, ClientToAMTokenIdentifier maliciousID)
{
Org.Apache.Hadoop.Security.Token.Token <ClientToAMTokenIdentifier> maliciousToken =
new Org.Apache.Hadoop.Security.Token.Token <ClientToAMTokenIdentifier>(maliciousID
.GetBytes(), token.GetPassword(), token.GetKind(), token.GetService());
ugi.AddToken(maliciousToken);
try
{
ugi.DoAs(new _PrivilegedExceptionAction_338(am, conf));
}
catch (Exception e)
{
NUnit.Framework.Assert.AreEqual(typeof(RemoteException).FullName, e.GetType().FullName
);
e = ((RemoteException)e).UnwrapRemoteException();
NUnit.Framework.Assert.AreEqual(typeof(SaslException).GetCanonicalName(), e.GetType
().GetCanonicalName());
NUnit.Framework.Assert.IsTrue(e.Message.Contains("DIGEST-MD5: digest response format violation. "
+ "Mismatched response."));
NUnit.Framework.Assert.IsFalse(am.pinged);
}
}
private static Org.Apache.Hadoop.Yarn.Api.Records.Token ConvertToProtoToken <_T0>(
Org.Apache.Hadoop.Security.Token.Token <_T0> token)
where _T0 : TokenIdentifier
{
return(Org.Apache.Hadoop.Yarn.Api.Records.Token.NewInstance(token.GetIdentifier()
, token.GetKind().ToString(), token.GetPassword(), token.GetService().ToString()
));
}
/// <returns>
/// true if this token corresponds to a logical nameservice
/// rather than a specific namenode.
/// </returns>
public static bool IsTokenForLogicalUri <_T0>(Org.Apache.Hadoop.Security.Token.Token
<_T0> token)
where _T0 : TokenIdentifier
{
return(token.GetService().ToString().StartsWith(HdfsConstants.HaDtServicePrefix));
}
/// <summary>Decode the given token's service field into an InetAddress</summary>
/// <param name="token">from which to obtain the service</param>
/// <returns>InetAddress for the service</returns>
public static IPEndPoint GetTokenServiceAddr <_T0>(Org.Apache.Hadoop.Security.Token.Token
<_T0> token)
where _T0 : TokenIdentifier
{
return(NetUtils.CreateSocketAddr(token.GetService().ToString()));
}
