/// <exception cref="System.Exception"/>
public virtual void TestDelegationTokenDFSApi()
{
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = GetDelegationToken
(fs, "JobTracker");
DelegationTokenIdentifier identifier = new DelegationTokenIdentifier();
byte[] tokenId = token.GetIdentifier();
identifier.ReadFields(new DataInputStream(new ByteArrayInputStream(tokenId)));
// Ensure that it's present in the NN's secret manager and can
// be renewed directly from there.
Log.Info("A valid token should have non-null password, " + "and should be renewed successfully"
);
NUnit.Framework.Assert.IsTrue(null != dtSecretManager.RetrievePassword(identifier
));
dtSecretManager.RenewToken(token, "JobTracker");
// Use the client conf with the failover info present to check
// renewal.
Configuration clientConf = dfs.GetConf();
DoRenewOrCancel(token, clientConf, TestDelegationTokensWithHA.TokenTestAction.Renew
);
// Using a configuration that doesn't have the logical nameservice
// configured should result in a reasonable error message.
Configuration emptyConf = new Configuration();
try
{
DoRenewOrCancel(token, emptyConf, TestDelegationTokensWithHA.TokenTestAction.Renew
);
NUnit.Framework.Assert.Fail("Did not throw trying to renew with an empty conf!");
}
catch (IOException ioe)
{
GenericTestUtils.AssertExceptionContains("Unable to map logical nameservice URI",
ioe);
}
// Ensure that the token can be renewed again after a failover.
cluster.TransitionToStandby(0);
cluster.TransitionToActive(1);
DoRenewOrCancel(token, clientConf, TestDelegationTokensWithHA.TokenTestAction.Renew
);
DoRenewOrCancel(token, clientConf, TestDelegationTokensWithHA.TokenTestAction.Cancel
);
}
public virtual void TestGetUgi()
{
conf.Set(DFSConfigKeys.FsDefaultNameKey, "hdfs://localhost:4321/");
HttpServletRequest request = Org.Mockito.Mockito.Mock <HttpServletRequest>();
ServletContext context = Org.Mockito.Mockito.Mock <ServletContext>();
string user = "******";
Text userText = new Text(user);
DelegationTokenIdentifier dtId = new DelegationTokenIdentifier(userText, userText
, null);
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token
<DelegationTokenIdentifier>(dtId, new TestJspHelper.DummySecretManager(0, 0, 0,
0));
string tokenString = token.EncodeToUrlString();
Org.Mockito.Mockito.When(request.GetParameter(JspHelper.DelegationParameterName))
.ThenReturn(tokenString);
Org.Mockito.Mockito.When(request.GetRemoteUser()).ThenReturn(user);
//Test attribute in the url to be used as service in the token.
Org.Mockito.Mockito.When(request.GetParameter(JspHelper.NamenodeAddress)).ThenReturn
("1.1.1.1:1111");
conf.Set(DFSConfigKeys.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
VerifyServiceInToken(context, request, "1.1.1.1:1111");
//Test attribute name.node.address
//Set the nnaddr url parameter to null.
Org.Mockito.Mockito.When(request.GetParameter(JspHelper.NamenodeAddress)).ThenReturn
(null);
IPEndPoint addr = new IPEndPoint("localhost", 2222);
Org.Mockito.Mockito.When(context.GetAttribute(NameNodeHttpServer.NamenodeAddressAttributeKey
)).ThenReturn(addr);
VerifyServiceInToken(context, request, addr.Address.GetHostAddress() + ":2222");
//Test service already set in the token
token.SetService(new Text("3.3.3.3:3333"));
tokenString = token.EncodeToUrlString();
//Set the name.node.address attribute in Servlet context to null
Org.Mockito.Mockito.When(context.GetAttribute(NameNodeHttpServer.NamenodeAddressAttributeKey
)).ThenReturn(null);
Org.Mockito.Mockito.When(request.GetParameter(JspHelper.DelegationParameterName))
.ThenReturn(tokenString);
VerifyServiceInToken(context, request, "3.3.3.3:3333");
}
/// <exception cref="System.IO.IOException"/>
private WebHdfsFileSystem GetWebHdfsFileSystem(UserGroupInformation ugi, Configuration
conf)
{
if (UserGroupInformation.IsSecurityEnabled())
{
DelegationTokenIdentifier dtId = new DelegationTokenIdentifier(new Text(ugi.GetUserName
()), null, null);
FSNamesystem namesystem = Org.Mockito.Mockito.Mock <FSNamesystem>();
DelegationTokenSecretManager dtSecretManager = new DelegationTokenSecretManager(86400000
, 86400000, 86400000, 86400000, namesystem);
dtSecretManager.StartThreads();
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token
<DelegationTokenIdentifier>(dtId, dtSecretManager);
SecurityUtil.SetTokenService(token, NetUtils.CreateSocketAddr(uri.GetAuthority())
);
token.SetKind(WebHdfsFileSystem.TokenKind);
ugi.AddToken(token);
}
return((WebHdfsFileSystem)FileSystem.Get(uri, conf));
}
public virtual void ExpectedTokenIsRetrievedFromDFS()
{
byte[] ident = new DelegationTokenIdentifier(new Text("owner"), new Text("renewer"
), new Text("realuser")).GetBytes();
byte[] pw = new byte[] { 42 };
Text service = new Text(uri.ToString());
// Create a token for the fetcher to fetch, wire NN to return it when asked
// for this particular user.
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> t = new Org.Apache.Hadoop.Security.Token.Token
<DelegationTokenIdentifier>(ident, pw, FakeRenewer.Kind, service);
Org.Mockito.Mockito.When(dfs.AddDelegationTokens(Matchers.Eq((string)null), Matchers.Any
<Credentials>())).ThenAnswer(new _Answer_77(service, t));
Org.Mockito.Mockito.When(dfs.GetUri()).ThenReturn(uri);
FakeRenewer.Reset();
FileSystem fileSys = FileSystem.GetLocal(conf);
try
{
DelegationTokenFetcher.Main(new string[] { "-fs", uri.ToString(), tokenFile });
Path p = new Path(fileSys.GetWorkingDirectory(), tokenFile);
Credentials creds = Credentials.ReadTokenStorageFile(p, conf);
IEnumerator <Org.Apache.Hadoop.Security.Token.Token <object> > itr = creds.GetAllTokens
().GetEnumerator();
// make sure we got back exactly the 1 token we expected
NUnit.Framework.Assert.IsTrue(itr.HasNext());
NUnit.Framework.Assert.AreEqual(t, itr.Next());
NUnit.Framework.Assert.IsTrue(!itr.HasNext());
DelegationTokenFetcher.Main(new string[] { "--print", tokenFile });
DelegationTokenFetcher.Main(new string[] { "--renew", tokenFile });
NUnit.Framework.Assert.AreEqual(t, FakeRenewer.lastRenewed);
FakeRenewer.Reset();
DelegationTokenFetcher.Main(new string[] { "--cancel", tokenFile });
NUnit.Framework.Assert.AreEqual(t, FakeRenewer.lastCanceled);
}
finally
{
fileSys.Delete(new Path(tokenFile), true);
}
}
/// <summary>
/// Test if StandbyException can be thrown from StandbyNN, when it's requested for
/// password.
/// </summary>
/// <remarks>
/// Test if StandbyException can be thrown from StandbyNN, when it's requested for
/// password. (HDFS-6475). With StandbyException, the client can failover to try
/// activeNN.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestDelegationTokenStandbyNNAppearFirst()
{
// make nn0 the standby NN, and nn1 the active NN
cluster.TransitionToStandby(0);
cluster.TransitionToActive(1);
DelegationTokenSecretManager stSecretManager = NameNodeAdapter.GetDtSecretManager
(nn1.GetNamesystem());
// create token
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = GetDelegationToken
(fs, "JobTracker");
DelegationTokenIdentifier identifier = new DelegationTokenIdentifier();
byte[] tokenId = token.GetIdentifier();
identifier.ReadFields(new DataInputStream(new ByteArrayInputStream(tokenId)));
NUnit.Framework.Assert.IsTrue(null != stSecretManager.RetrievePassword(identifier
));
UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser("JobTracker");
ugi.AddToken(token);
ugi.DoAs(new _PrivilegedExceptionAction_406(identifier));
}
public _PrivilegedExceptionAction_406(DelegationTokenIdentifier identifier)
{
this.identifier = identifier;
}
/// <summary>
/// Test if correct exception (StandbyException or RetriableException) can be
/// thrown during the NN failover.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestDelegationTokenDuringNNFailover()
{
EditLogTailer editLogTailer = nn1.GetNamesystem().GetEditLogTailer();
// stop the editLogTailer of nn1
editLogTailer.Stop();
Configuration conf = (Configuration)Whitebox.GetInternalState(editLogTailer, "conf"
);
nn1.GetNamesystem().SetEditLogTailerForTests(new TestDelegationTokensWithHA.EditLogTailerForTest
(this, nn1.GetNamesystem(), conf));
// create token
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = GetDelegationToken
(fs, "JobTracker");
DelegationTokenIdentifier identifier = new DelegationTokenIdentifier();
byte[] tokenId = token.GetIdentifier();
identifier.ReadFields(new DataInputStream(new ByteArrayInputStream(tokenId)));
// Ensure that it's present in the nn0 secret manager and can
// be renewed directly from there.
Log.Info("A valid token should have non-null password, " + "and should be renewed successfully"
);
NUnit.Framework.Assert.IsTrue(null != dtSecretManager.RetrievePassword(identifier
));
dtSecretManager.RenewToken(token, "JobTracker");
// transition nn0 to standby
cluster.TransitionToStandby(0);
try
{
cluster.GetNameNodeRpc(0).RenewDelegationToken(token);
NUnit.Framework.Assert.Fail("StandbyException is expected since nn0 is in standby state"
);
}
catch (StandbyException e)
{
GenericTestUtils.AssertExceptionContains(HAServiceProtocol.HAServiceState.Standby
.ToString(), e);
}
new _Thread_220().Start();
Sharpen.Thread.Sleep(1000);
try
{
nn1.GetNamesystem().VerifyToken(token.DecodeIdentifier(), token.GetPassword());
NUnit.Framework.Assert.Fail("RetriableException/StandbyException is expected since nn1 is in transition"
);
}
catch (IOException e)
{
NUnit.Framework.Assert.IsTrue(e is StandbyException || e is RetriableException);
Log.Info("Got expected exception", e);
}
catchup = true;
lock (this)
{
Sharpen.Runtime.NotifyAll(this);
}
Configuration clientConf = dfs.GetConf();
DoRenewOrCancel(token, clientConf, TestDelegationTokensWithHA.TokenTestAction.Renew
);
DoRenewOrCancel(token, clientConf, TestDelegationTokensWithHA.TokenTestAction.Cancel
);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void TestEditsForCancelOnTokenExpire()
{
long renewInterval = 2000;
Configuration conf = new Configuration();
conf.SetBoolean(DFSConfigKeys.DfsNamenodeDelegationTokenAlwaysUseKey, true);
conf.SetLong(DfsNamenodeDelegationTokenRenewIntervalKey, renewInterval);
conf.SetLong(DfsNamenodeDelegationTokenMaxLifetimeKey, renewInterval * 2);
Text renewer = new Text(UserGroupInformation.GetCurrentUser().GetUserName());
FSImage fsImage = Org.Mockito.Mockito.Mock <FSImage>();
FSEditLog log = Org.Mockito.Mockito.Mock <FSEditLog>();
Org.Mockito.Mockito.DoReturn(log).When(fsImage).GetEditLog();
FSNamesystem fsn = new FSNamesystem(conf, fsImage);
DelegationTokenSecretManager dtsm = fsn.GetDelegationTokenSecretManager();
try
{
dtsm.StartThreads();
// get two tokens
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token1 = fsn.GetDelegationToken
(renewer);
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token2 = fsn.GetDelegationToken
(renewer);
DelegationTokenIdentifier ident1 = token1.DecodeIdentifier();
DelegationTokenIdentifier ident2 = token2.DecodeIdentifier();
// verify we got the tokens
Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(1)).LogGetDelegationToken
(Eq(ident1), AnyLong());
Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(1)).LogGetDelegationToken
(Eq(ident2), AnyLong());
// this is a little tricky because DTSM doesn't let us set scan interval
// so need to periodically sleep, then stop/start threads to force scan
// renew first token 1/2 to expire
Sharpen.Thread.Sleep(renewInterval / 2);
fsn.RenewDelegationToken(token2);
Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(1)).LogRenewDelegationToken
(Eq(ident2), AnyLong());
// force scan and give it a little time to complete
dtsm.StopThreads();
dtsm.StartThreads();
Sharpen.Thread.Sleep(250);
// no token has expired yet
Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(0)).LogCancelDelegationToken
(Eq(ident1));
Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(0)).LogCancelDelegationToken
(Eq(ident2));
// sleep past expiration of 1st non-renewed token
Sharpen.Thread.Sleep(renewInterval / 2);
dtsm.StopThreads();
dtsm.StartThreads();
Sharpen.Thread.Sleep(250);
// non-renewed token should have implicitly been cancelled
Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(1)).LogCancelDelegationToken
(Eq(ident1));
Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(0)).LogCancelDelegationToken
(Eq(ident2));
// sleep past expiration of 2nd renewed token
Sharpen.Thread.Sleep(renewInterval / 2);
dtsm.StopThreads();
dtsm.StartThreads();
Sharpen.Thread.Sleep(250);
// both tokens should have been implicitly cancelled by now
Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(1)).LogCancelDelegationToken
(Eq(ident1));
Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(1)).LogCancelDelegationToken
(Eq(ident2));
}
finally
{
dtsm.StopThreads();
}
}
public virtual void TestGetUgiFromToken()
{
conf.Set(DFSConfigKeys.FsDefaultNameKey, "hdfs://localhost:4321/");
ServletContext context = Org.Mockito.Mockito.Mock <ServletContext>();
string realUser = "******";
string user = "******";
conf.Set(DFSConfigKeys.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
UserGroupInformation ugi;
HttpServletRequest request;
Text ownerText = new Text(user);
DelegationTokenIdentifier dtId = new DelegationTokenIdentifier(ownerText, ownerText
, new Text(realUser));
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token
<DelegationTokenIdentifier>(dtId, new TestJspHelper.DummySecretManager(0, 0, 0,
0));
string tokenString = token.EncodeToUrlString();
// token with no auth-ed user
request = GetMockRequest(null, null, null);
Org.Mockito.Mockito.When(request.GetParameter(JspHelper.DelegationParameterName))
.ThenReturn(tokenString);
ugi = JspHelper.GetUGI(context, request, conf);
NUnit.Framework.Assert.IsNotNull(ugi.GetRealUser());
NUnit.Framework.Assert.AreEqual(ugi.GetRealUser().GetShortUserName(), realUser);
NUnit.Framework.Assert.AreEqual(ugi.GetShortUserName(), user);
CheckUgiFromToken(ugi);
// token with auth-ed user
request = GetMockRequest(realUser, null, null);
Org.Mockito.Mockito.When(request.GetParameter(JspHelper.DelegationParameterName))
.ThenReturn(tokenString);
ugi = JspHelper.GetUGI(context, request, conf);
NUnit.Framework.Assert.IsNotNull(ugi.GetRealUser());
NUnit.Framework.Assert.AreEqual(ugi.GetRealUser().GetShortUserName(), realUser);
NUnit.Framework.Assert.AreEqual(ugi.GetShortUserName(), user);
CheckUgiFromToken(ugi);
// completely different user, token trumps auth
request = GetMockRequest("rogue", null, null);
Org.Mockito.Mockito.When(request.GetParameter(JspHelper.DelegationParameterName))
.ThenReturn(tokenString);
ugi = JspHelper.GetUGI(context, request, conf);
NUnit.Framework.Assert.IsNotNull(ugi.GetRealUser());
NUnit.Framework.Assert.AreEqual(ugi.GetRealUser().GetShortUserName(), realUser);
NUnit.Framework.Assert.AreEqual(ugi.GetShortUserName(), user);
CheckUgiFromToken(ugi);
// expected case
request = GetMockRequest(null, user, null);
Org.Mockito.Mockito.When(request.GetParameter(JspHelper.DelegationParameterName))
.ThenReturn(tokenString);
ugi = JspHelper.GetUGI(context, request, conf);
NUnit.Framework.Assert.IsNotNull(ugi.GetRealUser());
NUnit.Framework.Assert.AreEqual(ugi.GetRealUser().GetShortUserName(), realUser);
NUnit.Framework.Assert.AreEqual(ugi.GetShortUserName(), user);
CheckUgiFromToken(ugi);
// can't proxy with a token!
request = GetMockRequest(null, null, "rogue");
Org.Mockito.Mockito.When(request.GetParameter(JspHelper.DelegationParameterName))
.ThenReturn(tokenString);
try
{
JspHelper.GetUGI(context, request, conf);
NUnit.Framework.Assert.Fail("bad request allowed");
}
catch (IOException ioe)
{
NUnit.Framework.Assert.AreEqual("Usernames not matched: name=rogue != expected="
+ user, ioe.Message);
}
// can't proxy with a token!
request = GetMockRequest(null, user, "rogue");
Org.Mockito.Mockito.When(request.GetParameter(JspHelper.DelegationParameterName))
.ThenReturn(tokenString);
try
{
JspHelper.GetUGI(context, request, conf);
NUnit.Framework.Assert.Fail("bad request allowed");
}
catch (IOException ioe)
{
NUnit.Framework.Assert.AreEqual("Usernames not matched: name=rogue != expected="
+ user, ioe.Message);
}
}
