public virtual void TestDelegationTokenRpc()
{
ClientProtocol mockNN = Org.Mockito.Mockito.Mock <ClientProtocol>();
FSNamesystem mockNameSys = Org.Mockito.Mockito.Mock <FSNamesystem>();
DelegationTokenSecretManager sm = new DelegationTokenSecretManager(DFSConfigKeys.
DfsNamenodeDelegationKeyUpdateIntervalDefault, DFSConfigKeys.DfsNamenodeDelegationKeyUpdateIntervalDefault
, DFSConfigKeys.DfsNamenodeDelegationTokenMaxLifetimeDefault, 3600000, mockNameSys
);
sm.StartThreads();
Org.Apache.Hadoop.Ipc.Server server = new RPC.Builder(conf).SetProtocol(typeof(ClientProtocol
)).SetInstance(mockNN).SetBindAddress(Address).SetPort(0).SetNumHandlers(5).SetVerbose
(true).SetSecretManager(sm).Build();
server.Start();
UserGroupInformation current = UserGroupInformation.GetCurrentUser();
IPEndPoint addr = NetUtils.GetConnectAddress(server);
string user = current.GetUserName();
Text owner = new Text(user);
DelegationTokenIdentifier dtId = new DelegationTokenIdentifier(owner, owner, null
);
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token
<DelegationTokenIdentifier>(dtId, sm);
SecurityUtil.SetTokenService(token, addr);
Log.Info("Service for token is " + token.GetService());
current.AddToken(token);
current.DoAs(new _PrivilegedExceptionAction_100(addr, server));
}
/// <exception cref="System.IO.IOException"/>
public Void Run()
{
Credentials ts = DelegationTokenSecretManager.CreateCredentials(nn, ugi, renewerFinal
);
ts.Write(dosFinal);
return(null);
}
public virtual void SetUp()
{
config = new HdfsConfiguration();
config.SetBoolean(DFSConfigKeys.DfsWebhdfsEnabledKey, true);
config.SetLong(DFSConfigKeys.DfsNamenodeDelegationTokenMaxLifetimeKey, 10000);
config.SetLong(DFSConfigKeys.DfsNamenodeDelegationTokenRenewIntervalKey, 5000);
config.SetBoolean(DFSConfigKeys.DfsNamenodeDelegationTokenAlwaysUseKey, true);
config.Set("hadoop.security.auth_to_local", "RULE:[2:$1@$0](JobTracker@.*FOO.COM)s/@.*//"
+ "DEFAULT");
FileSystem.SetDefaultUri(config, "hdfs://localhost:" + "0");
cluster = new MiniDFSCluster.Builder(config).NumDataNodes(0).Build();
cluster.WaitActive();
dtSecretManager = NameNodeAdapter.GetDtSecretManager(cluster.GetNamesystem());
}
public virtual void SetupCluster()
{
SecurityUtilTestHelper.SetTokenServiceUseIp(true);
conf.SetBoolean(DFSConfigKeys.DfsNamenodeDelegationTokenAlwaysUseKey, true);
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthToLocal, "RULE:[2:$1@$0](JobTracker@.*FOO.COM)s/@.*//"
+ "DEFAULT");
cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology.SimpleHATopology
()).NumDataNodes(0).Build();
cluster.WaitActive();
string logicalName = HATestUtil.GetLogicalHostname(cluster);
HATestUtil.SetFailoverConfigurations(cluster, conf, logicalName, 0);
nn0 = cluster.GetNameNode(0);
nn1 = cluster.GetNameNode(1);
fs = HATestUtil.ConfigureFailoverFs(cluster, conf);
dfs = (DistributedFileSystem)fs;
cluster.TransitionToActive(0);
dtSecretManager = NameNodeAdapter.GetDtSecretManager(nn0.GetNamesystem());
}
/// <exception cref="System.IO.IOException"/>
private WebHdfsFileSystem GetWebHdfsFileSystem(UserGroupInformation ugi, Configuration
conf)
{
if (UserGroupInformation.IsSecurityEnabled())
{
DelegationTokenIdentifier dtId = new DelegationTokenIdentifier(new Text(ugi.GetUserName
()), null, null);
FSNamesystem namesystem = Org.Mockito.Mockito.Mock <FSNamesystem>();
DelegationTokenSecretManager dtSecretManager = new DelegationTokenSecretManager(86400000
, 86400000, 86400000, 86400000, namesystem);
dtSecretManager.StartThreads();
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token
<DelegationTokenIdentifier>(dtId, dtSecretManager);
SecurityUtil.SetTokenService(token, NetUtils.CreateSocketAddr(uri.GetAuthority())
);
token.SetKind(WebHdfsFileSystem.TokenKind);
ugi.AddToken(token);
}
return((WebHdfsFileSystem)FileSystem.Get(uri, conf));
}
public virtual void TestDTManagerInSafeMode()
{
cluster.StartDataNodes(config, 1, true, HdfsServerConstants.StartupOption.Regular
, null);
FileSystem fs = cluster.GetFileSystem();
for (int i = 0; i < 5; i++)
{
DFSTestUtil.CreateFile(fs, new Path("/test-" + i), 100, (short)1, 1L);
}
cluster.GetConfiguration(0).SetInt(DFSConfigKeys.DfsNamenodeDelegationKeyUpdateIntervalKey
, 500);
cluster.GetConfiguration(0).SetInt(DFSConfigKeys.DfsNamenodeSafemodeExtensionKey,
30000);
cluster.SetWaitSafeMode(false);
cluster.RestartNameNode();
NameNode nn = cluster.GetNameNode();
NUnit.Framework.Assert.IsTrue(nn.IsInSafeMode());
DelegationTokenSecretManager sm = NameNodeAdapter.GetDtSecretManager(nn.GetNamesystem
());
NUnit.Framework.Assert.IsFalse("Secret manager should not run in safe mode", sm.IsRunning
());
NameNodeAdapter.LeaveSafeMode(nn);
NUnit.Framework.Assert.IsTrue("Secret manager should start when safe mode is exited"
, sm.IsRunning());
Log.Info("========= entering safemode again");
NameNodeAdapter.EnterSafeMode(nn, false);
NUnit.Framework.Assert.IsFalse("Secret manager should stop again when safe mode "
+ "is manually entered", sm.IsRunning());
// Set the cluster to leave safemode quickly on its own.
cluster.GetConfiguration(0).SetInt(DFSConfigKeys.DfsNamenodeSafemodeExtensionKey,
0);
cluster.SetWaitSafeMode(true);
cluster.RestartNameNode();
nn = cluster.GetNameNode();
sm = NameNodeAdapter.GetDtSecretManager(nn.GetNamesystem());
NUnit.Framework.Assert.IsFalse(nn.IsInSafeMode());
NUnit.Framework.Assert.IsTrue(sm.IsRunning());
}
/// <summary>
/// Test if StandbyException can be thrown from StandbyNN, when it's requested for
/// password.
/// </summary>
/// <remarks>
/// Test if StandbyException can be thrown from StandbyNN, when it's requested for
/// password. (HDFS-6475). With StandbyException, the client can failover to try
/// activeNN.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestDelegationTokenStandbyNNAppearFirst()
{
// make nn0 the standby NN, and nn1 the active NN
cluster.TransitionToStandby(0);
cluster.TransitionToActive(1);
DelegationTokenSecretManager stSecretManager = NameNodeAdapter.GetDtSecretManager
(nn1.GetNamesystem());
// create token
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = GetDelegationToken
(fs, "JobTracker");
DelegationTokenIdentifier identifier = new DelegationTokenIdentifier();
byte[] tokenId = token.GetIdentifier();
identifier.ReadFields(new DataInputStream(new ByteArrayInputStream(tokenId)));
NUnit.Framework.Assert.IsTrue(null != stSecretManager.RetrievePassword(identifier
));
UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser("JobTracker");
ugi.AddToken(token);
ugi.DoAs(new _PrivilegedExceptionAction_406(identifier));
}
/// <summary>
/// Same as
/// <see cref="LogAuditEvent(bool, string, System.Net.IPAddress, string, string, string, Org.Apache.Hadoop.FS.FileStatus)
/// "/>
/// with additional parameters related to logging delegation token tracking
/// IDs.
/// </summary>
/// <param name="succeeded">Whether authorization succeeded.</param>
/// <param name="userName">Name of the user executing the request.</param>
/// <param name="addr">Remote address of the request.</param>
/// <param name="cmd">The requested command.</param>
/// <param name="src">Path of affected source file.</param>
/// <param name="dst">Path of affected destination file (if any).</param>
/// <param name="stat">
/// File information for operations that change the file's metadata
/// (permissions, owner, times, etc).
/// </param>
/// <param name="ugi">
/// UserGroupInformation of the current user, or null if not logging
/// token tracking information
/// </param>
/// <param name="dtSecretManager">
/// The token secret manager, or null if not logging
/// token tracking information
/// </param>
public abstract void LogAuditEvent(bool succeeded, string userName, IPAddress addr
, string cmd, string src, string dst, FileStatus stat, UserGroupInformation ugi,
DelegationTokenSecretManager dtSecretManager);
/// <exception cref="System.Exception"/>
public object Run()
{
if (print)
{
DelegationTokenIdentifier id = new DelegationTokenSecretManager(0, 0, 0, 0, null)
.CreateIdentifier();
foreach (Org.Apache.Hadoop.Security.Token.Token <object> token in DelegationTokenFetcher
.ReadTokens(tokenFile, conf))
{
DataInputStream @in = new DataInputStream(new ByteArrayInputStream(token.GetIdentifier
()));
id.ReadFields(@in);
System.Console.Out.WriteLine("Token (" + id + ") for " + token.GetService());
}
return(null);
}
if (renew)
{
foreach (Org.Apache.Hadoop.Security.Token.Token <object> token in DelegationTokenFetcher
.ReadTokens(tokenFile, conf))
{
if (token.IsManaged())
{
long result = token.Renew(conf);
if (DelegationTokenFetcher.Log.IsDebugEnabled())
{
DelegationTokenFetcher.Log.Debug("Renewed token for " + token.GetService() + " until: "
+ Sharpen.Extensions.CreateDate(result));
}
}
}
}
else
{
if (cancel)
{
foreach (Org.Apache.Hadoop.Security.Token.Token <object> token in DelegationTokenFetcher
.ReadTokens(tokenFile, conf))
{
if (token.IsManaged())
{
token.Cancel(conf);
if (DelegationTokenFetcher.Log.IsDebugEnabled())
{
DelegationTokenFetcher.Log.Debug("Cancelled token for " + token.GetService());
}
}
}
}
else
{
// otherwise we are fetching
if (webUrl != null)
{
Credentials creds = DelegationTokenFetcher.GetDTfromRemote(connectionFactory, new
URI(webUrl), renewer, null);
creds.WriteTokenStorageFile(tokenFile, conf);
foreach (Org.Apache.Hadoop.Security.Token.Token <object> token in creds.GetAllTokens
())
{
System.Console.Out.WriteLine("Fetched token via " + webUrl + " for " + token.GetService
() + " into " + tokenFile);
}
}
else
{
FileSystem fs = FileSystem.Get(conf);
Credentials cred = new Credentials();
Org.Apache.Hadoop.Security.Token.Token <object>[] tokens = fs.AddDelegationTokens(
renewer, cred);
cred.WriteTokenStorageFile(tokenFile, conf);
foreach (Org.Apache.Hadoop.Security.Token.Token <object> token in tokens)
{
System.Console.Out.WriteLine("Fetched token for " + token.GetService() + " into "
+ tokenFile);
}
}
}
}
return(null);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void TestEditsForCancelOnTokenExpire()
{
long renewInterval = 2000;
Configuration conf = new Configuration();
conf.SetBoolean(DFSConfigKeys.DfsNamenodeDelegationTokenAlwaysUseKey, true);
conf.SetLong(DfsNamenodeDelegationTokenRenewIntervalKey, renewInterval);
conf.SetLong(DfsNamenodeDelegationTokenMaxLifetimeKey, renewInterval * 2);
Text renewer = new Text(UserGroupInformation.GetCurrentUser().GetUserName());
FSImage fsImage = Org.Mockito.Mockito.Mock <FSImage>();
FSEditLog log = Org.Mockito.Mockito.Mock <FSEditLog>();
Org.Mockito.Mockito.DoReturn(log).When(fsImage).GetEditLog();
FSNamesystem fsn = new FSNamesystem(conf, fsImage);
DelegationTokenSecretManager dtsm = fsn.GetDelegationTokenSecretManager();
try
{
dtsm.StartThreads();
// get two tokens
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token1 = fsn.GetDelegationToken
(renewer);
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token2 = fsn.GetDelegationToken
(renewer);
DelegationTokenIdentifier ident1 = token1.DecodeIdentifier();
DelegationTokenIdentifier ident2 = token2.DecodeIdentifier();
// verify we got the tokens
Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(1)).LogGetDelegationToken
(Eq(ident1), AnyLong());
Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(1)).LogGetDelegationToken
(Eq(ident2), AnyLong());
// this is a little tricky because DTSM doesn't let us set scan interval
// so need to periodically sleep, then stop/start threads to force scan
// renew first token 1/2 to expire
Sharpen.Thread.Sleep(renewInterval / 2);
fsn.RenewDelegationToken(token2);
Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(1)).LogRenewDelegationToken
(Eq(ident2), AnyLong());
// force scan and give it a little time to complete
dtsm.StopThreads();
dtsm.StartThreads();
Sharpen.Thread.Sleep(250);
// no token has expired yet
Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(0)).LogCancelDelegationToken
(Eq(ident1));
Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(0)).LogCancelDelegationToken
(Eq(ident2));
// sleep past expiration of 1st non-renewed token
Sharpen.Thread.Sleep(renewInterval / 2);
dtsm.StopThreads();
dtsm.StartThreads();
Sharpen.Thread.Sleep(250);
// non-renewed token should have implicitly been cancelled
Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(1)).LogCancelDelegationToken
(Eq(ident1));
Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(0)).LogCancelDelegationToken
(Eq(ident2));
// sleep past expiration of 2nd renewed token
Sharpen.Thread.Sleep(renewInterval / 2);
dtsm.StopThreads();
dtsm.StartThreads();
Sharpen.Thread.Sleep(250);
// both tokens should have been implicitly cancelled by now
Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(1)).LogCancelDelegationToken
(Eq(ident1));
Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(1)).LogCancelDelegationToken
(Eq(ident2));
}
finally
{
dtsm.StopThreads();
}
}
