public virtual void TestDelegationTokenRpc() { ClientProtocol mockNN = Org.Mockito.Mockito.Mock <ClientProtocol>(); FSNamesystem mockNameSys = Org.Mockito.Mockito.Mock <FSNamesystem>(); DelegationTokenSecretManager sm = new DelegationTokenSecretManager(DFSConfigKeys. DfsNamenodeDelegationKeyUpdateIntervalDefault, DFSConfigKeys.DfsNamenodeDelegationKeyUpdateIntervalDefault , DFSConfigKeys.DfsNamenodeDelegationTokenMaxLifetimeDefault, 3600000, mockNameSys ); sm.StartThreads(); Org.Apache.Hadoop.Ipc.Server server = new RPC.Builder(conf).SetProtocol(typeof(ClientProtocol )).SetInstance(mockNN).SetBindAddress(Address).SetPort(0).SetNumHandlers(5).SetVerbose (true).SetSecretManager(sm).Build(); server.Start(); UserGroupInformation current = UserGroupInformation.GetCurrentUser(); IPEndPoint addr = NetUtils.GetConnectAddress(server); string user = current.GetUserName(); Text owner = new Text(user); DelegationTokenIdentifier dtId = new DelegationTokenIdentifier(owner, owner, null ); Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier>(dtId, sm); SecurityUtil.SetTokenService(token, addr); Log.Info("Service for token is " + token.GetService()); current.AddToken(token); current.DoAs(new _PrivilegedExceptionAction_100(addr, server)); }
/// <exception cref="System.IO.IOException"/> public Void Run() { Credentials ts = DelegationTokenSecretManager.CreateCredentials(nn, ugi, renewerFinal ); ts.Write(dosFinal); return(null); }
public virtual void SetUp() { config = new HdfsConfiguration(); config.SetBoolean(DFSConfigKeys.DfsWebhdfsEnabledKey, true); config.SetLong(DFSConfigKeys.DfsNamenodeDelegationTokenMaxLifetimeKey, 10000); config.SetLong(DFSConfigKeys.DfsNamenodeDelegationTokenRenewIntervalKey, 5000); config.SetBoolean(DFSConfigKeys.DfsNamenodeDelegationTokenAlwaysUseKey, true); config.Set("hadoop.security.auth_to_local", "RULE:[2:$1@$0](JobTracker@.*FOO.COM)s/@.*//" + "DEFAULT"); FileSystem.SetDefaultUri(config, "hdfs://localhost:" + "0"); cluster = new MiniDFSCluster.Builder(config).NumDataNodes(0).Build(); cluster.WaitActive(); dtSecretManager = NameNodeAdapter.GetDtSecretManager(cluster.GetNamesystem()); }
public virtual void SetupCluster() { SecurityUtilTestHelper.SetTokenServiceUseIp(true); conf.SetBoolean(DFSConfigKeys.DfsNamenodeDelegationTokenAlwaysUseKey, true); conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthToLocal, "RULE:[2:$1@$0](JobTracker@.*FOO.COM)s/@.*//" + "DEFAULT"); cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology.SimpleHATopology ()).NumDataNodes(0).Build(); cluster.WaitActive(); string logicalName = HATestUtil.GetLogicalHostname(cluster); HATestUtil.SetFailoverConfigurations(cluster, conf, logicalName, 0); nn0 = cluster.GetNameNode(0); nn1 = cluster.GetNameNode(1); fs = HATestUtil.ConfigureFailoverFs(cluster, conf); dfs = (DistributedFileSystem)fs; cluster.TransitionToActive(0); dtSecretManager = NameNodeAdapter.GetDtSecretManager(nn0.GetNamesystem()); }
/// <exception cref="System.IO.IOException"/> private WebHdfsFileSystem GetWebHdfsFileSystem(UserGroupInformation ugi, Configuration conf) { if (UserGroupInformation.IsSecurityEnabled()) { DelegationTokenIdentifier dtId = new DelegationTokenIdentifier(new Text(ugi.GetUserName ()), null, null); FSNamesystem namesystem = Org.Mockito.Mockito.Mock <FSNamesystem>(); DelegationTokenSecretManager dtSecretManager = new DelegationTokenSecretManager(86400000 , 86400000, 86400000, 86400000, namesystem); dtSecretManager.StartThreads(); Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier>(dtId, dtSecretManager); SecurityUtil.SetTokenService(token, NetUtils.CreateSocketAddr(uri.GetAuthority()) ); token.SetKind(WebHdfsFileSystem.TokenKind); ugi.AddToken(token); } return((WebHdfsFileSystem)FileSystem.Get(uri, conf)); }
public virtual void TestDTManagerInSafeMode() { cluster.StartDataNodes(config, 1, true, HdfsServerConstants.StartupOption.Regular , null); FileSystem fs = cluster.GetFileSystem(); for (int i = 0; i < 5; i++) { DFSTestUtil.CreateFile(fs, new Path("/test-" + i), 100, (short)1, 1L); } cluster.GetConfiguration(0).SetInt(DFSConfigKeys.DfsNamenodeDelegationKeyUpdateIntervalKey , 500); cluster.GetConfiguration(0).SetInt(DFSConfigKeys.DfsNamenodeSafemodeExtensionKey, 30000); cluster.SetWaitSafeMode(false); cluster.RestartNameNode(); NameNode nn = cluster.GetNameNode(); NUnit.Framework.Assert.IsTrue(nn.IsInSafeMode()); DelegationTokenSecretManager sm = NameNodeAdapter.GetDtSecretManager(nn.GetNamesystem ()); NUnit.Framework.Assert.IsFalse("Secret manager should not run in safe mode", sm.IsRunning ()); NameNodeAdapter.LeaveSafeMode(nn); NUnit.Framework.Assert.IsTrue("Secret manager should start when safe mode is exited" , sm.IsRunning()); Log.Info("========= entering safemode again"); NameNodeAdapter.EnterSafeMode(nn, false); NUnit.Framework.Assert.IsFalse("Secret manager should stop again when safe mode " + "is manually entered", sm.IsRunning()); // Set the cluster to leave safemode quickly on its own. cluster.GetConfiguration(0).SetInt(DFSConfigKeys.DfsNamenodeSafemodeExtensionKey, 0); cluster.SetWaitSafeMode(true); cluster.RestartNameNode(); nn = cluster.GetNameNode(); sm = NameNodeAdapter.GetDtSecretManager(nn.GetNamesystem()); NUnit.Framework.Assert.IsFalse(nn.IsInSafeMode()); NUnit.Framework.Assert.IsTrue(sm.IsRunning()); }
/// <summary> /// Test if StandbyException can be thrown from StandbyNN, when it's requested for /// password. /// </summary> /// <remarks> /// Test if StandbyException can be thrown from StandbyNN, when it's requested for /// password. (HDFS-6475). With StandbyException, the client can failover to try /// activeNN. /// </remarks> /// <exception cref="System.Exception"/> public virtual void TestDelegationTokenStandbyNNAppearFirst() { // make nn0 the standby NN, and nn1 the active NN cluster.TransitionToStandby(0); cluster.TransitionToActive(1); DelegationTokenSecretManager stSecretManager = NameNodeAdapter.GetDtSecretManager (nn1.GetNamesystem()); // create token Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = GetDelegationToken (fs, "JobTracker"); DelegationTokenIdentifier identifier = new DelegationTokenIdentifier(); byte[] tokenId = token.GetIdentifier(); identifier.ReadFields(new DataInputStream(new ByteArrayInputStream(tokenId))); NUnit.Framework.Assert.IsTrue(null != stSecretManager.RetrievePassword(identifier )); UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser("JobTracker"); ugi.AddToken(token); ugi.DoAs(new _PrivilegedExceptionAction_406(identifier)); }
/// <summary> /// Same as /// <see cref="LogAuditEvent(bool, string, System.Net.IPAddress, string, string, string, Org.Apache.Hadoop.FS.FileStatus) /// "/> /// with additional parameters related to logging delegation token tracking /// IDs. /// </summary> /// <param name="succeeded">Whether authorization succeeded.</param> /// <param name="userName">Name of the user executing the request.</param> /// <param name="addr">Remote address of the request.</param> /// <param name="cmd">The requested command.</param> /// <param name="src">Path of affected source file.</param> /// <param name="dst">Path of affected destination file (if any).</param> /// <param name="stat"> /// File information for operations that change the file's metadata /// (permissions, owner, times, etc). /// </param> /// <param name="ugi"> /// UserGroupInformation of the current user, or null if not logging /// token tracking information /// </param> /// <param name="dtSecretManager"> /// The token secret manager, or null if not logging /// token tracking information /// </param> public abstract void LogAuditEvent(bool succeeded, string userName, IPAddress addr , string cmd, string src, string dst, FileStatus stat, UserGroupInformation ugi, DelegationTokenSecretManager dtSecretManager);
/// <exception cref="System.Exception"/> public object Run() { if (print) { DelegationTokenIdentifier id = new DelegationTokenSecretManager(0, 0, 0, 0, null) .CreateIdentifier(); foreach (Org.Apache.Hadoop.Security.Token.Token <object> token in DelegationTokenFetcher .ReadTokens(tokenFile, conf)) { DataInputStream @in = new DataInputStream(new ByteArrayInputStream(token.GetIdentifier ())); id.ReadFields(@in); System.Console.Out.WriteLine("Token (" + id + ") for " + token.GetService()); } return(null); } if (renew) { foreach (Org.Apache.Hadoop.Security.Token.Token <object> token in DelegationTokenFetcher .ReadTokens(tokenFile, conf)) { if (token.IsManaged()) { long result = token.Renew(conf); if (DelegationTokenFetcher.Log.IsDebugEnabled()) { DelegationTokenFetcher.Log.Debug("Renewed token for " + token.GetService() + " until: " + Sharpen.Extensions.CreateDate(result)); } } } } else { if (cancel) { foreach (Org.Apache.Hadoop.Security.Token.Token <object> token in DelegationTokenFetcher .ReadTokens(tokenFile, conf)) { if (token.IsManaged()) { token.Cancel(conf); if (DelegationTokenFetcher.Log.IsDebugEnabled()) { DelegationTokenFetcher.Log.Debug("Cancelled token for " + token.GetService()); } } } } else { // otherwise we are fetching if (webUrl != null) { Credentials creds = DelegationTokenFetcher.GetDTfromRemote(connectionFactory, new URI(webUrl), renewer, null); creds.WriteTokenStorageFile(tokenFile, conf); foreach (Org.Apache.Hadoop.Security.Token.Token <object> token in creds.GetAllTokens ()) { System.Console.Out.WriteLine("Fetched token via " + webUrl + " for " + token.GetService () + " into " + tokenFile); } } else { FileSystem fs = FileSystem.Get(conf); Credentials cred = new Credentials(); Org.Apache.Hadoop.Security.Token.Token <object>[] tokens = fs.AddDelegationTokens( renewer, cred); cred.WriteTokenStorageFile(tokenFile, conf); foreach (Org.Apache.Hadoop.Security.Token.Token <object> token in tokens) { System.Console.Out.WriteLine("Fetched token for " + token.GetService() + " into " + tokenFile); } } } } return(null); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public virtual void TestEditsForCancelOnTokenExpire() { long renewInterval = 2000; Configuration conf = new Configuration(); conf.SetBoolean(DFSConfigKeys.DfsNamenodeDelegationTokenAlwaysUseKey, true); conf.SetLong(DfsNamenodeDelegationTokenRenewIntervalKey, renewInterval); conf.SetLong(DfsNamenodeDelegationTokenMaxLifetimeKey, renewInterval * 2); Text renewer = new Text(UserGroupInformation.GetCurrentUser().GetUserName()); FSImage fsImage = Org.Mockito.Mockito.Mock <FSImage>(); FSEditLog log = Org.Mockito.Mockito.Mock <FSEditLog>(); Org.Mockito.Mockito.DoReturn(log).When(fsImage).GetEditLog(); FSNamesystem fsn = new FSNamesystem(conf, fsImage); DelegationTokenSecretManager dtsm = fsn.GetDelegationTokenSecretManager(); try { dtsm.StartThreads(); // get two tokens Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token1 = fsn.GetDelegationToken (renewer); Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token2 = fsn.GetDelegationToken (renewer); DelegationTokenIdentifier ident1 = token1.DecodeIdentifier(); DelegationTokenIdentifier ident2 = token2.DecodeIdentifier(); // verify we got the tokens Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(1)).LogGetDelegationToken (Eq(ident1), AnyLong()); Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(1)).LogGetDelegationToken (Eq(ident2), AnyLong()); // this is a little tricky because DTSM doesn't let us set scan interval // so need to periodically sleep, then stop/start threads to force scan // renew first token 1/2 to expire Sharpen.Thread.Sleep(renewInterval / 2); fsn.RenewDelegationToken(token2); Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(1)).LogRenewDelegationToken (Eq(ident2), AnyLong()); // force scan and give it a little time to complete dtsm.StopThreads(); dtsm.StartThreads(); Sharpen.Thread.Sleep(250); // no token has expired yet Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(0)).LogCancelDelegationToken (Eq(ident1)); Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(0)).LogCancelDelegationToken (Eq(ident2)); // sleep past expiration of 1st non-renewed token Sharpen.Thread.Sleep(renewInterval / 2); dtsm.StopThreads(); dtsm.StartThreads(); Sharpen.Thread.Sleep(250); // non-renewed token should have implicitly been cancelled Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(1)).LogCancelDelegationToken (Eq(ident1)); Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(0)).LogCancelDelegationToken (Eq(ident2)); // sleep past expiration of 2nd renewed token Sharpen.Thread.Sleep(renewInterval / 2); dtsm.StopThreads(); dtsm.StartThreads(); Sharpen.Thread.Sleep(250); // both tokens should have been implicitly cancelled by now Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(1)).LogCancelDelegationToken (Eq(ident1)); Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(1)).LogCancelDelegationToken (Eq(ident2)); } finally { dtsm.StopThreads(); } }