/// <exception cref="System.Exception"/> public virtual void TestDelegationTokenDFSApi() { Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = GetDelegationToken (fs, "JobTracker"); DelegationTokenIdentifier identifier = new DelegationTokenIdentifier(); byte[] tokenId = token.GetIdentifier(); identifier.ReadFields(new DataInputStream(new ByteArrayInputStream(tokenId))); // Ensure that it's present in the NN's secret manager and can // be renewed directly from there. Log.Info("A valid token should have non-null password, " + "and should be renewed successfully" ); NUnit.Framework.Assert.IsTrue(null != dtSecretManager.RetrievePassword(identifier )); dtSecretManager.RenewToken(token, "JobTracker"); // Use the client conf with the failover info present to check // renewal. Configuration clientConf = dfs.GetConf(); DoRenewOrCancel(token, clientConf, TestDelegationTokensWithHA.TokenTestAction.Renew ); // Using a configuration that doesn't have the logical nameservice // configured should result in a reasonable error message. Configuration emptyConf = new Configuration(); try { DoRenewOrCancel(token, emptyConf, TestDelegationTokensWithHA.TokenTestAction.Renew ); NUnit.Framework.Assert.Fail("Did not throw trying to renew with an empty conf!"); } catch (IOException ioe) { GenericTestUtils.AssertExceptionContains("Unable to map logical nameservice URI", ioe); } // Ensure that the token can be renewed again after a failover. cluster.TransitionToStandby(0); cluster.TransitionToActive(1); DoRenewOrCancel(token, clientConf, TestDelegationTokensWithHA.TokenTestAction.Renew ); DoRenewOrCancel(token, clientConf, TestDelegationTokensWithHA.TokenTestAction.Cancel ); }
public virtual void TestGetUgi() { conf.Set(DFSConfigKeys.FsDefaultNameKey, "hdfs://localhost:4321/"); HttpServletRequest request = Org.Mockito.Mockito.Mock <HttpServletRequest>(); ServletContext context = Org.Mockito.Mockito.Mock <ServletContext>(); string user = "******"; Text userText = new Text(user); DelegationTokenIdentifier dtId = new DelegationTokenIdentifier(userText, userText , null); Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier>(dtId, new TestJspHelper.DummySecretManager(0, 0, 0, 0)); string tokenString = token.EncodeToUrlString(); Org.Mockito.Mockito.When(request.GetParameter(JspHelper.DelegationParameterName)) .ThenReturn(tokenString); Org.Mockito.Mockito.When(request.GetRemoteUser()).ThenReturn(user); //Test attribute in the url to be used as service in the token. Org.Mockito.Mockito.When(request.GetParameter(JspHelper.NamenodeAddress)).ThenReturn ("1.1.1.1:1111"); conf.Set(DFSConfigKeys.HadoopSecurityAuthentication, "kerberos"); UserGroupInformation.SetConfiguration(conf); VerifyServiceInToken(context, request, "1.1.1.1:1111"); //Test attribute name.node.address //Set the nnaddr url parameter to null. Org.Mockito.Mockito.When(request.GetParameter(JspHelper.NamenodeAddress)).ThenReturn (null); IPEndPoint addr = new IPEndPoint("localhost", 2222); Org.Mockito.Mockito.When(context.GetAttribute(NameNodeHttpServer.NamenodeAddressAttributeKey )).ThenReturn(addr); VerifyServiceInToken(context, request, addr.Address.GetHostAddress() + ":2222"); //Test service already set in the token token.SetService(new Text("3.3.3.3:3333")); tokenString = token.EncodeToUrlString(); //Set the name.node.address attribute in Servlet context to null Org.Mockito.Mockito.When(context.GetAttribute(NameNodeHttpServer.NamenodeAddressAttributeKey )).ThenReturn(null); Org.Mockito.Mockito.When(request.GetParameter(JspHelper.DelegationParameterName)) .ThenReturn(tokenString); VerifyServiceInToken(context, request, "3.3.3.3:3333"); }
/// <exception cref="System.IO.IOException"/> private WebHdfsFileSystem GetWebHdfsFileSystem(UserGroupInformation ugi, Configuration conf) { if (UserGroupInformation.IsSecurityEnabled()) { DelegationTokenIdentifier dtId = new DelegationTokenIdentifier(new Text(ugi.GetUserName ()), null, null); FSNamesystem namesystem = Org.Mockito.Mockito.Mock <FSNamesystem>(); DelegationTokenSecretManager dtSecretManager = new DelegationTokenSecretManager(86400000 , 86400000, 86400000, 86400000, namesystem); dtSecretManager.StartThreads(); Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier>(dtId, dtSecretManager); SecurityUtil.SetTokenService(token, NetUtils.CreateSocketAddr(uri.GetAuthority()) ); token.SetKind(WebHdfsFileSystem.TokenKind); ugi.AddToken(token); } return((WebHdfsFileSystem)FileSystem.Get(uri, conf)); }
public virtual void ExpectedTokenIsRetrievedFromDFS() { byte[] ident = new DelegationTokenIdentifier(new Text("owner"), new Text("renewer" ), new Text("realuser")).GetBytes(); byte[] pw = new byte[] { 42 }; Text service = new Text(uri.ToString()); // Create a token for the fetcher to fetch, wire NN to return it when asked // for this particular user. Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> t = new Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier>(ident, pw, FakeRenewer.Kind, service); Org.Mockito.Mockito.When(dfs.AddDelegationTokens(Matchers.Eq((string)null), Matchers.Any <Credentials>())).ThenAnswer(new _Answer_77(service, t)); Org.Mockito.Mockito.When(dfs.GetUri()).ThenReturn(uri); FakeRenewer.Reset(); FileSystem fileSys = FileSystem.GetLocal(conf); try { DelegationTokenFetcher.Main(new string[] { "-fs", uri.ToString(), tokenFile }); Path p = new Path(fileSys.GetWorkingDirectory(), tokenFile); Credentials creds = Credentials.ReadTokenStorageFile(p, conf); IEnumerator <Org.Apache.Hadoop.Security.Token.Token <object> > itr = creds.GetAllTokens ().GetEnumerator(); // make sure we got back exactly the 1 token we expected NUnit.Framework.Assert.IsTrue(itr.HasNext()); NUnit.Framework.Assert.AreEqual(t, itr.Next()); NUnit.Framework.Assert.IsTrue(!itr.HasNext()); DelegationTokenFetcher.Main(new string[] { "--print", tokenFile }); DelegationTokenFetcher.Main(new string[] { "--renew", tokenFile }); NUnit.Framework.Assert.AreEqual(t, FakeRenewer.lastRenewed); FakeRenewer.Reset(); DelegationTokenFetcher.Main(new string[] { "--cancel", tokenFile }); NUnit.Framework.Assert.AreEqual(t, FakeRenewer.lastCanceled); } finally { fileSys.Delete(new Path(tokenFile), true); } }
/// <summary> /// Test if StandbyException can be thrown from StandbyNN, when it's requested for /// password. /// </summary> /// <remarks> /// Test if StandbyException can be thrown from StandbyNN, when it's requested for /// password. (HDFS-6475). With StandbyException, the client can failover to try /// activeNN. /// </remarks> /// <exception cref="System.Exception"/> public virtual void TestDelegationTokenStandbyNNAppearFirst() { // make nn0 the standby NN, and nn1 the active NN cluster.TransitionToStandby(0); cluster.TransitionToActive(1); DelegationTokenSecretManager stSecretManager = NameNodeAdapter.GetDtSecretManager (nn1.GetNamesystem()); // create token Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = GetDelegationToken (fs, "JobTracker"); DelegationTokenIdentifier identifier = new DelegationTokenIdentifier(); byte[] tokenId = token.GetIdentifier(); identifier.ReadFields(new DataInputStream(new ByteArrayInputStream(tokenId))); NUnit.Framework.Assert.IsTrue(null != stSecretManager.RetrievePassword(identifier )); UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser("JobTracker"); ugi.AddToken(token); ugi.DoAs(new _PrivilegedExceptionAction_406(identifier)); }
public _PrivilegedExceptionAction_406(DelegationTokenIdentifier identifier) { this.identifier = identifier; }
/// <summary> /// Test if correct exception (StandbyException or RetriableException) can be /// thrown during the NN failover. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestDelegationTokenDuringNNFailover() { EditLogTailer editLogTailer = nn1.GetNamesystem().GetEditLogTailer(); // stop the editLogTailer of nn1 editLogTailer.Stop(); Configuration conf = (Configuration)Whitebox.GetInternalState(editLogTailer, "conf" ); nn1.GetNamesystem().SetEditLogTailerForTests(new TestDelegationTokensWithHA.EditLogTailerForTest (this, nn1.GetNamesystem(), conf)); // create token Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = GetDelegationToken (fs, "JobTracker"); DelegationTokenIdentifier identifier = new DelegationTokenIdentifier(); byte[] tokenId = token.GetIdentifier(); identifier.ReadFields(new DataInputStream(new ByteArrayInputStream(tokenId))); // Ensure that it's present in the nn0 secret manager and can // be renewed directly from there. Log.Info("A valid token should have non-null password, " + "and should be renewed successfully" ); NUnit.Framework.Assert.IsTrue(null != dtSecretManager.RetrievePassword(identifier )); dtSecretManager.RenewToken(token, "JobTracker"); // transition nn0 to standby cluster.TransitionToStandby(0); try { cluster.GetNameNodeRpc(0).RenewDelegationToken(token); NUnit.Framework.Assert.Fail("StandbyException is expected since nn0 is in standby state" ); } catch (StandbyException e) { GenericTestUtils.AssertExceptionContains(HAServiceProtocol.HAServiceState.Standby .ToString(), e); } new _Thread_220().Start(); Sharpen.Thread.Sleep(1000); try { nn1.GetNamesystem().VerifyToken(token.DecodeIdentifier(), token.GetPassword()); NUnit.Framework.Assert.Fail("RetriableException/StandbyException is expected since nn1 is in transition" ); } catch (IOException e) { NUnit.Framework.Assert.IsTrue(e is StandbyException || e is RetriableException); Log.Info("Got expected exception", e); } catchup = true; lock (this) { Sharpen.Runtime.NotifyAll(this); } Configuration clientConf = dfs.GetConf(); DoRenewOrCancel(token, clientConf, TestDelegationTokensWithHA.TokenTestAction.Renew ); DoRenewOrCancel(token, clientConf, TestDelegationTokensWithHA.TokenTestAction.Cancel ); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public virtual void TestEditsForCancelOnTokenExpire() { long renewInterval = 2000; Configuration conf = new Configuration(); conf.SetBoolean(DFSConfigKeys.DfsNamenodeDelegationTokenAlwaysUseKey, true); conf.SetLong(DfsNamenodeDelegationTokenRenewIntervalKey, renewInterval); conf.SetLong(DfsNamenodeDelegationTokenMaxLifetimeKey, renewInterval * 2); Text renewer = new Text(UserGroupInformation.GetCurrentUser().GetUserName()); FSImage fsImage = Org.Mockito.Mockito.Mock <FSImage>(); FSEditLog log = Org.Mockito.Mockito.Mock <FSEditLog>(); Org.Mockito.Mockito.DoReturn(log).When(fsImage).GetEditLog(); FSNamesystem fsn = new FSNamesystem(conf, fsImage); DelegationTokenSecretManager dtsm = fsn.GetDelegationTokenSecretManager(); try { dtsm.StartThreads(); // get two tokens Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token1 = fsn.GetDelegationToken (renewer); Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token2 = fsn.GetDelegationToken (renewer); DelegationTokenIdentifier ident1 = token1.DecodeIdentifier(); DelegationTokenIdentifier ident2 = token2.DecodeIdentifier(); // verify we got the tokens Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(1)).LogGetDelegationToken (Eq(ident1), AnyLong()); Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(1)).LogGetDelegationToken (Eq(ident2), AnyLong()); // this is a little tricky because DTSM doesn't let us set scan interval // so need to periodically sleep, then stop/start threads to force scan // renew first token 1/2 to expire Sharpen.Thread.Sleep(renewInterval / 2); fsn.RenewDelegationToken(token2); Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(1)).LogRenewDelegationToken (Eq(ident2), AnyLong()); // force scan and give it a little time to complete dtsm.StopThreads(); dtsm.StartThreads(); Sharpen.Thread.Sleep(250); // no token has expired yet Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(0)).LogCancelDelegationToken (Eq(ident1)); Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(0)).LogCancelDelegationToken (Eq(ident2)); // sleep past expiration of 1st non-renewed token Sharpen.Thread.Sleep(renewInterval / 2); dtsm.StopThreads(); dtsm.StartThreads(); Sharpen.Thread.Sleep(250); // non-renewed token should have implicitly been cancelled Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(1)).LogCancelDelegationToken (Eq(ident1)); Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(0)).LogCancelDelegationToken (Eq(ident2)); // sleep past expiration of 2nd renewed token Sharpen.Thread.Sleep(renewInterval / 2); dtsm.StopThreads(); dtsm.StartThreads(); Sharpen.Thread.Sleep(250); // both tokens should have been implicitly cancelled by now Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(1)).LogCancelDelegationToken (Eq(ident1)); Org.Mockito.Mockito.Verify(log, Org.Mockito.Mockito.Times(1)).LogCancelDelegationToken (Eq(ident2)); } finally { dtsm.StopThreads(); } }
public virtual void TestGetUgiFromToken() { conf.Set(DFSConfigKeys.FsDefaultNameKey, "hdfs://localhost:4321/"); ServletContext context = Org.Mockito.Mockito.Mock <ServletContext>(); string realUser = "******"; string user = "******"; conf.Set(DFSConfigKeys.HadoopSecurityAuthentication, "kerberos"); UserGroupInformation.SetConfiguration(conf); UserGroupInformation ugi; HttpServletRequest request; Text ownerText = new Text(user); DelegationTokenIdentifier dtId = new DelegationTokenIdentifier(ownerText, ownerText , new Text(realUser)); Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier>(dtId, new TestJspHelper.DummySecretManager(0, 0, 0, 0)); string tokenString = token.EncodeToUrlString(); // token with no auth-ed user request = GetMockRequest(null, null, null); Org.Mockito.Mockito.When(request.GetParameter(JspHelper.DelegationParameterName)) .ThenReturn(tokenString); ugi = JspHelper.GetUGI(context, request, conf); NUnit.Framework.Assert.IsNotNull(ugi.GetRealUser()); NUnit.Framework.Assert.AreEqual(ugi.GetRealUser().GetShortUserName(), realUser); NUnit.Framework.Assert.AreEqual(ugi.GetShortUserName(), user); CheckUgiFromToken(ugi); // token with auth-ed user request = GetMockRequest(realUser, null, null); Org.Mockito.Mockito.When(request.GetParameter(JspHelper.DelegationParameterName)) .ThenReturn(tokenString); ugi = JspHelper.GetUGI(context, request, conf); NUnit.Framework.Assert.IsNotNull(ugi.GetRealUser()); NUnit.Framework.Assert.AreEqual(ugi.GetRealUser().GetShortUserName(), realUser); NUnit.Framework.Assert.AreEqual(ugi.GetShortUserName(), user); CheckUgiFromToken(ugi); // completely different user, token trumps auth request = GetMockRequest("rogue", null, null); Org.Mockito.Mockito.When(request.GetParameter(JspHelper.DelegationParameterName)) .ThenReturn(tokenString); ugi = JspHelper.GetUGI(context, request, conf); NUnit.Framework.Assert.IsNotNull(ugi.GetRealUser()); NUnit.Framework.Assert.AreEqual(ugi.GetRealUser().GetShortUserName(), realUser); NUnit.Framework.Assert.AreEqual(ugi.GetShortUserName(), user); CheckUgiFromToken(ugi); // expected case request = GetMockRequest(null, user, null); Org.Mockito.Mockito.When(request.GetParameter(JspHelper.DelegationParameterName)) .ThenReturn(tokenString); ugi = JspHelper.GetUGI(context, request, conf); NUnit.Framework.Assert.IsNotNull(ugi.GetRealUser()); NUnit.Framework.Assert.AreEqual(ugi.GetRealUser().GetShortUserName(), realUser); NUnit.Framework.Assert.AreEqual(ugi.GetShortUserName(), user); CheckUgiFromToken(ugi); // can't proxy with a token! request = GetMockRequest(null, null, "rogue"); Org.Mockito.Mockito.When(request.GetParameter(JspHelper.DelegationParameterName)) .ThenReturn(tokenString); try { JspHelper.GetUGI(context, request, conf); NUnit.Framework.Assert.Fail("bad request allowed"); } catch (IOException ioe) { NUnit.Framework.Assert.AreEqual("Usernames not matched: name=rogue != expected=" + user, ioe.Message); } // can't proxy with a token! request = GetMockRequest(null, user, "rogue"); Org.Mockito.Mockito.When(request.GetParameter(JspHelper.DelegationParameterName)) .ThenReturn(tokenString); try { JspHelper.GetUGI(context, request, conf); NUnit.Framework.Assert.Fail("bad request allowed"); } catch (IOException ioe) { NUnit.Framework.Assert.AreEqual("Usernames not matched: name=rogue != expected=" + user, ioe.Message); } }