public virtual void TestBlockTokenInLastLocatedBlock() { Configuration conf = new HdfsConfiguration(); conf.SetBoolean(DFSConfigKeys.DfsBlockAccessTokenEnableKey, true); conf.SetInt(DFSConfigKeys.DfsBlockSizeKey, 512); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); cluster.WaitActive(); try { FileSystem fs = cluster.GetFileSystem(); string fileName = "/testBlockTokenInLastLocatedBlock"; Path filePath = new Path(fileName); FSDataOutputStream @out = fs.Create(filePath, (short)1); @out.Write(new byte[1000]); // ensure that the first block is written out (see FSOutputSummer#flush) @out.Flush(); LocatedBlocks locatedBlocks = cluster.GetNameNodeRpc().GetBlockLocations(fileName , 0, 1000); while (locatedBlocks.GetLastLocatedBlock() == null) { Sharpen.Thread.Sleep(100); locatedBlocks = cluster.GetNameNodeRpc().GetBlockLocations(fileName, 0, 1000); } Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token = locatedBlocks .GetLastLocatedBlock().GetBlockToken(); NUnit.Framework.Assert.AreEqual(BlockTokenIdentifier.KindName, token.GetKind()); @out.Close(); } finally { cluster.Shutdown(); } }
/// <exception cref="System.Exception"/> private void TestGetToken(string renewer, Text expectedTokenKind) { DelegationTokenAuthenticator.DelegationTokenOperation op = DelegationTokenAuthenticator.DelegationTokenOperation .Getdelegationtoken; HttpServletRequest request = Org.Mockito.Mockito.Mock <HttpServletRequest>(); HttpServletResponse response = Org.Mockito.Mockito.Mock <HttpServletResponse>(); Org.Mockito.Mockito.When(request.GetQueryString()).ThenReturn(DelegationTokenAuthenticator .OpParam + "=" + op.ToString()); Org.Mockito.Mockito.When(request.GetMethod()).ThenReturn(op.GetHttpMethod()); AuthenticationToken token = Org.Mockito.Mockito.Mock <AuthenticationToken>(); Org.Mockito.Mockito.When(token.GetUserName()).ThenReturn("user"); Org.Mockito.Mockito.When(response.GetWriter()).ThenReturn(new PrintWriter(new StringWriter ())); NUnit.Framework.Assert.IsFalse(handler.ManagementOperation(token, request, response )); Org.Mockito.Mockito.When(request.GetQueryString()).ThenReturn(DelegationTokenAuthenticator .OpParam + "=" + op.ToString() + "&" + DelegationTokenAuthenticator.RenewerParam + "=" + renewer); Org.Mockito.Mockito.Reset(response); Org.Mockito.Mockito.Reset(token); Org.Mockito.Mockito.When(token.GetUserName()).ThenReturn("user"); StringWriter writer = new StringWriter(); PrintWriter pwriter = new PrintWriter(writer); Org.Mockito.Mockito.When(response.GetWriter()).ThenReturn(pwriter); NUnit.Framework.Assert.IsFalse(handler.ManagementOperation(token, request, response )); if (renewer == null) { Org.Mockito.Mockito.Verify(token).GetUserName(); } else { Org.Mockito.Mockito.Verify(token).GetUserName(); } Org.Mockito.Mockito.Verify(response).SetStatus(HttpServletResponse.ScOk); Org.Mockito.Mockito.Verify(response).SetContentType(MediaType.ApplicationJson); pwriter.Close(); string responseOutput = writer.ToString(); string tokenLabel = DelegationTokenAuthenticator.DelegationTokenJson; Assert.True(responseOutput.Contains(tokenLabel)); Assert.True(responseOutput.Contains(DelegationTokenAuthenticator .DelegationTokenUrlStringJson)); ObjectMapper jsonMapper = new ObjectMapper(); IDictionary json = jsonMapper.ReadValue <IDictionary>(responseOutput); json = (IDictionary)json[tokenLabel]; string tokenStr; tokenStr = (string)json[DelegationTokenAuthenticator.DelegationTokenUrlStringJson ]; Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> dt = new Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier>(); dt.DecodeFromUrlString(tokenStr); handler.GetTokenManager().VerifyToken(dt); Assert.Equal(expectedTokenKind, dt.GetKind()); }
private static Org.Apache.Hadoop.Yarn.Api.Records.Token ConvertToProtoToken <_T0>( Org.Apache.Hadoop.Security.Token.Token <_T0> token) where _T0 : TokenIdentifier { return(Org.Apache.Hadoop.Yarn.Api.Records.Token.NewInstance(token.GetIdentifier() , token.GetKind().ToString(), token.GetPassword(), token.GetService().ToString() )); }
/// <exception cref="System.IO.IOException"/> private TokenAspect.TokenManagementDelegator GetInstance <_T0>(Org.Apache.Hadoop.Security.Token.Token <_T0> token, Configuration conf) where _T0 : TokenIdentifier { URI uri; string scheme = GetSchemeByKind(token.GetKind()); if (HAUtil.IsTokenForLogicalUri(token)) { uri = HAUtil.GetServiceUriFromToken(scheme, token); } else { IPEndPoint address = SecurityUtil.GetTokenServiceAddr(token); uri = URI.Create(scheme + "://" + NetUtils.GetHostPortString(address)); } return((TokenAspect.TokenManagementDelegator)FileSystem.Get(uri, conf)); }
public virtual void TestTokenCompatibilityFor203() { Configuration conf = new Configuration(); HftpFileSystem fs = new HftpFileSystem(); Org.Apache.Hadoop.Security.Token.Token <object> token = new Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier>(new byte[0], new byte[0], DelegationTokenIdentifier.HdfsDelegationKind , new Text("127.0.0.1:8020")); Credentials cred = new Credentials(); cred.AddToken(HftpFileSystem.TokenKind, token); ByteArrayOutputStream os = new ByteArrayOutputStream(); cred.Write(new DataOutputStream(os)); HttpURLConnection conn = Org.Mockito.Mockito.Mock <HttpURLConnection>(); Org.Mockito.Mockito.DoReturn(new ByteArrayInputStream(os.ToByteArray())).When(conn ).GetInputStream(); Org.Mockito.Mockito.DoReturn(HttpURLConnection.HttpOk).When(conn).GetResponseCode (); URLConnectionFactory factory = Org.Mockito.Mockito.Mock <URLConnectionFactory>(); Org.Mockito.Mockito.DoReturn(conn).When(factory).OpenConnection(Org.Mockito.Mockito .Any <Uri>(), Matchers.AnyBoolean()); URI uri = new URI("hftp://127.0.0.1:8020"); fs.Initialize(uri, conf); fs.connectionFactory = factory; UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting("foo", new string [] { "bar" }); TokenAspect <HftpFileSystem> tokenAspect = new TokenAspect <HftpFileSystem>(fs, SecurityUtil .BuildTokenService(uri), HftpFileSystem.TokenKind); tokenAspect.InitDelegationToken(ugi); tokenAspect.EnsureTokenInitialized(); NUnit.Framework.Assert.AreSame(HftpFileSystem.TokenKind, fs.GetRenewToken().GetKind ()); Org.Apache.Hadoop.Security.Token.Token <object> tok = (Org.Apache.Hadoop.Security.Token.Token <object>)Whitebox.GetInternalState(fs, "delegationToken"); NUnit.Framework.Assert.AreNotSame("Not making a copy of the remote token", token, tok); NUnit.Framework.Assert.AreEqual(token.GetKind(), tok.GetKind()); }
public virtual void TestSetTokenServiceAndKind() { MiniDFSCluster cluster = null; try { Configuration clusterConf = new HdfsConfiguration(conf); SecurityUtil.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Simple , clusterConf); clusterConf.SetBoolean(DFSConfigKeys.DfsNamenodeDelegationTokenAlwaysUseKey, true ); // trick the NN into thinking s[ecurity is enabled w/o it trying // to login from a keytab UserGroupInformation.SetConfiguration(clusterConf); cluster = new MiniDFSCluster.Builder(clusterConf).NumDataNodes(0).Build(); cluster.WaitActive(); SecurityUtil.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Kerberos , clusterConf); WebHdfsFileSystem fs = WebHdfsTestUtil.GetWebHdfsFileSystem(clusterConf, "webhdfs" ); Whitebox.SetInternalState(fs, "canRefreshDelegationToken", true); URLConnectionFactory factory = new _URLConnectionFactory_268(new _ConnectionConfigurator_262 ()); Whitebox.SetInternalState(fs, "connectionFactory", factory); Org.Apache.Hadoop.Security.Token.Token <object> token1 = fs.GetDelegationToken(); NUnit.Framework.Assert.AreEqual(new Text("bar"), token1.GetKind()); HttpOpParam.OP op = GetOpParam.OP.Getdelegationtoken; Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token2 = new _FsPathResponseRunner_281 (op, null, new RenewerParam(null)).Run(); NUnit.Framework.Assert.AreEqual(new Text("bar"), token2.GetKind()); NUnit.Framework.Assert.AreEqual(new Text("foo"), token2.GetService()); } finally { if (cluster != null) { cluster.Shutdown(); } } }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> public virtual AllocateResponse Allocate(AllocateRequest request) { NUnit.Framework.Assert.AreEqual("response ID mismatch", responseId, request.GetResponseId ()); ++responseId; Org.Apache.Hadoop.Yarn.Api.Records.Token yarnToken = null; if (amToken != null) { yarnToken = Org.Apache.Hadoop.Yarn.Api.Records.Token.NewInstance(amToken.GetIdentifier (), amToken.GetKind().ToString(), amToken.GetPassword(), amToken.GetService().ToString ()); } return(AllocateResponse.NewInstance(responseId, Collections.EmptyList <ContainerStatus >(), Collections.EmptyList <Container>(), Collections.EmptyList <NodeReport>(), Resources .None(), null, 1, null, Collections.EmptyList <NMToken>(), yarnToken, Collections .EmptyList <ContainerResourceIncrease>(), Collections.EmptyList <ContainerResourceDecrease >())); }
private void VerifyTamperedToken(Configuration conf, TestClientToAMTokens.CustomAM am, Org.Apache.Hadoop.Security.Token.Token <ClientToAMTokenIdentifier> token, UserGroupInformation ugi, ClientToAMTokenIdentifier maliciousID) { Org.Apache.Hadoop.Security.Token.Token <ClientToAMTokenIdentifier> maliciousToken = new Org.Apache.Hadoop.Security.Token.Token <ClientToAMTokenIdentifier>(maliciousID .GetBytes(), token.GetPassword(), token.GetKind(), token.GetService()); ugi.AddToken(maliciousToken); try { ugi.DoAs(new _PrivilegedExceptionAction_338(am, conf)); } catch (Exception e) { NUnit.Framework.Assert.AreEqual(typeof(RemoteException).FullName, e.GetType().FullName ); e = ((RemoteException)e).UnwrapRemoteException(); NUnit.Framework.Assert.AreEqual(typeof(SaslException).GetCanonicalName(), e.GetType ().GetCanonicalName()); NUnit.Framework.Assert.IsTrue(e.Message.Contains("DIGEST-MD5: digest response format violation. " + "Mismatched response.")); NUnit.Framework.Assert.IsFalse(am.pinged); } }
/// <exception cref="System.Exception"/> private void ValidateLazyTokenFetch(Configuration clusterConf) { string testUser = "******"; UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting(testUser, new string[] { "supergroup" }); WebHdfsFileSystem fs = ugi.DoAs(new _PrivilegedExceptionAction_304(this, clusterConf )); // verify token ops don't get a token NUnit.Framework.Assert.IsNull(fs.GetRenewToken()); Org.Apache.Hadoop.Security.Token.Token <object> token = ((Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier>)fs.GetDelegationToken(null)); fs.RenewDelegationToken(token); fs.CancelDelegationToken(token); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).GetDelegationToken(); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).ReplaceExpiredDelegationToken (); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).SetDelegationToken(Matchers.Any <Org.Apache.Hadoop.Security.Token.Token>()); NUnit.Framework.Assert.IsNull(fs.GetRenewToken()); Org.Mockito.Mockito.Reset(fs); // verify first non-token op gets a token Path p = new Path("/f"); fs.Create(p, (short)1).Close(); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Times(1)).GetDelegationToken(); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).ReplaceExpiredDelegationToken (); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Times(1)).GetDelegationToken(Matchers.AnyString ()); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Times(1)).SetDelegationToken(Matchers.Any <Org.Apache.Hadoop.Security.Token.Token>()); token = fs.GetRenewToken(); NUnit.Framework.Assert.IsNotNull(token); NUnit.Framework.Assert.AreEqual(testUser, GetTokenOwner(token)); NUnit.Framework.Assert.AreEqual(fs.GetTokenKind(), token.GetKind()); Org.Mockito.Mockito.Reset(fs); // verify prior token is reused fs.GetFileStatus(p); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Times(1)).GetDelegationToken(); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).ReplaceExpiredDelegationToken (); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).GetDelegationToken(Matchers.AnyString ()); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).SetDelegationToken(Matchers.Any <Org.Apache.Hadoop.Security.Token.Token>()); Org.Apache.Hadoop.Security.Token.Token <object> token2 = fs.GetRenewToken(); NUnit.Framework.Assert.IsNotNull(token2); NUnit.Framework.Assert.AreEqual(fs.GetTokenKind(), token.GetKind()); NUnit.Framework.Assert.AreSame(token, token2); Org.Mockito.Mockito.Reset(fs); // verify renew of expired token fails w/o getting a new token token = fs.GetRenewToken(); fs.CancelDelegationToken(token); try { fs.RenewDelegationToken(token); NUnit.Framework.Assert.Fail("should have failed"); } catch (SecretManager.InvalidToken) { } catch (Exception ex) { NUnit.Framework.Assert.Fail("wrong exception:" + ex); } Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).GetDelegationToken(); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).ReplaceExpiredDelegationToken (); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).GetDelegationToken(Matchers.AnyString ()); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).SetDelegationToken(Matchers.Any <Org.Apache.Hadoop.Security.Token.Token>()); token2 = fs.GetRenewToken(); NUnit.Framework.Assert.IsNotNull(token2); NUnit.Framework.Assert.AreEqual(fs.GetTokenKind(), token.GetKind()); NUnit.Framework.Assert.AreSame(token, token2); Org.Mockito.Mockito.Reset(fs); // verify cancel of expired token fails w/o getting a new token try { fs.CancelDelegationToken(token); NUnit.Framework.Assert.Fail("should have failed"); } catch (SecretManager.InvalidToken) { } catch (Exception ex) { NUnit.Framework.Assert.Fail("wrong exception:" + ex); } Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).GetDelegationToken(); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).ReplaceExpiredDelegationToken (); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).GetDelegationToken(Matchers.AnyString ()); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).SetDelegationToken(Matchers.Any <Org.Apache.Hadoop.Security.Token.Token>()); token2 = fs.GetRenewToken(); NUnit.Framework.Assert.IsNotNull(token2); NUnit.Framework.Assert.AreEqual(fs.GetTokenKind(), token.GetKind()); NUnit.Framework.Assert.AreSame(token, token2); Org.Mockito.Mockito.Reset(fs); // verify an expired token is replaced with a new token fs.Open(p).Close(); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Times(2)).GetDelegationToken(); // first bad, then good Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Times(1)).ReplaceExpiredDelegationToken (); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Times(1)).GetDelegationToken(null ); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Times(1)).SetDelegationToken(Matchers.Any <Org.Apache.Hadoop.Security.Token.Token>()); token2 = fs.GetRenewToken(); NUnit.Framework.Assert.IsNotNull(token2); NUnit.Framework.Assert.AreEqual(fs.GetTokenKind(), token.GetKind()); NUnit.Framework.Assert.AreNotSame(token, token2); NUnit.Framework.Assert.AreEqual(testUser, GetTokenOwner(token2)); Org.Mockito.Mockito.Reset(fs); // verify with open because it's a little different in how it // opens connections fs.CancelDelegationToken(fs.GetRenewToken()); InputStream @is = fs.Open(p); @is.Read(); @is.Close(); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Times(2)).GetDelegationToken(); // first bad, then good Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Times(1)).ReplaceExpiredDelegationToken (); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Times(1)).GetDelegationToken(null ); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Times(1)).SetDelegationToken(Matchers.Any <Org.Apache.Hadoop.Security.Token.Token>()); token2 = fs.GetRenewToken(); NUnit.Framework.Assert.IsNotNull(token2); NUnit.Framework.Assert.AreEqual(fs.GetTokenKind(), token.GetKind()); NUnit.Framework.Assert.AreNotSame(token, token2); NUnit.Framework.Assert.AreEqual(testUser, GetTokenOwner(token2)); Org.Mockito.Mockito.Reset(fs); // verify fs close cancels the token fs.Close(); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).GetDelegationToken(); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).ReplaceExpiredDelegationToken (); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).GetDelegationToken(Matchers.AnyString ()); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).SetDelegationToken(Matchers.Any <Org.Apache.Hadoop.Security.Token.Token>()); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Times(1)).CancelDelegationToken (Matchers.Eq(token2)); // add a token to ugi for a new fs, verify it uses that token token = ((Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier>)fs.GetDelegationToken (null)); ugi.AddToken(token); fs = ugi.DoAs(new _PrivilegedExceptionAction_426(this, clusterConf)); NUnit.Framework.Assert.IsNull(fs.GetRenewToken()); fs.GetFileStatus(new Path("/")); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Times(1)).GetDelegationToken(); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).ReplaceExpiredDelegationToken (); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).GetDelegationToken(Matchers.AnyString ()); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Times(1)).SetDelegationToken(Matchers.Eq (token)); token2 = fs.GetRenewToken(); NUnit.Framework.Assert.IsNotNull(token2); NUnit.Framework.Assert.AreEqual(fs.GetTokenKind(), token.GetKind()); NUnit.Framework.Assert.AreSame(token, token2); Org.Mockito.Mockito.Reset(fs); // verify it reuses the prior ugi token fs.GetFileStatus(new Path("/")); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Times(1)).GetDelegationToken(); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).ReplaceExpiredDelegationToken (); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).GetDelegationToken(Matchers.AnyString ()); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).SetDelegationToken(Matchers.Any <Org.Apache.Hadoop.Security.Token.Token>()); token2 = fs.GetRenewToken(); NUnit.Framework.Assert.IsNotNull(token2); NUnit.Framework.Assert.AreEqual(fs.GetTokenKind(), token.GetKind()); NUnit.Framework.Assert.AreSame(token, token2); Org.Mockito.Mockito.Reset(fs); // verify an expired ugi token is NOT replaced with a new token fs.CancelDelegationToken(token); for (int i = 0; i < 2; i++) { try { fs.GetFileStatus(new Path("/")); NUnit.Framework.Assert.Fail("didn't fail"); } catch (SecretManager.InvalidToken) { } catch (Exception ex) { NUnit.Framework.Assert.Fail("wrong exception:" + ex); } Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Times(1)).GetDelegationToken(); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Times(1)).ReplaceExpiredDelegationToken (); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).GetDelegationToken(Matchers.AnyString ()); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).SetDelegationToken(Matchers.Any <Org.Apache.Hadoop.Security.Token.Token>()); token2 = fs.GetRenewToken(); NUnit.Framework.Assert.IsNotNull(token2); NUnit.Framework.Assert.AreEqual(fs.GetTokenKind(), token.GetKind()); NUnit.Framework.Assert.AreSame(token, token2); Org.Mockito.Mockito.Reset(fs); } // verify fs close does NOT cancel the ugi token fs.Close(); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).GetDelegationToken(); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).ReplaceExpiredDelegationToken (); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).GetDelegationToken(Matchers.AnyString ()); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).SetDelegationToken(Matchers.Any <Org.Apache.Hadoop.Security.Token.Token>()); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).CancelDelegationToken (Matchers.Any <Org.Apache.Hadoop.Security.Token.Token>()); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public override void Cancel <_T0>(Org.Apache.Hadoop.Security.Token.Token <_T0> token , Configuration conf) { Org.Apache.Hadoop.Yarn.Api.Records.Token dToken = Org.Apache.Hadoop.Yarn.Api.Records.Token .NewInstance(token.GetIdentifier(), token.GetKind().ToString(), token.GetPassword (), token.GetService().ToString()); MRClientProtocol histProxy = InstantiateHistoryProxy(conf, SecurityUtil.GetTokenServiceAddr (token)); try { CancelDelegationTokenRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <CancelDelegationTokenRequest>(); request.SetDelegationToken(dToken); histProxy.CancelDelegationToken(request); } finally { StopHistoryProxy(histProxy); } }
public virtual void TestDelegationToken() { YarnConfiguration conf = new YarnConfiguration(); conf.Set(YarnConfiguration.RmPrincipal, "testuser/[email protected]"); conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos"); UserGroupInformation.SetConfiguration(conf); ResourceScheduler scheduler = CreateMockScheduler(conf); long initialInterval = 10000l; long maxLifetime = 20000l; long renewInterval = 10000l; RMDelegationTokenSecretManager rmDtSecretManager = CreateRMDelegationTokenSecretManager (initialInterval, maxLifetime, renewInterval); rmDtSecretManager.StartThreads(); Log.Info("Creating DelegationTokenSecretManager with initialInterval: " + initialInterval + ", maxLifetime: " + maxLifetime + ", renewInterval: " + renewInterval); ClientRMService clientRMService = new TestClientRMTokens.ClientRMServiceForTest(this , conf, scheduler, rmDtSecretManager); clientRMService.Init(conf); clientRMService.Start(); ApplicationClientProtocol clientRMWithDT = null; try { // Create a user for the renewr and fake the authentication-method UserGroupInformation loggedInUser = UserGroupInformation.CreateRemoteUser("*****@*****.**" ); NUnit.Framework.Assert.AreEqual("testrenewer", loggedInUser.GetShortUserName()); // Default realm is APACHE.ORG loggedInUser.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Kerberos ); Token token = GetDelegationToken(loggedInUser, clientRMService, loggedInUser.GetShortUserName ()); long tokenFetchTime = Runtime.CurrentTimeMillis(); Log.Info("Got delegation token at: " + tokenFetchTime); // Now try talking to RMService using the delegation token clientRMWithDT = GetClientRMProtocolWithDT(token, clientRMService.GetBindAddress( ), "loginuser1", conf); GetNewApplicationRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord < GetNewApplicationRequest>(); try { clientRMWithDT.GetNewApplication(request); } catch (IOException e) { NUnit.Framework.Assert.Fail("Unexpected exception" + e); } catch (YarnException e) { NUnit.Framework.Assert.Fail("Unexpected exception" + e); } // Renew after 50% of token age. while (Runtime.CurrentTimeMillis() < tokenFetchTime + initialInterval / 2) { Sharpen.Thread.Sleep(500l); } long nextExpTime = RenewDelegationToken(loggedInUser, clientRMService, token); long renewalTime = Runtime.CurrentTimeMillis(); Log.Info("Renewed token at: " + renewalTime + ", NextExpiryTime: " + nextExpTime); // Wait for first expiry, but before renewed expiry. while (Runtime.CurrentTimeMillis() > tokenFetchTime + initialInterval && Runtime. CurrentTimeMillis() < nextExpTime) { Sharpen.Thread.Sleep(500l); } Sharpen.Thread.Sleep(50l); // Valid token because of renewal. try { clientRMWithDT.GetNewApplication(request); } catch (IOException e) { NUnit.Framework.Assert.Fail("Unexpected exception" + e); } catch (YarnException e) { NUnit.Framework.Assert.Fail("Unexpected exception" + e); } // Wait for expiry. while (Runtime.CurrentTimeMillis() < renewalTime + renewInterval) { Sharpen.Thread.Sleep(500l); } Sharpen.Thread.Sleep(50l); Log.Info("At time: " + Runtime.CurrentTimeMillis() + ", token should be invalid"); // Token should have expired. try { clientRMWithDT.GetNewApplication(request); NUnit.Framework.Assert.Fail("Should not have succeeded with an expired token"); } catch (Exception e) { NUnit.Framework.Assert.AreEqual(typeof(SecretManager.InvalidToken).FullName, e.GetType ().FullName); NUnit.Framework.Assert.IsTrue(e.Message.Contains("is expired")); } // Test cancellation // Stop the existing proxy, start another. if (clientRMWithDT != null) { RPC.StopProxy(clientRMWithDT); clientRMWithDT = null; } token = GetDelegationToken(loggedInUser, clientRMService, loggedInUser.GetShortUserName ()); tokenFetchTime = Runtime.CurrentTimeMillis(); Log.Info("Got delegation token at: " + tokenFetchTime); // Now try talking to RMService using the delegation token clientRMWithDT = GetClientRMProtocolWithDT(token, clientRMService.GetBindAddress( ), "loginuser2", conf); request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <GetNewApplicationRequest> (); try { clientRMWithDT.GetNewApplication(request); } catch (IOException e) { NUnit.Framework.Assert.Fail("Unexpected exception" + e); } catch (YarnException e) { NUnit.Framework.Assert.Fail("Unexpected exception" + e); } CancelDelegationToken(loggedInUser, clientRMService, token); if (clientRMWithDT != null) { RPC.StopProxy(clientRMWithDT); clientRMWithDT = null; } // Creating a new connection. clientRMWithDT = GetClientRMProtocolWithDT(token, clientRMService.GetBindAddress( ), "loginuser2", conf); Log.Info("Cancelled delegation token at: " + Runtime.CurrentTimeMillis()); // Verify cancellation worked. try { clientRMWithDT.GetNewApplication(request); NUnit.Framework.Assert.Fail("Should not have succeeded with a cancelled delegation token" ); } catch (IOException) { } catch (YarnException) { } // Test new version token // Stop the existing proxy, start another. if (clientRMWithDT != null) { RPC.StopProxy(clientRMWithDT); clientRMWithDT = null; } token = GetDelegationToken(loggedInUser, clientRMService, loggedInUser.GetShortUserName ()); byte[] tokenIdentifierContent = ((byte[])token.GetIdentifier().Array()); RMDelegationTokenIdentifier tokenIdentifier = new RMDelegationTokenIdentifier(); DataInputBuffer dib = new DataInputBuffer(); dib.Reset(tokenIdentifierContent, tokenIdentifierContent.Length); tokenIdentifier.ReadFields(dib); // Construct new version RMDelegationTokenIdentifier with additional field RMDelegationTokenIdentifierForTest newVersionTokenIdentifier = new RMDelegationTokenIdentifierForTest (tokenIdentifier, "message"); Org.Apache.Hadoop.Security.Token.Token <RMDelegationTokenIdentifier> newRMDTtoken = new Org.Apache.Hadoop.Security.Token.Token <RMDelegationTokenIdentifier>(newVersionTokenIdentifier , rmDtSecretManager); Org.Apache.Hadoop.Yarn.Api.Records.Token newToken = BuilderUtils.NewDelegationToken (newRMDTtoken.GetIdentifier(), newRMDTtoken.GetKind().ToString(), newRMDTtoken.GetPassword (), newRMDTtoken.GetService().ToString()); // Now try talking to RMService using the new version delegation token clientRMWithDT = GetClientRMProtocolWithDT(newToken, clientRMService.GetBindAddress (), "loginuser3", conf); request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <GetNewApplicationRequest> (); try { clientRMWithDT.GetNewApplication(request); } catch (IOException e) { NUnit.Framework.Assert.Fail("Unexpected exception" + e); } catch (YarnException e) { NUnit.Framework.Assert.Fail("Unexpected exception" + e); } } finally { rmDtSecretManager.StopThreads(); // TODO PRECOMMIT Close proxies. if (clientRMWithDT != null) { RPC.StopProxy(clientRMWithDT); } } }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> public virtual AllocateResponse Allocate(AllocateRequest request) { AMRMTokenIdentifier amrmTokenIdentifier = AuthorizeRequest(); ApplicationAttemptId appAttemptId = amrmTokenIdentifier.GetApplicationAttemptId(); ApplicationId applicationId = appAttemptId.GetApplicationId(); this.amLivelinessMonitor.ReceivedPing(appAttemptId); /* check if its in cache */ ApplicationMasterService.AllocateResponseLock Lock = responseMap[appAttemptId]; if (Lock == null) { string message = "Application attempt " + appAttemptId + " doesn't exist in ApplicationMasterService cache."; Log.Error(message); throw new ApplicationAttemptNotFoundException(message); } lock (Lock) { AllocateResponse lastResponse = Lock.GetAllocateResponse(); if (!HasApplicationMasterRegistered(appAttemptId)) { string message = "AM is not registered for known application attempt: " + appAttemptId + " or RM had restarted after AM registered . AM should re-register."; Log.Info(message); RMAuditLogger.LogFailure(this.rmContext.GetRMApps()[appAttemptId.GetApplicationId ()].GetUser(), RMAuditLogger.AuditConstants.AmAllocate, string.Empty, "ApplicationMasterService" , message, applicationId, appAttemptId); throw new ApplicationMasterNotRegisteredException(message); } if ((request.GetResponseId() + 1) == lastResponse.GetResponseId()) { /* old heartbeat */ return(lastResponse); } else { if (request.GetResponseId() + 1 < lastResponse.GetResponseId()) { string message = "Invalid responseId in AllocateRequest from application attempt: " + appAttemptId + ", expect responseId to be " + (lastResponse.GetResponseId() + 1); throw new InvalidApplicationMasterRequestException(message); } } //filter illegal progress values float filteredProgress = request.GetProgress(); if (float.IsNaN(filteredProgress) || filteredProgress == float.NegativeInfinity || filteredProgress < 0) { request.SetProgress(0); } else { if (filteredProgress > 1 || filteredProgress == float.PositiveInfinity) { request.SetProgress(1); } } // Send the status update to the appAttempt. this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMAppAttemptStatusupdateEvent (appAttemptId, request.GetProgress())); IList <ResourceRequest> ask = request.GetAskList(); IList <ContainerId> release = request.GetReleaseList(); ResourceBlacklistRequest blacklistRequest = request.GetResourceBlacklistRequest(); IList <string> blacklistAdditions = (blacklistRequest != null) ? blacklistRequest. GetBlacklistAdditions() : Sharpen.Collections.EmptyList; IList <string> blacklistRemovals = (blacklistRequest != null) ? blacklistRequest.GetBlacklistRemovals () : Sharpen.Collections.EmptyList; RMApp app = this.rmContext.GetRMApps()[applicationId]; // set label expression for Resource Requests if resourceName=ANY ApplicationSubmissionContext asc = app.GetApplicationSubmissionContext(); foreach (ResourceRequest req in ask) { if (null == req.GetNodeLabelExpression() && ResourceRequest.Any.Equals(req.GetResourceName ())) { req.SetNodeLabelExpression(asc.GetNodeLabelExpression()); } } // sanity check try { RMServerUtils.NormalizeAndValidateRequests(ask, rScheduler.GetMaximumResourceCapability (), app.GetQueue(), rScheduler, rmContext); } catch (InvalidResourceRequestException e) { Log.Warn("Invalid resource ask by application " + appAttemptId, e); throw; } try { RMServerUtils.ValidateBlacklistRequest(blacklistRequest); } catch (InvalidResourceBlacklistRequestException e) { Log.Warn("Invalid blacklist request by application " + appAttemptId, e); throw; } // In the case of work-preserving AM restart, it's possible for the // AM to release containers from the earlier attempt. if (!app.GetApplicationSubmissionContext().GetKeepContainersAcrossApplicationAttempts ()) { try { RMServerUtils.ValidateContainerReleaseRequest(release, appAttemptId); } catch (InvalidContainerReleaseException e) { Log.Warn("Invalid container release by application " + appAttemptId, e); throw; } } // Send new requests to appAttempt. Allocation allocation = this.rScheduler.Allocate(appAttemptId, ask, release, blacklistAdditions , blacklistRemovals); if (!blacklistAdditions.IsEmpty() || !blacklistRemovals.IsEmpty()) { Log.Info("blacklist are updated in Scheduler." + "blacklistAdditions: " + blacklistAdditions + ", " + "blacklistRemovals: " + blacklistRemovals); } RMAppAttempt appAttempt = app.GetRMAppAttempt(appAttemptId); AllocateResponse allocateResponse = recordFactory.NewRecordInstance <AllocateResponse >(); if (!allocation.GetContainers().IsEmpty()) { allocateResponse.SetNMTokens(allocation.GetNMTokens()); } // update the response with the deltas of node status changes IList <RMNode> updatedNodes = new AList <RMNode>(); if (app.PullRMNodeUpdates(updatedNodes) > 0) { IList <NodeReport> updatedNodeReports = new AList <NodeReport>(); foreach (RMNode rmNode in updatedNodes) { SchedulerNodeReport schedulerNodeReport = rScheduler.GetNodeReport(rmNode.GetNodeID ()); Resource used = BuilderUtils.NewResource(0, 0); int numContainers = 0; if (schedulerNodeReport != null) { used = schedulerNodeReport.GetUsedResource(); numContainers = schedulerNodeReport.GetNumContainers(); } NodeId nodeId = rmNode.GetNodeID(); NodeReport report = BuilderUtils.NewNodeReport(nodeId, rmNode.GetState(), rmNode. GetHttpAddress(), rmNode.GetRackName(), used, rmNode.GetTotalCapability(), numContainers , rmNode.GetHealthReport(), rmNode.GetLastHealthReportTime(), rmNode.GetNodeLabels ()); updatedNodeReports.AddItem(report); } allocateResponse.SetUpdatedNodes(updatedNodeReports); } allocateResponse.SetAllocatedContainers(allocation.GetContainers()); allocateResponse.SetCompletedContainersStatuses(appAttempt.PullJustFinishedContainers ()); allocateResponse.SetResponseId(lastResponse.GetResponseId() + 1); allocateResponse.SetAvailableResources(allocation.GetResourceLimit()); allocateResponse.SetNumClusterNodes(this.rScheduler.GetNumClusterNodes()); // add preemption to the allocateResponse message (if any) allocateResponse.SetPreemptionMessage(GeneratePreemptionMessage(allocation)); // update AMRMToken if the token is rolled-up MasterKeyData nextMasterKey = this.rmContext.GetAMRMTokenSecretManager().GetNextMasterKeyData (); if (nextMasterKey != null && nextMasterKey.GetMasterKey().GetKeyId() != amrmTokenIdentifier .GetKeyId()) { RMAppAttemptImpl appAttemptImpl = (RMAppAttemptImpl)appAttempt; Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> amrmToken = appAttempt .GetAMRMToken(); if (nextMasterKey.GetMasterKey().GetKeyId() != appAttemptImpl.GetAMRMTokenKeyId()) { Log.Info("The AMRMToken has been rolled-over. Send new AMRMToken back" + " to application: " + applicationId); amrmToken = rmContext.GetAMRMTokenSecretManager().CreateAndGetAMRMToken(appAttemptId ); appAttemptImpl.SetAMRMToken(amrmToken); } allocateResponse.SetAMRMToken(Org.Apache.Hadoop.Yarn.Api.Records.Token.NewInstance (amrmToken.GetIdentifier(), amrmToken.GetKind().ToString(), amrmToken.GetPassword (), amrmToken.GetService().ToString())); } /* * As we are updating the response inside the lock object so we don't * need to worry about unregister call occurring in between (which * removes the lock object). */ Lock.SetAllocateResponse(allocateResponse); return(allocateResponse); } }