/// <exception cref="System.Exception"/> public virtual void TestGetCreds <T>() where T : TokenIdentifier { // from Mockito mocks UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser("someone"); Text service = new Text("service"); Org.Apache.Hadoop.Security.Token.Token <T> t1 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token >(); Org.Mockito.Mockito.When(t1.GetService()).ThenReturn(service); Org.Apache.Hadoop.Security.Token.Token <T> t2 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token >(); Org.Mockito.Mockito.When(t2.GetService()).ThenReturn(new Text("service2")); Org.Apache.Hadoop.Security.Token.Token <T> t3 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token >(); Org.Mockito.Mockito.When(t3.GetService()).ThenReturn(service); // add token to ugi ugi.AddToken(t1); ugi.AddToken(t2); CheckTokens(ugi, t1, t2); Credentials creds = ugi.GetCredentials(); creds.AddToken(t3.GetService(), t3); NUnit.Framework.Assert.AreSame(t3, creds.GetToken(service)); // check that ugi wasn't modified CheckTokens(ugi, t1, t2); }
/// <exception cref="System.Exception"/> public virtual void TestAddToken <T>() where T : TokenIdentifier { // from Mockito mocks UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser("someone"); Org.Apache.Hadoop.Security.Token.Token <T> t1 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token >(); Org.Apache.Hadoop.Security.Token.Token <T> t2 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token >(); Org.Apache.Hadoop.Security.Token.Token <T> t3 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token >(); // add token to ugi ugi.AddToken(t1); CheckTokens(ugi, t1); // replace token t1 with t2 - with same key (null) ugi.AddToken(t2); CheckTokens(ugi, t2); // change t1 service and add token Org.Mockito.Mockito.When(t1.GetService()).ThenReturn(new Text("t1")); ugi.AddToken(t1); CheckTokens(ugi, t1, t2); // overwrite t1 token with t3 - same key (!null) Org.Mockito.Mockito.When(t3.GetService()).ThenReturn(new Text("t1")); ugi.AddToken(t3); CheckTokens(ugi, t2, t3); // just try to re-add with new name Org.Mockito.Mockito.When(t1.GetService()).ThenReturn(new Text("t1.1")); ugi.AddToken(t1); CheckTokens(ugi, t1, t2, t3); // just try to re-add with new name again ugi.AddToken(t1); CheckTokens(ugi, t1, t2, t3); }
/// <exception cref="System.Exception"/> public virtual void TestAddCreds <T>() where T : TokenIdentifier { // from Mockito mocks UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser("someone"); Text service = new Text("service"); Org.Apache.Hadoop.Security.Token.Token <T> t1 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token >(); Org.Mockito.Mockito.When(t1.GetService()).ThenReturn(service); Org.Apache.Hadoop.Security.Token.Token <T> t2 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token >(); Org.Mockito.Mockito.When(t2.GetService()).ThenReturn(new Text("service2")); byte[] secret = new byte[] { }; Text secretKey = new Text("sshhh"); // fill credentials Credentials creds = new Credentials(); creds.AddToken(t1.GetService(), t1); creds.AddToken(t2.GetService(), t2); creds.AddSecretKey(secretKey, secret); // add creds to ugi, and check ugi ugi.AddCredentials(creds); CheckTokens(ugi, t1, t2); NUnit.Framework.Assert.AreSame(secret, ugi.GetCredentials().GetSecretKey(secretKey )); }
private bool CheckService <_T0>(Text service, Org.Apache.Hadoop.Security.Token.Token <_T0> token) where _T0 : TokenIdentifier { if (service == null || token.GetService() == null) { return(false); } return(token.GetService().ToString().Contains(service.ToString())); }
/// <exception cref="System.Exception"/> public Void Run() { // make sure it is not the same as the login user because we use the // same UGI object for every instantiation of the login user and you // won't run into the race condition otherwise Assert.AssertNotEquals(UserGroupInformation.GetLoginUser(), UserGroupInformation. GetCurrentUser()); TestUserGroupInformation.GetTokenThread thread = new TestUserGroupInformation.GetTokenThread (); try { thread.Start(); for (int i = 0; i < 100; i++) { Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> t = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token>(); Org.Mockito.Mockito.When(t.GetService()).ThenReturn(new Text("t" + i)); UserGroupInformation.GetCurrentUser().AddToken(t); NUnit.Framework.Assert.IsNull("ConcurrentModificationException encountered", thread .cme); } } catch (ConcurrentModificationException cme) { Runtime.PrintStackTrace(cme); NUnit.Framework.Assert.Fail("ConcurrentModificationException encountered"); } finally { thread.runThread = false; thread.Join(5 * 1000); } return(null); }
public virtual void TestGetTokensForNamenodes() { Path TestRootDir = new Path(Runtime.GetProperty("test.build.data", "test/build/data" )); // ick, but need fq path minus file:/ string binaryTokenFile = FileSystem.GetLocal(conf).MakeQualified(new Path(TestRootDir , "tokenFile")).ToUri().GetPath(); FileSystemTestHelper.MockFileSystem fs1 = CreateFileSystemForServiceName("service1" ); Credentials creds = new Credentials(); Org.Apache.Hadoop.Security.Token.Token <object> token1 = fs1.GetDelegationToken(renewer ); creds.AddToken(token1.GetService(), token1); // wait to set, else the obtain tokens call above will fail with FNF conf.Set(MRJobConfig.MapreduceJobCredentialsBinary, binaryTokenFile); creds.WriteTokenStorageFile(new Path(binaryTokenFile), conf); TokenCache.ObtainTokensForNamenodesInternal(fs1, creds, conf); string fs_addr = fs1.GetCanonicalServiceName(); Org.Apache.Hadoop.Security.Token.Token <object> nnt = TokenCache.GetDelegationToken (creds, fs_addr); NUnit.Framework.Assert.IsNotNull("Token for nn is null", nnt); }
/// <exception cref="System.IO.IOException"/> private void SetupTokens(ContainerLaunchContext container, ContainerId containerID ) { IDictionary <string, string> environment = container.GetEnvironment(); environment[ApplicationConstants.ApplicationWebProxyBaseEnv] = application.GetWebProxyBase (); // Set AppSubmitTime and MaxAppAttempts to be consumable by the AM. ApplicationId applicationId = application.GetAppAttemptId().GetApplicationId(); environment[ApplicationConstants.AppSubmitTimeEnv] = rmContext.GetRMApps()[applicationId ].GetSubmitTime().ToString(); environment[ApplicationConstants.MaxAppAttemptsEnv] = rmContext.GetRMApps()[applicationId ].GetMaxAppAttempts().ToString(); Credentials credentials = new Credentials(); DataInputByteBuffer dibb = new DataInputByteBuffer(); if (container.GetTokens() != null) { // TODO: Don't do this kind of checks everywhere. dibb.Reset(container.GetTokens()); credentials.ReadTokenStorageStream(dibb); } // Add AMRMToken Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> amrmToken = CreateAndSetAMRMToken (); if (amrmToken != null) { credentials.AddToken(amrmToken.GetService(), amrmToken); } DataOutputBuffer dob = new DataOutputBuffer(); credentials.WriteTokenStorageToStream(dob); container.SetTokens(ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength())); }
/// <exception cref="System.IO.IOException"/> private static ApplicationClientProtocol GetRmClient <_T0>(Org.Apache.Hadoop.Security.Token.Token <_T0> token, Configuration conf) where _T0 : TokenIdentifier { string[] services = token.GetService().ToString().Split(","); foreach (string service in services) { IPEndPoint addr = NetUtils.CreateSocketAddr(service); if (localSecretManager != null) { // return null if it's our token if (localServiceAddress.Address.IsAnyLocalAddress()) { if (NetUtils.IsLocalAddress(addr.Address) && addr.Port == localServiceAddress.Port) { return(null); } } else { if (addr.Equals(localServiceAddress)) { return(null); } } } } return(ClientRMProxy.CreateRMProxy <ApplicationClientProtocol>(conf)); }
/// <exception cref="System.IO.IOException"/> private void VerifyServiceInToken(ServletContext context, HttpServletRequest request , string expected) { UserGroupInformation ugi = JspHelper.GetUGI(context, request, conf); Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> tokenInUgi = ugi.GetTokens ().GetEnumerator().Next(); NUnit.Framework.Assert.AreEqual(expected, tokenInUgi.GetService().ToString()); }
/// <returns>a string representation of the token</returns> /// <exception cref="System.IO.IOException"/> public static string StringifyToken <_T0>(Org.Apache.Hadoop.Security.Token.Token <_T0 > token) where _T0 : TokenIdentifier { Org.Apache.Hadoop.Hdfs.Security.Token.Delegation.DelegationTokenIdentifier ident = new Org.Apache.Hadoop.Hdfs.Security.Token.Delegation.DelegationTokenIdentifier(); ByteArrayInputStream buf = new ByteArrayInputStream(token.GetIdentifier()); DataInputStream @in = new DataInputStream(buf); ident.ReadFields(@in); if (token.GetService().GetLength() > 0) { return(ident + " on " + token.GetService()); } else { return(ident.ToString()); } }
/// <exception cref="System.Exception"/> private void TestBinaryCredentials(bool hasScheme) { Path TestRootDir = new Path(Runtime.GetProperty("test.build.data", "test/build/data" )); // ick, but need fq path minus file:/ string binaryTokenFile = hasScheme ? FileSystem.GetLocal(conf).MakeQualified(new Path(TestRootDir, "tokenFile")).ToString() : FileSystem.GetLocal(conf).MakeQualified (new Path(TestRootDir, "tokenFile")).ToUri().GetPath(); FileSystemTestHelper.MockFileSystem fs1 = CreateFileSystemForServiceName("service1" ); FileSystemTestHelper.MockFileSystem fs2 = CreateFileSystemForServiceName("service2" ); FileSystemTestHelper.MockFileSystem fs3 = CreateFileSystemForServiceName("service3" ); // get the tokens for fs1 & fs2 and write out to binary creds file Credentials creds = new Credentials(); Org.Apache.Hadoop.Security.Token.Token <object> token1 = fs1.GetDelegationToken(renewer ); Org.Apache.Hadoop.Security.Token.Token <object> token2 = fs2.GetDelegationToken(renewer ); creds.AddToken(token1.GetService(), token1); creds.AddToken(token2.GetService(), token2); // wait to set, else the obtain tokens call above will fail with FNF conf.Set(MRJobConfig.MapreduceJobCredentialsBinary, binaryTokenFile); creds.WriteTokenStorageFile(new Path(binaryTokenFile), conf); // re-init creds and add a newer token for fs1 creds = new Credentials(); Org.Apache.Hadoop.Security.Token.Token <object> newerToken1 = fs1.GetDelegationToken (renewer); NUnit.Framework.Assert.AreNotSame(newerToken1, token1); creds.AddToken(newerToken1.GetService(), newerToken1); CheckToken(creds, newerToken1); // get token for fs1, see that fs2's token was loaded TokenCache.ObtainTokensForNamenodesInternal(fs1, creds, conf); CheckToken(creds, newerToken1, token2); // get token for fs2, nothing should change since already present TokenCache.ObtainTokensForNamenodesInternal(fs2, creds, conf); CheckToken(creds, newerToken1, token2); // get token for fs3, should only add token for fs3 TokenCache.ObtainTokensForNamenodesInternal(fs3, creds, conf); Org.Apache.Hadoop.Security.Token.Token <object> token3 = creds.GetToken(new Text(fs3 .GetCanonicalServiceName())); NUnit.Framework.Assert.IsTrue(token3 != null); CheckToken(creds, newerToken1, token2, token3); // be paranoid, check one last time that nothing changes TokenCache.ObtainTokensForNamenodesInternal(fs1, creds, conf); TokenCache.ObtainTokensForNamenodesInternal(fs2, creds, conf); TokenCache.ObtainTokensForNamenodesInternal(fs3, creds, conf); CheckToken(creds, newerToken1, token2, token3); }
/// <summary>Parse the file system URI out of the provided token.</summary> public static URI GetServiceUriFromToken <_T0>(string scheme, Org.Apache.Hadoop.Security.Token.Token <_T0> token) where _T0 : TokenIdentifier { string tokStr = token.GetService().ToString(); string prefix = BuildTokenServicePrefixForLogicalUri(scheme); if (tokStr.StartsWith(prefix)) { tokStr = tokStr.ReplaceFirst(prefix, string.Empty); } return(URI.Create(scheme + "://" + tokStr)); }
internal void InitDelegationToken(UserGroupInformation ugi) { lock (this) { Org.Apache.Hadoop.Security.Token.Token <object> token = SelectDelegationToken(ugi); if (token != null) { Log.Debug("Found existing DT for " + token.GetService()); fs.SetDelegationToken(token); hasInitedToken = true; } } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> private void VerifyNewVersionToken(Configuration conf, TestClientToAMTokens.CustomAM am, Org.Apache.Hadoop.Security.Token.Token <ClientToAMTokenIdentifier> token, MockRM rm) { UserGroupInformation ugi; ugi = UserGroupInformation.CreateRemoteUser("me"); Org.Apache.Hadoop.Security.Token.Token <ClientToAMTokenIdentifier> newToken = new Org.Apache.Hadoop.Security.Token.Token <ClientToAMTokenIdentifier>(new ClientToAMTokenIdentifierForTest (token.DecodeIdentifier(), "message"), am.GetClientToAMTokenSecretManager()); newToken.SetService(token.GetService()); ugi.AddToken(newToken); ugi.DoAs(new _PrivilegedExceptionAction_386(am, conf)); }
/// <exception cref="System.Exception"/> public virtual void TestUGITokens <T>() where T : TokenIdentifier { // from Mockito mocks UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting("TheDoctor", new string[] { "TheTARDIS" }); Org.Apache.Hadoop.Security.Token.Token <T> t1 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token >(); Org.Mockito.Mockito.When(t1.GetService()).ThenReturn(new Text("t1")); Org.Apache.Hadoop.Security.Token.Token <T> t2 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token >(); Org.Mockito.Mockito.When(t2.GetService()).ThenReturn(new Text("t2")); Credentials creds = new Credentials(); byte[] secretKey = new byte[] { }; Text secretName = new Text("shhh"); creds.AddSecretKey(secretName, secretKey); ugi.AddToken(t1); ugi.AddToken(t2); ugi.AddCredentials(creds); ICollection <Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> > z = ugi.GetTokens (); Assert.True(z.Contains(t1)); Assert.True(z.Contains(t2)); Assert.Equal(2, z.Count); Credentials ugiCreds = ugi.GetCredentials(); NUnit.Framework.Assert.AreSame(secretKey, ugiCreds.GetSecretKey(secretName)); Assert.Equal(1, ugiCreds.NumberOfSecretKeys()); try { z.Remove(t1); NUnit.Framework.Assert.Fail("Shouldn't be able to modify token collection from UGI" ); } catch (NotSupportedException) { } // Can't modify tokens // ensure that the tokens are passed through doAs ICollection <Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> > otherSet = ugi .DoAs(new _PrivilegedExceptionAction_612()); Assert.True(otherSet.Contains(t1)); Assert.True(otherSet.Contains(t2)); }
/// <exception cref="System.Exception"/> public virtual void TestHdfsGetCanonicalServiceName() { Configuration conf = dfs.GetConf(); URI haUri = HATestUtil.GetLogicalUri(cluster); AbstractFileSystem afs = AbstractFileSystem.CreateFileSystem(haUri, conf); string haService = HAUtil.BuildTokenServiceForLogicalUri(haUri, HdfsConstants.HdfsUriScheme ).ToString(); NUnit.Framework.Assert.AreEqual(haService, afs.GetCanonicalServiceName()); Org.Apache.Hadoop.Security.Token.Token <object> token = afs.GetDelegationTokens(UserGroupInformation .GetCurrentUser().GetShortUserName())[0]; NUnit.Framework.Assert.AreEqual(haService, token.GetService().ToString()); // make sure the logical uri is handled correctly token.Renew(conf); token.Cancel(conf); }
/// <summary> /// HDFS-3062: DistributedFileSystem.getCanonicalServiceName() throws an /// exception if the URI is a logical URI. /// </summary> /// <remarks> /// HDFS-3062: DistributedFileSystem.getCanonicalServiceName() throws an /// exception if the URI is a logical URI. This bug fails the combination of /// ha + mapred + security. /// </remarks> /// <exception cref="System.Exception"/> public virtual void TestDFSGetCanonicalServiceName() { URI hAUri = HATestUtil.GetLogicalUri(cluster); string haService = HAUtil.BuildTokenServiceForLogicalUri(hAUri, HdfsConstants.HdfsUriScheme ).ToString(); NUnit.Framework.Assert.AreEqual(haService, dfs.GetCanonicalServiceName()); string renewer = UserGroupInformation.GetCurrentUser().GetShortUserName(); Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = GetDelegationToken (dfs, renewer); NUnit.Framework.Assert.AreEqual(haService, token.GetService().ToString()); // make sure the logical uri is handled correctly token.Renew(dfs.GetConf()); token.Cancel(dfs.GetConf()); }
// check: // 1) buildTokenService honors use_ip setting // 2) setTokenService & getService works // 3) getTokenServiceAddr decodes to the identical socket addr private void VerifyTokenService(IPEndPoint addr, string host, string ip, int port , bool useIp) { //LOG.info("address:"+addr+" host:"+host+" ip:"+ip+" port:"+port); SecurityUtil.SetTokenServiceUseIp(useIp); string serviceHost = useIp ? ip : StringUtils.ToLowerCase(host); Org.Apache.Hadoop.Security.Token.Token <object> token = new Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier>(); Text service = new Text(serviceHost + ":" + port); Assert.Equal(service, SecurityUtil.BuildTokenService(addr)); SecurityUtil.SetTokenService(token, addr); Assert.Equal(service, token.GetService()); IPEndPoint serviceAddr = SecurityUtil.GetTokenServiceAddr(token); NUnit.Framework.Assert.IsNotNull(serviceAddr); VerifyValues(serviceAddr, serviceHost, ip, port); }
/// <exception cref="System.Exception"/> public virtual void TestAddNamedToken <T>() where T : TokenIdentifier { // from Mockito mocks UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser("someone"); Org.Apache.Hadoop.Security.Token.Token <T> t1 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token >(); Text service1 = new Text("t1"); Text service2 = new Text("t2"); Org.Mockito.Mockito.When(t1.GetService()).ThenReturn(service1); // add token ugi.AddToken(service1, t1); NUnit.Framework.Assert.AreSame(t1, ugi.GetCredentials().GetToken(service1)); // add token with another name ugi.AddToken(service2, t1); NUnit.Framework.Assert.AreSame(t1, ugi.GetCredentials().GetToken(service1)); NUnit.Framework.Assert.AreSame(t1, ugi.GetCredentials().GetToken(service2)); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public override void Cancel <_T0>(Org.Apache.Hadoop.Security.Token.Token <_T0> token , Configuration conf) { Org.Apache.Hadoop.Yarn.Api.Records.Token dToken = Org.Apache.Hadoop.Yarn.Api.Records.Token .NewInstance(token.GetIdentifier(), token.GetKind().ToString(), token.GetPassword (), token.GetService().ToString()); MRClientProtocol histProxy = InstantiateHistoryProxy(conf, SecurityUtil.GetTokenServiceAddr (token)); try { CancelDelegationTokenRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <CancelDelegationTokenRequest>(); request.SetDelegationToken(dToken); histProxy.CancelDelegationToken(request); } finally { StopHistoryProxy(histProxy); } }
/// <exception cref="System.IO.IOException"/> internal void EnsureTokenInitialized() { lock (this) { // we haven't inited yet, or we used to have a token but it expired if (!hasInitedToken || (action != null && !action.IsValid())) { //since we don't already have a token, go get one Org.Apache.Hadoop.Security.Token.Token <object> token = fs.GetDelegationToken(null ); // security might be disabled if (token != null) { fs.SetDelegationToken(token); AddRenewAction(fs); Log.Debug("Created new DT for " + token.GetService()); } hasInitedToken = true; } } }
public virtual void TestAMRMTokenUpdate() { Configuration conf = new Configuration(); ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(ApplicationId.NewInstance (1, 1), 1); AMRMTokenIdentifier oldTokenId = new AMRMTokenIdentifier(attemptId, 1); AMRMTokenIdentifier newTokenId = new AMRMTokenIdentifier(attemptId, 2); Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> oldToken = new Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier>(oldTokenId.GetBytes(), Sharpen.Runtime.GetBytesForString("oldpassword" ), oldTokenId.GetKind(), new Text()); Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> newToken = new Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier>(newTokenId.GetBytes(), Sharpen.Runtime.GetBytesForString("newpassword" ), newTokenId.GetKind(), new Text()); TestLocalContainerAllocator.MockScheduler scheduler = new TestLocalContainerAllocator.MockScheduler (); scheduler.amToken = newToken; LocalContainerAllocator lca = new TestLocalContainerAllocator.StubbedLocalContainerAllocator (scheduler); lca.Init(conf); lca.Start(); UserGroupInformation testUgi = UserGroupInformation.CreateUserForTesting("someuser" , new string[0]); testUgi.AddToken(oldToken); testUgi.DoAs(new _PrivilegedExceptionAction_144(lca)); lca.Close(); // verify there is only one AMRM token in the UGI and it matches the // updated token from the RM int tokenCount = 0; Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> ugiToken = null; foreach (Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> token in testUgi .GetTokens()) { if (AMRMTokenIdentifier.KindName.Equals(token.GetKind())) { ugiToken = token; ++tokenCount; } } NUnit.Framework.Assert.AreEqual("too many AMRM tokens", 1, tokenCount); Assert.AssertArrayEquals("token identifier not updated", newToken.GetIdentifier() , ugiToken.GetIdentifier()); Assert.AssertArrayEquals("token password not updated", newToken.GetPassword(), ugiToken .GetPassword()); NUnit.Framework.Assert.AreEqual("AMRM token service not updated", new Text(ClientRMProxy .GetAMRMTokenService(conf)), ugiToken.GetService()); }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> public virtual AllocateResponse Allocate(AllocateRequest request) { NUnit.Framework.Assert.AreEqual("response ID mismatch", responseId, request.GetResponseId ()); ++responseId; Org.Apache.Hadoop.Yarn.Api.Records.Token yarnToken = null; if (amToken != null) { yarnToken = Org.Apache.Hadoop.Yarn.Api.Records.Token.NewInstance(amToken.GetIdentifier (), amToken.GetKind().ToString(), amToken.GetPassword(), amToken.GetService().ToString ()); } return(AllocateResponse.NewInstance(responseId, Collections.EmptyList <ContainerStatus >(), Collections.EmptyList <Container>(), Collections.EmptyList <NodeReport>(), Resources .None(), null, 1, null, Collections.EmptyList <NMToken>(), yarnToken, Collections .EmptyList <ContainerResourceIncrease>(), Collections.EmptyList <ContainerResourceDecrease >())); }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> public virtual AllocateResponse Allocate(AllocateRequest request) { AMRMTokenIdentifier amrmTokenIdentifier = AuthorizeRequest(); ApplicationAttemptId appAttemptId = amrmTokenIdentifier.GetApplicationAttemptId(); ApplicationId applicationId = appAttemptId.GetApplicationId(); this.amLivelinessMonitor.ReceivedPing(appAttemptId); /* check if its in cache */ ApplicationMasterService.AllocateResponseLock Lock = responseMap[appAttemptId]; if (Lock == null) { string message = "Application attempt " + appAttemptId + " doesn't exist in ApplicationMasterService cache."; Log.Error(message); throw new ApplicationAttemptNotFoundException(message); } lock (Lock) { AllocateResponse lastResponse = Lock.GetAllocateResponse(); if (!HasApplicationMasterRegistered(appAttemptId)) { string message = "AM is not registered for known application attempt: " + appAttemptId + " or RM had restarted after AM registered . AM should re-register."; Log.Info(message); RMAuditLogger.LogFailure(this.rmContext.GetRMApps()[appAttemptId.GetApplicationId ()].GetUser(), RMAuditLogger.AuditConstants.AmAllocate, string.Empty, "ApplicationMasterService" , message, applicationId, appAttemptId); throw new ApplicationMasterNotRegisteredException(message); } if ((request.GetResponseId() + 1) == lastResponse.GetResponseId()) { /* old heartbeat */ return(lastResponse); } else { if (request.GetResponseId() + 1 < lastResponse.GetResponseId()) { string message = "Invalid responseId in AllocateRequest from application attempt: " + appAttemptId + ", expect responseId to be " + (lastResponse.GetResponseId() + 1); throw new InvalidApplicationMasterRequestException(message); } } //filter illegal progress values float filteredProgress = request.GetProgress(); if (float.IsNaN(filteredProgress) || filteredProgress == float.NegativeInfinity || filteredProgress < 0) { request.SetProgress(0); } else { if (filteredProgress > 1 || filteredProgress == float.PositiveInfinity) { request.SetProgress(1); } } // Send the status update to the appAttempt. this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMAppAttemptStatusupdateEvent (appAttemptId, request.GetProgress())); IList <ResourceRequest> ask = request.GetAskList(); IList <ContainerId> release = request.GetReleaseList(); ResourceBlacklistRequest blacklistRequest = request.GetResourceBlacklistRequest(); IList <string> blacklistAdditions = (blacklistRequest != null) ? blacklistRequest. GetBlacklistAdditions() : Sharpen.Collections.EmptyList; IList <string> blacklistRemovals = (blacklistRequest != null) ? blacklistRequest.GetBlacklistRemovals () : Sharpen.Collections.EmptyList; RMApp app = this.rmContext.GetRMApps()[applicationId]; // set label expression for Resource Requests if resourceName=ANY ApplicationSubmissionContext asc = app.GetApplicationSubmissionContext(); foreach (ResourceRequest req in ask) { if (null == req.GetNodeLabelExpression() && ResourceRequest.Any.Equals(req.GetResourceName ())) { req.SetNodeLabelExpression(asc.GetNodeLabelExpression()); } } // sanity check try { RMServerUtils.NormalizeAndValidateRequests(ask, rScheduler.GetMaximumResourceCapability (), app.GetQueue(), rScheduler, rmContext); } catch (InvalidResourceRequestException e) { Log.Warn("Invalid resource ask by application " + appAttemptId, e); throw; } try { RMServerUtils.ValidateBlacklistRequest(blacklistRequest); } catch (InvalidResourceBlacklistRequestException e) { Log.Warn("Invalid blacklist request by application " + appAttemptId, e); throw; } // In the case of work-preserving AM restart, it's possible for the // AM to release containers from the earlier attempt. if (!app.GetApplicationSubmissionContext().GetKeepContainersAcrossApplicationAttempts ()) { try { RMServerUtils.ValidateContainerReleaseRequest(release, appAttemptId); } catch (InvalidContainerReleaseException e) { Log.Warn("Invalid container release by application " + appAttemptId, e); throw; } } // Send new requests to appAttempt. Allocation allocation = this.rScheduler.Allocate(appAttemptId, ask, release, blacklistAdditions , blacklistRemovals); if (!blacklistAdditions.IsEmpty() || !blacklistRemovals.IsEmpty()) { Log.Info("blacklist are updated in Scheduler." + "blacklistAdditions: " + blacklistAdditions + ", " + "blacklistRemovals: " + blacklistRemovals); } RMAppAttempt appAttempt = app.GetRMAppAttempt(appAttemptId); AllocateResponse allocateResponse = recordFactory.NewRecordInstance <AllocateResponse >(); if (!allocation.GetContainers().IsEmpty()) { allocateResponse.SetNMTokens(allocation.GetNMTokens()); } // update the response with the deltas of node status changes IList <RMNode> updatedNodes = new AList <RMNode>(); if (app.PullRMNodeUpdates(updatedNodes) > 0) { IList <NodeReport> updatedNodeReports = new AList <NodeReport>(); foreach (RMNode rmNode in updatedNodes) { SchedulerNodeReport schedulerNodeReport = rScheduler.GetNodeReport(rmNode.GetNodeID ()); Resource used = BuilderUtils.NewResource(0, 0); int numContainers = 0; if (schedulerNodeReport != null) { used = schedulerNodeReport.GetUsedResource(); numContainers = schedulerNodeReport.GetNumContainers(); } NodeId nodeId = rmNode.GetNodeID(); NodeReport report = BuilderUtils.NewNodeReport(nodeId, rmNode.GetState(), rmNode. GetHttpAddress(), rmNode.GetRackName(), used, rmNode.GetTotalCapability(), numContainers , rmNode.GetHealthReport(), rmNode.GetLastHealthReportTime(), rmNode.GetNodeLabels ()); updatedNodeReports.AddItem(report); } allocateResponse.SetUpdatedNodes(updatedNodeReports); } allocateResponse.SetAllocatedContainers(allocation.GetContainers()); allocateResponse.SetCompletedContainersStatuses(appAttempt.PullJustFinishedContainers ()); allocateResponse.SetResponseId(lastResponse.GetResponseId() + 1); allocateResponse.SetAvailableResources(allocation.GetResourceLimit()); allocateResponse.SetNumClusterNodes(this.rScheduler.GetNumClusterNodes()); // add preemption to the allocateResponse message (if any) allocateResponse.SetPreemptionMessage(GeneratePreemptionMessage(allocation)); // update AMRMToken if the token is rolled-up MasterKeyData nextMasterKey = this.rmContext.GetAMRMTokenSecretManager().GetNextMasterKeyData (); if (nextMasterKey != null && nextMasterKey.GetMasterKey().GetKeyId() != amrmTokenIdentifier .GetKeyId()) { RMAppAttemptImpl appAttemptImpl = (RMAppAttemptImpl)appAttempt; Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> amrmToken = appAttempt .GetAMRMToken(); if (nextMasterKey.GetMasterKey().GetKeyId() != appAttemptImpl.GetAMRMTokenKeyId()) { Log.Info("The AMRMToken has been rolled-over. Send new AMRMToken back" + " to application: " + applicationId); amrmToken = rmContext.GetAMRMTokenSecretManager().CreateAndGetAMRMToken(appAttemptId ); appAttemptImpl.SetAMRMToken(amrmToken); } allocateResponse.SetAMRMToken(Org.Apache.Hadoop.Yarn.Api.Records.Token.NewInstance (amrmToken.GetIdentifier(), amrmToken.GetKind().ToString(), amrmToken.GetPassword (), amrmToken.GetService().ToString())); } /* * As we are updating the response inside the lock object so we don't * need to worry about unregister call occurring in between (which * removes the lock object). */ Lock.SetAllocateResponse(allocateResponse); return(allocateResponse); } }
public virtual void TestDelegationToken() { YarnConfiguration conf = new YarnConfiguration(); conf.Set(YarnConfiguration.RmPrincipal, "testuser/[email protected]"); conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos"); UserGroupInformation.SetConfiguration(conf); ResourceScheduler scheduler = CreateMockScheduler(conf); long initialInterval = 10000l; long maxLifetime = 20000l; long renewInterval = 10000l; RMDelegationTokenSecretManager rmDtSecretManager = CreateRMDelegationTokenSecretManager (initialInterval, maxLifetime, renewInterval); rmDtSecretManager.StartThreads(); Log.Info("Creating DelegationTokenSecretManager with initialInterval: " + initialInterval + ", maxLifetime: " + maxLifetime + ", renewInterval: " + renewInterval); ClientRMService clientRMService = new TestClientRMTokens.ClientRMServiceForTest(this , conf, scheduler, rmDtSecretManager); clientRMService.Init(conf); clientRMService.Start(); ApplicationClientProtocol clientRMWithDT = null; try { // Create a user for the renewr and fake the authentication-method UserGroupInformation loggedInUser = UserGroupInformation.CreateRemoteUser("*****@*****.**" ); NUnit.Framework.Assert.AreEqual("testrenewer", loggedInUser.GetShortUserName()); // Default realm is APACHE.ORG loggedInUser.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Kerberos ); Token token = GetDelegationToken(loggedInUser, clientRMService, loggedInUser.GetShortUserName ()); long tokenFetchTime = Runtime.CurrentTimeMillis(); Log.Info("Got delegation token at: " + tokenFetchTime); // Now try talking to RMService using the delegation token clientRMWithDT = GetClientRMProtocolWithDT(token, clientRMService.GetBindAddress( ), "loginuser1", conf); GetNewApplicationRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord < GetNewApplicationRequest>(); try { clientRMWithDT.GetNewApplication(request); } catch (IOException e) { NUnit.Framework.Assert.Fail("Unexpected exception" + e); } catch (YarnException e) { NUnit.Framework.Assert.Fail("Unexpected exception" + e); } // Renew after 50% of token age. while (Runtime.CurrentTimeMillis() < tokenFetchTime + initialInterval / 2) { Sharpen.Thread.Sleep(500l); } long nextExpTime = RenewDelegationToken(loggedInUser, clientRMService, token); long renewalTime = Runtime.CurrentTimeMillis(); Log.Info("Renewed token at: " + renewalTime + ", NextExpiryTime: " + nextExpTime); // Wait for first expiry, but before renewed expiry. while (Runtime.CurrentTimeMillis() > tokenFetchTime + initialInterval && Runtime. CurrentTimeMillis() < nextExpTime) { Sharpen.Thread.Sleep(500l); } Sharpen.Thread.Sleep(50l); // Valid token because of renewal. try { clientRMWithDT.GetNewApplication(request); } catch (IOException e) { NUnit.Framework.Assert.Fail("Unexpected exception" + e); } catch (YarnException e) { NUnit.Framework.Assert.Fail("Unexpected exception" + e); } // Wait for expiry. while (Runtime.CurrentTimeMillis() < renewalTime + renewInterval) { Sharpen.Thread.Sleep(500l); } Sharpen.Thread.Sleep(50l); Log.Info("At time: " + Runtime.CurrentTimeMillis() + ", token should be invalid"); // Token should have expired. try { clientRMWithDT.GetNewApplication(request); NUnit.Framework.Assert.Fail("Should not have succeeded with an expired token"); } catch (Exception e) { NUnit.Framework.Assert.AreEqual(typeof(SecretManager.InvalidToken).FullName, e.GetType ().FullName); NUnit.Framework.Assert.IsTrue(e.Message.Contains("is expired")); } // Test cancellation // Stop the existing proxy, start another. if (clientRMWithDT != null) { RPC.StopProxy(clientRMWithDT); clientRMWithDT = null; } token = GetDelegationToken(loggedInUser, clientRMService, loggedInUser.GetShortUserName ()); tokenFetchTime = Runtime.CurrentTimeMillis(); Log.Info("Got delegation token at: " + tokenFetchTime); // Now try talking to RMService using the delegation token clientRMWithDT = GetClientRMProtocolWithDT(token, clientRMService.GetBindAddress( ), "loginuser2", conf); request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <GetNewApplicationRequest> (); try { clientRMWithDT.GetNewApplication(request); } catch (IOException e) { NUnit.Framework.Assert.Fail("Unexpected exception" + e); } catch (YarnException e) { NUnit.Framework.Assert.Fail("Unexpected exception" + e); } CancelDelegationToken(loggedInUser, clientRMService, token); if (clientRMWithDT != null) { RPC.StopProxy(clientRMWithDT); clientRMWithDT = null; } // Creating a new connection. clientRMWithDT = GetClientRMProtocolWithDT(token, clientRMService.GetBindAddress( ), "loginuser2", conf); Log.Info("Cancelled delegation token at: " + Runtime.CurrentTimeMillis()); // Verify cancellation worked. try { clientRMWithDT.GetNewApplication(request); NUnit.Framework.Assert.Fail("Should not have succeeded with a cancelled delegation token" ); } catch (IOException) { } catch (YarnException) { } // Test new version token // Stop the existing proxy, start another. if (clientRMWithDT != null) { RPC.StopProxy(clientRMWithDT); clientRMWithDT = null; } token = GetDelegationToken(loggedInUser, clientRMService, loggedInUser.GetShortUserName ()); byte[] tokenIdentifierContent = ((byte[])token.GetIdentifier().Array()); RMDelegationTokenIdentifier tokenIdentifier = new RMDelegationTokenIdentifier(); DataInputBuffer dib = new DataInputBuffer(); dib.Reset(tokenIdentifierContent, tokenIdentifierContent.Length); tokenIdentifier.ReadFields(dib); // Construct new version RMDelegationTokenIdentifier with additional field RMDelegationTokenIdentifierForTest newVersionTokenIdentifier = new RMDelegationTokenIdentifierForTest (tokenIdentifier, "message"); Org.Apache.Hadoop.Security.Token.Token <RMDelegationTokenIdentifier> newRMDTtoken = new Org.Apache.Hadoop.Security.Token.Token <RMDelegationTokenIdentifier>(newVersionTokenIdentifier , rmDtSecretManager); Org.Apache.Hadoop.Yarn.Api.Records.Token newToken = BuilderUtils.NewDelegationToken (newRMDTtoken.GetIdentifier(), newRMDTtoken.GetKind().ToString(), newRMDTtoken.GetPassword (), newRMDTtoken.GetService().ToString()); // Now try talking to RMService using the new version delegation token clientRMWithDT = GetClientRMProtocolWithDT(newToken, clientRMService.GetBindAddress (), "loginuser3", conf); request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <GetNewApplicationRequest> (); try { clientRMWithDT.GetNewApplication(request); } catch (IOException e) { NUnit.Framework.Assert.Fail("Unexpected exception" + e); } catch (YarnException e) { NUnit.Framework.Assert.Fail("Unexpected exception" + e); } } finally { rmDtSecretManager.StopThreads(); // TODO PRECOMMIT Close proxies. if (clientRMWithDT != null) { RPC.StopProxy(clientRMWithDT); } } }
private void VerifyTamperedToken(Configuration conf, TestClientToAMTokens.CustomAM am, Org.Apache.Hadoop.Security.Token.Token <ClientToAMTokenIdentifier> token, UserGroupInformation ugi, ClientToAMTokenIdentifier maliciousID) { Org.Apache.Hadoop.Security.Token.Token <ClientToAMTokenIdentifier> maliciousToken = new Org.Apache.Hadoop.Security.Token.Token <ClientToAMTokenIdentifier>(maliciousID .GetBytes(), token.GetPassword(), token.GetKind(), token.GetService()); ugi.AddToken(maliciousToken); try { ugi.DoAs(new _PrivilegedExceptionAction_338(am, conf)); } catch (Exception e) { NUnit.Framework.Assert.AreEqual(typeof(RemoteException).FullName, e.GetType().FullName ); e = ((RemoteException)e).UnwrapRemoteException(); NUnit.Framework.Assert.AreEqual(typeof(SaslException).GetCanonicalName(), e.GetType ().GetCanonicalName()); NUnit.Framework.Assert.IsTrue(e.Message.Contains("DIGEST-MD5: digest response format violation. " + "Mismatched response.")); NUnit.Framework.Assert.IsFalse(am.pinged); } }
private static Org.Apache.Hadoop.Yarn.Api.Records.Token ConvertToProtoToken <_T0>( Org.Apache.Hadoop.Security.Token.Token <_T0> token) where _T0 : TokenIdentifier { return(Org.Apache.Hadoop.Yarn.Api.Records.Token.NewInstance(token.GetIdentifier() , token.GetKind().ToString(), token.GetPassword(), token.GetService().ToString() )); }
/// <returns> /// true if this token corresponds to a logical nameservice /// rather than a specific namenode. /// </returns> public static bool IsTokenForLogicalUri <_T0>(Org.Apache.Hadoop.Security.Token.Token <_T0> token) where _T0 : TokenIdentifier { return(token.GetService().ToString().StartsWith(HdfsConstants.HaDtServicePrefix)); }
/// <summary>Decode the given token's service field into an InetAddress</summary> /// <param name="token">from which to obtain the service</param> /// <returns>InetAddress for the service</returns> public static IPEndPoint GetTokenServiceAddr <_T0>(Org.Apache.Hadoop.Security.Token.Token <_T0> token) where _T0 : TokenIdentifier { return(NetUtils.CreateSocketAddr(token.GetService().ToString())); }