public virtual void TestJobTokenRpc() { TaskUmbilicalProtocol mockTT = Org.Mockito.Mockito.Mock <TaskUmbilicalProtocol>(); Org.Mockito.Mockito.DoReturn(TaskUmbilicalProtocol.versionID).When(mockTT).GetProtocolVersion (Matchers.AnyString(), Matchers.AnyLong()); Org.Mockito.Mockito.DoReturn(ProtocolSignature.GetProtocolSignature(mockTT, typeof( TaskUmbilicalProtocol).FullName, TaskUmbilicalProtocol.versionID, 0)).When(mockTT ).GetProtocolSignature(Matchers.AnyString(), Matchers.AnyLong(), Matchers.AnyInt ()); JobTokenSecretManager sm = new JobTokenSecretManager(); Server server = new RPC.Builder(conf).SetProtocol(typeof(TaskUmbilicalProtocol)). SetInstance(mockTT).SetBindAddress(Address).SetPort(0).SetNumHandlers(5).SetVerbose (true).SetSecretManager(sm).Build(); server.Start(); UserGroupInformation current = UserGroupInformation.GetCurrentUser(); IPEndPoint addr = NetUtils.GetConnectAddress(server); string jobId = current.GetUserName(); JobTokenIdentifier tokenId = new JobTokenIdentifier(new Text(jobId)); Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier>(tokenId, sm); sm.AddTokenForJob(jobId, token); SecurityUtil.SetTokenService(token, addr); Log.Info("Service address for token is " + token.GetService()); current.AddToken(token); current.DoAs(new _PrivilegedExceptionAction_110(addr, server)); }
public virtual void TestDelegationTokenRpc() { ClientProtocol mockNN = Org.Mockito.Mockito.Mock <ClientProtocol>(); FSNamesystem mockNameSys = Org.Mockito.Mockito.Mock <FSNamesystem>(); DelegationTokenSecretManager sm = new DelegationTokenSecretManager(DFSConfigKeys. DfsNamenodeDelegationKeyUpdateIntervalDefault, DFSConfigKeys.DfsNamenodeDelegationKeyUpdateIntervalDefault , DFSConfigKeys.DfsNamenodeDelegationTokenMaxLifetimeDefault, 3600000, mockNameSys ); sm.StartThreads(); Org.Apache.Hadoop.Ipc.Server server = new RPC.Builder(conf).SetProtocol(typeof(ClientProtocol )).SetInstance(mockNN).SetBindAddress(Address).SetPort(0).SetNumHandlers(5).SetVerbose (true).SetSecretManager(sm).Build(); server.Start(); UserGroupInformation current = UserGroupInformation.GetCurrentUser(); IPEndPoint addr = NetUtils.GetConnectAddress(server); string user = current.GetUserName(); Text owner = new Text(user); DelegationTokenIdentifier dtId = new DelegationTokenIdentifier(owner, owner, null ); Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier>(dtId, sm); SecurityUtil.SetTokenService(token, addr); Log.Info("Service for token is " + token.GetService()); current.AddToken(token); current.DoAs(new _PrivilegedExceptionAction_100(addr, server)); }
private void Compare(Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> expected, Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> actual) { NUnit.Framework.Assert.IsTrue(Arrays.Equals(expected.GetIdentifier(), actual.GetIdentifier ())); NUnit.Framework.Assert.IsTrue(Arrays.Equals(expected.GetPassword(), actual.GetPassword ())); NUnit.Framework.Assert.AreEqual(expected.GetKind(), actual.GetKind()); NUnit.Framework.Assert.AreEqual(expected.GetService(), actual.GetService()); }
/// <exception cref="System.IO.IOException"/> public virtual Org.Apache.Hadoop.Security.Token.Token <object>[] AddDelegationTokens (string renewer, Credentials credentials) { Org.Apache.Hadoop.Security.Token.Token <object>[] tokens = null; Text dtService = GetDelegationTokenService(); Org.Apache.Hadoop.Security.Token.Token <object> token = credentials.GetToken(dtService ); if (token == null) { Uri url = CreateURL(null, null, null, null); DelegationTokenAuthenticatedURL authUrl = new DelegationTokenAuthenticatedURL(configurator ); try { // 'actualUGI' is the UGI of the user creating the client // It is possible that the creator of the KMSClientProvier // calls this method on behalf of a proxyUser (the doAsUser). // In which case this call has to be made as the proxy user. UserGroupInformation currentUgi = UserGroupInformation.GetCurrentUser(); string doAsUser = (currentUgi.GetAuthenticationMethod() == UserGroupInformation.AuthenticationMethod .Proxy) ? currentUgi.GetShortUserName() : null; token = actualUgi.DoAs(new _PrivilegedExceptionAction_870(authUrl, url, renewer, doAsUser)); // Not using the cached token here.. Creating a new token here // everytime. if (token != null) { credentials.AddToken(token.GetService(), token); tokens = new Org.Apache.Hadoop.Security.Token.Token <object>[] { token }; } else { throw new IOException("Got NULL as delegation token"); } } catch (Exception) { Thread.CurrentThread().Interrupt(); } catch (Exception e) { throw new IOException(e); } } return(tokens); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> public override void CancelDelegationToken(Org.Apache.Hadoop.Security.Token.Token <TimelineDelegationTokenIdentifier> timelineDT) { bool isTokenServiceAddrEmpty = timelineDT.GetService().ToString().IsEmpty(); string scheme = isTokenServiceAddrEmpty ? null : (YarnConfiguration.UseHttps(this .GetConfig()) ? "https" : "http"); IPEndPoint address = isTokenServiceAddrEmpty ? null : SecurityUtil.GetTokenServiceAddr (timelineDT); PrivilegedExceptionAction <Void> cancelDTAction = new _PrivilegedExceptionAction_415 (this, timelineDT, isTokenServiceAddrEmpty, scheme, address); // If the timeline DT to cancel is different than cached, replace it. // Token to set every time for retry, because when exception happens, // DelegationTokenAuthenticatedURL will reset it to null; // If the token service address is not available, fall back to use // the configured service address. OperateDelegationToken(cancelDTAction); }
public virtual void TestGetDelegationTokensWithCredentials() { Credentials credentials = new Credentials(); IList <Org.Apache.Hadoop.Security.Token.Token <object> > delTokens = Arrays.AsList(fsView .AddDelegationTokens("sanjay", credentials)); int expectedTokenCount = GetExpectedDelegationTokenCountWithCredentials(); Assert.Equal(expectedTokenCount, delTokens.Count); Credentials newCredentials = new Credentials(); for (int i = 0; i < expectedTokenCount / 2; i++) { Org.Apache.Hadoop.Security.Token.Token <object> token = delTokens[i]; newCredentials.AddToken(token.GetService(), token); } IList <Org.Apache.Hadoop.Security.Token.Token <object> > delTokens2 = Arrays.AsList( fsView.AddDelegationTokens("sanjay", newCredentials)); Assert.Equal((expectedTokenCount + 1) / 2, delTokens2.Count); }
/// <exception cref="System.IO.IOException"/> private void RecordJobShuffleInfo(JobID jobId, string user, Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> jobToken) { if (stateDb != null) { SecurityProtos.TokenProto tokenProto = ((SecurityProtos.TokenProto)SecurityProtos.TokenProto .NewBuilder().SetIdentifier(ByteString.CopyFrom(jobToken.GetIdentifier())).SetPassword (ByteString.CopyFrom(jobToken.GetPassword())).SetKind(jobToken.GetKind().ToString ()).SetService(jobToken.GetService().ToString()).Build()); ShuffleHandlerRecoveryProtos.JobShuffleInfoProto proto = ((ShuffleHandlerRecoveryProtos.JobShuffleInfoProto )ShuffleHandlerRecoveryProtos.JobShuffleInfoProto.NewBuilder().SetUser(user).SetJobToken (tokenProto).Build()); try { stateDb.Put(JniDBFactory.Bytes(jobId.ToString()), proto.ToByteArray()); } catch (DBException e) { throw new IOException("Error storing " + jobId, e); } } AddJobToken(jobId, user, jobToken); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> public virtual void LaunchAM(ApplicationAttemptId attemptId) { Credentials credentials = new Credentials(); Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> token = rmClient.GetAMRMToken (attemptId.GetApplicationId()); // Service will be empty but that's okay, we are just passing down only // AMRMToken down to the real AM which eventually sets the correct // service-address. credentials.AddToken(token.GetService(), token); FilePath tokenFile = FilePath.CreateTempFile("unmanagedAMRMToken", string.Empty, new FilePath(Runtime.GetProperty("user.dir"))); try { FileUtil.Chmod(tokenFile.GetAbsolutePath(), "600"); } catch (Exception ex) { throw new RuntimeException(ex); } tokenFile.DeleteOnExit(); DataOutputStream os = new DataOutputStream(new FileOutputStream(tokenFile, true)); credentials.WriteTokenStorageToStream(os); os.Close(); IDictionary <string, string> env = Sharpen.Runtime.GetEnv(); AList <string> envAMList = new AList <string>(); bool setClasspath = false; foreach (KeyValuePair <string, string> entry in env) { string key = entry.Key; string value = entry.Value; if (key.Equals("CLASSPATH")) { setClasspath = true; if (classpath != null) { value = value + FilePath.pathSeparator + classpath; } } envAMList.AddItem(key + "=" + value); } if (!setClasspath && classpath != null) { envAMList.AddItem("CLASSPATH=" + classpath); } ContainerId containerId = ContainerId.NewContainerId(attemptId, 0); string hostname = Sharpen.Runtime.GetLocalHost().GetHostName(); envAMList.AddItem(ApplicationConstants.Environment.ContainerId.ToString() + "=" + containerId); envAMList.AddItem(ApplicationConstants.Environment.NmHost.ToString() + "=" + hostname ); envAMList.AddItem(ApplicationConstants.Environment.NmHttpPort.ToString() + "=0"); envAMList.AddItem(ApplicationConstants.Environment.NmPort.ToString() + "=0"); envAMList.AddItem(ApplicationConstants.Environment.LocalDirs.ToString() + "= /tmp" ); envAMList.AddItem(ApplicationConstants.AppSubmitTimeEnv + "=" + Runtime.CurrentTimeMillis ()); envAMList.AddItem(ApplicationConstants.ContainerTokenFileEnvName + "=" + tokenFile .GetAbsolutePath()); string[] envAM = new string[envAMList.Count]; SystemProcess amProc = Runtime.GetRuntime().Exec(amCmd, Sharpen.Collections.ToArray (envAMList, envAM)); BufferedReader errReader = new BufferedReader(new InputStreamReader(amProc.GetErrorStream (), Sharpen.Extensions.GetEncoding("UTF-8"))); BufferedReader inReader = new BufferedReader(new InputStreamReader(amProc.GetInputStream (), Sharpen.Extensions.GetEncoding("UTF-8"))); // read error and input streams as this would free up the buffers // free the error stream buffer Sharpen.Thread errThread = new _Thread_244(errReader); Sharpen.Thread outThread = new _Thread_258(inReader); try { errThread.Start(); outThread.Start(); } catch (InvalidOperationException) { } // wait for the process to finish and check the exit code try { int exitCode = amProc.WaitFor(); Log.Info("AM process exited with value: " + exitCode); } catch (Exception e) { Sharpen.Runtime.PrintStackTrace(e); } finally { amCompleted = true; } try { // make sure that the error thread exits // on Windows these threads sometimes get stuck and hang the execution // timeout and join later after destroying the process. errThread.Join(); outThread.Join(); errReader.Close(); inReader.Close(); } catch (Exception ie) { Log.Info("ShellExecutor: Interrupted while reading the error/out stream", ie); } catch (IOException ioe) { Log.Warn("Error while closing the error/out stream", ioe); } amProc.Destroy(); }
/// <exception cref="System.IO.IOException"/> public virtual GetDelegationTokenResponse GetDelegationToken(GetDelegationTokenRequest request) { UserGroupInformation ugi = UserGroupInformation.GetCurrentUser(); // Verify that the connection is kerberos authenticated if (!this.IsAllowedDelegationTokenOp()) { throw new IOException("Delegation Token can be issued only with kerberos authentication" ); } GetDelegationTokenResponse response = this.recordFactory.NewRecordInstance <GetDelegationTokenResponse >(); string user = ugi.GetUserName(); Text owner = new Text(user); Text realUser = null; if (ugi.GetRealUser() != null) { realUser = new Text(ugi.GetRealUser().GetUserName()); } MRDelegationTokenIdentifier tokenIdentifier = new MRDelegationTokenIdentifier(owner , new Text(request.GetRenewer()), realUser); Org.Apache.Hadoop.Security.Token.Token <MRDelegationTokenIdentifier> realJHSToken = new Org.Apache.Hadoop.Security.Token.Token <MRDelegationTokenIdentifier>(tokenIdentifier , this._enclosing.jhsDTSecretManager); Org.Apache.Hadoop.Yarn.Api.Records.Token mrDToken = Org.Apache.Hadoop.Yarn.Api.Records.Token .NewInstance(realJHSToken.GetIdentifier(), realJHSToken.GetKind().ToString(), realJHSToken .GetPassword(), realJHSToken.GetService().ToString()); response.SetDelegationToken(mrDToken); return(response); }
/// <exception cref="System.Exception"/> public LocatedBlocks Answer(InvocationOnMock arg0) { LocatedBlocks locatedBlocks = (LocatedBlocks)arg0.CallRealMethod(); foreach (LocatedBlock lb in locatedBlocks.GetLocatedBlocks()) { Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token = lb.GetBlockToken (); BlockTokenIdentifier id = lb.GetBlockToken().DecodeIdentifier(); id.SetExpiryDate(Time.Now() + 10); Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> newToken = new Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier>(id.GetBytes(), token.GetPassword(), token.GetKind(), token .GetService()); lb.SetBlockToken(newToken); } return(locatedBlocks); }
internal static bool CheckEqual(Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier > a, Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> b) { return(Arrays.Equals(a.GetIdentifier(), b.GetIdentifier()) && Arrays.Equals(a.GetPassword (), b.GetPassword()) && IsEqual(a.GetKind(), b.GetKind()) && IsEqual(a.GetService (), b.GetService())); }
public virtual void TestDelegationTokenOperations() { TimelineClient httpUserClient = KerberosTestUtils.DoAs(HttpUser + "/localhost", new _Callable_221(this)); UserGroupInformation httpUser = KerberosTestUtils.DoAs(HttpUser + "/localhost", new _Callable_228()); // Let HTTP user to get the delegation for itself Org.Apache.Hadoop.Security.Token.Token <TimelineDelegationTokenIdentifier> token = httpUserClient.GetDelegationToken(httpUser.GetShortUserName()); NUnit.Framework.Assert.IsNotNull(token); TimelineDelegationTokenIdentifier tDT = token.DecodeIdentifier(); NUnit.Framework.Assert.IsNotNull(tDT); NUnit.Framework.Assert.AreEqual(new Text(HttpUser), tDT.GetOwner()); // Renew token NUnit.Framework.Assert.IsFalse(token.GetService().ToString().IsEmpty()); // Renew the token from the token service address long renewTime1 = httpUserClient.RenewDelegationToken(token); Sharpen.Thread.Sleep(100); token.SetService(new Text()); NUnit.Framework.Assert.IsTrue(token.GetService().ToString().IsEmpty()); // If the token service address is not avaiable, it still can be renewed // from the configured address long renewTime2 = httpUserClient.RenewDelegationToken(token); NUnit.Framework.Assert.IsTrue(renewTime1 < renewTime2); // Cancel token NUnit.Framework.Assert.IsTrue(token.GetService().ToString().IsEmpty()); // If the token service address is not avaiable, it still can be canceled // from the configured address httpUserClient.CancelDelegationToken(token); // Renew should not be successful because the token is canceled try { httpUserClient.RenewDelegationToken(token); NUnit.Framework.Assert.Fail(); } catch (Exception e) { NUnit.Framework.Assert.IsTrue(e.Message.Contains("Renewal request for unknown token" )); } // Let HTTP user to get the delegation token for FOO user UserGroupInformation fooUgi = UserGroupInformation.CreateProxyUser(FooUser, httpUser ); TimelineClient fooUserClient = fooUgi.DoAs(new _PrivilegedExceptionAction_272(this )); token = fooUserClient.GetDelegationToken(httpUser.GetShortUserName()); NUnit.Framework.Assert.IsNotNull(token); tDT = token.DecodeIdentifier(); NUnit.Framework.Assert.IsNotNull(tDT); NUnit.Framework.Assert.AreEqual(new Text(FooUser), tDT.GetOwner()); NUnit.Framework.Assert.AreEqual(new Text(HttpUser), tDT.GetRealUser()); // Renew token as the renewer Org.Apache.Hadoop.Security.Token.Token <TimelineDelegationTokenIdentifier> tokenToRenew = token; renewTime1 = httpUserClient.RenewDelegationToken(tokenToRenew); renewTime2 = httpUserClient.RenewDelegationToken(tokenToRenew); NUnit.Framework.Assert.IsTrue(renewTime1 < renewTime2); // Cancel token NUnit.Framework.Assert.IsFalse(tokenToRenew.GetService().ToString().IsEmpty()); // Cancel the token from the token service address fooUserClient.CancelDelegationToken(tokenToRenew); // Renew should not be successful because the token is canceled try { httpUserClient.RenewDelegationToken(tokenToRenew); NUnit.Framework.Assert.Fail(); } catch (Exception e) { NUnit.Framework.Assert.IsTrue(e.Message.Contains("Renewal request for unknown token" )); } // Let HTTP user to get the delegation token for BAR user UserGroupInformation barUgi = UserGroupInformation.CreateProxyUser(BarUser, httpUser ); TimelineClient barUserClient = barUgi.DoAs(new _PrivilegedExceptionAction_309(this )); try { barUserClient.GetDelegationToken(httpUser.GetShortUserName()); NUnit.Framework.Assert.Fail(); } catch (Exception e) { NUnit.Framework.Assert.IsTrue(e.InnerException is AuthorizationException || e.InnerException is AuthenticationException); } }