public virtual void TestJobTokenRpc()
{
TaskUmbilicalProtocol mockTT = Org.Mockito.Mockito.Mock <TaskUmbilicalProtocol>();
Org.Mockito.Mockito.DoReturn(TaskUmbilicalProtocol.versionID).When(mockTT).GetProtocolVersion
(Matchers.AnyString(), Matchers.AnyLong());
Org.Mockito.Mockito.DoReturn(ProtocolSignature.GetProtocolSignature(mockTT, typeof(
TaskUmbilicalProtocol).FullName, TaskUmbilicalProtocol.versionID, 0)).When(mockTT
).GetProtocolSignature(Matchers.AnyString(), Matchers.AnyLong(), Matchers.AnyInt
());
JobTokenSecretManager sm = new JobTokenSecretManager();
Server server = new RPC.Builder(conf).SetProtocol(typeof(TaskUmbilicalProtocol)).
SetInstance(mockTT).SetBindAddress(Address).SetPort(0).SetNumHandlers(5).SetVerbose
(true).SetSecretManager(sm).Build();
server.Start();
UserGroupInformation current = UserGroupInformation.GetCurrentUser();
IPEndPoint addr = NetUtils.GetConnectAddress(server);
string jobId = current.GetUserName();
JobTokenIdentifier tokenId = new JobTokenIdentifier(new Text(jobId));
Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token
<JobTokenIdentifier>(tokenId, sm);
sm.AddTokenForJob(jobId, token);
SecurityUtil.SetTokenService(token, addr);
Log.Info("Service address for token is " + token.GetService());
current.AddToken(token);
current.DoAs(new _PrivilegedExceptionAction_110(addr, server));
}
public virtual void TestDelegationTokenRpc()
{
ClientProtocol mockNN = Org.Mockito.Mockito.Mock <ClientProtocol>();
FSNamesystem mockNameSys = Org.Mockito.Mockito.Mock <FSNamesystem>();
DelegationTokenSecretManager sm = new DelegationTokenSecretManager(DFSConfigKeys.
DfsNamenodeDelegationKeyUpdateIntervalDefault, DFSConfigKeys.DfsNamenodeDelegationKeyUpdateIntervalDefault
, DFSConfigKeys.DfsNamenodeDelegationTokenMaxLifetimeDefault, 3600000, mockNameSys
);
sm.StartThreads();
Org.Apache.Hadoop.Ipc.Server server = new RPC.Builder(conf).SetProtocol(typeof(ClientProtocol
)).SetInstance(mockNN).SetBindAddress(Address).SetPort(0).SetNumHandlers(5).SetVerbose
(true).SetSecretManager(sm).Build();
server.Start();
UserGroupInformation current = UserGroupInformation.GetCurrentUser();
IPEndPoint addr = NetUtils.GetConnectAddress(server);
string user = current.GetUserName();
Text owner = new Text(user);
DelegationTokenIdentifier dtId = new DelegationTokenIdentifier(owner, owner, null
);
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token
<DelegationTokenIdentifier>(dtId, sm);
SecurityUtil.SetTokenService(token, addr);
Log.Info("Service for token is " + token.GetService());
current.AddToken(token);
current.DoAs(new _PrivilegedExceptionAction_100(addr, server));
}
private void Compare(Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier>
expected, Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> actual)
{
NUnit.Framework.Assert.IsTrue(Arrays.Equals(expected.GetIdentifier(), actual.GetIdentifier
()));
NUnit.Framework.Assert.IsTrue(Arrays.Equals(expected.GetPassword(), actual.GetPassword
()));
NUnit.Framework.Assert.AreEqual(expected.GetKind(), actual.GetKind());
NUnit.Framework.Assert.AreEqual(expected.GetService(), actual.GetService());
}
/// <exception cref="System.IO.IOException"/>
public virtual Org.Apache.Hadoop.Security.Token.Token <object>[] AddDelegationTokens
(string renewer, Credentials credentials)
{
Org.Apache.Hadoop.Security.Token.Token <object>[] tokens = null;
Text dtService = GetDelegationTokenService();
Org.Apache.Hadoop.Security.Token.Token <object> token = credentials.GetToken(dtService
);
if (token == null)
{
Uri url = CreateURL(null, null, null, null);
DelegationTokenAuthenticatedURL authUrl = new DelegationTokenAuthenticatedURL(configurator
);
try
{
// 'actualUGI' is the UGI of the user creating the client
// It is possible that the creator of the KMSClientProvier
// calls this method on behalf of a proxyUser (the doAsUser).
// In which case this call has to be made as the proxy user.
UserGroupInformation currentUgi = UserGroupInformation.GetCurrentUser();
string doAsUser = (currentUgi.GetAuthenticationMethod() == UserGroupInformation.AuthenticationMethod
.Proxy) ? currentUgi.GetShortUserName() : null;
token = actualUgi.DoAs(new _PrivilegedExceptionAction_870(authUrl, url, renewer,
doAsUser));
// Not using the cached token here.. Creating a new token here
// everytime.
if (token != null)
{
credentials.AddToken(token.GetService(), token);
tokens = new Org.Apache.Hadoop.Security.Token.Token <object>[] { token };
}
else
{
throw new IOException("Got NULL as delegation token");
}
}
catch (Exception)
{
Thread.CurrentThread().Interrupt();
}
catch (Exception e)
{
throw new IOException(e);
}
}
return(tokens);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public override void CancelDelegationToken(Org.Apache.Hadoop.Security.Token.Token
<TimelineDelegationTokenIdentifier> timelineDT)
{
bool isTokenServiceAddrEmpty = timelineDT.GetService().ToString().IsEmpty();
string scheme = isTokenServiceAddrEmpty ? null : (YarnConfiguration.UseHttps(this
.GetConfig()) ? "https" : "http");
IPEndPoint address = isTokenServiceAddrEmpty ? null : SecurityUtil.GetTokenServiceAddr
(timelineDT);
PrivilegedExceptionAction <Void> cancelDTAction = new _PrivilegedExceptionAction_415
(this, timelineDT, isTokenServiceAddrEmpty, scheme, address);
// If the timeline DT to cancel is different than cached, replace it.
// Token to set every time for retry, because when exception happens,
// DelegationTokenAuthenticatedURL will reset it to null;
// If the token service address is not available, fall back to use
// the configured service address.
OperateDelegationToken(cancelDTAction);
}
public virtual void TestGetDelegationTokensWithCredentials()
{
Credentials credentials = new Credentials();
IList <Org.Apache.Hadoop.Security.Token.Token <object> > delTokens = Arrays.AsList(fsView
.AddDelegationTokens("sanjay", credentials));
int expectedTokenCount = GetExpectedDelegationTokenCountWithCredentials();
Assert.Equal(expectedTokenCount, delTokens.Count);
Credentials newCredentials = new Credentials();
for (int i = 0; i < expectedTokenCount / 2; i++)
{
Org.Apache.Hadoop.Security.Token.Token <object> token = delTokens[i];
newCredentials.AddToken(token.GetService(), token);
}
IList <Org.Apache.Hadoop.Security.Token.Token <object> > delTokens2 = Arrays.AsList(
fsView.AddDelegationTokens("sanjay", newCredentials));
Assert.Equal((expectedTokenCount + 1) / 2, delTokens2.Count);
}
/// <exception cref="System.IO.IOException"/>
private void RecordJobShuffleInfo(JobID jobId, string user, Org.Apache.Hadoop.Security.Token.Token
<JobTokenIdentifier> jobToken)
{
if (stateDb != null)
{
SecurityProtos.TokenProto tokenProto = ((SecurityProtos.TokenProto)SecurityProtos.TokenProto
.NewBuilder().SetIdentifier(ByteString.CopyFrom(jobToken.GetIdentifier())).SetPassword
(ByteString.CopyFrom(jobToken.GetPassword())).SetKind(jobToken.GetKind().ToString
()).SetService(jobToken.GetService().ToString()).Build());
ShuffleHandlerRecoveryProtos.JobShuffleInfoProto proto = ((ShuffleHandlerRecoveryProtos.JobShuffleInfoProto
)ShuffleHandlerRecoveryProtos.JobShuffleInfoProto.NewBuilder().SetUser(user).SetJobToken
(tokenProto).Build());
try
{
stateDb.Put(JniDBFactory.Bytes(jobId.ToString()), proto.ToByteArray());
}
catch (DBException e)
{
throw new IOException("Error storing " + jobId, e);
}
}
AddJobToken(jobId, user, jobToken);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public virtual void LaunchAM(ApplicationAttemptId attemptId)
{
Credentials credentials = new Credentials();
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> token = rmClient.GetAMRMToken
(attemptId.GetApplicationId());
// Service will be empty but that's okay, we are just passing down only
// AMRMToken down to the real AM which eventually sets the correct
// service-address.
credentials.AddToken(token.GetService(), token);
FilePath tokenFile = FilePath.CreateTempFile("unmanagedAMRMToken", string.Empty,
new FilePath(Runtime.GetProperty("user.dir")));
try
{
FileUtil.Chmod(tokenFile.GetAbsolutePath(), "600");
}
catch (Exception ex)
{
throw new RuntimeException(ex);
}
tokenFile.DeleteOnExit();
DataOutputStream os = new DataOutputStream(new FileOutputStream(tokenFile, true));
credentials.WriteTokenStorageToStream(os);
os.Close();
IDictionary <string, string> env = Sharpen.Runtime.GetEnv();
AList <string> envAMList = new AList <string>();
bool setClasspath = false;
foreach (KeyValuePair <string, string> entry in env)
{
string key = entry.Key;
string value = entry.Value;
if (key.Equals("CLASSPATH"))
{
setClasspath = true;
if (classpath != null)
{
value = value + FilePath.pathSeparator + classpath;
}
}
envAMList.AddItem(key + "=" + value);
}
if (!setClasspath && classpath != null)
{
envAMList.AddItem("CLASSPATH=" + classpath);
}
ContainerId containerId = ContainerId.NewContainerId(attemptId, 0);
string hostname = Sharpen.Runtime.GetLocalHost().GetHostName();
envAMList.AddItem(ApplicationConstants.Environment.ContainerId.ToString() + "=" +
containerId);
envAMList.AddItem(ApplicationConstants.Environment.NmHost.ToString() + "=" + hostname
);
envAMList.AddItem(ApplicationConstants.Environment.NmHttpPort.ToString() + "=0");
envAMList.AddItem(ApplicationConstants.Environment.NmPort.ToString() + "=0");
envAMList.AddItem(ApplicationConstants.Environment.LocalDirs.ToString() + "= /tmp"
);
envAMList.AddItem(ApplicationConstants.AppSubmitTimeEnv + "=" + Runtime.CurrentTimeMillis
());
envAMList.AddItem(ApplicationConstants.ContainerTokenFileEnvName + "=" + tokenFile
.GetAbsolutePath());
string[] envAM = new string[envAMList.Count];
SystemProcess amProc = Runtime.GetRuntime().Exec(amCmd, Sharpen.Collections.ToArray
(envAMList, envAM));
BufferedReader errReader = new BufferedReader(new InputStreamReader(amProc.GetErrorStream
(), Sharpen.Extensions.GetEncoding("UTF-8")));
BufferedReader inReader = new BufferedReader(new InputStreamReader(amProc.GetInputStream
(), Sharpen.Extensions.GetEncoding("UTF-8")));
// read error and input streams as this would free up the buffers
// free the error stream buffer
Sharpen.Thread errThread = new _Thread_244(errReader);
Sharpen.Thread outThread = new _Thread_258(inReader);
try
{
errThread.Start();
outThread.Start();
}
catch (InvalidOperationException)
{
}
// wait for the process to finish and check the exit code
try
{
int exitCode = amProc.WaitFor();
Log.Info("AM process exited with value: " + exitCode);
}
catch (Exception e)
{
Sharpen.Runtime.PrintStackTrace(e);
}
finally
{
amCompleted = true;
}
try
{
// make sure that the error thread exits
// on Windows these threads sometimes get stuck and hang the execution
// timeout and join later after destroying the process.
errThread.Join();
outThread.Join();
errReader.Close();
inReader.Close();
}
catch (Exception ie)
{
Log.Info("ShellExecutor: Interrupted while reading the error/out stream", ie);
}
catch (IOException ioe)
{
Log.Warn("Error while closing the error/out stream", ioe);
}
amProc.Destroy();
}
/// <exception cref="System.IO.IOException"/>
public virtual GetDelegationTokenResponse GetDelegationToken(GetDelegationTokenRequest
request)
{
UserGroupInformation ugi = UserGroupInformation.GetCurrentUser();
// Verify that the connection is kerberos authenticated
if (!this.IsAllowedDelegationTokenOp())
{
throw new IOException("Delegation Token can be issued only with kerberos authentication"
);
}
GetDelegationTokenResponse response = this.recordFactory.NewRecordInstance <GetDelegationTokenResponse
>();
string user = ugi.GetUserName();
Text owner = new Text(user);
Text realUser = null;
if (ugi.GetRealUser() != null)
{
realUser = new Text(ugi.GetRealUser().GetUserName());
}
MRDelegationTokenIdentifier tokenIdentifier = new MRDelegationTokenIdentifier(owner
, new Text(request.GetRenewer()), realUser);
Org.Apache.Hadoop.Security.Token.Token <MRDelegationTokenIdentifier> realJHSToken =
new Org.Apache.Hadoop.Security.Token.Token <MRDelegationTokenIdentifier>(tokenIdentifier
, this._enclosing.jhsDTSecretManager);
Org.Apache.Hadoop.Yarn.Api.Records.Token mrDToken = Org.Apache.Hadoop.Yarn.Api.Records.Token
.NewInstance(realJHSToken.GetIdentifier(), realJHSToken.GetKind().ToString(), realJHSToken
.GetPassword(), realJHSToken.GetService().ToString());
response.SetDelegationToken(mrDToken);
return(response);
}
/// <exception cref="System.Exception"/>
public LocatedBlocks Answer(InvocationOnMock arg0)
{
LocatedBlocks locatedBlocks = (LocatedBlocks)arg0.CallRealMethod();
foreach (LocatedBlock lb in locatedBlocks.GetLocatedBlocks())
{
Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token = lb.GetBlockToken
();
BlockTokenIdentifier id = lb.GetBlockToken().DecodeIdentifier();
id.SetExpiryDate(Time.Now() + 10);
Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> newToken = new Org.Apache.Hadoop.Security.Token.Token
<BlockTokenIdentifier>(id.GetBytes(), token.GetPassword(), token.GetKind(), token
.GetService());
lb.SetBlockToken(newToken);
}
return(locatedBlocks);
}
internal static bool CheckEqual(Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier
> a, Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> b)
{
return(Arrays.Equals(a.GetIdentifier(), b.GetIdentifier()) && Arrays.Equals(a.GetPassword
(), b.GetPassword()) && IsEqual(a.GetKind(), b.GetKind()) && IsEqual(a.GetService
(), b.GetService()));
}
public virtual void TestDelegationTokenOperations()
{
TimelineClient httpUserClient = KerberosTestUtils.DoAs(HttpUser + "/localhost", new
_Callable_221(this));
UserGroupInformation httpUser = KerberosTestUtils.DoAs(HttpUser + "/localhost", new
_Callable_228());
// Let HTTP user to get the delegation for itself
Org.Apache.Hadoop.Security.Token.Token <TimelineDelegationTokenIdentifier> token =
httpUserClient.GetDelegationToken(httpUser.GetShortUserName());
NUnit.Framework.Assert.IsNotNull(token);
TimelineDelegationTokenIdentifier tDT = token.DecodeIdentifier();
NUnit.Framework.Assert.IsNotNull(tDT);
NUnit.Framework.Assert.AreEqual(new Text(HttpUser), tDT.GetOwner());
// Renew token
NUnit.Framework.Assert.IsFalse(token.GetService().ToString().IsEmpty());
// Renew the token from the token service address
long renewTime1 = httpUserClient.RenewDelegationToken(token);
Sharpen.Thread.Sleep(100);
token.SetService(new Text());
NUnit.Framework.Assert.IsTrue(token.GetService().ToString().IsEmpty());
// If the token service address is not avaiable, it still can be renewed
// from the configured address
long renewTime2 = httpUserClient.RenewDelegationToken(token);
NUnit.Framework.Assert.IsTrue(renewTime1 < renewTime2);
// Cancel token
NUnit.Framework.Assert.IsTrue(token.GetService().ToString().IsEmpty());
// If the token service address is not avaiable, it still can be canceled
// from the configured address
httpUserClient.CancelDelegationToken(token);
// Renew should not be successful because the token is canceled
try
{
httpUserClient.RenewDelegationToken(token);
NUnit.Framework.Assert.Fail();
}
catch (Exception e)
{
NUnit.Framework.Assert.IsTrue(e.Message.Contains("Renewal request for unknown token"
));
}
// Let HTTP user to get the delegation token for FOO user
UserGroupInformation fooUgi = UserGroupInformation.CreateProxyUser(FooUser, httpUser
);
TimelineClient fooUserClient = fooUgi.DoAs(new _PrivilegedExceptionAction_272(this
));
token = fooUserClient.GetDelegationToken(httpUser.GetShortUserName());
NUnit.Framework.Assert.IsNotNull(token);
tDT = token.DecodeIdentifier();
NUnit.Framework.Assert.IsNotNull(tDT);
NUnit.Framework.Assert.AreEqual(new Text(FooUser), tDT.GetOwner());
NUnit.Framework.Assert.AreEqual(new Text(HttpUser), tDT.GetRealUser());
// Renew token as the renewer
Org.Apache.Hadoop.Security.Token.Token <TimelineDelegationTokenIdentifier> tokenToRenew
= token;
renewTime1 = httpUserClient.RenewDelegationToken(tokenToRenew);
renewTime2 = httpUserClient.RenewDelegationToken(tokenToRenew);
NUnit.Framework.Assert.IsTrue(renewTime1 < renewTime2);
// Cancel token
NUnit.Framework.Assert.IsFalse(tokenToRenew.GetService().ToString().IsEmpty());
// Cancel the token from the token service address
fooUserClient.CancelDelegationToken(tokenToRenew);
// Renew should not be successful because the token is canceled
try
{
httpUserClient.RenewDelegationToken(tokenToRenew);
NUnit.Framework.Assert.Fail();
}
catch (Exception e)
{
NUnit.Framework.Assert.IsTrue(e.Message.Contains("Renewal request for unknown token"
));
}
// Let HTTP user to get the delegation token for BAR user
UserGroupInformation barUgi = UserGroupInformation.CreateProxyUser(BarUser, httpUser
);
TimelineClient barUserClient = barUgi.DoAs(new _PrivilegedExceptionAction_309(this
));
try
{
barUserClient.GetDelegationToken(httpUser.GetShortUserName());
NUnit.Framework.Assert.Fail();
}
catch (Exception e)
{
NUnit.Framework.Assert.IsTrue(e.InnerException is AuthorizationException || e.InnerException
is AuthenticationException);
}
}
