/// <exception cref="System.Exception"/>
public virtual void TestGetCreds <T>()
where T : TokenIdentifier
{
// from Mockito mocks
UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser("someone");
Text service = new Text("service");
Org.Apache.Hadoop.Security.Token.Token <T> t1 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token
>();
Org.Mockito.Mockito.When(t1.GetService()).ThenReturn(service);
Org.Apache.Hadoop.Security.Token.Token <T> t2 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token
>();
Org.Mockito.Mockito.When(t2.GetService()).ThenReturn(new Text("service2"));
Org.Apache.Hadoop.Security.Token.Token <T> t3 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token
>();
Org.Mockito.Mockito.When(t3.GetService()).ThenReturn(service);
// add token to ugi
ugi.AddToken(t1);
ugi.AddToken(t2);
CheckTokens(ugi, t1, t2);
Credentials creds = ugi.GetCredentials();
creds.AddToken(t3.GetService(), t3);
NUnit.Framework.Assert.AreSame(t3, creds.GetToken(service));
// check that ugi wasn't modified
CheckTokens(ugi, t1, t2);
}
/// <exception cref="System.Exception"/>
public virtual void TestUGITokens <T>()
where T : TokenIdentifier
{
// from Mockito mocks
UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting("TheDoctor",
new string[] { "TheTARDIS" });
Org.Apache.Hadoop.Security.Token.Token <T> t1 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token
>();
Org.Mockito.Mockito.When(t1.GetService()).ThenReturn(new Text("t1"));
Org.Apache.Hadoop.Security.Token.Token <T> t2 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token
>();
Org.Mockito.Mockito.When(t2.GetService()).ThenReturn(new Text("t2"));
Credentials creds = new Credentials();
byte[] secretKey = new byte[] { };
Text secretName = new Text("shhh");
creds.AddSecretKey(secretName, secretKey);
ugi.AddToken(t1);
ugi.AddToken(t2);
ugi.AddCredentials(creds);
ICollection <Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> > z = ugi.GetTokens
();
Assert.True(z.Contains(t1));
Assert.True(z.Contains(t2));
Assert.Equal(2, z.Count);
Credentials ugiCreds = ugi.GetCredentials();
NUnit.Framework.Assert.AreSame(secretKey, ugiCreds.GetSecretKey(secretName));
Assert.Equal(1, ugiCreds.NumberOfSecretKeys());
try
{
z.Remove(t1);
NUnit.Framework.Assert.Fail("Shouldn't be able to modify token collection from UGI"
);
}
catch (NotSupportedException)
{
}
// Can't modify tokens
// ensure that the tokens are passed through doAs
ICollection <Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> > otherSet = ugi
.DoAs(new _PrivilegedExceptionAction_612());
Assert.True(otherSet.Contains(t1));
Assert.True(otherSet.Contains(t2));
}
public virtual void TestTokenBySuperUser()
{
TestSaslRPC.TestTokenSecretManager sm = new TestSaslRPC.TestTokenSecretManager();
Configuration newConf = new Configuration(masterConf);
SecurityUtil.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Kerberos
, newConf);
UserGroupInformation.SetConfiguration(newConf);
Server server = new RPC.Builder(newConf).SetProtocol(typeof(TestDoAsEffectiveUser.TestProtocol
)).SetInstance(new TestDoAsEffectiveUser.TestImpl(this)).SetBindAddress(Address)
.SetPort(0).SetNumHandlers(5).SetVerbose(true).SetSecretManager(sm).Build();
server.Start();
UserGroupInformation current = UserGroupInformation.CreateUserForTesting(RealUserName
, GroupNames);
RefreshConf(newConf);
IPEndPoint addr = NetUtils.GetConnectAddress(server);
TestSaslRPC.TestTokenIdentifier tokenId = new TestSaslRPC.TestTokenIdentifier(new
Org.Apache.Hadoop.IO.Text(current.GetUserName()), new Org.Apache.Hadoop.IO.Text(
"SomeSuperUser"));
Org.Apache.Hadoop.Security.Token.Token <TestSaslRPC.TestTokenIdentifier> token = new
Org.Apache.Hadoop.Security.Token.Token <TestSaslRPC.TestTokenIdentifier>(tokenId,
sm);
SecurityUtil.SetTokenService(token, addr);
current.AddToken(token);
string retVal = current.DoAs(new _PrivilegedExceptionAction_509(this, addr, newConf
, server));
string expected = RealUserName + " (auth:TOKEN) via SomeSuperUser (auth:SIMPLE)";
Assert.Equal(retVal + "!=" + expected, expected, retVal);
}
/// <exception cref="System.Exception"/>
public Void Run()
{
DelegationTokenAuthenticatedURL.Token token = new DelegationTokenAuthenticatedURL.Token
();
DelegationTokenAuthenticatedURL aUrl = new DelegationTokenAuthenticatedURL();
HttpURLConnection conn = aUrl.OpenConnection(url, token, TestWebDelegationToken.OkUser
);
Assert.Equal(HttpURLConnection.HttpOk, conn.GetResponseCode());
IList <string> ret = IOUtils.ReadLines(conn.GetInputStream());
Assert.Equal(1, ret.Count);
Assert.Equal(TestWebDelegationToken.OkUser, ret[0]);
conn = aUrl.OpenConnection(url, token, TestWebDelegationToken.FailUser);
Assert.Equal(HttpURLConnection.HttpForbidden, conn.GetResponseCode
());
aUrl.GetDelegationToken(url, token, TestWebDelegationToken.FooUser);
UserGroupInformation ugi = UserGroupInformation.GetCurrentUser();
ugi.AddToken(token.GetDelegationToken());
token = new DelegationTokenAuthenticatedURL.Token();
conn = aUrl.OpenConnection(url, token, TestWebDelegationToken.OkUser);
Assert.Equal(HttpURLConnection.HttpOk, conn.GetResponseCode());
ret = IOUtils.ReadLines(conn.GetInputStream());
Assert.Equal(1, ret.Count);
Assert.Equal(TestWebDelegationToken.FooUser, ret[0]);
return(null);
}
/// <exception cref="System.IO.IOException"/>
internal static ClientDatanodeProtocolPB CreateClientDatanodeProtocolProxy(DatanodeID
datanodeid, Configuration conf, int socketTimeout, bool connectToDnViaHostname,
LocatedBlock locatedBlock)
{
string dnAddr = datanodeid.GetIpcAddr(connectToDnViaHostname);
IPEndPoint addr = NetUtils.CreateSocketAddr(dnAddr);
if (Log.IsDebugEnabled())
{
Log.Debug("Connecting to datanode " + dnAddr + " addr=" + addr);
}
// Since we're creating a new UserGroupInformation here, we know that no
// future RPC proxies will be able to re-use the same connection. And
// usages of this proxy tend to be one-off calls.
//
// This is a temporary fix: callers should really achieve this by using
// RPC.stopProxy() on the resulting object, but this is currently not
// working in trunk. See the discussion on HDFS-1965.
Configuration confWithNoIpcIdle = new Configuration(conf);
confWithNoIpcIdle.SetInt(CommonConfigurationKeysPublic.IpcClientConnectionMaxidletimeKey
, 0);
UserGroupInformation ticket = UserGroupInformation.CreateRemoteUser(locatedBlock.
GetBlock().GetLocalBlock().ToString());
ticket.AddToken(locatedBlock.GetBlockToken());
return(CreateClientDatanodeProtocolProxy(addr, ticket, confWithNoIpcIdle, NetUtils
.GetDefaultSocketFactory(conf), socketTimeout));
}
/// <summary>
/// Locate a delegation token associated with the given HA cluster URI, and if
/// one is found, clone it to also represent the underlying namenode address.
/// </summary>
/// <param name="ugi">the UGI to modify</param>
/// <param name="haUri">the logical URI for the cluster</param>
/// <param name="nnAddrs">
/// collection of NNs in the cluster to which the token
/// applies
/// </param>
public static void CloneDelegationTokenForLogicalUri(UserGroupInformation ugi, URI
haUri, ICollection <IPEndPoint> nnAddrs)
{
// this cloning logic is only used by hdfs
Text haService = Org.Apache.Hadoop.Hdfs.HAUtil.BuildTokenServiceForLogicalUri(haUri
, HdfsConstants.HdfsUriScheme);
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> haToken = tokenSelector
.SelectToken(haService, ugi.GetTokens());
if (haToken != null)
{
foreach (IPEndPoint singleNNAddr in nnAddrs)
{
// this is a minor hack to prevent physical HA tokens from being
// exposed to the user via UGI.getCredentials(), otherwise these
// cloned tokens may be inadvertently propagated to jobs
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> specificToken =
new Token.PrivateToken <DelegationTokenIdentifier>(haToken);
SecurityUtil.SetTokenService(specificToken, singleNNAddr);
Text alias = new Text(BuildTokenServicePrefixForLogicalUri(HdfsConstants.HdfsUriScheme
) + "//" + specificToken.GetService());
ugi.AddToken(alias, specificToken);
Log.Debug("Mapped HA service delegation token for logical URI " + haUri + " to namenode "
+ singleNNAddr);
}
}
else
{
Log.Debug("No HA service delegation token found for logical URI " + haUri);
}
}
/// <exception cref="System.IO.IOException"/>
private static UserGroupInformation GetTokenUGI(ServletContext context, HttpServletRequest
request, string tokenString, Configuration conf)
{
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token
<DelegationTokenIdentifier>();
token.DecodeFromUrlString(tokenString);
IPEndPoint serviceAddress = GetNNServiceAddress(context, request);
if (serviceAddress != null)
{
SecurityUtil.SetTokenService(token, serviceAddress);
token.SetKind(DelegationTokenIdentifier.HdfsDelegationKind);
}
ByteArrayInputStream buf = new ByteArrayInputStream(token.GetIdentifier());
DataInputStream @in = new DataInputStream(buf);
DelegationTokenIdentifier id = new DelegationTokenIdentifier();
id.ReadFields(@in);
if (context != null)
{
NameNode nn = NameNodeHttpServer.GetNameNodeFromContext(context);
if (nn != null)
{
// Verify the token.
nn.GetNamesystem().VerifyToken(id, token.GetPassword());
}
}
UserGroupInformation ugi = id.GetUser();
ugi.AddToken(token);
return(ugi);
}
public virtual void TestCacheForUgi <T>()
where T : TokenIdentifier
{
Configuration conf = new Configuration();
conf.Set("fs.cachedfile.impl", FileSystem.GetFileSystemClass("file", null).FullName
);
UserGroupInformation ugiA = UserGroupInformation.CreateRemoteUser("foo");
UserGroupInformation ugiB = UserGroupInformation.CreateRemoteUser("bar");
FileSystem fsA = ugiA.DoAs(new _PrivilegedExceptionAction_171(conf));
FileSystem fsA1 = ugiA.DoAs(new _PrivilegedExceptionAction_177(conf));
//Since the UGIs are the same, we should have the same filesystem for both
NUnit.Framework.Assert.AreSame(fsA, fsA1);
FileSystem fsB = ugiB.DoAs(new _PrivilegedExceptionAction_186(conf));
//Since the UGIs are different, we should end up with different filesystems
//corresponding to the two UGIs
NUnit.Framework.Assert.AreNotSame(fsA, fsB);
Org.Apache.Hadoop.Security.Token.Token <T> t1 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token
>();
UserGroupInformation ugiA2 = UserGroupInformation.CreateRemoteUser("foo");
fsA = ugiA2.DoAs(new _PrivilegedExceptionAction_199(conf));
// Although the users in the UGI are same, they have different subjects
// and so are different.
NUnit.Framework.Assert.AreNotSame(fsA, fsA1);
ugiA.AddToken(t1);
fsA = ugiA.DoAs(new _PrivilegedExceptionAction_211(conf));
// Make sure that different UGI's with the same subject lead to the same
// file system.
NUnit.Framework.Assert.AreSame(fsA, fsA1);
}
public virtual void TestJobTokenRpc()
{
TaskUmbilicalProtocol mockTT = Org.Mockito.Mockito.Mock <TaskUmbilicalProtocol>();
Org.Mockito.Mockito.DoReturn(TaskUmbilicalProtocol.versionID).When(mockTT).GetProtocolVersion
(Matchers.AnyString(), Matchers.AnyLong());
Org.Mockito.Mockito.DoReturn(ProtocolSignature.GetProtocolSignature(mockTT, typeof(
TaskUmbilicalProtocol).FullName, TaskUmbilicalProtocol.versionID, 0)).When(mockTT
).GetProtocolSignature(Matchers.AnyString(), Matchers.AnyLong(), Matchers.AnyInt
());
JobTokenSecretManager sm = new JobTokenSecretManager();
Server server = new RPC.Builder(conf).SetProtocol(typeof(TaskUmbilicalProtocol)).
SetInstance(mockTT).SetBindAddress(Address).SetPort(0).SetNumHandlers(5).SetVerbose
(true).SetSecretManager(sm).Build();
server.Start();
UserGroupInformation current = UserGroupInformation.GetCurrentUser();
IPEndPoint addr = NetUtils.GetConnectAddress(server);
string jobId = current.GetUserName();
JobTokenIdentifier tokenId = new JobTokenIdentifier(new Text(jobId));
Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token
<JobTokenIdentifier>(tokenId, sm);
sm.AddTokenForJob(jobId, token);
SecurityUtil.SetTokenService(token, addr);
Log.Info("Service address for token is " + token.GetService());
current.AddToken(token);
current.DoAs(new _PrivilegedExceptionAction_110(addr, server));
}
public virtual void TestDelegationTokenRpc()
{
ClientProtocol mockNN = Org.Mockito.Mockito.Mock <ClientProtocol>();
FSNamesystem mockNameSys = Org.Mockito.Mockito.Mock <FSNamesystem>();
DelegationTokenSecretManager sm = new DelegationTokenSecretManager(DFSConfigKeys.
DfsNamenodeDelegationKeyUpdateIntervalDefault, DFSConfigKeys.DfsNamenodeDelegationKeyUpdateIntervalDefault
, DFSConfigKeys.DfsNamenodeDelegationTokenMaxLifetimeDefault, 3600000, mockNameSys
);
sm.StartThreads();
Org.Apache.Hadoop.Ipc.Server server = new RPC.Builder(conf).SetProtocol(typeof(ClientProtocol
)).SetInstance(mockNN).SetBindAddress(Address).SetPort(0).SetNumHandlers(5).SetVerbose
(true).SetSecretManager(sm).Build();
server.Start();
UserGroupInformation current = UserGroupInformation.GetCurrentUser();
IPEndPoint addr = NetUtils.GetConnectAddress(server);
string user = current.GetUserName();
Text owner = new Text(user);
DelegationTokenIdentifier dtId = new DelegationTokenIdentifier(owner, owner, null
);
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token
<DelegationTokenIdentifier>(dtId, sm);
SecurityUtil.SetTokenService(token, addr);
Log.Info("Service for token is " + token.GetService());
current.AddToken(token);
current.DoAs(new _PrivilegedExceptionAction_100(addr, server));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public static void StartContainer(NodeManager nm, ContainerId cId, FileContext localFS
, FilePath scriptFileDir, FilePath processStartFile)
{
FilePath scriptFile = CreateUnhaltingScriptFile(cId, scriptFileDir, processStartFile
);
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
NodeId nodeId = BuilderUtils.NewNodeId(Sharpen.Extensions.GetAddressByName("localhost"
).ToString(), 12345);
URL localResourceUri = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new
Path(scriptFile.GetAbsolutePath())));
LocalResource localResource = recordFactory.NewRecordInstance <LocalResource>();
localResource.SetResource(localResourceUri);
localResource.SetSize(-1);
localResource.SetVisibility(LocalResourceVisibility.Application);
localResource.SetType(LocalResourceType.File);
localResource.SetTimestamp(scriptFile.LastModified());
string destinationFile = "dest_file";
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
localResources[destinationFile] = localResource;
containerLaunchContext.SetLocalResources(localResources);
IList <string> commands = Arrays.AsList(Shell.GetRunScriptCommand(scriptFile));
containerLaunchContext.SetCommands(commands);
IPEndPoint containerManagerBindAddress = NetUtils.CreateSocketAddrForHost("127.0.0.1"
, 12345);
UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(cId.ToString
());
Org.Apache.Hadoop.Security.Token.Token <NMTokenIdentifier> nmToken = ConverterUtils
.ConvertFromYarn(nm.GetNMContext().GetNMTokenSecretManager().CreateNMToken(cId.GetApplicationAttemptId
(), nodeId, user), containerManagerBindAddress);
currentUser.AddToken(nmToken);
ContainerManagementProtocol containerManager = currentUser.DoAs(new _PrivilegedAction_229
());
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, TestContainerManager.CreateContainerToken(cId, 0, nodeId, user, nm.GetNMContext
().GetContainerTokenSecretManager()));
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
containerManager.StartContainers(allRequests);
IList <ContainerId> containerIds = new AList <ContainerId>();
containerIds.AddItem(cId);
GetContainerStatusesRequest request = GetContainerStatusesRequest.NewInstance(containerIds
);
ContainerStatus containerStatus = containerManager.GetContainerStatuses(request).
GetContainerStatuses()[0];
NUnit.Framework.Assert.AreEqual(ContainerState.Running, containerStatus.GetState(
));
}
/// <exception cref="System.IO.IOException"/>
private void InjectToken()
{
if (UserGroupInformation.IsSecurityEnabled())
{
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = @params
.DelegationToken();
token.SetKind(DelegationTokenIdentifier.HdfsDelegationKind);
ugi.AddToken(token);
}
}
/// <exception cref="System.IO.IOException"/>
private void UpdateAMRMToken(Token token)
{
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> amrmToken = new Org.Apache.Hadoop.Security.Token.Token
<AMRMTokenIdentifier>(((byte[])token.GetIdentifier().Array()), ((byte[])token.GetPassword
().Array()), new Text(token.GetKind()), new Text(token.GetService()));
UserGroupInformation currentUGI = UserGroupInformation.GetCurrentUser();
currentUGI.AddToken(amrmToken);
amrmToken.SetService(ClientRMProxy.GetAMRMTokenService(GetConfig()));
}
/// <exception cref="System.Exception"/>
public virtual void TestHAUtilClonesDelegationTokens()
{
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = GetDelegationToken
(fs, "JobTracker");
UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser("test");
URI haUri = new URI("hdfs://my-ha-uri/");
token.SetService(HAUtil.BuildTokenServiceForLogicalUri(haUri, HdfsConstants.HdfsUriScheme
));
ugi.AddToken(token);
ICollection <IPEndPoint> nnAddrs = new HashSet <IPEndPoint>();
nnAddrs.AddItem(new IPEndPoint("localhost", nn0.GetNameNodeAddress().Port));
nnAddrs.AddItem(new IPEndPoint("localhost", nn1.GetNameNodeAddress().Port));
HAUtil.CloneDelegationTokenForLogicalUri(ugi, haUri, nnAddrs);
ICollection <Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> > tokens = ugi
.GetTokens();
NUnit.Framework.Assert.AreEqual(3, tokens.Count);
Log.Info("Tokens:\n" + Joiner.On("\n").Join(tokens));
DelegationTokenSelector dts = new DelegationTokenSelector();
// check that the token selected for one of the physical IPC addresses
// matches the one we received
foreach (IPEndPoint addr in nnAddrs)
{
Text ipcDtService = SecurityUtil.BuildTokenService(addr);
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token2 = dts.SelectToken
(ipcDtService, ugi.GetTokens());
NUnit.Framework.Assert.IsNotNull(token2);
Assert.AssertArrayEquals(token.GetIdentifier(), token2.GetIdentifier());
Assert.AssertArrayEquals(token.GetPassword(), token2.GetPassword());
}
// switch to host-based tokens, shouldn't match existing tokens
SecurityUtilTestHelper.SetTokenServiceUseIp(false);
foreach (IPEndPoint addr_1 in nnAddrs)
{
Text ipcDtService = SecurityUtil.BuildTokenService(addr_1);
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token2 = dts.SelectToken
(ipcDtService, ugi.GetTokens());
NUnit.Framework.Assert.IsNull(token2);
}
// reclone the tokens, and see if they match now
HAUtil.CloneDelegationTokenForLogicalUri(ugi, haUri, nnAddrs);
foreach (IPEndPoint addr_2 in nnAddrs)
{
Text ipcDtService = SecurityUtil.BuildTokenService(addr_2);
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token2 = dts.SelectToken
(ipcDtService, ugi.GetTokens());
NUnit.Framework.Assert.IsNotNull(token2);
Assert.AssertArrayEquals(token.GetIdentifier(), token2.GetIdentifier());
Assert.AssertArrayEquals(token.GetPassword(), token2.GetPassword());
}
}
public virtual void TestValidateResourceBlacklistRequest()
{
TestAMAuthorization.MyContainerManager containerManager = new TestAMAuthorization.MyContainerManager
();
TestAMAuthorization.MockRMWithAMS rm = new TestAMAuthorization.MockRMWithAMS(new
YarnConfiguration(), containerManager);
rm.Start();
MockNM nm1 = rm.RegisterNode("localhost:1234", 5120);
IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
, string>(2);
acls[ApplicationAccessType.ViewApp] = "*";
RMApp app = rm.SubmitApp(1024, "appname", "appuser", acls);
nm1.NodeHeartbeat(true);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
ApplicationAttemptId applicationAttemptId = attempt.GetAppAttemptId();
WaitForLaunchedState(attempt);
// Create a client to the RM.
Configuration conf = rm.GetConfig();
YarnRPC rpc = YarnRPC.Create(conf);
UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(applicationAttemptId
.ToString());
Credentials credentials = containerManager.GetContainerCredentials();
IPEndPoint rmBindAddress = rm.GetApplicationMasterService().GetBindAddress();
Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> amRMToken = TestAMAuthorization.MockRMWithAMS
.SetupAndReturnAMRMToken(rmBindAddress, credentials.GetAllTokens());
currentUser.AddToken(amRMToken);
ApplicationMasterProtocol client = currentUser.DoAs(new _PrivilegedAction_626(rpc
, rmBindAddress, conf));
RegisterApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<RegisterApplicationMasterRequest>();
client.RegisterApplicationMaster(request);
ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.NewInstance(
Sharpen.Collections.SingletonList(ResourceRequest.Any), null);
AllocateRequest allocateRequest = AllocateRequest.NewInstance(0, 0.0f, null, null
, blacklistRequest);
bool error = false;
try
{
client.Allocate(allocateRequest);
}
catch (InvalidResourceBlacklistRequestException)
{
error = true;
}
rm.Stop();
NUnit.Framework.Assert.IsTrue("Didn't not catch InvalidResourceBlacklistRequestException"
, error);
}
/// <exception cref="System.Exception"/>
public virtual void TestAddNamedToken <T>()
where T : TokenIdentifier
{
// from Mockito mocks
UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser("someone");
Org.Apache.Hadoop.Security.Token.Token <T> t1 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token
>();
Text service1 = new Text("t1");
Text service2 = new Text("t2");
Org.Mockito.Mockito.When(t1.GetService()).ThenReturn(service1);
// add token
ugi.AddToken(service1, t1);
NUnit.Framework.Assert.AreSame(t1, ugi.GetCredentials().GetToken(service1));
// add token with another name
ugi.AddToken(service2, t1);
NUnit.Framework.Assert.AreSame(t1, ugi.GetCredentials().GetToken(service1));
NUnit.Framework.Assert.AreSame(t1, ugi.GetCredentials().GetToken(service2));
}
public virtual void TestAuthorizedAccess()
{
TestAMAuthorization.MyContainerManager containerManager = new TestAMAuthorization.MyContainerManager
();
rm = new TestAMAuthorization.MockRMWithAMS(conf, containerManager);
rm.Start();
MockNM nm1 = rm.RegisterNode("localhost:1234", 5120);
IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
, string>(2);
acls[ApplicationAccessType.ViewApp] = "*";
RMApp app = rm.SubmitApp(1024, "appname", "appuser", acls);
nm1.NodeHeartbeat(true);
int waitCount = 0;
while (containerManager.containerTokens == null && waitCount++ < 20)
{
Log.Info("Waiting for AM Launch to happen..");
Sharpen.Thread.Sleep(1000);
}
NUnit.Framework.Assert.IsNotNull(containerManager.containerTokens);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
ApplicationAttemptId applicationAttemptId = attempt.GetAppAttemptId();
WaitForLaunchedState(attempt);
// Create a client to the RM.
Configuration conf = rm.GetConfig();
YarnRPC rpc = YarnRPC.Create(conf);
UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(applicationAttemptId
.ToString());
Credentials credentials = containerManager.GetContainerCredentials();
IPEndPoint rmBindAddress = rm.GetApplicationMasterService().GetBindAddress();
Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> amRMToken = TestAMAuthorization.MockRMWithAMS
.SetupAndReturnAMRMToken(rmBindAddress, credentials.GetAllTokens());
currentUser.AddToken(amRMToken);
ApplicationMasterProtocol client = currentUser.DoAs(new _PrivilegedAction_206(this
, rpc, conf));
RegisterApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<RegisterApplicationMasterRequest>();
RegisterApplicationMasterResponse response = client.RegisterApplicationMaster(request
);
NUnit.Framework.Assert.IsNotNull(response.GetClientToAMTokenMasterKey());
if (UserGroupInformation.IsSecurityEnabled())
{
NUnit.Framework.Assert.IsTrue(((byte[])response.GetClientToAMTokenMasterKey().Array
()).Length > 0);
}
NUnit.Framework.Assert.AreEqual("Register response has bad ACLs", "*", response.GetApplicationACLs
()[ApplicationAccessType.ViewApp]);
}
public virtual void TestPrivateTokenExclusion()
{
UserGroupInformation ugi = UserGroupInformation.GetCurrentUser();
TestSaslRPC.TestTokenIdentifier tokenId = new TestSaslRPC.TestTokenIdentifier();
Org.Apache.Hadoop.Security.Token.Token <TestSaslRPC.TestTokenIdentifier> token = new
Org.Apache.Hadoop.Security.Token.Token <TestSaslRPC.TestTokenIdentifier>(tokenId.
GetBytes(), Runtime.GetBytesForString("password"), tokenId.GetKind(), null
);
ugi.AddToken(new Text("regular-token"), token);
// Now add cloned private token
ugi.AddToken(new Text("private-token"), new Token.PrivateToken <TestSaslRPC.TestTokenIdentifier
>(token));
ugi.AddToken(new Text("private-token1"), new Token.PrivateToken <TestSaslRPC.TestTokenIdentifier
>(token));
// Ensure only non-private tokens are returned
ICollection <Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> > tokens = ugi
.GetCredentials().GetAllTokens();
Assert.Equal(1, tokens.Count);
}
private MRClientProtocol GetMRClientProtocol(Token token, IPEndPoint hsAddress, string
user, Configuration conf)
{
UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser(user);
ugi.AddToken(ConverterUtils.ConvertFromYarn(token, hsAddress));
YarnRPC rpc = YarnRPC.Create(conf);
MRClientProtocol hsWithDT = ugi.DoAs(new _PrivilegedAction_288(rpc, hsAddress, conf
));
return(hsWithDT);
}
/// <exception cref="System.Exception"/>
public virtual void TestAddToken <T>()
where T : TokenIdentifier
{
// from Mockito mocks
UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser("someone");
Org.Apache.Hadoop.Security.Token.Token <T> t1 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token
>();
Org.Apache.Hadoop.Security.Token.Token <T> t2 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token
>();
Org.Apache.Hadoop.Security.Token.Token <T> t3 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token
>();
// add token to ugi
ugi.AddToken(t1);
CheckTokens(ugi, t1);
// replace token t1 with t2 - with same key (null)
ugi.AddToken(t2);
CheckTokens(ugi, t2);
// change t1 service and add token
Org.Mockito.Mockito.When(t1.GetService()).ThenReturn(new Text("t1"));
ugi.AddToken(t1);
CheckTokens(ugi, t1, t2);
// overwrite t1 token with t3 - same key (!null)
Org.Mockito.Mockito.When(t3.GetService()).ThenReturn(new Text("t1"));
ugi.AddToken(t3);
CheckTokens(ugi, t2, t3);
// just try to re-add with new name
Org.Mockito.Mockito.When(t1.GetService()).ThenReturn(new Text("t1.1"));
ugi.AddToken(t1);
CheckTokens(ugi, t1, t2, t3);
// just try to re-add with new name again
ugi.AddToken(t1);
CheckTokens(ugi, t1, t2, t3);
}
/// <exception cref="System.IO.IOException"/>
private void UpdateAMRMToken(Token token)
{
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> amrmToken = new Org.Apache.Hadoop.Security.Token.Token
<AMRMTokenIdentifier>(((byte[])token.GetIdentifier().Array()), ((byte[])token.GetPassword
().Array()), new Text(token.GetKind()), new Text(token.GetService()));
// Preserve the token service sent by the RM when adding the token
// to ensure we replace the previous token setup by the RM.
// Afterwards we can update the service address for the RPC layer.
UserGroupInformation currentUGI = UserGroupInformation.GetCurrentUser();
currentUGI.AddToken(amrmToken);
amrmToken.SetService(ClientRMProxy.GetAMRMTokenService(GetConfig()));
}
/// <exception cref="System.IO.IOException"/>
private UserGroupInformation TokenUGI()
{
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = @params
.DelegationToken();
ByteArrayInputStream buf = new ByteArrayInputStream(token.GetIdentifier());
DataInputStream @in = new DataInputStream(buf);
DelegationTokenIdentifier id = new DelegationTokenIdentifier();
id.ReadFields(@in);
UserGroupInformation ugi = id.GetUser();
ugi.AddToken(token);
return(ugi);
}
public virtual void TestAMRMTokenUpdate()
{
Configuration conf = new Configuration();
ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(ApplicationId.NewInstance
(1, 1), 1);
AMRMTokenIdentifier oldTokenId = new AMRMTokenIdentifier(attemptId, 1);
AMRMTokenIdentifier newTokenId = new AMRMTokenIdentifier(attemptId, 2);
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> oldToken = new Org.Apache.Hadoop.Security.Token.Token
<AMRMTokenIdentifier>(oldTokenId.GetBytes(), Sharpen.Runtime.GetBytesForString("oldpassword"
), oldTokenId.GetKind(), new Text());
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> newToken = new Org.Apache.Hadoop.Security.Token.Token
<AMRMTokenIdentifier>(newTokenId.GetBytes(), Sharpen.Runtime.GetBytesForString("newpassword"
), newTokenId.GetKind(), new Text());
TestLocalContainerAllocator.MockScheduler scheduler = new TestLocalContainerAllocator.MockScheduler
();
scheduler.amToken = newToken;
LocalContainerAllocator lca = new TestLocalContainerAllocator.StubbedLocalContainerAllocator
(scheduler);
lca.Init(conf);
lca.Start();
UserGroupInformation testUgi = UserGroupInformation.CreateUserForTesting("someuser"
, new string[0]);
testUgi.AddToken(oldToken);
testUgi.DoAs(new _PrivilegedExceptionAction_144(lca));
lca.Close();
// verify there is only one AMRM token in the UGI and it matches the
// updated token from the RM
int tokenCount = 0;
Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> ugiToken = null;
foreach (Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> token in testUgi
.GetTokens())
{
if (AMRMTokenIdentifier.KindName.Equals(token.GetKind()))
{
ugiToken = token;
++tokenCount;
}
}
NUnit.Framework.Assert.AreEqual("too many AMRM tokens", 1, tokenCount);
Assert.AssertArrayEquals("token identifier not updated", newToken.GetIdentifier()
, ugiToken.GetIdentifier());
Assert.AssertArrayEquals("token password not updated", newToken.GetPassword(), ugiToken
.GetPassword());
NUnit.Framework.Assert.AreEqual("AMRM token service not updated", new Text(ClientRMProxy
.GetAMRMTokenService(conf)), ugiToken.GetService());
}
private ApplicationClientProtocol GetClientRMProtocolWithDT(Org.Apache.Hadoop.Yarn.Api.Records.Token
token, IPEndPoint rmAddress, string user, Configuration conf)
{
// Maybe consider converting to Hadoop token, serialize de-serialize etc
// before trying to renew the token.
UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser(user);
ugi.AddToken(ConverterUtils.ConvertFromYarn(token, rmAddress));
YarnRPC rpc = YarnRPC.Create(conf);
ApplicationClientProtocol clientRMWithDT = ugi.DoAs(new _PrivilegedAction_464(rpc
, rmAddress, conf));
return(clientRMWithDT);
}
protected internal virtual ContainerManagementProtocol GetContainerManagementProtocolProxy
(YarnRPC rpc, Org.Apache.Hadoop.Yarn.Api.Records.Token nmToken, NodeId nodeId, string
user)
{
ContainerManagementProtocol proxy;
UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser(user);
IPEndPoint addr = NetUtils.CreateSocketAddr(nodeId.GetHost(), nodeId.GetPort());
if (nmToken != null)
{
ugi.AddToken(ConverterUtils.ConvertFromYarn(nmToken, addr));
}
proxy = NMProxy.CreateNMProxy <ContainerManagementProtocol>(conf, ugi, rpc, addr);
return(proxy);
}
// Protected. For tests.
protected internal virtual ContainerManagementProtocol GetContainerMgrProxy(ContainerId
containerId)
{
NodeId node = masterContainer.GetNodeId();
IPEndPoint containerManagerConnectAddress = NetUtils.CreateSocketAddrForHost(node
.GetHost(), node.GetPort());
YarnRPC rpc = GetYarnRPC();
UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(containerId
.GetApplicationAttemptId().ToString());
string user = rmContext.GetRMApps()[containerId.GetApplicationAttemptId().GetApplicationId
()].GetUser();
Token token = rmContext.GetNMTokenSecretManager().CreateNMToken(containerId.GetApplicationAttemptId
(), node, user);
currentUser.AddToken(ConverterUtils.ConvertFromYarn(token, containerManagerConnectAddress
));
return(NMProxy.CreateNMProxy <ContainerManagementProtocol>(conf, currentUser, rpc,
containerManagerConnectAddress));
}
/// <exception cref="System.IO.IOException"/>
private WebHdfsFileSystem GetWebHdfsFileSystem(UserGroupInformation ugi, Configuration
conf)
{
if (UserGroupInformation.IsSecurityEnabled())
{
DelegationTokenIdentifier dtId = new DelegationTokenIdentifier(new Text(ugi.GetUserName
()), null, null);
FSNamesystem namesystem = Org.Mockito.Mockito.Mock <FSNamesystem>();
DelegationTokenSecretManager dtSecretManager = new DelegationTokenSecretManager(86400000
, 86400000, 86400000, 86400000, namesystem);
dtSecretManager.StartThreads();
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token
<DelegationTokenIdentifier>(dtId, dtSecretManager);
SecurityUtil.SetTokenService(token, NetUtils.CreateSocketAddr(uri.GetAuthority())
);
token.SetKind(WebHdfsFileSystem.TokenKind);
ugi.AddToken(token);
}
return((WebHdfsFileSystem)FileSystem.Get(uri, conf));
}
protected internal virtual ContainerManagementProtocol NewProxy(YarnRPC rpc, string
containerManagerBindAddr, ContainerId containerId, Token token)
{
if (token == null)
{
throw new SecretManager.InvalidToken("No NMToken sent for " + containerManagerBindAddr
);
}
IPEndPoint cmAddr = NetUtils.CreateSocketAddr(containerManagerBindAddr);
ContainerManagementProtocolProxy.Log.Info("Opening proxy : " + containerManagerBindAddr
);
// the user in createRemoteUser in this context has to be ContainerID
UserGroupInformation user = UserGroupInformation.CreateRemoteUser(containerId.GetApplicationAttemptId
().ToString());
Org.Apache.Hadoop.Security.Token.Token <NMTokenIdentifier> nmToken = ConverterUtils
.ConvertFromYarn(token, cmAddr);
user.AddToken(nmToken);
return(NMProxy.CreateNMProxy <ContainerManagementProtocol>(this._enclosing.conf, user
, rpc, cmAddr));
}
/// <summary>
/// Test if StandbyException can be thrown from StandbyNN, when it's requested for
/// password.
/// </summary>
/// <remarks>
/// Test if StandbyException can be thrown from StandbyNN, when it's requested for
/// password. (HDFS-6475). With StandbyException, the client can failover to try
/// activeNN.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestDelegationTokenStandbyNNAppearFirst()
{
// make nn0 the standby NN, and nn1 the active NN
cluster.TransitionToStandby(0);
cluster.TransitionToActive(1);
DelegationTokenSecretManager stSecretManager = NameNodeAdapter.GetDtSecretManager
(nn1.GetNamesystem());
// create token
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = GetDelegationToken
(fs, "JobTracker");
DelegationTokenIdentifier identifier = new DelegationTokenIdentifier();
byte[] tokenId = token.GetIdentifier();
identifier.ReadFields(new DataInputStream(new ByteArrayInputStream(tokenId)));
NUnit.Framework.Assert.IsTrue(null != stSecretManager.RetrievePassword(identifier
));
UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser("JobTracker");
ugi.AddToken(token);
ugi.DoAs(new _PrivilegedExceptionAction_406(identifier));
}
/// <exception cref="System.Exception"/>
public virtual void TestNMProxyRetry()
{
containerManager.Start();
containerManager.SetBlockNewContainerRequests(false);
StartContainersRequest allRequests = Records.NewRecord <StartContainersRequest>();
ApplicationId appId = ApplicationId.NewInstance(1, 1);
ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1);
Token nmToken = context.GetNMTokenSecretManager().CreateNMToken(attemptId, context
.GetNodeId(), user);
IPEndPoint address = conf.GetSocketAddr(YarnConfiguration.NmBindHost, YarnConfiguration
.NmAddress, YarnConfiguration.DefaultNmAddress, YarnConfiguration.DefaultNmPort);
Org.Apache.Hadoop.Security.Token.Token <NMTokenIdentifier> token = ConverterUtils.
ConvertFromYarn(nmToken, SecurityUtil.BuildTokenService(address));
UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser(user);
ugi.AddToken(token);
ContainerManagementProtocol proxy = NMProxy.CreateNMProxy <ContainerManagementProtocol
>(conf, ugi, YarnRPC.Create(conf), address);
retryCount = 0;
shouldThrowNMNotYetReadyException = false;
proxy.StartContainers(allRequests);
NUnit.Framework.Assert.AreEqual(5, retryCount);
retryCount = 0;
shouldThrowNMNotYetReadyException = false;
proxy.StopContainers(Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <StopContainersRequest
>());
NUnit.Framework.Assert.AreEqual(5, retryCount);
retryCount = 0;
shouldThrowNMNotYetReadyException = false;
proxy.GetContainerStatuses(Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <GetContainerStatusesRequest
>());
NUnit.Framework.Assert.AreEqual(5, retryCount);
retryCount = 0;
shouldThrowNMNotYetReadyException = true;
proxy.StartContainers(allRequests);
NUnit.Framework.Assert.AreEqual(5, retryCount);
}
