public virtual void TestUGIForLogAndJobRefresh()
{
UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting("test", new
string[] { "grp" });
UserGroupInformation loginUGI = Org.Mockito.Mockito.Spy(hsAdminServer.GetLoginUGI
());
hsAdminServer.SetLoginUGI(loginUGI);
// Run refresh log retention settings with test user
ugi.DoAs(new _PrivilegedAction_303(this));
// Verify if AggregatedLogDeletionService#refreshLogRetentionSettings was
// called with login UGI, instead of the UGI command was run with.
Org.Mockito.Mockito.Verify(loginUGI).DoAs(Matchers.Any <PrivilegedExceptionAction>
());
Org.Mockito.Mockito.Verify(alds).RefreshLogRetentionSettings();
// Reset for refresh job retention settings
Org.Mockito.Mockito.Reset(loginUGI);
// Run refresh job retention settings with test user
ugi.DoAs(new _PrivilegedAction_325(this));
// Verify if JobHistory#refreshJobRetentionSettings was called with
// login UGI, instead of the UGI command was run with.
Org.Mockito.Mockito.Verify(loginUGI).DoAs(Matchers.Any <PrivilegedExceptionAction>
());
Org.Mockito.Mockito.Verify(jobHistoryService).RefreshJobRetentionSettings();
}
public virtual void TestProxyUsersWithProviderOverride()
{
Configuration conf = new Configuration();
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityImpersonationProviderClass,
"org.apache.hadoop.security.authorize.TestProxyUsers$TestDummyImpersonationProvider"
);
ProxyUsers.RefreshSuperUserGroupsConfiguration(conf);
// First try proxying a group that's allowed
UserGroupInformation realUserUgi = UserGroupInformation.CreateUserForTesting(RealUserName
, SudoGroupNames);
UserGroupInformation proxyUserUgi = UserGroupInformation.CreateProxyUserForTesting
(ProxyUserName, realUserUgi, GroupNames);
// From good IP
AssertAuthorized(proxyUserUgi, "1.2.3.4");
// From bad IP
AssertAuthorized(proxyUserUgi, "1.2.3.5");
// Now try proxying a group that's not allowed
realUserUgi = UserGroupInformation.CreateUserForTesting(RealUserName, GroupNames);
proxyUserUgi = UserGroupInformation.CreateProxyUserForTesting(ProxyUserName, realUserUgi
, GroupNames);
// From good IP
AssertNotAuthorized(proxyUserUgi, "1.2.3.4");
// From bad IP
AssertNotAuthorized(proxyUserUgi, "1.2.3.5");
}
/// <exception cref="System.Exception"/>
public virtual void TestHDFSRMStateStore()
{
HdfsConfiguration conf = new HdfsConfiguration();
UserGroupInformation yarnAdmin = UserGroupInformation.CreateUserForTesting("yarn"
, new string[] { "admin" });
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.GetFileSystem().Mkdir(new Path("/yarn"), FsPermission.ValueOf("-rwxrwxrwx"
));
cluster.GetFileSystem().SetOwner(new Path("/yarn"), "yarn", "admin");
UserGroupInformation hdfsAdmin = UserGroupInformation.GetCurrentUser();
RMStateStoreTestBase.StoreStateVerifier verifier = new _StoreStateVerifier_200(this
, hdfsAdmin, cluster);
// Wait for things to settle
// Wait for things to settle
try
{
yarnAdmin.DoAs(new _PrivilegedExceptionAction_243(this, cluster, verifier));
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestDelegationTokenWebHdfsApi()
{
((Log4JLogger)NamenodeWebHdfsMethods.Log).GetLogger().SetLevel(Level.All);
string uri = WebHdfsFileSystem.Scheme + "://" + config.Get(DFSConfigKeys.DfsNamenodeHttpAddressKey
);
//get file system as JobTracker
UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting("JobTracker"
, new string[] { "user" });
WebHdfsFileSystem webhdfs = ugi.DoAs(new _PrivilegedExceptionAction_180(this, uri
));
{
//test addDelegationTokens(..)
Credentials creds = new Credentials();
Org.Apache.Hadoop.Security.Token.Token <object>[] tokens = webhdfs.AddDelegationTokens
("JobTracker", creds);
NUnit.Framework.Assert.AreEqual(1, tokens.Length);
NUnit.Framework.Assert.AreEqual(1, creds.NumberOfTokens());
NUnit.Framework.Assert.AreSame(tokens[0], creds.GetAllTokens().GetEnumerator().Next
());
CheckTokenIdentifier(ugi, tokens[0]);
Org.Apache.Hadoop.Security.Token.Token <object>[] tokens2 = webhdfs.AddDelegationTokens
("JobTracker", creds);
NUnit.Framework.Assert.AreEqual(0, tokens2.Length);
}
}
public virtual void TestTokenBySuperUser()
{
TestSaslRPC.TestTokenSecretManager sm = new TestSaslRPC.TestTokenSecretManager();
Configuration newConf = new Configuration(masterConf);
SecurityUtil.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Kerberos
, newConf);
UserGroupInformation.SetConfiguration(newConf);
Server server = new RPC.Builder(newConf).SetProtocol(typeof(TestDoAsEffectiveUser.TestProtocol
)).SetInstance(new TestDoAsEffectiveUser.TestImpl(this)).SetBindAddress(Address)
.SetPort(0).SetNumHandlers(5).SetVerbose(true).SetSecretManager(sm).Build();
server.Start();
UserGroupInformation current = UserGroupInformation.CreateUserForTesting(RealUserName
, GroupNames);
RefreshConf(newConf);
IPEndPoint addr = NetUtils.GetConnectAddress(server);
TestSaslRPC.TestTokenIdentifier tokenId = new TestSaslRPC.TestTokenIdentifier(new
Org.Apache.Hadoop.IO.Text(current.GetUserName()), new Org.Apache.Hadoop.IO.Text(
"SomeSuperUser"));
Org.Apache.Hadoop.Security.Token.Token <TestSaslRPC.TestTokenIdentifier> token = new
Org.Apache.Hadoop.Security.Token.Token <TestSaslRPC.TestTokenIdentifier>(tokenId,
sm);
SecurityUtil.SetTokenService(token, addr);
current.AddToken(token);
string retVal = current.DoAs(new _PrivilegedExceptionAction_509(this, addr, newConf
, server));
string expected = RealUserName + " (auth:TOKEN) via SomeSuperUser (auth:SIMPLE)";
Assert.Equal(retVal + "!=" + expected, expected, retVal);
}
public virtual void TestClusterNoAdmins()
{
IDictionary <JobACL, AccessControlList> tmpJobACLs = new Dictionary <JobACL, AccessControlList
>();
Configuration conf = new Configuration();
string jobOwner = "testuser";
conf.Set(JobACL.ViewJob.GetAclName(), string.Empty);
conf.SetBoolean(MRConfig.MrAclsEnabled, true);
string noAdminUser = "******";
JobACLsManager aclsManager = new JobACLsManager(conf);
tmpJobACLs = aclsManager.ConstructJobACLs(conf);
IDictionary <JobACL, AccessControlList> jobACLs = tmpJobACLs;
UserGroupInformation callerUGI = UserGroupInformation.CreateUserForTesting(noAdminUser
, new string[] { });
// random user should not have access
bool val = aclsManager.CheckAccess(callerUGI, JobACL.ViewJob, jobOwner, jobACLs[JobACL
.ViewJob]);
NUnit.Framework.Assert.IsFalse("random user should not have view access", val);
val = aclsManager.CheckAccess(callerUGI, JobACL.ModifyJob, jobOwner, jobACLs[JobACL
.ModifyJob]);
NUnit.Framework.Assert.IsFalse("random user should not have modify access", val);
callerUGI = UserGroupInformation.CreateUserForTesting(jobOwner, new string[] { }
);
// Owner should have access
val = aclsManager.CheckAccess(callerUGI, JobACL.ViewJob, jobOwner, jobACLs[JobACL
.ViewJob]);
NUnit.Framework.Assert.IsTrue("owner should have view access", val);
val = aclsManager.CheckAccess(callerUGI, JobACL.ModifyJob, jobOwner, jobACLs[JobACL
.ModifyJob]);
NUnit.Framework.Assert.IsTrue("owner should have modify access", val);
}
public virtual void SetupCluster()
{
// must configure prior to instantiating the namesystem because it
// will reconfigure the logger if async is enabled
ConfigureAuditLogs();
conf = new HdfsConfiguration();
long precision = 1L;
conf.SetLong(DFSConfigKeys.DfsNamenodeAccesstimePrecisionKey, precision);
conf.SetLong(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 10000L);
conf.SetBoolean(DFSConfigKeys.DfsWebhdfsEnabledKey, true);
conf.SetBoolean(DFSConfigKeys.DfsNamenodeAuditLogAsyncKey, useAsyncLog);
util = new DFSTestUtil.Builder().SetName("TestAuditAllowed").SetNumFiles(20).Build
();
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(4).Build();
fs = cluster.GetFileSystem();
util.CreateFiles(fs, fileName);
// make sure the appender is what it's supposed to be
Logger logger = ((Log4JLogger)FSNamesystem.auditLog).GetLogger();
IList <Appender> appenders = Sharpen.Collections.List(logger.GetAllAppenders());
NUnit.Framework.Assert.AreEqual(1, appenders.Count);
NUnit.Framework.Assert.AreEqual(useAsyncLog, appenders[0] is AsyncAppender);
fnames = util.GetFileNames(fileName);
util.WaitReplication(fs, fileName, (short)3);
userGroupInfo = UserGroupInformation.CreateUserForTesting(username, groups);
}
public static void SetupCluster()
{
Configuration conf = new Configuration();
conf.SetBoolean(DFSConfigKeys.DfsWebhdfsEnabledKey, true);
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, 1024);
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build();
cluster.WaitActive();
//change root permission to 777
cluster.GetFileSystem().SetPermission(new Path("/"), new FsPermission((short)0x1ff
));
string uri = WebHdfsFileSystem.Scheme + "://" + conf.Get(DFSConfigKeys.DfsNamenodeHttpAddressKey
);
//get file system as a non-superuser
UserGroupInformation current = UserGroupInformation.GetCurrentUser();
UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting(current.GetShortUserName
() + "x", new string[] { "user" });
fileSystem = ugi.DoAs(new _PrivilegedExceptionAction_91(uri, conf));
defaultWorkingDirectory = fileSystem.GetWorkingDirectory();
}
catch (Exception e)
{
throw new RuntimeException(e);
}
}
public virtual void TestGroups()
{
IDictionary <JobACL, AccessControlList> tmpJobACLs = new Dictionary <JobACL, AccessControlList
>();
Configuration conf = new Configuration();
string jobOwner = "testuser";
conf.Set(JobACL.ViewJob.GetAclName(), jobOwner);
conf.SetBoolean(MRConfig.MrAclsEnabled, true);
string user = "******";
string adminGroup = "adminGroup";
conf.Set(MRConfig.MrAdmins, " " + adminGroup);
JobACLsManager aclsManager = new JobACLsManager(conf);
tmpJobACLs = aclsManager.ConstructJobACLs(conf);
IDictionary <JobACL, AccessControlList> jobACLs = tmpJobACLs;
UserGroupInformation callerUGI = UserGroupInformation.CreateUserForTesting(user,
new string[] { adminGroup });
// acls off so anyone should have access
bool val = aclsManager.CheckAccess(callerUGI, JobACL.ViewJob, jobOwner, jobACLs[JobACL
.ViewJob]);
NUnit.Framework.Assert.IsTrue("user in admin group should have access", val);
}
public virtual void TestMachineList()
{
UserGroupInformation drwho = UserGroupInformation.CreateUserForTesting("*****@*****.**"
, new string[] { "group1", "group2" });
ServiceAuthorizationManager serviceAuthorizationManager = new ServiceAuthorizationManager
();
Configuration conf = new Configuration();
conf.Set(HostConfig, "1.2.3.4");
serviceAuthorizationManager.Refresh(conf, new TestServiceAuthorization.TestPolicyProvider
());
try
{
serviceAuthorizationManager.Authorize(drwho, typeof(TestRPC.TestProtocol), conf,
Extensions.GetAddressByName(AuthorizedIp));
}
catch (AuthorizationException)
{
NUnit.Framework.Assert.Fail();
}
try
{
serviceAuthorizationManager.Authorize(drwho, typeof(TestRPC.TestProtocol), conf,
Extensions.GetAddressByName(UnauthorizedIp));
NUnit.Framework.Assert.Fail();
}
catch (AuthorizationException)
{
}
}
public virtual void TestTokenRaceCondition()
{
UserGroupInformation userGroupInfo = UserGroupInformation.CreateUserForTesting(UserName
, GroupNames);
userGroupInfo.DoAs(new _PrivilegedExceptionAction_844());
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
private DistributedFileSystem CreateFsAsOtherUser(MiniDFSCluster cluster, Configuration
conf)
{
return((DistributedFileSystem)UserGroupInformation.CreateUserForTesting("otheruser"
, new string[] { "othergroup" }).DoAs(new _PrivilegedExceptionAction_508(cluster
, conf)));
}
/// <exception cref="System.Exception"/>
public virtual void TestAdminAccessOnly()
{
Path zone = new Path("zone");
Path slashZone = new Path("/", zone);
fs.Mkdirs(slashZone);
dfsAdmin.CreateEncryptionZone(slashZone, TestKey);
Path @base = new Path("base");
Path reservedRaw = new Path("/.reserved/raw");
int len = 8192;
/* Test failure of create file in reserved/raw as non admin */
UserGroupInformation user = UserGroupInformation.CreateUserForTesting("user", new
string[] { "mygroup" });
user.DoAs(new _PrivilegedExceptionAction_233(this, reservedRaw, zone, @base, len)
);
/* Test failure of getFileStatus in reserved/raw as non admin */
Path ezRawEncFile = new Path(new Path(reservedRaw, zone), @base);
DFSTestUtil.CreateFile(fs, ezRawEncFile, len, (short)1, unchecked ((int)(0xFEED)));
user.DoAs(new _PrivilegedExceptionAction_252(this, ezRawEncFile));
/* Test failure of listStatus in reserved/raw as non admin */
user.DoAs(new _PrivilegedExceptionAction_267(this, ezRawEncFile));
fs.SetPermission(new Path("/"), new FsPermission((short)0x1ff));
/* Test failure of mkdir in reserved/raw as non admin */
user.DoAs(new _PrivilegedExceptionAction_283(this, reservedRaw));
}
public virtual void Setup()
{
user1 = UserGroupInformation.CreateUserForTesting("alice", new string[] { "users" });
user2 = UserGroupInformation.CreateUserForTesting("bob", new string[] { "users" }
);
cluster = new MiniMRCluster(0, 0, 1, "file:///", 1);
}
/// <exception cref="System.Exception"/>
public virtual void TestLargeDirectory()
{
Configuration conf = WebHdfsTestUtil.CreateConf();
int listLimit = 2;
// force small chunking of directory listing
conf.SetInt(DFSConfigKeys.DfsListLimit, listLimit);
// force paths to be only owner-accessible to ensure ugi isn't changing
// during listStatus
FsPermission.SetUMask(conf, new FsPermission((short)0x3f));
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build();
try
{
cluster.WaitActive();
WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem.Scheme).SetPermission
(new Path("/"), new FsPermission(FsAction.All, FsAction.All, FsAction.All));
// trick the NN into not believing it's not the superuser so we can
// tell if the correct user is used by listStatus
UserGroupInformation.SetLoginUser(UserGroupInformation.CreateUserForTesting("not-superuser"
, new string[] { "not-supergroup" }));
UserGroupInformation.CreateUserForTesting("me", new string[] { "my-group" }).DoAs
(new _PrivilegedExceptionAction_263(conf, listLimit));
}
finally
{
cluster.Shutdown();
}
}
/// <summary>
/// Returns the reference to a new instance of FileSystem created
/// with different user name
/// </summary>
/// <param name="conf">current Configuration</param>
/// <returns>FileSystem instance</returns>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"></exception>
public static FileSystem CreateHdfsWithDifferentUsername(Configuration conf)
{
string username = UserGroupInformation.GetCurrentUser().GetShortUserName() + "_XXX";
UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting(username, new
string[] { "supergroup" });
return(DFSTestUtil.GetFileSystemAs(ugi, conf));
}
/// <summary>
/// Test the listing with different user names to make sure only directories
/// that are owned by the user are listed.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestListWithDifferentUser()
{
cluster.GetNamesystem().GetSnapshotManager().SetAllowNestedSnapshots(true);
// first make dir1 and dir2 snapshottable
hdfs.AllowSnapshot(dir1);
hdfs.AllowSnapshot(dir2);
hdfs.SetPermission(root, FsPermission.ValueOf("-rwxrwxrwx"));
// create two dirs and make them snapshottable under the name of user1
UserGroupInformation ugi1 = UserGroupInformation.CreateUserForTesting("user1", new
string[] { "group1" });
DistributedFileSystem fs1 = (DistributedFileSystem)DFSTestUtil.GetFileSystemAs(ugi1
, conf);
Path dir1_user1 = new Path("/dir1_user1");
Path dir2_user1 = new Path("/dir2_user1");
fs1.Mkdirs(dir1_user1);
fs1.Mkdirs(dir2_user1);
hdfs.AllowSnapshot(dir1_user1);
hdfs.AllowSnapshot(dir2_user1);
// user2
UserGroupInformation ugi2 = UserGroupInformation.CreateUserForTesting("user2", new
string[] { "group2" });
DistributedFileSystem fs2 = (DistributedFileSystem)DFSTestUtil.GetFileSystemAs(ugi2
, conf);
Path dir_user2 = new Path("/dir_user2");
Path subdir_user2 = new Path(dir_user2, "subdir");
fs2.Mkdirs(dir_user2);
fs2.Mkdirs(subdir_user2);
hdfs.AllowSnapshot(dir_user2);
hdfs.AllowSnapshot(subdir_user2);
// super user
string supergroup = conf.Get(DFSConfigKeys.DfsPermissionsSuperusergroupKey, DFSConfigKeys
.DfsPermissionsSuperusergroupDefault);
UserGroupInformation superUgi = UserGroupInformation.CreateUserForTesting("superuser"
, new string[] { supergroup });
DistributedFileSystem fs3 = (DistributedFileSystem)DFSTestUtil.GetFileSystemAs(superUgi
, conf);
// list the snapshottable dirs for superuser
SnapshottableDirectoryStatus[] dirs = fs3.GetSnapshottableDirListing();
// 6 snapshottable dirs: dir1, dir2, dir1_user1, dir2_user1, dir_user2, and
// subdir_user2
NUnit.Framework.Assert.AreEqual(6, dirs.Length);
// list the snapshottable dirs for user1
dirs = fs1.GetSnapshottableDirListing();
// 2 dirs owned by user1: dir1_user1 and dir2_user1
NUnit.Framework.Assert.AreEqual(2, dirs.Length);
NUnit.Framework.Assert.AreEqual(dir1_user1, dirs[0].GetFullPath());
NUnit.Framework.Assert.AreEqual(dir2_user1, dirs[1].GetFullPath());
// list the snapshottable dirs for user2
dirs = fs2.GetSnapshottableDirListing();
// 2 dirs owned by user2: dir_user2 and subdir_user2
NUnit.Framework.Assert.AreEqual(2, dirs.Length);
NUnit.Framework.Assert.AreEqual(dir_user2, dirs[0].GetFullPath());
NUnit.Framework.Assert.AreEqual(subdir_user2, dirs[1].GetFullPath());
}
public static void SetUp()
{
conf = new Configuration();
SecurityUtil.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Kerberos
, conf);
UserGroupInformation.SetConfiguration(conf);
UserGroupInformation.SetLoginUser(UserGroupInformation.CreateUserForTesting("LoginUser"
, new string[] { "supergroup" }));
}
private void TestConstructorSuccess(string principal, string shortName)
{
UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting(principal, GroupNames
);
// make sure the short and full user names are correct
Assert.Equal(principal, ugi.GetUserName());
Assert.Equal(shortName, ugi.GetShortUserName());
}
/// <exception cref="System.Exception"/>
public virtual void TestGettingGroups()
{
UserGroupInformation uugi = UserGroupInformation.CreateUserForTesting(UserName, GroupNames
);
Assert.Equal(UserName, uugi.GetUserName());
Assert.AssertArrayEquals(new string[] { Group1Name, Group2Name, Group3Name }, uugi
.GetGroupNames());
}
// To avoid using cached client
/// <exception cref="System.Exception"/>
public virtual void TestAMRMMasterKeysUpdate()
{
AtomicReference <AMRMTokenSecretManager> spySecretMgrRef = new AtomicReference <AMRMTokenSecretManager
>();
MockRM rm = new _MockRM_349(this, spySecretMgrRef, conf);
// Skip the login.
rm.Start();
MockNM nm = rm.RegisterNode("127.0.0.1:1234", 8000);
RMApp app = rm.SubmitApp(200);
MockAM am = MockRM.LaunchAndRegisterAM(app, rm, nm);
AMRMTokenSecretManager spySecretMgr = spySecretMgrRef.Get();
// Do allocate. Should not update AMRMToken
AllocateResponse response = am.Allocate(Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<AllocateRequest>());
NUnit.Framework.Assert.IsNull(response.GetAMRMToken());
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> oldToken = rm.GetRMContext
().GetRMApps()[app.GetApplicationId()].GetRMAppAttempt(am.GetApplicationAttemptId
()).GetAMRMToken();
// roll over the master key
// Do allocate again. the AM should get the latest AMRMToken
rm.GetRMContext().GetAMRMTokenSecretManager().RollMasterKey();
response = am.Allocate(Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest
>());
NUnit.Framework.Assert.IsNotNull(response.GetAMRMToken());
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> amrmToken = ConverterUtils
.ConvertFromYarn(response.GetAMRMToken(), new Text(response.GetAMRMToken().GetService
()));
NUnit.Framework.Assert.AreEqual(amrmToken.DecodeIdentifier().GetKeyId(), rm.GetRMContext
().GetAMRMTokenSecretManager().GetMasterKey().GetMasterKey().GetKeyId());
// Do allocate again with the same old token and verify the RM sends
// back the last generated token instead of generating it again.
Org.Mockito.Mockito.Reset(spySecretMgr);
UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting(am.GetApplicationAttemptId
().ToString(), new string[0]);
ugi.AddTokenIdentifier(oldToken.DecodeIdentifier());
response = am.DoAllocateAs(ugi, Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest
>());
NUnit.Framework.Assert.IsNotNull(response.GetAMRMToken());
Org.Mockito.Mockito.Verify(spySecretMgr, Org.Mockito.Mockito.Never()).CreateAndGetAMRMToken
(Matchers.IsA <ApplicationAttemptId>());
// Do allocate again with the updated token and verify we do not
// receive a new token to use.
response = am.Allocate(Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest
>());
NUnit.Framework.Assert.IsNull(response.GetAMRMToken());
// Activate the next master key. Since there is new master key generated
// in AMRMTokenSecretManager. The AMRMToken will not get updated for AM
rm.GetRMContext().GetAMRMTokenSecretManager().ActivateNextMasterKey();
response = am.Allocate(Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest
>());
NUnit.Framework.Assert.IsNull(response.GetAMRMToken());
rm.Stop();
}
/// <exception cref="System.Exception"/>
protected override void SetUp()
{
//get file system as a non-superuser
UserGroupInformation current = UserGroupInformation.GetCurrentUser();
ugi = UserGroupInformation.CreateUserForTesting(current.GetShortUserName() + "x",
new string[] { "user" });
fs = WebHdfsTestUtil.GetWebHdfsFileSystemAs(ugi, conf, WebHdfsFileSystem.Scheme);
defaultWorkingDirectory = fs.GetWorkingDirectory().ToUri().GetPath();
}
public virtual void TestRefreshUserToGroupsMappingsWithFileSystemBasedConfigurationProvider
()
{
configuration.Set(YarnConfiguration.RmConfigurationProviderClass, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"
);
string[] defaultTestUserGroups = new string[] { "dummy_group1", "dummy_group2" };
UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting("dummyUser",
defaultTestUserGroups);
string user = ugi.GetUserName();
IList <string> groupWithInit = new AList <string>(2);
for (int i = 0; i < ugi.GetGroupNames().Length; i++)
{
groupWithInit.AddItem(ugi.GetGroupNames()[i]);
}
// upload default configurations
UploadDefaultConfiguration();
Configuration conf = new Configuration();
conf.SetClass(CommonConfigurationKeys.HadoopSecurityGroupMapping, typeof(TestRMAdminService.MockUnixGroupsMapping
), typeof(GroupMappingServiceProvider));
UploadConfiguration(conf, "core-site.xml");
try
{
rm = new MockRM(configuration);
rm.Init(configuration);
rm.Start();
}
catch (Exception)
{
NUnit.Framework.Assert.Fail("Should not get any exceptions");
}
// Make sure RM will use the updated GroupMappingServiceProvider
IList <string> groupBefore = new AList <string>(Groups.GetUserToGroupsMappingService
(configuration).GetGroups(user));
NUnit.Framework.Assert.IsTrue(groupBefore.Contains("test_group_A") && groupBefore
.Contains("test_group_B") && groupBefore.Contains("test_group_C") && groupBefore
.Count == 3);
NUnit.Framework.Assert.IsTrue(groupWithInit.Count != groupBefore.Count);
NUnit.Framework.Assert.IsFalse(groupWithInit.Contains("test_group_A") || groupWithInit
.Contains("test_group_B") || groupWithInit.Contains("test_group_C"));
// update the groups
TestRMAdminService.MockUnixGroupsMapping.UpdateGroups();
rm.adminService.RefreshUserToGroupsMappings(RefreshUserToGroupsMappingsRequest.NewInstance
());
IList <string> groupAfter = Groups.GetUserToGroupsMappingService(configuration).GetGroups
(user);
// should get the updated groups
NUnit.Framework.Assert.IsTrue(groupAfter.Contains("test_group_D") && groupAfter.Contains
("test_group_E") && groupAfter.Contains("test_group_F") && groupAfter.Count == 3
);
}
public virtual void TestMultipleKMSInstancesWithZKSigner()
{
FilePath testDir = TestKMS.GetTestDir();
Configuration conf = CreateBaseKMSConf(testDir);
TestingServer zkServer = new TestingServer();
zkServer.Start();
MiniKMS kms1 = null;
MiniKMS kms2 = null;
conf.Set(KMSAuthenticationFilter.ConfigPrefix + AuthenticationFilter.SignerSecretProvider
, "zookeeper");
conf.Set(KMSAuthenticationFilter.ConfigPrefix + ZKSignerSecretProvider.ZookeeperConnectionString
, zkServer.GetConnectString());
conf.Set(KMSAuthenticationFilter.ConfigPrefix + ZKSignerSecretProvider.ZookeeperPath
, "/secret");
TestKMS.WriteConf(testDir, conf);
try
{
kms1 = new MiniKMS.Builder().SetKmsConfDir(testDir).SetLog4jConfFile("log4j.properties"
).Build();
kms1.Start();
kms2 = new MiniKMS.Builder().SetKmsConfDir(testDir).SetLog4jConfFile("log4j.properties"
).Build();
kms2.Start();
Uri url1 = new Uri(kms1.GetKMSUrl().ToExternalForm() + KMSRESTConstants.ServiceVersion
+ "/" + KMSRESTConstants.KeysNamesResource);
Uri url2 = new Uri(kms2.GetKMSUrl().ToExternalForm() + KMSRESTConstants.ServiceVersion
+ "/" + KMSRESTConstants.KeysNamesResource);
DelegationTokenAuthenticatedURL.Token token = new DelegationTokenAuthenticatedURL.Token
();
DelegationTokenAuthenticatedURL aUrl = new DelegationTokenAuthenticatedURL();
UserGroupInformation ugiFoo = UserGroupInformation.CreateUserForTesting("foo", new
string[] { "gfoo" });
UserGroupInformation ugiBar = UserGroupInformation.CreateUserForTesting("bar", new
string[] { "gBar" });
ugiFoo.DoAs(new _PrivilegedExceptionAction_135(aUrl, url1, token));
ugiBar.DoAs(new _PrivilegedExceptionAction_145(aUrl, url2, token));
ugiBar.DoAs(new _PrivilegedExceptionAction_155(aUrl, url2));
}
finally
{
if (kms2 != null)
{
kms2.Stop();
}
if (kms1 != null)
{
kms1.Stop();
}
zkServer.Stop();
}
}
public virtual void TestDelegationToProvider()
{
NUnit.Framework.Assert.IsTrue(Called.Contains("start"));
FileSystem fs = FileSystem.Get(miniDFS.GetConfiguration(0));
fs.Mkdirs(new Path("/tmp"));
fs.SetPermission(new Path("/tmp"), new FsPermission((short)0x1ff));
UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting("u1", new string
[] { "g1" });
ugi.DoAs(new _PrivilegedExceptionAction_201(this));
}
/// <exception cref="System.Exception"/>
public virtual void TestEqualsWithRealUser()
{
UserGroupInformation realUgi1 = UserGroupInformation.CreateUserForTesting("RealUser"
, GroupNames);
UserGroupInformation proxyUgi1 = UserGroupInformation.CreateProxyUser(UserName, realUgi1
);
UserGroupInformation proxyUgi2 = new UserGroupInformation(proxyUgi1.GetSubject());
UserGroupInformation remoteUgi = UserGroupInformation.CreateRemoteUser(UserName);
Assert.Equal(proxyUgi1, proxyUgi2);
NUnit.Framework.Assert.IsFalse(remoteUgi.Equals(proxyUgi1));
}
public virtual void SetUpUsers()
{
// Make sure the current user's info is in the list of test users.
UserGroupInformation currentUser = UserGroupInformation.GetCurrentUser();
UserGroupInformation.CreateUserForTesting(currentUser.GetUserName(), currentUser.
GetGroupNames());
testUser1 = UserGroupInformation.CreateUserForTesting("foo", new string[] { "bar"
, "baz" });
testUser2 = UserGroupInformation.CreateUserForTesting("fiz", new string[] { "buz"
, "boz" });
}
public virtual void Pipeline_02_03()
{
Configuration conf = new HdfsConfiguration();
conf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, 1);
// create cluster
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(4).Build();
try
{
//change the lease limits.
cluster.SetLeasePeriod(SoftLeaseLimit, HardLeaseLimit);
//wait for the cluster
cluster.WaitActive();
FileSystem fs = cluster.GetFileSystem();
Path p = new Path(Dir, "file1");
int half = BlockSize / 2;
{
//a. On Machine M1, Create file. Write half block of data.
// Invoke DFSOutputStream.hflush() on the dfs file handle.
// Do not close file yet.
FSDataOutputStream @out = fs.Create(p, true, fs.GetConf().GetInt(CommonConfigurationKeys
.IoFileBufferSizeKey, 4096), (short)3, BlockSize);
Write(@out, 0, half);
//hflush
((DFSOutputStream)@out.GetWrappedStream()).Hflush();
}
//b. On another machine M2, open file and verify that the half-block
// of data can be read successfully.
CheckFile(p, half, conf);
AppendTestUtil.Log.Info("leasechecker.interruptAndJoin()");
((DistributedFileSystem)fs).dfs.GetLeaseRenewer().InterruptAndJoin();
{
//c. On M1, append another half block of data. Close file on M1.
//sleep to let the lease is expired.
Sharpen.Thread.Sleep(2 * SoftLeaseLimit);
UserGroupInformation current = UserGroupInformation.GetCurrentUser();
UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting(current.GetShortUserName
() + "x", new string[] { "supergroup" });
DistributedFileSystem dfs = ugi.DoAs(new _PrivilegedExceptionAction_102(conf));
FSDataOutputStream @out = Append(dfs, p);
Write(@out, 0, half);
@out.Close();
}
//d. On M2, open file and read 1 block of data from it. Close file.
CheckFile(p, 2 * half, conf);
}
finally
{
cluster.Shutdown();
}
}
public static void AddUserConf(Configuration conf)
{
conf.Set("hadoop.security.authentication", "simple");
conf.Set("hadoop.proxyuser." + HadoopUsersConfTestHelper.GetHadoopProxyUser() + ".hosts"
, HadoopUsersConfTestHelper.GetHadoopProxyUserHosts());
conf.Set("hadoop.proxyuser." + HadoopUsersConfTestHelper.GetHadoopProxyUser() + ".groups"
, HadoopUsersConfTestHelper.GetHadoopProxyUserGroups());
foreach (string user in HadoopUsersConfTestHelper.GetHadoopUsers())
{
string[] groups = HadoopUsersConfTestHelper.GetHadoopUserGroups(user);
UserGroupInformation.CreateUserForTesting(user, groups);
}
}
/// <exception cref="System.IO.IOException"/>
public MiniRPCBenchmark.MiniProtocol Run()
{
MiniRPCBenchmark.MiniProtocol p = RPC.GetProxy <MiniRPCBenchmark.MiniProtocol>(MiniRPCBenchmark.MiniProtocol
.versionID, addr, conf);
Org.Apache.Hadoop.Security.Token.Token <TestDelegationToken.TestDelegationTokenIdentifier
> token;
token = p.GetDelegationToken(new Text(MiniRPCBenchmark.Renewer));
this._enclosing.currentUgi = UserGroupInformation.CreateUserForTesting(MiniRPCBenchmark
.MiniUser, MiniRPCBenchmark.GroupNames);
SecurityUtil.SetTokenService(token, addr);
this._enclosing.currentUgi.AddToken(token);
return(p);
}