/// <exception cref="System.IO.IOException"/>
public override FileStatus GetFileStatus(Path f)
{
CheckPathIsSlash(f);
return(new FileStatus(0, true, 0, 0, creationTime, creationTime, Constants.Permission555
, ugi.GetUserName(), ugi.GetGroupNames()[0], new Path(theInternalDir.fullPath).MakeQualified
(myUri, RootPath)));
}
public virtual void TestRefreshUserToGroupsMappingsWithFileSystemBasedConfigurationProvider
()
{
configuration.Set(YarnConfiguration.RmConfigurationProviderClass, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"
);
string[] defaultTestUserGroups = new string[] { "dummy_group1", "dummy_group2" };
UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting("dummyUser",
defaultTestUserGroups);
string user = ugi.GetUserName();
IList <string> groupWithInit = new AList <string>(2);
for (int i = 0; i < ugi.GetGroupNames().Length; i++)
{
groupWithInit.AddItem(ugi.GetGroupNames()[i]);
}
// upload default configurations
UploadDefaultConfiguration();
Configuration conf = new Configuration();
conf.SetClass(CommonConfigurationKeys.HadoopSecurityGroupMapping, typeof(TestRMAdminService.MockUnixGroupsMapping
), typeof(GroupMappingServiceProvider));
UploadConfiguration(conf, "core-site.xml");
try
{
rm = new MockRM(configuration);
rm.Init(configuration);
rm.Start();
}
catch (Exception)
{
NUnit.Framework.Assert.Fail("Should not get any exceptions");
}
// Make sure RM will use the updated GroupMappingServiceProvider
IList <string> groupBefore = new AList <string>(Groups.GetUserToGroupsMappingService
(configuration).GetGroups(user));
NUnit.Framework.Assert.IsTrue(groupBefore.Contains("test_group_A") && groupBefore
.Contains("test_group_B") && groupBefore.Contains("test_group_C") && groupBefore
.Count == 3);
NUnit.Framework.Assert.IsTrue(groupWithInit.Count != groupBefore.Count);
NUnit.Framework.Assert.IsFalse(groupWithInit.Contains("test_group_A") || groupWithInit
.Contains("test_group_B") || groupWithInit.Contains("test_group_C"));
// update the groups
TestRMAdminService.MockUnixGroupsMapping.UpdateGroups();
rm.adminService.RefreshUserToGroupsMappings(RefreshUserToGroupsMappingsRequest.NewInstance
());
IList <string> groupAfter = Groups.GetUserToGroupsMappingService(configuration).GetGroups
(user);
// should get the updated groups
NUnit.Framework.Assert.IsTrue(groupAfter.Contains("test_group_D") && groupAfter.Contains
("test_group_E") && groupAfter.Contains("test_group_F") && groupAfter.Count == 3
);
}
/// <exception cref="System.IO.IOException"/>
private static void CheckStat(FilePath f, string owner, string group, string expectedOwner
, string expectedGroup)
{
bool success = true;
if (expectedOwner != null && !expectedOwner.Equals(owner))
{
if (Path.Windows)
{
UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser(expectedOwner);
string adminsGroupString = "Administrators";
success = owner.Equals(adminsGroupString) && Arrays.AsList(ugi.GetGroupNames()).Contains
(adminsGroupString);
}
else
{
success = false;
}
}
if (!success)
{
throw new IOException("Owner '" + owner + "' for path " + f + " did not match " +
"expected owner '" + expectedOwner + "'");
}
}
public virtual void TestRefreshSuperUserGroups()
{
UserGroupInformation ugi = Org.Mockito.Mockito.Mock <UserGroupInformation>();
UserGroupInformation superUser = Org.Mockito.Mockito.Mock <UserGroupInformation>();
Org.Mockito.Mockito.When(ugi.GetRealUser()).ThenReturn(superUser);
Org.Mockito.Mockito.When(superUser.GetShortUserName()).ThenReturn("superuser");
Org.Mockito.Mockito.When(superUser.GetUserName()).ThenReturn("superuser");
Org.Mockito.Mockito.When(ugi.GetGroupNames()).ThenReturn(new string[] { "group3" }
);
Org.Mockito.Mockito.When(ugi.GetUserName()).ThenReturn("regularUser");
// Set super user groups not to include groups of regularUser
conf.Set("hadoop.proxyuser.superuser.groups", "group1,group2");
conf.Set("hadoop.proxyuser.superuser.hosts", "127.0.0.1");
string[] args = new string[1];
args[0] = "-refreshSuperUserGroupsConfiguration";
hsAdminClient.Run(args);
Exception th = null;
try
{
ProxyUsers.Authorize(ugi, "127.0.0.1");
}
catch (Exception e)
{
th = e;
}
// Exception should be thrown
NUnit.Framework.Assert.IsTrue(th is AuthorizationException);
// Now add regularUser group to superuser group but not execute
// refreshSuperUserGroupMapping
conf.Set("hadoop.proxyuser.superuser.groups", "group1,group2,group3");
// Again,lets run ProxyUsers.authorize and see if regularUser can be
// impersonated
// resetting th
th = null;
try
{
ProxyUsers.Authorize(ugi, "127.0.0.1");
}
catch (Exception e)
{
th = e;
}
// Exception should be thrown again since we didn't refresh the configs
NUnit.Framework.Assert.IsTrue(th is AuthorizationException);
// Lets refresh the config by running refreshSuperUserGroupsConfiguration
hsAdminClient.Run(args);
th = null;
try
{
ProxyUsers.Authorize(ugi, "127.0.0.1");
}
catch (Exception e)
{
th = e;
}
// No exception thrown since regularUser can be impersonated.
NUnit.Framework.Assert.IsNull("Unexpected exception thrown: " + th, th);
}
/// <exception cref="System.Exception"/>
public virtual void TestFstat()
{
FileOutputStream fos = new FileOutputStream(new FilePath(TestDir, "testfstat"));
NativeIO.POSIX.Stat stat = NativeIO.POSIX.GetFstat(fos.GetFD());
fos.Close();
Log.Info("Stat: " + stat.ToString());
string owner = stat.GetOwner();
string expectedOwner = Runtime.GetProperty("user.name");
if (Path.Windows)
{
UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser(expectedOwner);
string adminsGroupString = "Administrators";
if (Arrays.AsList(ugi.GetGroupNames()).Contains(adminsGroupString))
{
expectedOwner = adminsGroupString;
}
}
Assert.Equal(expectedOwner, owner);
NUnit.Framework.Assert.IsNotNull(stat.GetGroup());
Assert.True(!stat.GetGroup().IsEmpty());
Assert.Equal("Stat mode field should indicate a regular file",
NativeIO.POSIX.Stat.SIfreg, stat.GetMode() & NativeIO.POSIX.Stat.SIfmt);
}
/// <exception cref="System.IO.IOException"/>
public object Run()
{
UserGroupInformation current = UserGroupInformation.GetCurrentUser();
NUnit.Framework.Assert.IsFalse(current.Equals(login));
Assert.Equal(current, fakeUser);
Assert.Equal(0, current.GetGroupNames().Length);
return(null);
}
private static string GetExpectedOutput(UserGroupInformation user)
{
string expectedOutput = user.GetUserName() + " :";
foreach (string group in user.GetGroupNames())
{
expectedOutput += " " + group;
}
return(expectedOutput + Runtime.GetProperty("line.separator"));
}
/// <summary>
/// Authorize a user (superuser) to impersonate another user (user1) if the
/// superuser belongs to the group "sudo_user1" .
/// </summary>
/// <exception cref="Org.Apache.Hadoop.Security.Authorize.AuthorizationException"/>
public virtual void Authorize(UserGroupInformation user, string remoteAddress)
{
UserGroupInformation superUser = user.GetRealUser();
string sudoGroupName = "sudo_" + user.GetShortUserName();
if (!Arrays.AsList(superUser.GetGroupNames()).Contains(sudoGroupName))
{
throw new AuthorizationException("User: "******" is not allowed to impersonate "
+ user.GetUserName());
}
}
/// <summary>given user name - get all the groups.</summary>
/// <remarks>
/// given user name - get all the groups.
/// Needs to happen before creating the test users
/// </remarks>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void TestGetServerSideGroups()
{
// get the user name
SystemProcess pp = Runtime.GetRuntime().Exec("whoami");
BufferedReader br = new BufferedReader(new InputStreamReader(pp.GetInputStream())
);
string userName = br.ReadLine().Trim();
// If on windows domain, token format is DOMAIN\\user and we want to
// extract only the user name
if (Shell.Windows)
{
int sp = userName.LastIndexOf('\\');
if (sp != -1)
{
userName = Runtime.Substring(userName, sp + 1);
}
// user names are case insensitive on Windows. Make consistent
userName = StringUtils.ToLowerCase(userName);
}
// get the groups
pp = Runtime.GetRuntime().Exec(Shell.Windows ? Shell.Winutils + " groups -F" : "id -Gn"
);
br = new BufferedReader(new InputStreamReader(pp.GetInputStream()));
string line = br.ReadLine();
System.Console.Out.WriteLine(userName + ":" + line);
ICollection <string> groups = new LinkedHashSet <string>();
string[] tokens = line.Split(Shell.TokenSeparatorRegex);
foreach (string s in tokens)
{
groups.AddItem(s);
}
UserGroupInformation login = UserGroupInformation.GetCurrentUser();
string loginUserName = login.GetShortUserName();
if (Shell.Windows)
{
// user names are case insensitive on Windows. Make consistent
loginUserName = StringUtils.ToLowerCase(loginUserName);
}
Assert.Equal(userName, loginUserName);
string[] gi = login.GetGroupNames();
Assert.Equal(groups.Count, gi.Length);
for (int i = 0; i < gi.Length; i++)
{
Assert.True(groups.Contains(gi[i]));
}
UserGroupInformation fakeUser = UserGroupInformation.CreateRemoteUser("foo.bar");
fakeUser.DoAs(new _PrivilegedExceptionAction_248(login, fakeUser));
}
public virtual void SetUpUsers()
{
// Make sure the current user's info is in the list of test users.
UserGroupInformation currentUser = UserGroupInformation.GetCurrentUser();
UserGroupInformation.CreateUserForTesting(currentUser.GetUserName(), currentUser.
GetGroupNames());
testUser1 = UserGroupInformation.CreateUserForTesting("foo", new string[] { "bar"
, "baz" });
testUser2 = UserGroupInformation.CreateUserForTesting("fiz", new string[] { "buz"
, "boz" });
}
public virtual void TestInternalGetAclStatus()
{
UserGroupInformation currentUser = UserGroupInformation.GetCurrentUser();
AclStatus aclStatus = fsView.GetAclStatus(new Path("/internalDir"));
Assert.Equal(aclStatus.GetOwner(), currentUser.GetUserName());
Assert.Equal(aclStatus.GetGroup(), currentUser.GetGroupNames()
[0]);
Assert.Equal(aclStatus.GetEntries(), AclUtil.GetMinimalAcl(Constants
.Permission555));
NUnit.Framework.Assert.IsFalse(aclStatus.IsStickyBit());
}
// Expected
/// <exception cref="System.IO.IOException"/>
public virtual void TestDanglingLink()
{
Assume.AssumeTrue(!Path.Windows);
Path fileAbs = new Path(TestBaseDir1() + "/file");
Path fileQual = new Path(TestURI().ToString(), fileAbs);
Path link = new Path(TestBaseDir1() + "/linkToFile");
Path linkQual = new Path(TestURI().ToString(), link.ToString());
wrapper.CreateSymlink(fileAbs, link, false);
// Deleting the link using FileContext currently fails because
// resolve looks up LocalFs rather than RawLocalFs for the path
// so we call ChecksumFs delete (which doesn't delete dangling
// links) instead of delegating to delete in RawLocalFileSystem
// which deletes via fullyDelete. testDeleteLink above works
// because the link is not dangling.
//assertTrue(fc.delete(link, false));
FileUtil.FullyDelete(new FilePath(link.ToUri().GetPath()));
wrapper.CreateSymlink(fileAbs, link, false);
try
{
wrapper.GetFileStatus(link);
NUnit.Framework.Assert.Fail("Got FileStatus for dangling link");
}
catch (FileNotFoundException)
{
}
// Expected. File's exists method returns false for dangling links
// We can stat a dangling link
UserGroupInformation user = UserGroupInformation.GetCurrentUser();
FileStatus fsd = wrapper.GetFileLinkStatus(link);
Assert.Equal(fileQual, fsd.GetSymlink());
Assert.True(fsd.IsSymlink());
NUnit.Framework.Assert.IsFalse(fsd.IsDirectory());
Assert.Equal(user.GetUserName(), fsd.GetOwner());
// Compare against user's primary group
Assert.Equal(user.GetGroupNames()[0], fsd.GetGroup());
Assert.Equal(linkQual, fsd.GetPath());
// Accessing the link
try
{
ReadFile(link);
NUnit.Framework.Assert.Fail("Got FileStatus for dangling link");
}
catch (FileNotFoundException)
{
}
// Ditto.
// Creating the file makes the link work
CreateAndWriteFile(fileAbs);
wrapper.GetFileStatus(link);
}
internal FSPermissionChecker(string fsOwner, string supergroup, UserGroupInformation
callerUgi, INodeAttributeProvider attributeProvider)
{
this.fsOwner = fsOwner;
this.supergroup = supergroup;
this.callerUgi = callerUgi;
HashSet <string> s = new HashSet <string>(Arrays.AsList(callerUgi.GetGroupNames()));
groups = Sharpen.Collections.UnmodifiableSet(s);
user = callerUgi.GetShortUserName();
isSuper = user.Equals(fsOwner) || groups.Contains(supergroup);
this.attributeProvider = attributeProvider;
}
/// <summary>
/// Checks if a user represented by the provided
/// <see cref="Org.Apache.Hadoop.Security.UserGroupInformation"/>
/// is a member of the Access Control List
/// </summary>
/// <param name="ugi">UserGroupInformation to check if contained in the ACL</param>
/// <returns>true if ugi is member of the list</returns>
public bool IsUserInList(UserGroupInformation ugi)
{
if (allAllowed || users.Contains(ugi.GetShortUserName()))
{
return(true);
}
else
{
if (!groups.IsEmpty())
{
foreach (string group in ugi.GetGroupNames())
{
if (groups.Contains(group))
{
return(true);
}
}
}
}
return(false);
}
/// <summary>Test login method</summary>
/// <exception cref="System.Exception"/>
public virtual void TestLogin()
{
conf.Set(CommonConfigurationKeys.HadoopUserGroupMetricsPercentilesIntervals, PercentilesInterval
.ToString());
UserGroupInformation.SetConfiguration(conf);
// login from unix
UserGroupInformation ugi = UserGroupInformation.GetCurrentUser();
Assert.Equal(UserGroupInformation.GetCurrentUser(), UserGroupInformation
.GetLoginUser());
Assert.True(ugi.GetGroupNames().Length >= 1);
VerifyGroupMetrics(1);
// ensure that doAs works correctly
UserGroupInformation userGroupInfo = UserGroupInformation.CreateUserForTesting(UserName
, GroupNames);
UserGroupInformation curUGI = userGroupInfo.DoAs(new _PrivilegedExceptionAction_185
());
// make sure in the scope of the doAs, the right user is current
Assert.Equal(curUGI, userGroupInfo);
// make sure it is not the same as the login user
NUnit.Framework.Assert.IsFalse(curUGI.Equals(UserGroupInformation.GetLoginUser())
);
}
public virtual void TestSimpleAppend()
{
Configuration conf = new HdfsConfiguration();
conf.SetInt(DFSConfigKeys.DfsDatanodeHandlerCountKey, 50);
fileContents = AppendTestUtil.InitBuffer(AppendTestUtil.FileSize);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
FileSystem fs = cluster.GetFileSystem();
try
{
{
// test appending to a file.
// create a new file.
Path file1 = new Path("/simpleAppend.dat");
FSDataOutputStream stm = AppendTestUtil.CreateFile(fs, file1, 1);
System.Console.Out.WriteLine("Created file simpleAppend.dat");
// write to file
int mid = 186;
// io.bytes.per.checksum bytes
System.Console.Out.WriteLine("Writing " + mid + " bytes to file " + file1);
stm.Write(fileContents, 0, mid);
stm.Close();
System.Console.Out.WriteLine("Wrote and Closed first part of file.");
// write to file
int mid2 = 607;
// io.bytes.per.checksum bytes
System.Console.Out.WriteLine("Writing " + mid + " bytes to file " + file1);
stm = fs.Append(file1);
stm.Write(fileContents, mid, mid2 - mid);
stm.Close();
System.Console.Out.WriteLine("Wrote and Closed second part of file.");
// write the remainder of the file
stm = fs.Append(file1);
// ensure getPos is set to reflect existing size of the file
NUnit.Framework.Assert.IsTrue(stm.GetPos() > 0);
System.Console.Out.WriteLine("Writing " + (AppendTestUtil.FileSize - mid2) + " bytes to file "
+ file1);
stm.Write(fileContents, mid2, AppendTestUtil.FileSize - mid2);
System.Console.Out.WriteLine("Written second part of file");
stm.Close();
System.Console.Out.WriteLine("Wrote and Closed second part of file.");
// verify that entire file is good
AppendTestUtil.CheckFullFile(fs, file1, AppendTestUtil.FileSize, fileContents, "Read 2"
);
}
{
// test appending to an non-existing file.
FSDataOutputStream @out = null;
try
{
@out = fs.Append(new Path("/non-existing.dat"));
NUnit.Framework.Assert.Fail("Expected to have FileNotFoundException");
}
catch (FileNotFoundException fnfe)
{
System.Console.Out.WriteLine("Good: got " + fnfe);
Sharpen.Runtime.PrintStackTrace(fnfe, System.Console.Out);
}
finally
{
IOUtils.CloseStream(@out);
}
}
{
// test append permission.
//set root to all writable
Path root = new Path("/");
fs.SetPermission(root, new FsPermission((short)0x1ff));
fs.Close();
// login as a different user
UserGroupInformation superuser = UserGroupInformation.GetCurrentUser();
string username = "******";
string group = "testappendgroup";
NUnit.Framework.Assert.IsFalse(superuser.GetShortUserName().Equals(username));
NUnit.Framework.Assert.IsFalse(Arrays.AsList(superuser.GetGroupNames()).Contains(
group));
UserGroupInformation appenduser = UserGroupInformation.CreateUserForTesting(username
, new string[] { group });
fs = DFSTestUtil.GetFileSystemAs(appenduser, conf);
// create a file
Path dir = new Path(root, GetType().Name);
Path foo = new Path(dir, "foo.dat");
FSDataOutputStream @out = null;
int offset = 0;
try
{
@out = fs.Create(foo);
int len = 10 + AppendTestUtil.NextInt(100);
@out.Write(fileContents, offset, len);
offset += len;
}
finally
{
IOUtils.CloseStream(@out);
}
// change dir and foo to minimal permissions.
fs.SetPermission(dir, new FsPermission((short)0x40));
fs.SetPermission(foo, new FsPermission((short)0x80));
// try append, should success
@out = null;
try
{
@out = fs.Append(foo);
int len = 10 + AppendTestUtil.NextInt(100);
@out.Write(fileContents, offset, len);
offset += len;
}
finally
{
IOUtils.CloseStream(@out);
}
// change dir and foo to all but no write on foo.
fs.SetPermission(foo, new FsPermission((short)0x17f));
fs.SetPermission(dir, new FsPermission((short)0x1ff));
// try append, should fail
@out = null;
try
{
@out = fs.Append(foo);
NUnit.Framework.Assert.Fail("Expected to have AccessControlException");
}
catch (AccessControlException ace)
{
System.Console.Out.WriteLine("Good: got " + ace);
Sharpen.Runtime.PrintStackTrace(ace, System.Console.Out);
}
finally
{
IOUtils.CloseStream(@out);
}
}
}
catch (IOException e)
{
System.Console.Out.WriteLine("Exception :" + e);
throw;
}
catch (Exception e)
{
System.Console.Out.WriteLine("Throwable :" + e);
Sharpen.Runtime.PrintStackTrace(e);
throw new IOException("Throwable : " + e);
}
finally
{
fs.Close();
cluster.Shutdown();
}
}
/// <summary>
/// test QueueManager
/// configuration from file
/// </summary>
/// <exception cref="System.IO.IOException"/>
public virtual void TestQueue()
{
FilePath f = null;
try
{
f = WriteFile();
QueueManager manager = new QueueManager(f.GetCanonicalPath(), true);
manager.SetSchedulerInfo("first", "queueInfo");
manager.SetSchedulerInfo("second", "queueInfoqueueInfo");
Queue root = manager.GetRoot();
NUnit.Framework.Assert.IsTrue(root.GetChildren().Count == 2);
IEnumerator <Queue> iterator = root.GetChildren().GetEnumerator();
Queue firstSubQueue = iterator.Next();
NUnit.Framework.Assert.IsTrue(firstSubQueue.GetName().Equals("first"));
NUnit.Framework.Assert.AreEqual(firstSubQueue.GetAcls()["mapred.queue.first.acl-submit-job"
].ToString(), "Users [user1, user2] and members of the groups [group1, group2] are allowed"
);
Queue secondSubQueue = iterator.Next();
NUnit.Framework.Assert.IsTrue(secondSubQueue.GetName().Equals("second"));
NUnit.Framework.Assert.AreEqual(secondSubQueue.GetProperties().GetProperty("key")
, "value");
NUnit.Framework.Assert.AreEqual(secondSubQueue.GetProperties().GetProperty("key1"
), "value1");
// test status
NUnit.Framework.Assert.AreEqual(firstSubQueue.GetState().GetStateName(), "running"
);
NUnit.Framework.Assert.AreEqual(secondSubQueue.GetState().GetStateName(), "stopped"
);
ICollection <string> template = new HashSet <string>();
template.AddItem("first");
template.AddItem("second");
NUnit.Framework.Assert.AreEqual(manager.GetLeafQueueNames(), template);
// test user access
UserGroupInformation mockUGI = Org.Mockito.Mockito.Mock <UserGroupInformation>();
Org.Mockito.Mockito.When(mockUGI.GetShortUserName()).ThenReturn("user1");
string[] groups = new string[] { "group1" };
Org.Mockito.Mockito.When(mockUGI.GetGroupNames()).ThenReturn(groups);
NUnit.Framework.Assert.IsTrue(manager.HasAccess("first", QueueACL.SubmitJob, mockUGI
));
NUnit.Framework.Assert.IsFalse(manager.HasAccess("second", QueueACL.SubmitJob, mockUGI
));
NUnit.Framework.Assert.IsFalse(manager.HasAccess("first", QueueACL.AdministerJobs
, mockUGI));
Org.Mockito.Mockito.When(mockUGI.GetShortUserName()).ThenReturn("user3");
NUnit.Framework.Assert.IsTrue(manager.HasAccess("first", QueueACL.AdministerJobs,
mockUGI));
QueueAclsInfo[] qai = manager.GetQueueAcls(mockUGI);
NUnit.Framework.Assert.AreEqual(qai.Length, 1);
// test refresh queue
manager.RefreshQueues(GetConfiguration(), null);
iterator = root.GetChildren().GetEnumerator();
Queue firstSubQueue1 = iterator.Next();
Queue secondSubQueue1 = iterator.Next();
// tets equal method
NUnit.Framework.Assert.IsTrue(firstSubQueue.Equals(firstSubQueue1));
NUnit.Framework.Assert.AreEqual(firstSubQueue1.GetState().GetStateName(), "running"
);
NUnit.Framework.Assert.AreEqual(secondSubQueue1.GetState().GetStateName(), "stopped"
);
NUnit.Framework.Assert.AreEqual(firstSubQueue1.GetSchedulingInfo(), "queueInfo");
NUnit.Framework.Assert.AreEqual(secondSubQueue1.GetSchedulingInfo(), "queueInfoqueueInfo"
);
// test JobQueueInfo
NUnit.Framework.Assert.AreEqual(firstSubQueue.GetJobQueueInfo().GetQueueName(), "first"
);
NUnit.Framework.Assert.AreEqual(firstSubQueue.GetJobQueueInfo().GetQueueState(),
"running");
NUnit.Framework.Assert.AreEqual(firstSubQueue.GetJobQueueInfo().GetSchedulingInfo
(), "queueInfo");
NUnit.Framework.Assert.AreEqual(secondSubQueue.GetJobQueueInfo().GetChildren().Count
, 0);
// test
NUnit.Framework.Assert.AreEqual(manager.GetSchedulerInfo("first"), "queueInfo");
ICollection <string> queueJobQueueInfos = new HashSet <string>();
foreach (JobQueueInfo jobInfo in manager.GetJobQueueInfos())
{
queueJobQueueInfos.AddItem(jobInfo.GetQueueName());
}
ICollection <string> rootJobQueueInfos = new HashSet <string>();
foreach (Queue queue in root.GetChildren())
{
rootJobQueueInfos.AddItem(queue.GetJobQueueInfo().GetQueueName());
}
NUnit.Framework.Assert.AreEqual(queueJobQueueInfos, rootJobQueueInfos);
// test getJobQueueInfoMapping
NUnit.Framework.Assert.AreEqual(manager.GetJobQueueInfoMapping()["first"].GetQueueName
(), "first");
// test dumpConfiguration
TextWriter writer = new StringWriter();
Configuration conf = GetConfiguration();
conf.Unset(DeprecatedQueueConfigurationParser.MapredQueueNamesKey);
QueueManager.DumpConfiguration(writer, f.GetAbsolutePath(), conf);
string result = writer.ToString();
NUnit.Framework.Assert.IsTrue(result.IndexOf("\"name\":\"first\",\"state\":\"running\",\"acl_submit_job\":\"user1,user2 group1,group2\",\"acl_administer_jobs\":\"user3,user4 group3,group4\",\"properties\":[],\"children\":[]"
) > 0);
writer = new StringWriter();
QueueManager.DumpConfiguration(writer, conf);
result = writer.ToString();
NUnit.Framework.Assert.AreEqual("{\"queues\":[{\"name\":\"default\",\"state\":\"running\",\"acl_submit_job\":\"*\",\"acl_administer_jobs\":\"*\",\"properties\":[],\"children\":[]},{\"name\":\"q1\",\"state\":\"running\",\"acl_submit_job\":\" \",\"acl_administer_jobs\":\" \",\"properties\":[],\"children\":[{\"name\":\"q1:q2\",\"state\":\"running\",\"acl_submit_job\":\" \",\"acl_administer_jobs\":\" \",\"properties\":[{\"key\":\"capacity\",\"value\":\"20\"},{\"key\":\"user-limit\",\"value\":\"30\"}],\"children\":[]}]}]}"
, result);
// test constructor QueueAclsInfo
QueueAclsInfo qi = new QueueAclsInfo();
NUnit.Framework.Assert.IsNull(qi.GetQueueName());
}
finally
{
if (f != null)
{
f.Delete();
}
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestContainerLogsFileAccess()
{
// This test will run only if NativeIO is enabled as SecureIOUtils
// require it to be enabled.
Assume.AssumeTrue(NativeIO.IsAvailable());
Configuration conf = new Configuration();
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
FilePath workDir = new FilePath(testWorkDir, "testContainerLogsFileAccess1");
Path remoteAppLogFile = new Path(workDir.GetAbsolutePath(), "aggregatedLogFile");
Path srcFileRoot = new Path(workDir.GetAbsolutePath(), "srcFiles");
string data = "Log File content for container : ";
// Creating files for container1. Log aggregator will try to read log files
// with illegal user.
ApplicationId applicationId = ApplicationId.NewInstance(1, 1);
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.NewInstance(applicationId
, 1);
ContainerId testContainerId1 = ContainerId.NewContainerId(applicationAttemptId, 1
);
Path appDir = new Path(srcFileRoot, testContainerId1.GetApplicationAttemptId().GetApplicationId
().ToString());
Path srcFilePath1 = new Path(appDir, testContainerId1.ToString());
string stdout = "stdout";
string stderr = "stderr";
WriteSrcFile(srcFilePath1, stdout, data + testContainerId1.ToString() + stdout);
WriteSrcFile(srcFilePath1, stderr, data + testContainerId1.ToString() + stderr);
UserGroupInformation ugi = UserGroupInformation.GetCurrentUser();
AggregatedLogFormat.LogWriter logWriter = new AggregatedLogFormat.LogWriter(conf,
remoteAppLogFile, ugi);
AggregatedLogFormat.LogKey logKey = new AggregatedLogFormat.LogKey(testContainerId1
);
string randomUser = "******";
AggregatedLogFormat.LogValue logValue = Org.Mockito.Mockito.Spy(new AggregatedLogFormat.LogValue
(Collections.SingletonList(srcFileRoot.ToString()), testContainerId1, randomUser
));
// It is trying simulate a situation where first log file is owned by
// different user (probably symlink) and second one by the user itself.
// The first file should not be aggregated. Because this log file has the invalid
// user name.
Org.Mockito.Mockito.When(logValue.GetUser()).ThenReturn(randomUser).ThenReturn(ugi
.GetShortUserName());
logWriter.Append(logKey, logValue);
logWriter.Close();
BufferedReader @in = new BufferedReader(new FileReader(new FilePath(remoteAppLogFile
.ToUri().GetRawPath())));
string line;
StringBuilder sb = new StringBuilder(string.Empty);
while ((line = @in.ReadLine()) != null)
{
Log.Info(line);
sb.Append(line);
}
line = sb.ToString();
string expectedOwner = ugi.GetShortUserName();
if (Path.Windows)
{
string adminsGroupString = "Administrators";
if (Arrays.AsList(ugi.GetGroupNames()).Contains(adminsGroupString))
{
expectedOwner = adminsGroupString;
}
}
// This file: stderr should not be aggregated.
// And we will not aggregate the log message.
string stdoutFile1 = StringUtils.Join(FilePath.separator, Arrays.AsList(new string
[] { workDir.GetAbsolutePath(), "srcFiles", testContainerId1.GetApplicationAttemptId
().GetApplicationId().ToString(), testContainerId1.ToString(), stderr }));
// The file: stdout is expected to be aggregated.
string stdoutFile2 = StringUtils.Join(FilePath.separator, Arrays.AsList(new string
[] { workDir.GetAbsolutePath(), "srcFiles", testContainerId1.GetApplicationAttemptId
().GetApplicationId().ToString(), testContainerId1.ToString(), stdout }));
string message2 = "Owner '" + expectedOwner + "' for path " + stdoutFile2 + " did not match expected owner '"
+ ugi.GetShortUserName() + "'";
NUnit.Framework.Assert.IsFalse(line.Contains(message2));
NUnit.Framework.Assert.IsFalse(line.Contains(data + testContainerId1.ToString() +
stderr));
NUnit.Framework.Assert.IsTrue(line.Contains(data + testContainerId1.ToString() +
stdout));
}
/// <exception cref="System.Exception"/>
public virtual void TestGettingGroups()
{
UserGroupInformation uugi = UserGroupInformation.CreateUserForTesting(UserName, GroupNames
);
Assert.Equal(UserName, uugi.GetUserName());
Assert.AssertArrayEquals(new string[] { Group1Name, Group2Name, Group3Name }, uugi
.GetGroupNames());
}
public virtual void TestRefreshSuperUserGroupsConfiguration()
{
string SuperUser = "******";
string[] GroupNames1 = new string[] { "gr1", "gr2" };
string[] GroupNames2 = new string[] { "gr3", "gr4" };
//keys in conf
string userKeyGroups = DefaultImpersonationProvider.GetTestProvider().GetProxySuperuserGroupConfKey
(SuperUser);
string userKeyHosts = DefaultImpersonationProvider.GetTestProvider().GetProxySuperuserIpConfKey
(SuperUser);
config.Set(userKeyGroups, "gr3,gr4,gr5");
// superuser can proxy for this group
config.Set(userKeyHosts, "127.0.0.1");
ProxyUsers.RefreshSuperUserGroupsConfiguration(config);
UserGroupInformation ugi1 = Org.Mockito.Mockito.Mock <UserGroupInformation>();
UserGroupInformation ugi2 = Org.Mockito.Mockito.Mock <UserGroupInformation>();
UserGroupInformation suUgi = Org.Mockito.Mockito.Mock <UserGroupInformation>();
Org.Mockito.Mockito.When(ugi1.GetRealUser()).ThenReturn(suUgi);
Org.Mockito.Mockito.When(ugi2.GetRealUser()).ThenReturn(suUgi);
Org.Mockito.Mockito.When(suUgi.GetShortUserName()).ThenReturn(SuperUser);
// super user
Org.Mockito.Mockito.When(suUgi.GetUserName()).ThenReturn(SuperUser + "L");
// super user
Org.Mockito.Mockito.When(ugi1.GetShortUserName()).ThenReturn("user1");
Org.Mockito.Mockito.When(ugi2.GetShortUserName()).ThenReturn("user2");
Org.Mockito.Mockito.When(ugi1.GetUserName()).ThenReturn("userL1");
Org.Mockito.Mockito.When(ugi2.GetUserName()).ThenReturn("userL2");
// set groups for users
Org.Mockito.Mockito.When(ugi1.GetGroupNames()).ThenReturn(GroupNames1);
Org.Mockito.Mockito.When(ugi2.GetGroupNames()).ThenReturn(GroupNames2);
// check before
try
{
ProxyUsers.Authorize(ugi1, "127.0.0.1");
NUnit.Framework.Assert.Fail("first auth for " + ugi1.GetShortUserName() + " should've failed "
);
}
catch (AuthorizationException)
{
// expected
System.Console.Error.WriteLine("auth for " + ugi1.GetUserName() + " failed");
}
try
{
ProxyUsers.Authorize(ugi2, "127.0.0.1");
System.Console.Error.WriteLine("auth for " + ugi2.GetUserName() + " succeeded");
}
catch (AuthorizationException e)
{
// expected
NUnit.Framework.Assert.Fail("first auth for " + ugi2.GetShortUserName() + " should've succeeded: "
+ e.GetLocalizedMessage());
}
// refresh will look at configuration on the server side
// add additional resource with the new value
// so the server side will pick it up
string rsrc = "testGroupMappingRefresh_rsrc.xml";
AddNewConfigResource(rsrc, userKeyGroups, "gr2", userKeyHosts, "127.0.0.1");
DFSAdmin admin = new DFSAdmin(config);
string[] args = new string[] { "-refreshSuperUserGroupsConfiguration" };
admin.Run(args);
try
{
ProxyUsers.Authorize(ugi2, "127.0.0.1");
NUnit.Framework.Assert.Fail("second auth for " + ugi2.GetShortUserName() + " should've failed "
);
}
catch (AuthorizationException)
{
// expected
System.Console.Error.WriteLine("auth for " + ugi2.GetUserName() + " failed");
}
try
{
ProxyUsers.Authorize(ugi1, "127.0.0.1");
System.Console.Error.WriteLine("auth for " + ugi1.GetUserName() + " succeeded");
}
catch (AuthorizationException e)
{
// expected
NUnit.Framework.Assert.Fail("second auth for " + ugi1.GetShortUserName() + " should've succeeded: "
+ e.GetLocalizedMessage());
}
}
