/// <summary>
/// Returns the reference to a new instance of FileSystem created
/// with different user name
/// </summary>
/// <param name="conf">current Configuration</param>
/// <returns>FileSystem instance</returns>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"></exception>
public static FileSystem CreateHdfsWithDifferentUsername(Configuration conf)
{
string username = UserGroupInformation.GetCurrentUser().GetShortUserName() + "_XXX";
UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting(username, new
string[] { "supergroup" });
return(DFSTestUtil.GetFileSystemAs(ugi, conf));
}
/// <summary>
/// Test the listing with different user names to make sure only directories
/// that are owned by the user are listed.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestListWithDifferentUser()
{
cluster.GetNamesystem().GetSnapshotManager().SetAllowNestedSnapshots(true);
// first make dir1 and dir2 snapshottable
hdfs.AllowSnapshot(dir1);
hdfs.AllowSnapshot(dir2);
hdfs.SetPermission(root, FsPermission.ValueOf("-rwxrwxrwx"));
// create two dirs and make them snapshottable under the name of user1
UserGroupInformation ugi1 = UserGroupInformation.CreateUserForTesting("user1", new
string[] { "group1" });
DistributedFileSystem fs1 = (DistributedFileSystem)DFSTestUtil.GetFileSystemAs(ugi1
, conf);
Path dir1_user1 = new Path("/dir1_user1");
Path dir2_user1 = new Path("/dir2_user1");
fs1.Mkdirs(dir1_user1);
fs1.Mkdirs(dir2_user1);
hdfs.AllowSnapshot(dir1_user1);
hdfs.AllowSnapshot(dir2_user1);
// user2
UserGroupInformation ugi2 = UserGroupInformation.CreateUserForTesting("user2", new
string[] { "group2" });
DistributedFileSystem fs2 = (DistributedFileSystem)DFSTestUtil.GetFileSystemAs(ugi2
, conf);
Path dir_user2 = new Path("/dir_user2");
Path subdir_user2 = new Path(dir_user2, "subdir");
fs2.Mkdirs(dir_user2);
fs2.Mkdirs(subdir_user2);
hdfs.AllowSnapshot(dir_user2);
hdfs.AllowSnapshot(subdir_user2);
// super user
string supergroup = conf.Get(DFSConfigKeys.DfsPermissionsSuperusergroupKey, DFSConfigKeys
.DfsPermissionsSuperusergroupDefault);
UserGroupInformation superUgi = UserGroupInformation.CreateUserForTesting("superuser"
, new string[] { supergroup });
DistributedFileSystem fs3 = (DistributedFileSystem)DFSTestUtil.GetFileSystemAs(superUgi
, conf);
// list the snapshottable dirs for superuser
SnapshottableDirectoryStatus[] dirs = fs3.GetSnapshottableDirListing();
// 6 snapshottable dirs: dir1, dir2, dir1_user1, dir2_user1, dir_user2, and
// subdir_user2
NUnit.Framework.Assert.AreEqual(6, dirs.Length);
// list the snapshottable dirs for user1
dirs = fs1.GetSnapshottableDirListing();
// 2 dirs owned by user1: dir1_user1 and dir2_user1
NUnit.Framework.Assert.AreEqual(2, dirs.Length);
NUnit.Framework.Assert.AreEqual(dir1_user1, dirs[0].GetFullPath());
NUnit.Framework.Assert.AreEqual(dir2_user1, dirs[1].GetFullPath());
// list the snapshottable dirs for user2
dirs = fs2.GetSnapshottableDirListing();
// 2 dirs owned by user2: dir_user2 and subdir_user2
NUnit.Framework.Assert.AreEqual(2, dirs.Length);
NUnit.Framework.Assert.AreEqual(dir_user2, dirs[0].GetFullPath());
NUnit.Framework.Assert.AreEqual(subdir_user2, dirs[1].GetFullPath());
}
/// <exception cref="System.Exception"/>
private static void InitCluster(bool format)
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(4).Format(format).Build();
hdfs = cluster.GetFileSystem();
NUnit.Framework.Assert.IsTrue(hdfs is DistributedFileSystem);
hdfsAsUser1 = DFSTestUtil.GetFileSystemAs(user1, conf);
NUnit.Framework.Assert.IsTrue(hdfsAsUser1 is DistributedFileSystem);
hdfsAsUser2 = DFSTestUtil.GetFileSystemAs(user2, conf);
NUnit.Framework.Assert.IsTrue(hdfsAsUser2 is DistributedFileSystem);
}
public virtual void TestAuditAllowedStat()
{
Path file = new Path(fnames[0]);
FileSystem userfs = DFSTestUtil.GetFileSystemAs(userGroupInfo, conf);
SetupAuditLogs();
FileStatus st = userfs.GetFileStatus(file);
VerifyAuditLogs(true);
NUnit.Framework.Assert.IsTrue("failed to stat file", st != null && st.IsFile());
}
public virtual void TestAuditAllowed()
{
Path file = new Path(fnames[0]);
FileSystem userfs = DFSTestUtil.GetFileSystemAs(userGroupInfo, conf);
SetupAuditLogs();
InputStream istream = userfs.Open(file);
int val = istream.Read();
istream.Close();
VerifyAuditLogs(true);
NUnit.Framework.Assert.IsTrue("failed to read from file", val >= 0);
}
public virtual void TestAuditDenied()
{
Path file = new Path(fnames[0]);
FileSystem userfs = DFSTestUtil.GetFileSystemAs(userGroupInfo, conf);
fs.SetPermission(file, new FsPermission((short)0x180));
fs.SetOwner(file, "root", null);
SetupAuditLogs();
try
{
userfs.Open(file);
NUnit.Framework.Assert.Fail("open must not succeed");
}
catch (AccessControlException)
{
System.Console.Out.WriteLine("got access denied, as expected.");
}
VerifyAuditLogs(false);
}
//check the file
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
internal static void CheckFile(Path p, int expectedsize, Configuration conf)
{
//open the file with another user account
string username = UserGroupInformation.GetCurrentUser().GetShortUserName() + "_"
+ ++userCount;
UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting(username, new
string[] { "supergroup" });
FileSystem fs = DFSTestUtil.GetFileSystemAs(ugi, conf);
HdfsDataInputStream @in = (HdfsDataInputStream)fs.Open(p);
//Check visible length
NUnit.Framework.Assert.IsTrue(@in.GetVisibleLength() >= expectedsize);
//Able to read?
for (int i = 0; i < expectedsize; i++)
{
NUnit.Framework.Assert.AreEqual(unchecked ((byte)i), unchecked ((byte)@in.Read()));
}
@in.Close();
}
public virtual void TestFilePermision()
{
Configuration conf = new HdfsConfiguration();
conf.SetBoolean(DFSConfigKeys.DfsPermissionsEnabledKey, true);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build();
cluster.WaitActive();
try
{
FileSystem nnfs = FileSystem.Get(conf);
// test permissions on files that do not exist
NUnit.Framework.Assert.IsFalse(nnfs.Exists(ChildFile1));
try
{
nnfs.SetOwner(ChildFile1, "foo", "bar");
NUnit.Framework.Assert.IsTrue(false);
}
catch (FileNotFoundException e)
{
Log.Info("GOOD: got " + e);
}
try
{
nnfs.SetPermission(ChildFile1, new FsPermission((short)0x1ff));
NUnit.Framework.Assert.IsTrue(false);
}
catch (FileNotFoundException e)
{
Log.Info("GOOD: got " + e);
}
// make sure nn can take user specified permission (with default fs
// permission umask applied)
FSDataOutputStream @out = nnfs.Create(ChildFile1, new FsPermission((short)0x1ff),
true, 1024, (short)1, 1024, null);
FileStatus status = nnfs.GetFileStatus(ChildFile1);
// FS_PERMISSIONS_UMASK_DEFAULT is 0022
NUnit.Framework.Assert.IsTrue(status.GetPermission().ToString().Equals("rwxr-xr-x"
));
nnfs.Delete(ChildFile1, false);
// following dir/file creations are legal
nnfs.Mkdirs(ChildDir1);
@out = nnfs.Create(ChildFile1);
status = nnfs.GetFileStatus(ChildFile1);
NUnit.Framework.Assert.IsTrue(status.GetPermission().ToString().Equals("rw-r--r--"
));
byte[] data = new byte[FileLen];
Ran.NextBytes(data);
@out.Write(data);
@out.Close();
nnfs.SetPermission(ChildFile1, new FsPermission("700"));
status = nnfs.GetFileStatus(ChildFile1);
NUnit.Framework.Assert.IsTrue(status.GetPermission().ToString().Equals("rwx------"
));
// following read is legal
byte[] dataIn = new byte[FileLen];
FSDataInputStream fin = nnfs.Open(ChildFile1);
int bytesRead = fin.Read(dataIn);
NUnit.Framework.Assert.IsTrue(bytesRead == FileLen);
for (int i = 0; i < FileLen; i++)
{
NUnit.Framework.Assert.AreEqual(data[i], dataIn[i]);
}
// test execution bit support for files
nnfs.SetPermission(ChildFile1, new FsPermission("755"));
status = nnfs.GetFileStatus(ChildFile1);
NUnit.Framework.Assert.IsTrue(status.GetPermission().ToString().Equals("rwxr-xr-x"
));
nnfs.SetPermission(ChildFile1, new FsPermission("744"));
status = nnfs.GetFileStatus(ChildFile1);
NUnit.Framework.Assert.IsTrue(status.GetPermission().ToString().Equals("rwxr--r--"
));
nnfs.SetPermission(ChildFile1, new FsPermission("700"));
////////////////////////////////////////////////////////////////
// test illegal file/dir creation
UserGroupInformation userGroupInfo = UserGroupInformation.CreateUserForTesting(UserName
, GroupNames);
FileSystem userfs = DFSTestUtil.GetFileSystemAs(userGroupInfo, conf);
// make sure mkdir of a existing directory that is not owned by
// this user does not throw an exception.
userfs.Mkdirs(ChildDir1);
// illegal mkdir
NUnit.Framework.Assert.IsTrue(!CanMkdirs(userfs, ChildDir2));
// illegal file creation
NUnit.Framework.Assert.IsTrue(!CanCreate(userfs, ChildFile2));
// illegal file open
NUnit.Framework.Assert.IsTrue(!CanOpen(userfs, ChildFile1));
nnfs.SetPermission(RootPath, new FsPermission((short)0x1ed));
nnfs.SetPermission(ChildDir1, new FsPermission("777"));
nnfs.SetPermission(new Path("/"), new FsPermission((short)0x1ff));
Path RenamePath = new Path("/foo/bar");
userfs.Mkdirs(RenamePath);
NUnit.Framework.Assert.IsTrue(CanRename(userfs, RenamePath, ChildDir1));
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestSimpleAppend()
{
Configuration conf = new HdfsConfiguration();
conf.SetInt(DFSConfigKeys.DfsDatanodeHandlerCountKey, 50);
fileContents = AppendTestUtil.InitBuffer(AppendTestUtil.FileSize);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
FileSystem fs = cluster.GetFileSystem();
try
{
{
// test appending to a file.
// create a new file.
Path file1 = new Path("/simpleAppend.dat");
FSDataOutputStream stm = AppendTestUtil.CreateFile(fs, file1, 1);
System.Console.Out.WriteLine("Created file simpleAppend.dat");
// write to file
int mid = 186;
// io.bytes.per.checksum bytes
System.Console.Out.WriteLine("Writing " + mid + " bytes to file " + file1);
stm.Write(fileContents, 0, mid);
stm.Close();
System.Console.Out.WriteLine("Wrote and Closed first part of file.");
// write to file
int mid2 = 607;
// io.bytes.per.checksum bytes
System.Console.Out.WriteLine("Writing " + mid + " bytes to file " + file1);
stm = fs.Append(file1);
stm.Write(fileContents, mid, mid2 - mid);
stm.Close();
System.Console.Out.WriteLine("Wrote and Closed second part of file.");
// write the remainder of the file
stm = fs.Append(file1);
// ensure getPos is set to reflect existing size of the file
NUnit.Framework.Assert.IsTrue(stm.GetPos() > 0);
System.Console.Out.WriteLine("Writing " + (AppendTestUtil.FileSize - mid2) + " bytes to file "
+ file1);
stm.Write(fileContents, mid2, AppendTestUtil.FileSize - mid2);
System.Console.Out.WriteLine("Written second part of file");
stm.Close();
System.Console.Out.WriteLine("Wrote and Closed second part of file.");
// verify that entire file is good
AppendTestUtil.CheckFullFile(fs, file1, AppendTestUtil.FileSize, fileContents, "Read 2"
);
}
{
// test appending to an non-existing file.
FSDataOutputStream @out = null;
try
{
@out = fs.Append(new Path("/non-existing.dat"));
NUnit.Framework.Assert.Fail("Expected to have FileNotFoundException");
}
catch (FileNotFoundException fnfe)
{
System.Console.Out.WriteLine("Good: got " + fnfe);
Sharpen.Runtime.PrintStackTrace(fnfe, System.Console.Out);
}
finally
{
IOUtils.CloseStream(@out);
}
}
{
// test append permission.
//set root to all writable
Path root = new Path("/");
fs.SetPermission(root, new FsPermission((short)0x1ff));
fs.Close();
// login as a different user
UserGroupInformation superuser = UserGroupInformation.GetCurrentUser();
string username = "******";
string group = "testappendgroup";
NUnit.Framework.Assert.IsFalse(superuser.GetShortUserName().Equals(username));
NUnit.Framework.Assert.IsFalse(Arrays.AsList(superuser.GetGroupNames()).Contains(
group));
UserGroupInformation appenduser = UserGroupInformation.CreateUserForTesting(username
, new string[] { group });
fs = DFSTestUtil.GetFileSystemAs(appenduser, conf);
// create a file
Path dir = new Path(root, GetType().Name);
Path foo = new Path(dir, "foo.dat");
FSDataOutputStream @out = null;
int offset = 0;
try
{
@out = fs.Create(foo);
int len = 10 + AppendTestUtil.NextInt(100);
@out.Write(fileContents, offset, len);
offset += len;
}
finally
{
IOUtils.CloseStream(@out);
}
// change dir and foo to minimal permissions.
fs.SetPermission(dir, new FsPermission((short)0x40));
fs.SetPermission(foo, new FsPermission((short)0x80));
// try append, should success
@out = null;
try
{
@out = fs.Append(foo);
int len = 10 + AppendTestUtil.NextInt(100);
@out.Write(fileContents, offset, len);
offset += len;
}
finally
{
IOUtils.CloseStream(@out);
}
// change dir and foo to all but no write on foo.
fs.SetPermission(foo, new FsPermission((short)0x17f));
fs.SetPermission(dir, new FsPermission((short)0x1ff));
// try append, should fail
@out = null;
try
{
@out = fs.Append(foo);
NUnit.Framework.Assert.Fail("Expected to have AccessControlException");
}
catch (AccessControlException ace)
{
System.Console.Out.WriteLine("Good: got " + ace);
Sharpen.Runtime.PrintStackTrace(ace, System.Console.Out);
}
finally
{
IOUtils.CloseStream(@out);
}
}
}
catch (IOException e)
{
System.Console.Out.WriteLine("Exception :" + e);
throw;
}
catch (Exception e)
{
System.Console.Out.WriteLine("Throwable :" + e);
Sharpen.Runtime.PrintStackTrace(e);
throw new IOException("Throwable : " + e);
}
finally
{
fs.Close();
cluster.Shutdown();
}
}
public virtual void TestSoftLeaseRecovery()
{
IDictionary <string, string[]> u2g_map = new Dictionary <string, string[]>(1);
u2g_map[fakeUsername] = new string[] { fakeGroup };
DFSTestUtil.UpdateConfWithFakeGroupMapping(conf, u2g_map);
// Reset default lease periods
cluster.SetLeasePeriod(HdfsConstants.LeaseSoftlimitPeriod, HdfsConstants.LeaseHardlimitPeriod
);
//create a file
// create a random file name
string filestr = "/foo" + AppendTestUtil.NextInt();
AppendTestUtil.Log.Info("filestr=" + filestr);
Path filepath = new Path(filestr);
FSDataOutputStream stm = dfs.Create(filepath, true, BufSize, ReplicationNum, BlockSize
);
NUnit.Framework.Assert.IsTrue(dfs.dfs.Exists(filestr));
// write random number of bytes into it.
int size = AppendTestUtil.NextInt(FileSize);
AppendTestUtil.Log.Info("size=" + size);
stm.Write(buffer, 0, size);
// hflush file
AppendTestUtil.Log.Info("hflush");
stm.Hflush();
AppendTestUtil.Log.Info("leasechecker.interruptAndJoin()");
dfs.dfs.GetLeaseRenewer().InterruptAndJoin();
// set the soft limit to be 1 second so that the
// namenode triggers lease recovery on next attempt to write-for-open.
cluster.SetLeasePeriod(ShortLeasePeriod, LongLeasePeriod);
{
// try to re-open the file before closing the previous handle. This
// should fail but will trigger lease recovery.
UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting(fakeUsername
, new string[] { fakeGroup });
FileSystem dfs2 = DFSTestUtil.GetFileSystemAs(ugi, conf);
bool done = false;
for (int i = 0; i < 10 && !done; i++)
{
AppendTestUtil.Log.Info("i=" + i);
try
{
dfs2.Create(filepath, false, BufSize, ReplicationNum, BlockSize);
NUnit.Framework.Assert.Fail("Creation of an existing file should never succeed.");
}
catch (FileAlreadyExistsException)
{
done = true;
}
catch (AlreadyBeingCreatedException ex)
{
AppendTestUtil.Log.Info("GOOD! got " + ex.Message);
}
catch (IOException ioe)
{
AppendTestUtil.Log.Warn("UNEXPECTED IOException", ioe);
}
if (!done)
{
AppendTestUtil.Log.Info("sleep " + 5000 + "ms");
try
{
Sharpen.Thread.Sleep(5000);
}
catch (Exception)
{
}
}
}
NUnit.Framework.Assert.IsTrue(done);
}
AppendTestUtil.Log.Info("Lease for file " + filepath + " is recovered. " + "Validating its contents now..."
);
// verify that file-size matches
long fileSize = dfs.GetFileStatus(filepath).GetLen();
NUnit.Framework.Assert.IsTrue("File should be " + size + " bytes, but is actually "
+ " found to be " + fileSize + " bytes", fileSize == size);
// verify data
AppendTestUtil.Log.Info("File size is good. " + "Now validating data and sizes from datanodes..."
);
AppendTestUtil.CheckFullFile(dfs, filepath, size, buffer, filestr);
}
public virtual void TestConcat()
{
int numFiles = 10;
long fileLen = blockSize * 3;
HdfsFileStatus fStatus;
FSDataInputStream stm;
string trg = "/trg";
Path trgPath = new Path(trg);
DFSTestUtil.CreateFile(dfs, trgPath, fileLen, ReplFactor, 1);
fStatus = nn.GetFileInfo(trg);
long trgLen = fStatus.GetLen();
long trgBlocks = nn.GetBlockLocations(trg, 0, trgLen).LocatedBlockCount();
Path[] files = new Path[numFiles];
byte[][] bytes = new byte[][] { new byte[(int)fileLen], new byte[(int)fileLen], new
byte[(int)fileLen], new byte[(int)fileLen], new byte[(int)fileLen], new byte[(int
)fileLen], new byte[(int)fileLen], new byte[(int)fileLen], new byte[(int)fileLen
], new byte[(int)fileLen], new byte[(int)fileLen] };
LocatedBlocks[] lblocks = new LocatedBlocks[numFiles];
long[] lens = new long[numFiles];
stm = dfs.Open(trgPath);
stm.ReadFully(0, bytes[0]);
stm.Close();
int i;
for (i = 0; i < files.Length; i++)
{
files[i] = new Path("/file" + i);
Path path = files[i];
System.Console.Out.WriteLine("Creating file " + path);
// make files with different content
DFSTestUtil.CreateFile(dfs, path, fileLen, ReplFactor, i);
fStatus = nn.GetFileInfo(path.ToUri().GetPath());
lens[i] = fStatus.GetLen();
NUnit.Framework.Assert.AreEqual(trgLen, lens[i]);
// file of the same length.
lblocks[i] = nn.GetBlockLocations(path.ToUri().GetPath(), 0, lens[i]);
//read the file
stm = dfs.Open(path);
stm.ReadFully(0, bytes[i + 1]);
//bytes[i][10] = 10;
stm.Close();
}
// check permissions -try the operation with the "wrong" user
UserGroupInformation user1 = UserGroupInformation.CreateUserForTesting("theDoctor"
, new string[] { "tardis" });
DistributedFileSystem hdfs = (DistributedFileSystem)DFSTestUtil.GetFileSystemAs(user1
, conf);
try
{
hdfs.Concat(trgPath, files);
NUnit.Framework.Assert.Fail("Permission exception expected");
}
catch (IOException ie)
{
System.Console.Out.WriteLine("Got expected exception for permissions:" + ie.GetLocalizedMessage
());
}
// expected
// check count update
ContentSummary cBefore = dfs.GetContentSummary(trgPath.GetParent());
// resort file array, make INode id not sorted.
for (int j = 0; j < files.Length / 2; j++)
{
Path tempPath = files[j];
files[j] = files[files.Length - 1 - j];
files[files.Length - 1 - j] = tempPath;
byte[] tempBytes = bytes[1 + j];
bytes[1 + j] = bytes[files.Length - 1 - j + 1];
bytes[files.Length - 1 - j + 1] = tempBytes;
}
// now concatenate
dfs.Concat(trgPath, files);
// verify count
ContentSummary cAfter = dfs.GetContentSummary(trgPath.GetParent());
NUnit.Framework.Assert.AreEqual(cBefore.GetFileCount(), cAfter.GetFileCount() + files
.Length);
// verify other stuff
long totalLen = trgLen;
long totalBlocks = trgBlocks;
for (i = 0; i < files.Length; i++)
{
totalLen += lens[i];
totalBlocks += lblocks[i].LocatedBlockCount();
}
System.Console.Out.WriteLine("total len=" + totalLen + "; totalBlocks=" + totalBlocks
);
fStatus = nn.GetFileInfo(trg);
trgLen = fStatus.GetLen();
// new length
// read the resulting file
stm = dfs.Open(trgPath);
byte[] byteFileConcat = new byte[(int)trgLen];
stm.ReadFully(0, byteFileConcat);
stm.Close();
trgBlocks = nn.GetBlockLocations(trg, 0, trgLen).LocatedBlockCount();
//verifications
// 1. number of blocks
NUnit.Framework.Assert.AreEqual(trgBlocks, totalBlocks);
// 2. file lengths
NUnit.Framework.Assert.AreEqual(trgLen, totalLen);
// 3. removal of the src file
foreach (Path p in files)
{
fStatus = nn.GetFileInfo(p.ToUri().GetPath());
NUnit.Framework.Assert.IsNull("File " + p + " still exists", fStatus);
// file shouldn't exist
// try to create fie with the same name
DFSTestUtil.CreateFile(dfs, p, fileLen, ReplFactor, 1);
}
// 4. content
CheckFileContent(byteFileConcat, bytes);
// add a small file (less then a block)
Path smallFile = new Path("/sfile");
int sFileLen = 10;
DFSTestUtil.CreateFile(dfs, smallFile, sFileLen, ReplFactor, 1);
dfs.Concat(trgPath, new Path[] { smallFile });
fStatus = nn.GetFileInfo(trg);
trgLen = fStatus.GetLen();
// new length
// check number of blocks
trgBlocks = nn.GetBlockLocations(trg, 0, trgLen).LocatedBlockCount();
NUnit.Framework.Assert.AreEqual(trgBlocks, totalBlocks + 1);
// and length
NUnit.Framework.Assert.AreEqual(trgLen, totalLen + sFileLen);
}