public static void Setup()
{
// start a cluster with single datanode
cluster = new MiniDFSCluster.Builder(Conf).Build();
cluster.WaitActive();
fs = cluster.GetFileSystem();
string str = "hftp://" + Conf.Get(DFSConfigKeys.DfsNamenodeHttpAddressKey);
hftpURI = new URI(str);
hftpFs = cluster.GetHftpFileSystem(0);
}
public static void TestSetUp()
{
conf = new HdfsConfiguration();
conf.SetInt(DFSConfigKeys.DfsListLimit, 2);
cluster = new MiniDFSCluster.Builder(conf).Build();
fs = cluster.GetFileSystem();
fc = FileContext.GetFileContext(cluster.GetURI(0), conf);
hftpfs = cluster.GetHftpFileSystem(0);
dfsClient = new DFSClient(NameNode.GetAddress(conf), conf);
file1 = new Path("filestatus.dat");
WriteFile(fs, file1, 1, fileSize, blockSize);
}
public virtual void TestListStatus()
{
// Empty root directory
CheckStatus("/");
// Root directory with files and directories
CreateFile("/a", 1);
CreateFile("/b", 1);
Mkdirs("/dir");
CheckFile(new Path("/a"));
CheckFile(new Path("/b"));
CheckStatus("/");
// A directory with files and directories
CreateFile("/dir/.a.crc", 1);
CreateFile("/dir/b", 1);
Mkdirs("/dir/dir1");
CheckFile(new Path("/dir/.a.crc"));
CheckFile(new Path("/dir/b"));
CheckStatus("/dir");
// Non existent path
CheckStatus("/nonexistent");
CheckStatus("/nonexistent/a");
string username = UserGroupInformation.GetCurrentUser().GetShortUserName() + "1";
HftpFileSystem hftp2 = cluster.GetHftpFileSystemAs(username, Conf, 0, "somegroup"
);
{
//test file not found on hftp
Path nonexistent = new Path("/nonexistent");
try
{
hftp2.GetFileStatus(nonexistent);
NUnit.Framework.Assert.Fail();
}
catch (IOException ioe)
{
FileSystem.Log.Info("GOOD: getting an exception", ioe);
}
}
{
//test permission error on hftp
Path dir = new Path("/dir");
fs.SetPermission(dir, new FsPermission((short)0));
try
{
hftp2.GetFileStatus(new Path(dir, "a"));
NUnit.Framework.Assert.Fail();
}
catch (IOException ioe)
{
FileSystem.Log.Info("GOOD: getting an exception", ioe);
}
}
}
public virtual void TestAuditHftp()
{
Path file = new Path(fnames[0]);
string hftpUri = "hftp://" + conf.Get(DFSConfigKeys.DfsNamenodeHttpAddressKey);
HftpFileSystem hftpFs = null;
SetupAuditLogs();
try
{
hftpFs = (HftpFileSystem) new Path(hftpUri).GetFileSystem(conf);
InputStream istream = hftpFs.Open(file);
int val = istream.Read();
istream.Close();
VerifyAuditLogs(true);
}
finally
{
if (hftpFs != null)
{
hftpFs.Close();
}
}
}
public virtual void TestGetFileStatusOnDir()
{
// Create the directory
Path dir = new Path("/test/mkdirs");
NUnit.Framework.Assert.IsTrue("mkdir failed", fs.Mkdirs(dir));
NUnit.Framework.Assert.IsTrue("mkdir failed", fs.Exists(dir));
// test getFileStatus on an empty directory
FileStatus status = fs.GetFileStatus(dir);
NUnit.Framework.Assert.IsTrue(dir + " should be a directory", status.IsDirectory(
));
NUnit.Framework.Assert.IsTrue(dir + " should be zero size ", status.GetLen() == 0
);
NUnit.Framework.Assert.AreEqual(dir.MakeQualified(fs.GetUri(), fs.GetWorkingDirectory
()).ToString(), status.GetPath().ToString());
// test listStatus on an empty directory
FileStatus[] stats = fs.ListStatus(dir);
NUnit.Framework.Assert.AreEqual(dir + " should be empty", 0, stats.Length);
NUnit.Framework.Assert.AreEqual(dir + " should be zero size ", 0, fs.GetContentSummary
(dir).GetLength());
NUnit.Framework.Assert.AreEqual(dir + " should be zero size using hftp", 0, hftpfs
.GetContentSummary(dir).GetLength());
RemoteIterator <FileStatus> itor = fc.ListStatus(dir);
NUnit.Framework.Assert.IsFalse(dir + " should be empty", itor.HasNext());
itor = fs.ListStatusIterator(dir);
NUnit.Framework.Assert.IsFalse(dir + " should be empty", itor.HasNext());
// create another file that is smaller than a block.
Path file2 = new Path(dir, "filestatus2.dat");
WriteFile(fs, file2, 1, blockSize / 4, blockSize);
CheckFile(fs, file2, 1);
// verify file attributes
status = fs.GetFileStatus(file2);
NUnit.Framework.Assert.AreEqual(blockSize, status.GetBlockSize());
NUnit.Framework.Assert.AreEqual(1, status.GetReplication());
file2 = fs.MakeQualified(file2);
NUnit.Framework.Assert.AreEqual(file2.ToString(), status.GetPath().ToString());
// Create another file in the same directory
Path file3 = new Path(dir, "filestatus3.dat");
WriteFile(fs, file3, 1, blockSize / 4, blockSize);
CheckFile(fs, file3, 1);
file3 = fs.MakeQualified(file3);
// Verify that the size of the directory increased by the size
// of the two files
int expected = blockSize / 2;
NUnit.Framework.Assert.AreEqual(dir + " size should be " + expected, expected, fs
.GetContentSummary(dir).GetLength());
NUnit.Framework.Assert.AreEqual(dir + " size should be " + expected + " using hftp"
, expected, hftpfs.GetContentSummary(dir).GetLength());
// Test listStatus on a non-empty directory
stats = fs.ListStatus(dir);
NUnit.Framework.Assert.AreEqual(dir + " should have two entries", 2, stats.Length
);
NUnit.Framework.Assert.AreEqual(file2.ToString(), stats[0].GetPath().ToString());
NUnit.Framework.Assert.AreEqual(file3.ToString(), stats[1].GetPath().ToString());
itor = fc.ListStatus(dir);
NUnit.Framework.Assert.AreEqual(file2.ToString(), itor.Next().GetPath().ToString(
));
NUnit.Framework.Assert.AreEqual(file3.ToString(), itor.Next().GetPath().ToString(
));
NUnit.Framework.Assert.IsFalse("Unexpected addtional file", itor.HasNext());
itor = fs.ListStatusIterator(dir);
NUnit.Framework.Assert.AreEqual(file2.ToString(), itor.Next().GetPath().ToString(
));
NUnit.Framework.Assert.AreEqual(file3.ToString(), itor.Next().GetPath().ToString(
));
NUnit.Framework.Assert.IsFalse("Unexpected addtional file", itor.HasNext());
// Test iterative listing. Now dir has 2 entries, create one more.
Path dir3 = fs.MakeQualified(new Path(dir, "dir3"));
fs.Mkdirs(dir3);
dir3 = fs.MakeQualified(dir3);
stats = fs.ListStatus(dir);
NUnit.Framework.Assert.AreEqual(dir + " should have three entries", 3, stats.Length
);
NUnit.Framework.Assert.AreEqual(dir3.ToString(), stats[0].GetPath().ToString());
NUnit.Framework.Assert.AreEqual(file2.ToString(), stats[1].GetPath().ToString());
NUnit.Framework.Assert.AreEqual(file3.ToString(), stats[2].GetPath().ToString());
itor = fc.ListStatus(dir);
NUnit.Framework.Assert.AreEqual(dir3.ToString(), itor.Next().GetPath().ToString()
);
NUnit.Framework.Assert.AreEqual(file2.ToString(), itor.Next().GetPath().ToString(
));
NUnit.Framework.Assert.AreEqual(file3.ToString(), itor.Next().GetPath().ToString(
));
NUnit.Framework.Assert.IsFalse("Unexpected addtional file", itor.HasNext());
itor = fs.ListStatusIterator(dir);
NUnit.Framework.Assert.AreEqual(dir3.ToString(), itor.Next().GetPath().ToString()
);
NUnit.Framework.Assert.AreEqual(file2.ToString(), itor.Next().GetPath().ToString(
));
NUnit.Framework.Assert.AreEqual(file3.ToString(), itor.Next().GetPath().ToString(
));
NUnit.Framework.Assert.IsFalse("Unexpected addtional file", itor.HasNext());
// Now dir has 3 entries, create two more
Path dir4 = fs.MakeQualified(new Path(dir, "dir4"));
fs.Mkdirs(dir4);
dir4 = fs.MakeQualified(dir4);
Path dir5 = fs.MakeQualified(new Path(dir, "dir5"));
fs.Mkdirs(dir5);
dir5 = fs.MakeQualified(dir5);
stats = fs.ListStatus(dir);
NUnit.Framework.Assert.AreEqual(dir + " should have five entries", 5, stats.Length
);
NUnit.Framework.Assert.AreEqual(dir3.ToString(), stats[0].GetPath().ToString());
NUnit.Framework.Assert.AreEqual(dir4.ToString(), stats[1].GetPath().ToString());
NUnit.Framework.Assert.AreEqual(dir5.ToString(), stats[2].GetPath().ToString());
NUnit.Framework.Assert.AreEqual(file2.ToString(), stats[3].GetPath().ToString());
NUnit.Framework.Assert.AreEqual(file3.ToString(), stats[4].GetPath().ToString());
itor = fc.ListStatus(dir);
NUnit.Framework.Assert.AreEqual(dir3.ToString(), itor.Next().GetPath().ToString()
);
NUnit.Framework.Assert.AreEqual(dir4.ToString(), itor.Next().GetPath().ToString()
);
NUnit.Framework.Assert.AreEqual(dir5.ToString(), itor.Next().GetPath().ToString()
);
NUnit.Framework.Assert.AreEqual(file2.ToString(), itor.Next().GetPath().ToString(
));
NUnit.Framework.Assert.AreEqual(file3.ToString(), itor.Next().GetPath().ToString(
));
NUnit.Framework.Assert.IsFalse(itor.HasNext());
itor = fs.ListStatusIterator(dir);
NUnit.Framework.Assert.AreEqual(dir3.ToString(), itor.Next().GetPath().ToString()
);
NUnit.Framework.Assert.AreEqual(dir4.ToString(), itor.Next().GetPath().ToString()
);
NUnit.Framework.Assert.AreEqual(dir5.ToString(), itor.Next().GetPath().ToString()
);
NUnit.Framework.Assert.AreEqual(file2.ToString(), itor.Next().GetPath().ToString(
));
NUnit.Framework.Assert.AreEqual(file3.ToString(), itor.Next().GetPath().ToString(
));
NUnit.Framework.Assert.IsFalse(itor.HasNext());
{
//test permission error on hftp
fs.SetPermission(dir, new FsPermission((short)0));
try
{
string username = UserGroupInformation.GetCurrentUser().GetShortUserName() + "1";
HftpFileSystem hftp2 = cluster.GetHftpFileSystemAs(username, conf, 0, "somegroup"
);
hftp2.GetContentSummary(dir);
NUnit.Framework.Assert.Fail();
}
catch (IOException ioe)
{
FileSystem.Log.Info("GOOD: getting an exception", ioe);
}
}
fs.Delete(dir, true);
}
protected override SimpleDateFormat InitialValue()
{
return(HftpFileSystem.GetDateFormat());
}
