//Do Nothing
/// <exception cref="System.Exception"/>
public virtual void TestWebHdfsDoAs()
{
WebHdfsTestUtil.Log.Info("START: testWebHdfsDoAs()");
WebHdfsTestUtil.Log.Info("ugi.getShortUserName()=" + ugi.GetShortUserName());
WebHdfsFileSystem webhdfs = WebHdfsTestUtil.GetWebHdfsFileSystemAs(ugi, config, WebHdfsFileSystem
.Scheme);
Path root = new Path("/");
cluster.GetFileSystem().SetPermission(root, new FsPermission((short)0x1ff));
Whitebox.SetInternalState(webhdfs, "ugi", proxyUgi);
{
Path responsePath = webhdfs.GetHomeDirectory();
WebHdfsTestUtil.Log.Info("responsePath=" + responsePath);
NUnit.Framework.Assert.AreEqual(webhdfs.GetUri() + "/user/" + ProxyUser, responsePath
.ToString());
}
Path f = new Path("/testWebHdfsDoAs/a.txt");
{
FSDataOutputStream @out = webhdfs.Create(f);
@out.Write(Sharpen.Runtime.GetBytesForString("Hello, webhdfs user!"));
@out.Close();
FileStatus status = webhdfs.GetFileStatus(f);
WebHdfsTestUtil.Log.Info("status.getOwner()=" + status.GetOwner());
NUnit.Framework.Assert.AreEqual(ProxyUser, status.GetOwner());
}
{
FSDataOutputStream @out = webhdfs.Append(f);
@out.Write(Sharpen.Runtime.GetBytesForString("\nHello again!"));
@out.Close();
FileStatus status = webhdfs.GetFileStatus(f);
WebHdfsTestUtil.Log.Info("status.getOwner()=" + status.GetOwner());
WebHdfsTestUtil.Log.Info("status.getLen() =" + status.GetLen());
NUnit.Framework.Assert.AreEqual(ProxyUser, status.GetOwner());
}
}
public static void BeforeClassSetup()
{
Configuration conf = new HdfsConfiguration();
conf.SetBoolean(DFSConfigKeys.DfsWebhdfsEnabledKey, true);
conf.Set(FsPermission.UmaskLabel, "000");
conf.SetInt(DFSConfigKeys.DfsNamenodeMaxComponentLengthKey, 0);
cluster = new MiniDFSCluster.Builder(conf).Build();
webhdfs = WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem.Scheme);
dfs = cluster.GetFileSystem();
}
public virtual void TestAuditWebHdfsOpen()
{
Path file = new Path(fnames[0]);
fs.SetPermission(file, new FsPermission((short)0x1a4));
fs.SetOwner(file, "root", null);
SetupAuditLogs();
WebHdfsFileSystem webfs = WebHdfsTestUtil.GetWebHdfsFileSystemAs(userGroupInfo, conf
, WebHdfsFileSystem.Scheme);
webfs.Open(file);
VerifyAuditLogsCheckPattern(true, 3, webOpenPattern);
}
public virtual void TestAuditWebHdfsStat()
{
Path file = new Path(fnames[0]);
fs.SetPermission(file, new FsPermission((short)0x1a4));
fs.SetOwner(file, "root", null);
SetupAuditLogs();
WebHdfsFileSystem webfs = WebHdfsTestUtil.GetWebHdfsFileSystemAs(userGroupInfo, conf
, WebHdfsFileSystem.Scheme);
FileStatus st = webfs.GetFileStatus(file);
VerifyAuditLogs(true);
NUnit.Framework.Assert.IsTrue("failed to stat file", st != null && st.IsFile());
}
public virtual void TestAuditWebHdfs()
{
Path file = new Path(fnames[0]);
fs.SetPermission(file, new FsPermission((short)0x1a4));
fs.SetOwner(file, "root", null);
SetupAuditLogs();
WebHdfsFileSystem webfs = WebHdfsTestUtil.GetWebHdfsFileSystemAs(userGroupInfo, conf
, WebHdfsFileSystem.Scheme);
InputStream istream = webfs.Open(file);
int val = istream.Read();
istream.Close();
VerifyAuditLogsRepeat(true, 3);
NUnit.Framework.Assert.IsTrue("failed to read from file", val >= 0);
}
public virtual void TestAuditWebHdfsDenied()
{
Path file = new Path(fnames[0]);
fs.SetPermission(file, new FsPermission((short)0x180));
fs.SetOwner(file, "root", null);
SetupAuditLogs();
try
{
WebHdfsFileSystem webfs = WebHdfsTestUtil.GetWebHdfsFileSystemAs(userGroupInfo, conf
, WebHdfsFileSystem.Scheme);
InputStream istream = webfs.Open(file);
int val = istream.Read();
NUnit.Framework.Assert.Fail("open+read must not succeed, got " + val);
}
catch (AccessControlException)
{
System.Console.Out.WriteLine("got access denied, as expected.");
}
VerifyAuditLogsRepeat(false, 2);
}
public virtual void TestDataLocality()
{
Configuration conf = WebHdfsTestUtil.CreateConf();
string[] racks = new string[] { Rack0, Rack0, Rack1, Rack1, Rack2, Rack2 };
int nDataNodes = racks.Length;
Log.Info("nDataNodes=" + nDataNodes + ", racks=" + Arrays.AsList(racks));
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(nDataNodes
).Racks(racks).Build();
try
{
cluster.WaitActive();
DistributedFileSystem dfs = cluster.GetFileSystem();
NameNode namenode = cluster.GetNameNode();
DatanodeManager dm = namenode.GetNamesystem().GetBlockManager().GetDatanodeManager
();
Log.Info("dm=" + dm);
long blocksize = DFSConfigKeys.DfsBlockSizeDefault;
string f = "/foo";
{
//test CREATE
for (int i = 0; i < nDataNodes; i++)
{
//set client address to a particular datanode
DataNode dn = cluster.GetDataNodes()[i];
string ipAddr = dm.GetDatanode(dn.GetDatanodeId()).GetIpAddr();
//The chosen datanode must be the same as the client address
DatanodeInfo chosen = NamenodeWebHdfsMethods.ChooseDatanode(namenode, f, PutOpParam.OP
.Create, -1L, blocksize, null);
NUnit.Framework.Assert.AreEqual(ipAddr, chosen.GetIpAddr());
}
}
//create a file with one replica.
Path p = new Path(f);
FSDataOutputStream @out = dfs.Create(p, (short)1);
@out.Write(1);
@out.Close();
//get replica location.
LocatedBlocks locatedblocks = NameNodeAdapter.GetBlockLocations(namenode, f, 0, 1
);
IList <LocatedBlock> lb = locatedblocks.GetLocatedBlocks();
NUnit.Framework.Assert.AreEqual(1, lb.Count);
DatanodeInfo[] locations = lb[0].GetLocations();
NUnit.Framework.Assert.AreEqual(1, locations.Length);
DatanodeInfo expected = locations[0];
{
//For GETFILECHECKSUM, OPEN and APPEND,
//the chosen datanode must be the same as the replica location.
//test GETFILECHECKSUM
DatanodeInfo chosen = NamenodeWebHdfsMethods.ChooseDatanode(namenode, f, GetOpParam.OP
.Getfilechecksum, -1L, blocksize, null);
NUnit.Framework.Assert.AreEqual(expected, chosen);
}
{
//test OPEN
DatanodeInfo chosen = NamenodeWebHdfsMethods.ChooseDatanode(namenode, f, GetOpParam.OP
.Open, 0, blocksize, null);
NUnit.Framework.Assert.AreEqual(expected, chosen);
}
{
//test APPEND
DatanodeInfo chosen = NamenodeWebHdfsMethods.ChooseDatanode(namenode, f, PostOpParam.OP
.Append, -1L, blocksize, null);
NUnit.Framework.Assert.AreEqual(expected, chosen);
}
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestExcludeDataNodes()
{
Configuration conf = WebHdfsTestUtil.CreateConf();
string[] racks = new string[] { Rack0, Rack0, Rack1, Rack1, Rack2, Rack2 };
string[] hosts = new string[] { "DataNode1", "DataNode2", "DataNode3", "DataNode4"
, "DataNode5", "DataNode6" };
int nDataNodes = hosts.Length;
Log.Info("nDataNodes=" + nDataNodes + ", racks=" + Arrays.AsList(racks) + ", hosts="
+ Arrays.AsList(hosts));
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Hosts(hosts).NumDataNodes
(nDataNodes).Racks(racks).Build();
try
{
cluster.WaitActive();
DistributedFileSystem dfs = cluster.GetFileSystem();
NameNode namenode = cluster.GetNameNode();
DatanodeManager dm = namenode.GetNamesystem().GetBlockManager().GetDatanodeManager
();
Log.Info("dm=" + dm);
long blocksize = DFSConfigKeys.DfsBlockSizeDefault;
string f = "/foo";
//create a file with three replica.
Path p = new Path(f);
FSDataOutputStream @out = dfs.Create(p, (short)3);
@out.Write(1);
@out.Close();
//get replica location.
LocatedBlocks locatedblocks = NameNodeAdapter.GetBlockLocations(namenode, f, 0, 1
);
IList <LocatedBlock> lb = locatedblocks.GetLocatedBlocks();
NUnit.Framework.Assert.AreEqual(1, lb.Count);
DatanodeInfo[] locations = lb[0].GetLocations();
NUnit.Framework.Assert.AreEqual(3, locations.Length);
//For GETFILECHECKSUM, OPEN and APPEND,
//the chosen datanode must be different with exclude nodes.
StringBuilder sb = new StringBuilder();
for (int i = 0; i < 2; i++)
{
sb.Append(locations[i].GetXferAddr());
{
// test GETFILECHECKSUM
DatanodeInfo chosen = NamenodeWebHdfsMethods.ChooseDatanode(namenode, f, GetOpParam.OP
.Getfilechecksum, -1L, blocksize, sb.ToString());
for (int j = 0; j <= i; j++)
{
Assert.AssertNotEquals(locations[j].GetHostName(), chosen.GetHostName());
}
}
{
// test OPEN
DatanodeInfo chosen = NamenodeWebHdfsMethods.ChooseDatanode(namenode, f, GetOpParam.OP
.Open, 0, blocksize, sb.ToString());
for (int j = 0; j <= i; j++)
{
Assert.AssertNotEquals(locations[j].GetHostName(), chosen.GetHostName());
}
}
{
// test APPEND
DatanodeInfo chosen = NamenodeWebHdfsMethods.ChooseDatanode(namenode, f, PostOpParam.OP
.Append, -1L, blocksize, sb.ToString());
for (int j = 0; j <= i; j++)
{
Assert.AssertNotEquals(locations[j].GetHostName(), chosen.GetHostName());
}
}
sb.Append(",");
}
}
finally
{
cluster.Shutdown();
}
}
