public virtual void TestExportPoint()
{
NfsConfiguration config = new NfsConfiguration();
MiniDFSCluster cluster = null;
string exportPoint = "/myexport1";
config.SetStrings(NfsConfigKeys.DfsNfsExportPointKey, exportPoint);
// Use emphral port in case tests are running in parallel
config.SetInt("nfs3.mountd.port", 0);
config.SetInt("nfs3.server.port", 0);
try
{
cluster = new MiniDFSCluster.Builder(config).NumDataNodes(1).Build();
cluster.WaitActive();
// Start nfs
Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3 nfsServer = new Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3
(config);
nfsServer.StartServiceInternal(false);
Mountd mountd = nfsServer.GetMountd();
RpcProgramMountd rpcMount = (RpcProgramMountd)mountd.GetRpcProgram();
NUnit.Framework.Assert.IsTrue(rpcMount.GetExports().Count == 1);
string exportInMountd = rpcMount.GetExports()[0];
NUnit.Framework.Assert.IsTrue(exportInMountd.Equals(exportPoint));
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public static void Setup()
{
string currentUser = Runtime.GetProperty("user.name");
config.Set(DefaultImpersonationProvider.GetTestProvider().GetProxySuperuserGroupConfKey
(currentUser), "*");
config.Set(DefaultImpersonationProvider.GetTestProvider().GetProxySuperuserIpConfKey
(currentUser), "*");
ProxyUsers.RefreshSuperUserGroupsConfiguration(config);
cluster = new MiniDFSCluster.Builder(config).NumDataNodes(1).Build();
cluster.WaitActive();
hdfs = cluster.GetFileSystem();
nn = cluster.GetNameNode();
// Use emphral port in case tests are running in parallel
config.SetInt("nfs3.mountd.port", 0);
config.SetInt("nfs3.server.port", 0);
// Start nfs
Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3 nfs3 = new Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3
(config);
nfs3.StartServiceInternal(false);
nfsd = (RpcProgramNfs3)nfs3.GetRpcProgram();
securityHandler = Org.Mockito.Mockito.Mock <SecurityHandler>();
Org.Mockito.Mockito.When(securityHandler.GetUser()).ThenReturn(Runtime.GetProperty
("user.name"));
}
public static void SetUp()
{
conf.Set(DFSConfigKeys.DfsHttpPolicyKey, HttpConfig.Policy.HttpAndHttps.ToString(
));
conf.Set(NfsConfigKeys.NfsHttpAddressKey, "localhost:0");
conf.Set(NfsConfigKeys.NfsHttpsAddressKey, "localhost:0");
// Use emphral port in case tests are running in parallel
conf.SetInt(NfsConfigKeys.DfsNfsServerPortKey, 0);
conf.SetInt(NfsConfigKeys.DfsNfsMountdPortKey, 0);
FilePath @base = new FilePath(Basedir);
FileUtil.FullyDelete(@base);
@base.Mkdirs();
keystoresDir = new FilePath(Basedir).GetAbsolutePath();
sslConfDir = KeyStoreTestUtil.GetClasspathDir(typeof(TestNfs3HttpServer));
KeyStoreTestUtil.SetupSSLConfig(keystoresDir, sslConfDir, conf, false);
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
}
public static void Setup()
{
string currentUser = Runtime.GetProperty("user.name");
config.Set("fs.permissions.umask-mode", "u=rwx,g=,o=");
config.Set(DefaultImpersonationProvider.GetTestProvider().GetProxySuperuserGroupConfKey
(currentUser), "*");
config.Set(DefaultImpersonationProvider.GetTestProvider().GetProxySuperuserIpConfKey
(currentUser), "*");
fsHelper = new FileSystemTestHelper();
// Set up java key store
string testRoot = fsHelper.GetTestRootDir();
testRootDir = new FilePath(testRoot).GetAbsoluteFile();
Path jksPath = new Path(testRootDir.ToString(), "test.jks");
config.Set(DFSConfigKeys.DfsEncryptionKeyProviderUri, JavaKeyStoreProvider.SchemeName
+ "://file" + jksPath.ToUri());
ProxyUsers.RefreshSuperUserGroupsConfiguration(config);
cluster = new MiniDFSCluster.Builder(config).NumDataNodes(1).Build();
cluster.WaitActive();
hdfs = cluster.GetFileSystem();
nn = cluster.GetNameNode();
dfsAdmin = new HdfsAdmin(cluster.GetURI(), config);
// Use ephemeral ports in case tests are running in parallel
config.SetInt("nfs3.mountd.port", 0);
config.SetInt("nfs3.server.port", 0);
// Start NFS with allowed.hosts set to "* rw"
config.Set("dfs.nfs.exports.allowed.hosts", "* rw");
nfs = new Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3(config);
nfs.StartServiceInternal(false);
nfsd = (RpcProgramNfs3)nfs.GetRpcProgram();
hdfs.GetClient().SetKeyProvider(nn.GetNamesystem().GetProvider());
DFSTestUtil.CreateKey(TestKey, cluster, config);
// Mock SecurityHandler which returns system user.name
securityHandler = Org.Mockito.Mockito.Mock <SecurityHandler>();
Org.Mockito.Mockito.When(securityHandler.GetUser()).ThenReturn(currentUser);
// Mock SecurityHandler which returns a dummy username "harry"
securityHandlerUnpriviledged = Org.Mockito.Mockito.Mock <SecurityHandler>();
Org.Mockito.Mockito.When(securityHandlerUnpriviledged.GetUser()).ThenReturn("harry"
);
}
public virtual void TestDeprecatedKeys()
{
NfsConfiguration conf = new NfsConfiguration();
conf.SetInt("nfs3.server.port", 998);
NUnit.Framework.Assert.IsTrue(conf.GetInt(NfsConfigKeys.DfsNfsServerPortKey, 0) ==
998);
conf.SetInt("nfs3.mountd.port", 999);
NUnit.Framework.Assert.IsTrue(conf.GetInt(NfsConfigKeys.DfsNfsMountdPortKey, 0) ==
999);
conf.Set("dfs.nfs.exports.allowed.hosts", "host1");
NUnit.Framework.Assert.IsTrue(conf.Get(CommonConfigurationKeys.NfsExportsAllowedHostsKey
).Equals("host1"));
conf.SetInt("dfs.nfs.exports.cache.expirytime.millis", 1000);
NUnit.Framework.Assert.IsTrue(conf.GetInt(Nfs3Constant.NfsExportsCacheExpirytimeMillisKey
, 0) == 1000);
conf.SetInt("hadoop.nfs.userupdate.milly", 10);
NUnit.Framework.Assert.IsTrue(conf.GetInt(IdMappingConstant.UsergroupidUpdateMillisKey
, 0) == 10);
conf.Set("dfs.nfs3.dump.dir", "/nfs/tmp");
NUnit.Framework.Assert.IsTrue(conf.Get(NfsConfigKeys.DfsNfsFileDumpDirKey).Equals
("/nfs/tmp"));
conf.SetBoolean("dfs.nfs3.enableDump", false);
NUnit.Framework.Assert.IsTrue(conf.GetBoolean(NfsConfigKeys.DfsNfsFileDumpKey, true
) == false);
conf.SetInt("dfs.nfs3.max.open.files", 500);
NUnit.Framework.Assert.IsTrue(conf.GetInt(NfsConfigKeys.DfsNfsMaxOpenFilesKey, 0)
== 500);
conf.SetInt("dfs.nfs3.stream.timeout", 6000);
NUnit.Framework.Assert.IsTrue(conf.GetInt(NfsConfigKeys.DfsNfsStreamTimeoutKey, 0
) == 6000);
conf.Set("dfs.nfs3.export.point", "/dir1");
NUnit.Framework.Assert.IsTrue(conf.Get(NfsConfigKeys.DfsNfsExportPointKey).Equals
("/dir1"));
}
public virtual void TestEviction()
{
NfsConfiguration conf = new NfsConfiguration();
// Only two entries will be in the cache
conf.SetInt(NfsConfigKeys.DfsNfsMaxOpenFilesKey, 2);
DFSClient dfsClient = Org.Mockito.Mockito.Mock <DFSClient>();
Nfs3FileAttributes attr = new Nfs3FileAttributes();
HdfsDataOutputStream fos = Org.Mockito.Mockito.Mock <HdfsDataOutputStream>();
Org.Mockito.Mockito.When(fos.GetPos()).ThenReturn((long)0);
OpenFileCtx context1 = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, new
ShellBasedIdMapping(new NfsConfiguration()));
OpenFileCtx context2 = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, new
ShellBasedIdMapping(new NfsConfiguration()));
OpenFileCtx context3 = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, new
ShellBasedIdMapping(new NfsConfiguration()));
OpenFileCtx context4 = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, new
ShellBasedIdMapping(new NfsConfiguration()));
OpenFileCtx context5 = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, new
ShellBasedIdMapping(new NfsConfiguration()));
OpenFileCtxCache cache = new OpenFileCtxCache(conf, 10 * 60 * 100);
bool ret = cache.Put(new FileHandle(1), context1);
NUnit.Framework.Assert.IsTrue(ret);
Sharpen.Thread.Sleep(1000);
ret = cache.Put(new FileHandle(2), context2);
NUnit.Framework.Assert.IsTrue(ret);
ret = cache.Put(new FileHandle(3), context3);
NUnit.Framework.Assert.IsFalse(ret);
NUnit.Framework.Assert.IsTrue(cache.Size() == 2);
// Wait for the oldest stream to be evict-able, insert again
Sharpen.Thread.Sleep(NfsConfigKeys.DfsNfsStreamTimeoutMinDefault);
NUnit.Framework.Assert.IsTrue(cache.Size() == 2);
ret = cache.Put(new FileHandle(3), context3);
NUnit.Framework.Assert.IsTrue(ret);
NUnit.Framework.Assert.IsTrue(cache.Size() == 2);
NUnit.Framework.Assert.IsTrue(cache.Get(new FileHandle(1)) == null);
// Test inactive entry is evicted immediately
context3.SetActiveStatusForTest(false);
ret = cache.Put(new FileHandle(4), context4);
NUnit.Framework.Assert.IsTrue(ret);
// Now the cache has context2 and context4
// Test eviction failure if all entries have pending work.
context2.GetPendingWritesForTest()[new OffsetRange(0, 100)] = new WriteCtx(null,
0, 0, 0, null, null, null, 0, false, null);
context4.GetPendingCommitsForTest()[System.Convert.ToInt64(100)] = new OpenFileCtx.CommitCtx
(0, null, 0, attr);
Sharpen.Thread.Sleep(NfsConfigKeys.DfsNfsStreamTimeoutMinDefault);
ret = cache.Put(new FileHandle(5), context5);
NUnit.Framework.Assert.IsFalse(ret);
}
public virtual void TestStart()
{
// Start minicluster
NfsConfiguration config = new NfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(config).NumDataNodes(1).Build
();
cluster.WaitActive();
// Use emphral port in case tests are running in parallel
config.SetInt("nfs3.mountd.port", 0);
config.SetInt("nfs3.server.port", 0);
// Start nfs
Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3 nfs3 = new Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3
(config);
nfs3.StartServiceInternal(false);
RpcProgramMountd mountd = (RpcProgramMountd)nfs3.GetMountd().GetRpcProgram();
mountd.NullOp(new XDR(), 1234, Sharpen.Extensions.GetAddressByName("localhost"));
RpcProgramNfs3 nfsd = (RpcProgramNfs3)nfs3.GetRpcProgram();
nfsd.NullProcedure();
cluster.Shutdown();
}
public virtual void TestScan()
{
NfsConfiguration conf = new NfsConfiguration();
// Only two entries will be in the cache
conf.SetInt(NfsConfigKeys.DfsNfsMaxOpenFilesKey, 2);
DFSClient dfsClient = Org.Mockito.Mockito.Mock <DFSClient>();
Nfs3FileAttributes attr = new Nfs3FileAttributes();
HdfsDataOutputStream fos = Org.Mockito.Mockito.Mock <HdfsDataOutputStream>();
Org.Mockito.Mockito.When(fos.GetPos()).ThenReturn((long)0);
OpenFileCtx context1 = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, new
ShellBasedIdMapping(new NfsConfiguration()));
OpenFileCtx context2 = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, new
ShellBasedIdMapping(new NfsConfiguration()));
OpenFileCtx context3 = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, new
ShellBasedIdMapping(new NfsConfiguration()));
OpenFileCtx context4 = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, new
ShellBasedIdMapping(new NfsConfiguration()));
OpenFileCtxCache cache = new OpenFileCtxCache(conf, 10 * 60 * 100);
// Test cleaning expired entry
bool ret = cache.Put(new FileHandle(1), context1);
NUnit.Framework.Assert.IsTrue(ret);
ret = cache.Put(new FileHandle(2), context2);
NUnit.Framework.Assert.IsTrue(ret);
Sharpen.Thread.Sleep(NfsConfigKeys.DfsNfsStreamTimeoutMinDefault + 1);
cache.Scan(NfsConfigKeys.DfsNfsStreamTimeoutMinDefault);
NUnit.Framework.Assert.IsTrue(cache.Size() == 0);
// Test cleaning inactive entry
ret = cache.Put(new FileHandle(3), context3);
NUnit.Framework.Assert.IsTrue(ret);
ret = cache.Put(new FileHandle(4), context4);
NUnit.Framework.Assert.IsTrue(ret);
context3.SetActiveStatusForTest(false);
cache.Scan(NfsConfigKeys.DfsNfsStreamTimeoutDefault);
NUnit.Framework.Assert.IsTrue(cache.Size() == 1);
NUnit.Framework.Assert.IsTrue(cache.Get(new FileHandle(3)) == null);
NUnit.Framework.Assert.IsTrue(cache.Get(new FileHandle(4)) != null);
}
public virtual void TestOOOWrites()
{
NfsConfiguration config = new NfsConfiguration();
MiniDFSCluster cluster = null;
RpcProgramNfs3 nfsd;
int bufSize = 32;
int numOOO = 3;
SecurityHandler securityHandler = Org.Mockito.Mockito.Mock <SecurityHandler>();
Org.Mockito.Mockito.When(securityHandler.GetUser()).ThenReturn(Runtime.GetProperty
("user.name"));
string currentUser = Runtime.GetProperty("user.name");
config.Set(DefaultImpersonationProvider.GetTestProvider().GetProxySuperuserGroupConfKey
(currentUser), "*");
config.Set(DefaultImpersonationProvider.GetTestProvider().GetProxySuperuserIpConfKey
(currentUser), "*");
ProxyUsers.RefreshSuperUserGroupsConfiguration(config);
// Use emphral port in case tests are running in parallel
config.SetInt("nfs3.mountd.port", 0);
config.SetInt("nfs3.server.port", 0);
try
{
cluster = new MiniDFSCluster.Builder(config).NumDataNodes(1).Build();
cluster.WaitActive();
Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3 nfs3 = new Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3
(config);
nfs3.StartServiceInternal(false);
nfsd = (RpcProgramNfs3)nfs3.GetRpcProgram();
DFSClient dfsClient = new DFSClient(NameNode.GetAddress(config), config);
HdfsFileStatus status = dfsClient.GetFileInfo("/");
FileHandle rootHandle = new FileHandle(status.GetFileId());
CREATE3Request createReq = new CREATE3Request(rootHandle, "out-of-order-write" +
Runtime.CurrentTimeMillis(), Nfs3Constant.CreateUnchecked, new SetAttr3(), 0);
XDR createXdr = new XDR();
createReq.Serialize(createXdr);
CREATE3Response createRsp = nfsd.Create(createXdr.AsReadOnlyWrap(), securityHandler
, new IPEndPoint("localhost", 1234));
FileHandle handle = createRsp.GetObjHandle();
byte[][] oooBuf = new byte[][] { new byte[bufSize], new byte[bufSize], new byte[bufSize
] };
for (int i = 0; i < numOOO; i++)
{
Arrays.Fill(oooBuf[i], unchecked ((byte)i));
}
for (int i_1 = 0; i_1 < numOOO; i_1++)
{
long offset = (numOOO - 1 - i_1) * bufSize;
WRITE3Request writeReq = new WRITE3Request(handle, offset, bufSize, Nfs3Constant.WriteStableHow
.Unstable, ByteBuffer.Wrap(oooBuf[i_1]));
XDR writeXdr = new XDR();
writeReq.Serialize(writeXdr);
nfsd.Write(writeXdr.AsReadOnlyWrap(), null, 1, securityHandler, new IPEndPoint("localhost"
, 1234));
}
WaitWrite(nfsd, handle, 60000);
READ3Request readReq = new READ3Request(handle, bufSize, bufSize);
XDR readXdr = new XDR();
readReq.Serialize(readXdr);
READ3Response readRsp = nfsd.Read(readXdr.AsReadOnlyWrap(), securityHandler, new
IPEndPoint("localhost", config.GetInt(NfsConfigKeys.DfsNfsServerPortKey, NfsConfigKeys
.DfsNfsServerPortDefault)));
NUnit.Framework.Assert.IsTrue(Arrays.Equals(oooBuf[1], ((byte[])readRsp.GetData()
.Array())));
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestWriteStableHow()
{
NfsConfiguration config = new NfsConfiguration();
DFSClient client = null;
MiniDFSCluster cluster = null;
RpcProgramNfs3 nfsd;
SecurityHandler securityHandler = Org.Mockito.Mockito.Mock <SecurityHandler>();
Org.Mockito.Mockito.When(securityHandler.GetUser()).ThenReturn(Runtime.GetProperty
("user.name"));
string currentUser = Runtime.GetProperty("user.name");
config.Set(DefaultImpersonationProvider.GetTestProvider().GetProxySuperuserGroupConfKey
(currentUser), "*");
config.Set(DefaultImpersonationProvider.GetTestProvider().GetProxySuperuserIpConfKey
(currentUser), "*");
ProxyUsers.RefreshSuperUserGroupsConfiguration(config);
try
{
cluster = new MiniDFSCluster.Builder(config).NumDataNodes(1).Build();
cluster.WaitActive();
client = new DFSClient(NameNode.GetAddress(config), config);
// Use emphral port in case tests are running in parallel
config.SetInt("nfs3.mountd.port", 0);
config.SetInt("nfs3.server.port", 0);
// Start nfs
Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3 nfs3 = new Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3
(config);
nfs3.StartServiceInternal(false);
nfsd = (RpcProgramNfs3)nfs3.GetRpcProgram();
HdfsFileStatus status = client.GetFileInfo("/");
FileHandle rootHandle = new FileHandle(status.GetFileId());
// Create file1
CREATE3Request createReq = new CREATE3Request(rootHandle, "file1", Nfs3Constant.CreateUnchecked
, new SetAttr3(), 0);
XDR createXdr = new XDR();
createReq.Serialize(createXdr);
CREATE3Response createRsp = nfsd.Create(createXdr.AsReadOnlyWrap(), securityHandler
, new IPEndPoint("localhost", 1234));
FileHandle handle = createRsp.GetObjHandle();
// Test DATA_SYNC
byte[] buffer = new byte[10];
for (int i = 0; i < 10; i++)
{
buffer[i] = unchecked ((byte)i);
}
WRITE3Request writeReq = new WRITE3Request(handle, 0, 10, Nfs3Constant.WriteStableHow
.DataSync, ByteBuffer.Wrap(buffer));
XDR writeXdr = new XDR();
writeReq.Serialize(writeXdr);
nfsd.Write(writeXdr.AsReadOnlyWrap(), null, 1, securityHandler, new IPEndPoint("localhost"
, 1234));
WaitWrite(nfsd, handle, 60000);
// Readback
READ3Request readReq = new READ3Request(handle, 0, 10);
XDR readXdr = new XDR();
readReq.Serialize(readXdr);
READ3Response readRsp = nfsd.Read(readXdr.AsReadOnlyWrap(), securityHandler, new
IPEndPoint("localhost", 1234));
NUnit.Framework.Assert.IsTrue(Arrays.Equals(buffer, ((byte[])readRsp.GetData().Array
())));
// Test FILE_SYNC
// Create file2
CREATE3Request createReq2 = new CREATE3Request(rootHandle, "file2", Nfs3Constant.
CreateUnchecked, new SetAttr3(), 0);
XDR createXdr2 = new XDR();
createReq2.Serialize(createXdr2);
CREATE3Response createRsp2 = nfsd.Create(createXdr2.AsReadOnlyWrap(), securityHandler
, new IPEndPoint("localhost", 1234));
FileHandle handle2 = createRsp2.GetObjHandle();
WRITE3Request writeReq2 = new WRITE3Request(handle2, 0, 10, Nfs3Constant.WriteStableHow
.FileSync, ByteBuffer.Wrap(buffer));
XDR writeXdr2 = new XDR();
writeReq2.Serialize(writeXdr2);
nfsd.Write(writeXdr2.AsReadOnlyWrap(), null, 1, securityHandler, new IPEndPoint("localhost"
, 1234));
WaitWrite(nfsd, handle2, 60000);
// Readback
READ3Request readReq2 = new READ3Request(handle2, 0, 10);
XDR readXdr2 = new XDR();
readReq2.Serialize(readXdr2);
READ3Response readRsp2 = nfsd.Read(readXdr2.AsReadOnlyWrap(), securityHandler, new
IPEndPoint("localhost", 1234));
NUnit.Framework.Assert.IsTrue(Arrays.Equals(buffer, ((byte[])readRsp2.GetData().Array
())));
// FILE_SYNC should sync the file size
status = client.GetFileInfo("/file2");
NUnit.Framework.Assert.IsTrue(status.GetLen() == 10);
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
