public virtual void SetupCluster()
{
conf = new Configuration();
conf.SetInt(DFSConfigKeys.DfsHaTaileditsPeriodKey, 1);
HAUtil.SetAllowStandbyReads(conf, true);
fsHelper = new FileSystemTestHelper();
string testRoot = fsHelper.GetTestRootDir();
testRootDir = new FilePath(testRoot).GetAbsoluteFile();
conf.Set(DFSConfigKeys.DfsEncryptionKeyProviderUri, JavaKeyStoreProvider.SchemeName
+ "://file" + new Path(testRootDir.ToString(), "test.jks").ToUri());
cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology.SimpleHATopology
()).NumDataNodes(1).Build();
cluster.WaitActive();
cluster.TransitionToActive(0);
fs = (DistributedFileSystem)HATestUtil.ConfigureFailoverFs(cluster, conf);
DFSTestUtil.CreateKey(TestKey, cluster, 0, conf);
DFSTestUtil.CreateKey(TestKey, cluster, 1, conf);
nn0 = cluster.GetNameNode(0);
nn1 = cluster.GetNameNode(1);
dfsAdmin0 = new HdfsAdmin(cluster.GetURI(0), conf);
dfsAdmin1 = new HdfsAdmin(cluster.GetURI(1), conf);
KeyProviderCryptoExtension nn0Provider = cluster.GetNameNode(0).GetNamesystem().GetProvider
();
fs.GetClient().SetKeyProvider(nn0Provider);
}
public static void Setup()
{
string currentUser = Runtime.GetProperty("user.name");
config.Set("fs.permissions.umask-mode", "u=rwx,g=,o=");
config.Set(DefaultImpersonationProvider.GetTestProvider().GetProxySuperuserGroupConfKey
(currentUser), "*");
config.Set(DefaultImpersonationProvider.GetTestProvider().GetProxySuperuserIpConfKey
(currentUser), "*");
fsHelper = new FileSystemTestHelper();
// Set up java key store
string testRoot = fsHelper.GetTestRootDir();
testRootDir = new FilePath(testRoot).GetAbsoluteFile();
Path jksPath = new Path(testRootDir.ToString(), "test.jks");
config.Set(DFSConfigKeys.DfsEncryptionKeyProviderUri, JavaKeyStoreProvider.SchemeName
+ "://file" + jksPath.ToUri());
ProxyUsers.RefreshSuperUserGroupsConfiguration(config);
cluster = new MiniDFSCluster.Builder(config).NumDataNodes(1).Build();
cluster.WaitActive();
hdfs = cluster.GetFileSystem();
nn = cluster.GetNameNode();
dfsAdmin = new HdfsAdmin(cluster.GetURI(), config);
// Use ephemeral ports in case tests are running in parallel
config.SetInt("nfs3.mountd.port", 0);
config.SetInt("nfs3.server.port", 0);
// Start NFS with allowed.hosts set to "* rw"
config.Set("dfs.nfs.exports.allowed.hosts", "* rw");
nfs = new Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3(config);
nfs.StartServiceInternal(false);
nfsd = (RpcProgramNfs3)nfs.GetRpcProgram();
hdfs.GetClient().SetKeyProvider(nn.GetNamesystem().GetProvider());
DFSTestUtil.CreateKey(TestKey, cluster, config);
// Mock SecurityHandler which returns system user.name
securityHandler = Org.Mockito.Mockito.Mock <SecurityHandler>();
Org.Mockito.Mockito.When(securityHandler.GetUser()).ThenReturn(currentUser);
// Mock SecurityHandler which returns a dummy username "harry"
securityHandlerUnpriviledged = Org.Mockito.Mockito.Mock <SecurityHandler>();
Org.Mockito.Mockito.When(securityHandlerUnpriviledged.GetUser()).ThenReturn("harry"
);
}
public virtual void TestHdfsAdminSetQuota()
{
HdfsAdmin dfsAdmin = new HdfsAdmin(FileSystem.GetDefaultUri(conf), conf);
FileSystem fs = null;
try
{
fs = FileSystem.Get(conf);
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(TestPath));
NUnit.Framework.Assert.AreEqual(-1, fs.GetContentSummary(TestPath).GetQuota());
NUnit.Framework.Assert.AreEqual(-1, fs.GetContentSummary(TestPath).GetSpaceQuota(
));
dfsAdmin.SetSpaceQuota(TestPath, 10);
NUnit.Framework.Assert.AreEqual(-1, fs.GetContentSummary(TestPath).GetQuota());
NUnit.Framework.Assert.AreEqual(10, fs.GetContentSummary(TestPath).GetSpaceQuota(
));
dfsAdmin.SetQuota(TestPath, 10);
NUnit.Framework.Assert.AreEqual(10, fs.GetContentSummary(TestPath).GetQuota());
NUnit.Framework.Assert.AreEqual(10, fs.GetContentSummary(TestPath).GetSpaceQuota(
));
dfsAdmin.ClearSpaceQuota(TestPath);
NUnit.Framework.Assert.AreEqual(10, fs.GetContentSummary(TestPath).GetQuota());
NUnit.Framework.Assert.AreEqual(-1, fs.GetContentSummary(TestPath).GetSpaceQuota(
));
dfsAdmin.ClearQuota(TestPath);
NUnit.Framework.Assert.AreEqual(-1, fs.GetContentSummary(TestPath).GetQuota());
NUnit.Framework.Assert.AreEqual(-1, fs.GetContentSummary(TestPath).GetSpaceQuota(
));
}
finally
{
if (fs != null)
{
fs.Close();
}
}
}
public virtual void Setup()
{
conf = new HdfsConfiguration();
fsHelper = new FileSystemTestHelper();
// Set up java key store
string testRoot = fsHelper.GetTestRootDir();
FilePath testRootDir = new FilePath(testRoot).GetAbsoluteFile();
Path jksPath = new Path(testRootDir.ToString(), "test.jks");
conf.Set(DFSConfigKeys.DfsEncryptionKeyProviderUri, JavaKeyStoreProvider.SchemeName
+ "://file" + jksPath.ToUri());
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
Logger.GetLogger(typeof(EncryptionZoneManager)).SetLevel(Level.Trace);
fs = cluster.GetFileSystem();
fsWrapper = new FileSystemTestWrapper(cluster.GetFileSystem());
fcWrapper = new FileContextTestWrapper(FileContext.GetFileContext(cluster.GetURI(
), conf));
dfsAdmin = new HdfsAdmin(cluster.GetURI(), conf);
// Need to set the client's KeyProvider to the NN's for JKS,
// else the updates do not get flushed properly
fs.GetClient().SetKeyProvider(cluster.GetNameNode().GetNamesystem().GetProvider()
);
DFSTestUtil.CreateKey(TestKey, cluster, conf);
}
public virtual void TestCheckpoint()
{
MiniDFSCluster cluster = null;
SecondaryNameNode secondary = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).Build();
cluster.WaitActive();
secondary = new SecondaryNameNode(conf);
SnapshotManager nnSnapshotManager = cluster.GetNamesystem().GetSnapshotManager();
SnapshotManager secondarySnapshotManager = secondary.GetFSNamesystem().GetSnapshotManager
();
FileSystem fs = cluster.GetFileSystem();
HdfsAdmin admin = new HdfsAdmin(FileSystem.GetDefaultUri(conf), conf);
NUnit.Framework.Assert.AreEqual(0, nnSnapshotManager.GetNumSnapshots());
NUnit.Framework.Assert.AreEqual(0, nnSnapshotManager.GetNumSnapshottableDirs());
NUnit.Framework.Assert.AreEqual(0, secondarySnapshotManager.GetNumSnapshots());
NUnit.Framework.Assert.AreEqual(0, secondarySnapshotManager.GetNumSnapshottableDirs
());
// 1. Create a snapshottable directory foo on the NN.
fs.Mkdirs(TestPath);
admin.AllowSnapshot(TestPath);
NUnit.Framework.Assert.AreEqual(0, nnSnapshotManager.GetNumSnapshots());
NUnit.Framework.Assert.AreEqual(1, nnSnapshotManager.GetNumSnapshottableDirs());
// 2. Create a snapshot of the dir foo. This will be referenced both in
// the SnapshotManager as well as in the file system tree. The snapshot
// count will go up to 1.
Path snapshotPath = fs.CreateSnapshot(TestPath);
NUnit.Framework.Assert.AreEqual(1, nnSnapshotManager.GetNumSnapshots());
NUnit.Framework.Assert.AreEqual(1, nnSnapshotManager.GetNumSnapshottableDirs());
// 3. Start up a 2NN and have it do a checkpoint. It will have foo and its
// snapshot in its list of snapshottable dirs referenced from the
// SnapshotManager, as well as in the file system tree.
secondary.DoCheckpoint();
NUnit.Framework.Assert.AreEqual(1, secondarySnapshotManager.GetNumSnapshots());
NUnit.Framework.Assert.AreEqual(1, secondarySnapshotManager.GetNumSnapshottableDirs
());
// 4. Disallow snapshots on and delete foo on the NN. The snapshot count
// will go down to 0 and the snapshottable dir will be removed from the fs
// tree.
fs.DeleteSnapshot(TestPath, snapshotPath.GetName());
admin.DisallowSnapshot(TestPath);
NUnit.Framework.Assert.AreEqual(0, nnSnapshotManager.GetNumSnapshots());
NUnit.Framework.Assert.AreEqual(0, nnSnapshotManager.GetNumSnapshottableDirs());
// 5. Have the NN do a saveNamespace, writing out a new fsimage with
// snapshot count 0.
NameNodeAdapter.EnterSafeMode(cluster.GetNameNode(), false);
NameNodeAdapter.SaveNamespace(cluster.GetNameNode());
NameNodeAdapter.LeaveSafeMode(cluster.GetNameNode());
// 6. Have the still-running 2NN do a checkpoint. It will notice that the
// fsimage has changed on the NN and redownload/reload from that image.
// This will replace all INodes in the file system tree as well as reset
// the snapshot counter to 0 in the SnapshotManager. However, it will not
// clear the list of snapshottable dirs referenced from the
// SnapshotManager. When it writes out an fsimage, the 2NN will write out
// 0 for the snapshot count, but still serialize the snapshottable dir
// referenced in the SnapshotManager even though it no longer appears in
// the file system tree. The NN will not be able to start up with this.
secondary.DoCheckpoint();
NUnit.Framework.Assert.AreEqual(0, secondarySnapshotManager.GetNumSnapshots());
NUnit.Framework.Assert.AreEqual(0, secondarySnapshotManager.GetNumSnapshottableDirs
());
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
if (secondary != null)
{
secondary.Shutdown();
}
}
}