public virtual void SetupCluster() { conf = new Configuration(); conf.SetInt(DFSConfigKeys.DfsHaTaileditsPeriodKey, 1); HAUtil.SetAllowStandbyReads(conf, true); fsHelper = new FileSystemTestHelper(); string testRoot = fsHelper.GetTestRootDir(); testRootDir = new FilePath(testRoot).GetAbsoluteFile(); conf.Set(DFSConfigKeys.DfsEncryptionKeyProviderUri, JavaKeyStoreProvider.SchemeName + "://file" + new Path(testRootDir.ToString(), "test.jks").ToUri()); cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology.SimpleHATopology ()).NumDataNodes(1).Build(); cluster.WaitActive(); cluster.TransitionToActive(0); fs = (DistributedFileSystem)HATestUtil.ConfigureFailoverFs(cluster, conf); DFSTestUtil.CreateKey(TestKey, cluster, 0, conf); DFSTestUtil.CreateKey(TestKey, cluster, 1, conf); nn0 = cluster.GetNameNode(0); nn1 = cluster.GetNameNode(1); dfsAdmin0 = new HdfsAdmin(cluster.GetURI(0), conf); dfsAdmin1 = new HdfsAdmin(cluster.GetURI(1), conf); KeyProviderCryptoExtension nn0Provider = cluster.GetNameNode(0).GetNamesystem().GetProvider (); fs.GetClient().SetKeyProvider(nn0Provider); }
public static void Setup() { string currentUser = Runtime.GetProperty("user.name"); config.Set("fs.permissions.umask-mode", "u=rwx,g=,o="); config.Set(DefaultImpersonationProvider.GetTestProvider().GetProxySuperuserGroupConfKey (currentUser), "*"); config.Set(DefaultImpersonationProvider.GetTestProvider().GetProxySuperuserIpConfKey (currentUser), "*"); fsHelper = new FileSystemTestHelper(); // Set up java key store string testRoot = fsHelper.GetTestRootDir(); testRootDir = new FilePath(testRoot).GetAbsoluteFile(); Path jksPath = new Path(testRootDir.ToString(), "test.jks"); config.Set(DFSConfigKeys.DfsEncryptionKeyProviderUri, JavaKeyStoreProvider.SchemeName + "://file" + jksPath.ToUri()); ProxyUsers.RefreshSuperUserGroupsConfiguration(config); cluster = new MiniDFSCluster.Builder(config).NumDataNodes(1).Build(); cluster.WaitActive(); hdfs = cluster.GetFileSystem(); nn = cluster.GetNameNode(); dfsAdmin = new HdfsAdmin(cluster.GetURI(), config); // Use ephemeral ports in case tests are running in parallel config.SetInt("nfs3.mountd.port", 0); config.SetInt("nfs3.server.port", 0); // Start NFS with allowed.hosts set to "* rw" config.Set("dfs.nfs.exports.allowed.hosts", "* rw"); nfs = new Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3(config); nfs.StartServiceInternal(false); nfsd = (RpcProgramNfs3)nfs.GetRpcProgram(); hdfs.GetClient().SetKeyProvider(nn.GetNamesystem().GetProvider()); DFSTestUtil.CreateKey(TestKey, cluster, config); // Mock SecurityHandler which returns system user.name securityHandler = Org.Mockito.Mockito.Mock <SecurityHandler>(); Org.Mockito.Mockito.When(securityHandler.GetUser()).ThenReturn(currentUser); // Mock SecurityHandler which returns a dummy username "harry" securityHandlerUnpriviledged = Org.Mockito.Mockito.Mock <SecurityHandler>(); Org.Mockito.Mockito.When(securityHandlerUnpriviledged.GetUser()).ThenReturn("harry" ); }
public virtual void TestHdfsAdminSetQuota() { HdfsAdmin dfsAdmin = new HdfsAdmin(FileSystem.GetDefaultUri(conf), conf); FileSystem fs = null; try { fs = FileSystem.Get(conf); NUnit.Framework.Assert.IsTrue(fs.Mkdirs(TestPath)); NUnit.Framework.Assert.AreEqual(-1, fs.GetContentSummary(TestPath).GetQuota()); NUnit.Framework.Assert.AreEqual(-1, fs.GetContentSummary(TestPath).GetSpaceQuota( )); dfsAdmin.SetSpaceQuota(TestPath, 10); NUnit.Framework.Assert.AreEqual(-1, fs.GetContentSummary(TestPath).GetQuota()); NUnit.Framework.Assert.AreEqual(10, fs.GetContentSummary(TestPath).GetSpaceQuota( )); dfsAdmin.SetQuota(TestPath, 10); NUnit.Framework.Assert.AreEqual(10, fs.GetContentSummary(TestPath).GetQuota()); NUnit.Framework.Assert.AreEqual(10, fs.GetContentSummary(TestPath).GetSpaceQuota( )); dfsAdmin.ClearSpaceQuota(TestPath); NUnit.Framework.Assert.AreEqual(10, fs.GetContentSummary(TestPath).GetQuota()); NUnit.Framework.Assert.AreEqual(-1, fs.GetContentSummary(TestPath).GetSpaceQuota( )); dfsAdmin.ClearQuota(TestPath); NUnit.Framework.Assert.AreEqual(-1, fs.GetContentSummary(TestPath).GetQuota()); NUnit.Framework.Assert.AreEqual(-1, fs.GetContentSummary(TestPath).GetSpaceQuota( )); } finally { if (fs != null) { fs.Close(); } } }
public virtual void Setup() { conf = new HdfsConfiguration(); fsHelper = new FileSystemTestHelper(); // Set up java key store string testRoot = fsHelper.GetTestRootDir(); FilePath testRootDir = new FilePath(testRoot).GetAbsoluteFile(); Path jksPath = new Path(testRootDir.ToString(), "test.jks"); conf.Set(DFSConfigKeys.DfsEncryptionKeyProviderUri, JavaKeyStoreProvider.SchemeName + "://file" + jksPath.ToUri()); cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); Logger.GetLogger(typeof(EncryptionZoneManager)).SetLevel(Level.Trace); fs = cluster.GetFileSystem(); fsWrapper = new FileSystemTestWrapper(cluster.GetFileSystem()); fcWrapper = new FileContextTestWrapper(FileContext.GetFileContext(cluster.GetURI( ), conf)); dfsAdmin = new HdfsAdmin(cluster.GetURI(), conf); // Need to set the client's KeyProvider to the NN's for JKS, // else the updates do not get flushed properly fs.GetClient().SetKeyProvider(cluster.GetNameNode().GetNamesystem().GetProvider() ); DFSTestUtil.CreateKey(TestKey, cluster, conf); }
public virtual void TestCheckpoint() { MiniDFSCluster cluster = null; SecondaryNameNode secondary = null; try { cluster = new MiniDFSCluster.Builder(conf).Build(); cluster.WaitActive(); secondary = new SecondaryNameNode(conf); SnapshotManager nnSnapshotManager = cluster.GetNamesystem().GetSnapshotManager(); SnapshotManager secondarySnapshotManager = secondary.GetFSNamesystem().GetSnapshotManager (); FileSystem fs = cluster.GetFileSystem(); HdfsAdmin admin = new HdfsAdmin(FileSystem.GetDefaultUri(conf), conf); NUnit.Framework.Assert.AreEqual(0, nnSnapshotManager.GetNumSnapshots()); NUnit.Framework.Assert.AreEqual(0, nnSnapshotManager.GetNumSnapshottableDirs()); NUnit.Framework.Assert.AreEqual(0, secondarySnapshotManager.GetNumSnapshots()); NUnit.Framework.Assert.AreEqual(0, secondarySnapshotManager.GetNumSnapshottableDirs ()); // 1. Create a snapshottable directory foo on the NN. fs.Mkdirs(TestPath); admin.AllowSnapshot(TestPath); NUnit.Framework.Assert.AreEqual(0, nnSnapshotManager.GetNumSnapshots()); NUnit.Framework.Assert.AreEqual(1, nnSnapshotManager.GetNumSnapshottableDirs()); // 2. Create a snapshot of the dir foo. This will be referenced both in // the SnapshotManager as well as in the file system tree. The snapshot // count will go up to 1. Path snapshotPath = fs.CreateSnapshot(TestPath); NUnit.Framework.Assert.AreEqual(1, nnSnapshotManager.GetNumSnapshots()); NUnit.Framework.Assert.AreEqual(1, nnSnapshotManager.GetNumSnapshottableDirs()); // 3. Start up a 2NN and have it do a checkpoint. It will have foo and its // snapshot in its list of snapshottable dirs referenced from the // SnapshotManager, as well as in the file system tree. secondary.DoCheckpoint(); NUnit.Framework.Assert.AreEqual(1, secondarySnapshotManager.GetNumSnapshots()); NUnit.Framework.Assert.AreEqual(1, secondarySnapshotManager.GetNumSnapshottableDirs ()); // 4. Disallow snapshots on and delete foo on the NN. The snapshot count // will go down to 0 and the snapshottable dir will be removed from the fs // tree. fs.DeleteSnapshot(TestPath, snapshotPath.GetName()); admin.DisallowSnapshot(TestPath); NUnit.Framework.Assert.AreEqual(0, nnSnapshotManager.GetNumSnapshots()); NUnit.Framework.Assert.AreEqual(0, nnSnapshotManager.GetNumSnapshottableDirs()); // 5. Have the NN do a saveNamespace, writing out a new fsimage with // snapshot count 0. NameNodeAdapter.EnterSafeMode(cluster.GetNameNode(), false); NameNodeAdapter.SaveNamespace(cluster.GetNameNode()); NameNodeAdapter.LeaveSafeMode(cluster.GetNameNode()); // 6. Have the still-running 2NN do a checkpoint. It will notice that the // fsimage has changed on the NN and redownload/reload from that image. // This will replace all INodes in the file system tree as well as reset // the snapshot counter to 0 in the SnapshotManager. However, it will not // clear the list of snapshottable dirs referenced from the // SnapshotManager. When it writes out an fsimage, the 2NN will write out // 0 for the snapshot count, but still serialize the snapshottable dir // referenced in the SnapshotManager even though it no longer appears in // the file system tree. The NN will not be able to start up with this. secondary.DoCheckpoint(); NUnit.Framework.Assert.AreEqual(0, secondarySnapshotManager.GetNumSnapshots()); NUnit.Framework.Assert.AreEqual(0, secondarySnapshotManager.GetNumSnapshottableDirs ()); } finally { if (cluster != null) { cluster.Shutdown(); } if (secondary != null) { secondary.Shutdown(); } } }