/// <summary>Test FileStatus of snapshot file before/after rename</summary> /// <exception cref="System.Exception"/> public virtual void TestSnapshotRename() { DFSTestUtil.CreateFile(hdfs, file1, Blocksize, Replication, seed); // Create snapshot for sub1 Path snapshotRoot = SnapshotTestHelper.CreateSnapshot(hdfs, sub1, "s1"); Path ssPath = new Path(snapshotRoot, file1.GetName()); NUnit.Framework.Assert.IsTrue(hdfs.Exists(ssPath)); FileStatus statusBeforeRename = hdfs.GetFileStatus(ssPath); // Rename the snapshot hdfs.RenameSnapshot(sub1, "s1", "s2"); // <sub1>/.snapshot/s1/file1 should no longer exist NUnit.Framework.Assert.IsFalse(hdfs.Exists(ssPath)); snapshotRoot = SnapshotTestHelper.GetSnapshotRoot(sub1, "s2"); ssPath = new Path(snapshotRoot, file1.GetName()); // Instead, <sub1>/.snapshot/s2/file1 should exist NUnit.Framework.Assert.IsTrue(hdfs.Exists(ssPath)); FileStatus statusAfterRename = hdfs.GetFileStatus(ssPath); // FileStatus of the snapshot should not change except the path NUnit.Framework.Assert.IsFalse(statusBeforeRename.Equals(statusAfterRename)); statusBeforeRename.SetPath(statusAfterRename.GetPath()); NUnit.Framework.Assert.AreEqual(statusBeforeRename.ToString(), statusAfterRename. ToString()); }
public virtual void TestConcatInEditLog() { Path TestDir = new Path("/testConcatInEditLog"); long FileLen = blockSize; // 1. Concat some files Path[] srcFiles = new Path[3]; for (int i = 0; i < srcFiles.Length; i++) { Path path = new Path(TestDir, "src-" + i); DFSTestUtil.CreateFile(dfs, path, FileLen, ReplFactor, 1); srcFiles[i] = path; } Path targetFile = new Path(TestDir, "target"); DFSTestUtil.CreateFile(dfs, targetFile, FileLen, ReplFactor, 1); dfs.Concat(targetFile, srcFiles); // 2. Verify the concat operation basically worked, and record // file status. NUnit.Framework.Assert.IsTrue(dfs.Exists(targetFile)); FileStatus origStatus = dfs.GetFileStatus(targetFile); // 3. Restart NN to force replay from edit log cluster.RestartNameNode(true); // 4. Verify concat operation was replayed correctly and file status // did not change. NUnit.Framework.Assert.IsTrue(dfs.Exists(targetFile)); NUnit.Framework.Assert.IsFalse(dfs.Exists(srcFiles[0])); FileStatus statusAfterRestart = dfs.GetFileStatus(targetFile); NUnit.Framework.Assert.AreEqual(origStatus.GetModificationTime(), statusAfterRestart .GetModificationTime()); }
public virtual void TestSocketFactory() { // Create a standard mini-cluster Configuration sconf = new Configuration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(sconf).NumDataNodes(1).Build( ); int nameNodePort = cluster.GetNameNodePort(); // Get a reference to its DFS directly FileSystem fs = cluster.GetFileSystem(); NUnit.Framework.Assert.IsTrue(fs is DistributedFileSystem); DistributedFileSystem directDfs = (DistributedFileSystem)fs; Configuration cconf = GetCustomSocketConfigs(nameNodePort); fs = FileSystem.Get(cconf); NUnit.Framework.Assert.IsTrue(fs is DistributedFileSystem); DistributedFileSystem dfs = (DistributedFileSystem)fs; JobClient client = null; MiniMRYarnCluster miniMRYarnCluster = null; try { // This will test RPC to the NameNode only. // could we test Client-DataNode connections? Path filePath = new Path("/dir"); NUnit.Framework.Assert.IsFalse(directDfs.Exists(filePath)); NUnit.Framework.Assert.IsFalse(dfs.Exists(filePath)); directDfs.Mkdirs(filePath); NUnit.Framework.Assert.IsTrue(directDfs.Exists(filePath)); NUnit.Framework.Assert.IsTrue(dfs.Exists(filePath)); // This will test RPC to a Resource Manager fs = FileSystem.Get(sconf); JobConf jobConf = new JobConf(); FileSystem.SetDefaultUri(jobConf, fs.GetUri().ToString()); miniMRYarnCluster = InitAndStartMiniMRYarnCluster(jobConf); JobConf jconf = new JobConf(miniMRYarnCluster.GetConfig()); jconf.Set("hadoop.rpc.socket.factory.class.default", "org.apache.hadoop.ipc.DummySocketFactory" ); jconf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName); string rmAddress = jconf.Get("yarn.resourcemanager.address"); string[] split = rmAddress.Split(":"); jconf.Set("yarn.resourcemanager.address", split[0] + ':' + (System.Convert.ToInt32 (split[1]) + 10)); client = new JobClient(jconf); JobStatus[] jobs = client.JobsToComplete(); NUnit.Framework.Assert.IsTrue(jobs.Length == 0); } finally { CloseClient(client); CloseDfs(dfs); CloseDfs(directDfs); StopMiniMRYarnCluster(miniMRYarnCluster); ShutdownDFSCluster(cluster); } }
public virtual void TestRollbackWithQJM() { Configuration conf = new HdfsConfiguration(); MiniJournalCluster mjc = null; MiniDFSCluster cluster = null; Path foo = new Path("/foo"); Path bar = new Path("/bar"); try { mjc = new MiniJournalCluster.Builder(conf).NumJournalNodes(NumJournalNodes).Build (); conf.Set(DFSConfigKeys.DfsNamenodeEditsDirKey, mjc.GetQuorumJournalURI(JournalId) .ToString()); cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build(); cluster.WaitActive(); DistributedFileSystem dfs = cluster.GetFileSystem(); DFSAdmin dfsadmin = new DFSAdmin(conf); dfs.Mkdirs(foo); // start rolling upgrade dfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter); NUnit.Framework.Assert.AreEqual(0, dfsadmin.Run(new string[] { "-rollingUpgrade", "prepare" })); dfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave); // create new directory dfs.Mkdirs(bar); dfs.Close(); // rollback cluster.RestartNameNode("-rollingUpgrade", "rollback"); // make sure /foo is still there, but /bar is not dfs = cluster.GetFileSystem(); NUnit.Framework.Assert.IsTrue(dfs.Exists(foo)); NUnit.Framework.Assert.IsFalse(dfs.Exists(bar)); // check storage in JNs for (int i = 0; i < NumJournalNodes; i++) { FilePath dir = mjc.GetCurrentDir(0, JournalId); // segments:(startSegment, mkdir, endSegment), (startSegment, upgrade // marker, mkdir, endSegment) CheckJNStorage(dir, 4, 7); } } finally { if (cluster != null) { cluster.Shutdown(); } if (mjc != null) { mjc.Shutdown(); } } }
/// <exception cref="System.IO.IOException"/> private static void AssertFile(Path s1, Path s2, Path file, params bool[] expected ) { Path[] paths = new Path[] { file, new Path(s1, "bar/" + file.GetName()), new Path (s2, file.GetName()) }; NUnit.Framework.Assert.AreEqual(expected.Length, paths.Length); for (int i = 0; i < paths.Length; i++) { bool computed = hdfs.Exists(paths[i]); NUnit.Framework.Assert.AreEqual("Failed on " + paths[i], expected[i], computed); } }
/// <exception cref="System.IO.IOException"/> private static void RollbackRollingUpgrade(Path foo, Path bar, Path file, byte[] data, MiniDFSCluster cluster) { MiniDFSCluster.DataNodeProperties dnprop = cluster.StopDataNode(0); cluster.RestartNameNode("-rollingUpgrade", "rollback"); cluster.RestartDataNode(dnprop, true); DistributedFileSystem dfs = cluster.GetFileSystem(); NUnit.Framework.Assert.IsTrue(dfs.Exists(foo)); NUnit.Framework.Assert.IsFalse(dfs.Exists(bar)); AppendTestUtil.CheckFullFile(dfs, file, data.Length, data); }
/// <summary>Ensures that the blocks belonging to the deleted file are in trash</summary> /// <exception cref="System.Exception"/> private void DeleteAndEnsureInTrash(Path pathToDelete, FilePath blockFile, FilePath trashFile) { NUnit.Framework.Assert.IsTrue(blockFile.Exists()); NUnit.Framework.Assert.IsFalse(trashFile.Exists()); // Now delete the file and ensure the corresponding block in trash Log.Info("Deleting file " + pathToDelete + " during rolling upgrade"); fs.Delete(pathToDelete, false); System.Diagnostics.Debug.Assert((!fs.Exists(pathToDelete))); TriggerHeartBeats(); NUnit.Framework.Assert.IsTrue(trashFile.Exists()); NUnit.Framework.Assert.IsFalse(blockFile.Exists()); }
/// <exception cref="System.IO.IOException"/> private void TestPersistHelper(Configuration conf) { MiniDFSCluster cluster = null; try { cluster = new MiniDFSCluster.Builder(conf).Build(); cluster.WaitActive(); FSNamesystem fsn = cluster.GetNamesystem(); DistributedFileSystem fs = cluster.GetFileSystem(); Path dir = new Path("/abc/def"); Path file1 = new Path(dir, "f1"); Path file2 = new Path(dir, "f2"); // create an empty file f1 fs.Create(file1).Close(); // create an under-construction file f2 FSDataOutputStream @out = fs.Create(file2); @out.WriteBytes("hello"); ((DFSOutputStream)@out.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag .UpdateLength)); // checkpoint fs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter); fs.SaveNamespace(); fs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave); cluster.RestartNameNode(); cluster.WaitActive(); fs = cluster.GetFileSystem(); NUnit.Framework.Assert.IsTrue(fs.IsDirectory(dir)); NUnit.Framework.Assert.IsTrue(fs.Exists(file1)); NUnit.Framework.Assert.IsTrue(fs.Exists(file2)); // check internals of file2 INodeFile file2Node = fsn.dir.GetINode4Write(file2.ToString()).AsFile(); NUnit.Framework.Assert.AreEqual("hello".Length, file2Node.ComputeFileSize()); NUnit.Framework.Assert.IsTrue(file2Node.IsUnderConstruction()); BlockInfoContiguous[] blks = file2Node.GetBlocks(); NUnit.Framework.Assert.AreEqual(1, blks.Length); NUnit.Framework.Assert.AreEqual(HdfsServerConstants.BlockUCState.UnderConstruction , blks[0].GetBlockUCState()); // check lease manager LeaseManager.Lease lease = fsn.leaseManager.GetLeaseByPath(file2.ToString()); NUnit.Framework.Assert.IsNotNull(lease); } finally { if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TearDown() { if (fs.Exists(new Path("/sub1"))) { if (fs.Exists(new Path("/sub1/.snapshot"))) { foreach (FileStatus st in fs.ListStatus(new Path("/sub1/.snapshot"))) { fs.DeleteSnapshot(new Path("/sub1"), st.GetPath().GetName()); } fs.DisallowSnapshot(new Path("/sub1")); } fs.Delete(new Path("/sub1"), true); } }
/// <summary> /// The idea for making sure that there is no more than one instance /// running in an HDFS is to create a file in the HDFS, writes the hostname /// of the machine on which the instance is running to the file, but did not /// close the file until it exits. /// </summary> /// <remarks> /// The idea for making sure that there is no more than one instance /// running in an HDFS is to create a file in the HDFS, writes the hostname /// of the machine on which the instance is running to the file, but did not /// close the file until it exits. /// This prevents the second instance from running because it can not /// creates the file while the first one is running. /// This method checks if there is any running instance. If no, mark yes. /// Note that this is an atomic operation. /// </remarks> /// <returns> /// null if there is a running instance; /// otherwise, the output stream to the newly created file. /// </returns> /// <exception cref="System.IO.IOException"/> private OutputStream CheckAndMarkRunning() { try { if (fs.Exists(idPath)) { // try appending to it so that it will fail fast if another balancer is // running. IOUtils.CloseStream(fs.Append(idPath)); fs.Delete(idPath, true); } FSDataOutputStream fsout = fs.Create(idPath, false); // mark balancer idPath to be deleted during filesystem closure fs.DeleteOnExit(idPath); if (write2IdFile) { fsout.WriteBytes(Sharpen.Runtime.GetLocalHost().GetHostName()); fsout.Hflush(); } return(fsout); } catch (RemoteException e) { if (typeof(AlreadyBeingCreatedException).FullName.Equals(e.GetClassName())) { return(null); } else { throw; } } }
/// <summary> /// Check the blocks of dst file are cleaned after rename with overwrite /// Restart NN to check the rename successfully /// </summary> /// <exception cref="System.Exception"/> public virtual void TestRenameWithOverwrite() { short replFactor = 2; long blockSize = 512; Configuration conf = new Configuration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(replFactor ).Build(); DistributedFileSystem dfs = cluster.GetFileSystem(); try { long fileLen = blockSize * 3; string src = "/foo/src"; string dst = "/foo/dst"; Path srcPath = new Path(src); Path dstPath = new Path(dst); DFSTestUtil.CreateFile(dfs, srcPath, fileLen, replFactor, 1); DFSTestUtil.CreateFile(dfs, dstPath, fileLen, replFactor, 1); LocatedBlocks lbs = NameNodeAdapter.GetBlockLocations(cluster.GetNameNode(), dst, 0, fileLen); BlockManager bm = NameNodeAdapter.GetNamesystem(cluster.GetNameNode()).GetBlockManager (); NUnit.Framework.Assert.IsTrue(bm.GetStoredBlock(lbs.GetLocatedBlocks()[0].GetBlock ().GetLocalBlock()) != null); dfs.Rename(srcPath, dstPath, Options.Rename.Overwrite); NUnit.Framework.Assert.IsTrue(bm.GetStoredBlock(lbs.GetLocatedBlocks()[0].GetBlock ().GetLocalBlock()) == null); // Restart NN and check the rename successfully cluster.RestartNameNodes(); NUnit.Framework.Assert.IsFalse(dfs.Exists(srcPath)); NUnit.Framework.Assert.IsTrue(dfs.Exists(dstPath)); } finally { if (dfs != null) { dfs.Close(); } if (cluster != null) { cluster.Shutdown(); } } }
/// <exception cref="System.Exception"/> public virtual void TestDowngrade() { Configuration conf = new HdfsConfiguration(); MiniQJMHACluster cluster = null; Path foo = new Path("/foo"); Path bar = new Path("/bar"); try { cluster = new MiniQJMHACluster.Builder(conf).Build(); MiniDFSCluster dfsCluster = cluster.GetDfsCluster(); dfsCluster.WaitActive(); // let NN1 tail editlog every 1s dfsCluster.GetConfiguration(1).SetInt(DFSConfigKeys.DfsHaTaileditsPeriodKey, 1); dfsCluster.RestartNameNode(1); dfsCluster.TransitionToActive(0); DistributedFileSystem dfs = dfsCluster.GetFileSystem(0); dfs.Mkdirs(foo); // start rolling upgrade RollingUpgradeInfo info = dfs.RollingUpgrade(HdfsConstants.RollingUpgradeAction.Prepare ); NUnit.Framework.Assert.IsTrue(info.IsStarted()); dfs.Mkdirs(bar); TestRollingUpgrade.QueryForPreparation(dfs); dfs.Close(); dfsCluster.RestartNameNode(0, true, "-rollingUpgrade", "downgrade"); // Once downgraded, there should be no more fsimage for rollbacks. NUnit.Framework.Assert.IsFalse(dfsCluster.GetNamesystem(0).GetFSImage().HasRollbackFSImage ()); // shutdown NN1 dfsCluster.ShutdownNameNode(1); dfsCluster.TransitionToActive(0); dfs = dfsCluster.GetFileSystem(0); NUnit.Framework.Assert.IsTrue(dfs.Exists(foo)); NUnit.Framework.Assert.IsTrue(dfs.Exists(bar)); } finally { if (cluster != null) { cluster.Shutdown(); } } }
/// <exception cref="System.IO.IOException"/> private static void StartRollingUpgrade(Path foo, Path bar, Path file, byte[] data , MiniDFSCluster cluster) { DistributedFileSystem dfs = cluster.GetFileSystem(); //start rolling upgrade dfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter); dfs.RollingUpgrade(HdfsConstants.RollingUpgradeAction.Prepare); dfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave); dfs.Mkdirs(bar); NUnit.Framework.Assert.IsTrue(dfs.Exists(foo)); NUnit.Framework.Assert.IsTrue(dfs.Exists(bar)); //truncate a file int newLength = DFSUtil.GetRandom().Next(data.Length - 1) + 1; dfs.Truncate(file, newLength); TestFileTruncate.CheckBlockRecovery(file, dfs); AppendTestUtil.CheckFullFile(dfs, file, newLength, data); }
/// <summary>Check the functionality of a snapshot.</summary> /// <param name="hdfs">DistributedFileSystem instance</param> /// <param name="snapshotRoot">The root of the snapshot</param> /// <param name="snapshottedDir">The snapshotted directory</param> /// <exception cref="System.Exception"/> public static void CheckSnapshotCreation(DistributedFileSystem hdfs, Path snapshotRoot , Path snapshottedDir) { // Currently we only check if the snapshot was created successfully NUnit.Framework.Assert.IsTrue(hdfs.Exists(snapshotRoot)); // Compare the snapshot with the current dir FileStatus[] currentFiles = hdfs.ListStatus(snapshottedDir); FileStatus[] snapshotFiles = hdfs.ListStatus(snapshotRoot); NUnit.Framework.Assert.AreEqual("snapshottedDir=" + snapshottedDir + ", snapshotRoot=" + snapshotRoot, currentFiles.Length, snapshotFiles.Length); }
/// <summary>Create snapshot for a dir using a given snapshot name</summary> /// <param name="hdfs">DistributedFileSystem instance</param> /// <param name="snapshotRoot">The dir to be snapshotted</param> /// <param name="snapshotName">The name of the snapshot</param> /// <returns>The path of the snapshot root</returns> /// <exception cref="System.Exception"/> public static Path CreateSnapshot(DistributedFileSystem hdfs, Path snapshotRoot, string snapshotName) { Log.Info("createSnapshot " + snapshotName + " for " + snapshotRoot); NUnit.Framework.Assert.IsTrue(hdfs.Exists(snapshotRoot)); hdfs.AllowSnapshot(snapshotRoot); hdfs.CreateSnapshot(snapshotRoot, snapshotName); // set quota to a large value for testing counts hdfs.SetQuota(snapshotRoot, long.MaxValue - 1, long.MaxValue - 1); return(Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.SnapshotTestHelper.GetSnapshotRoot (snapshotRoot, snapshotName)); }
public virtual void TestEditsLogRename() { DistributedFileSystem fs = cluster.GetFileSystem(); Path src1 = GetTestRootPath(fc, "testEditsLogRename/srcdir/src1"); Path dst1 = GetTestRootPath(fc, "testEditsLogRename/dstdir/dst1"); CreateFile(src1); fs.Mkdirs(dst1.GetParent()); CreateFile(dst1); // Set quota so that dst1 parent cannot allow under it new files/directories fs.SetQuota(dst1.GetParent(), 2, HdfsConstants.QuotaDontSet); // Free up quota for a subsequent rename fs.Delete(dst1, true); Rename(src1, dst1, true, true, false, Options.Rename.Overwrite); // Restart the cluster and ensure the above operations can be // loaded from the edits log RestartCluster(); fs = cluster.GetFileSystem(); src1 = GetTestRootPath(fc, "testEditsLogRename/srcdir/src1"); dst1 = GetTestRootPath(fc, "testEditsLogRename/dstdir/dst1"); NUnit.Framework.Assert.IsFalse(fs.Exists(src1)); // ensure src1 is already renamed NUnit.Framework.Assert.IsTrue(fs.Exists(dst1)); }
/// <exception cref="System.IO.IOException"/> protected internal LocatedBlocks EnsureFileReplicasOnStorageType(Path path, StorageType storageType) { // Ensure that returned block locations returned are correct! Log.Info("Ensure path: " + path + " is on StorageType: " + storageType); Assert.AssertThat(fs.Exists(path), IS.Is(true)); long fileLength = client.GetFileInfo(path.ToString()).GetLen(); LocatedBlocks locatedBlocks = client.GetLocatedBlocks(path.ToString(), 0, fileLength ); foreach (LocatedBlock locatedBlock in locatedBlocks.GetLocatedBlocks()) { Assert.AssertThat(locatedBlock.GetStorageTypes()[0], IS.Is(storageType)); } return(locatedBlocks); }
/// <exception cref="System.Exception"/> public virtual void TestFinalize() { Configuration conf = new HdfsConfiguration(); MiniQJMHACluster cluster = null; Path foo = new Path("/foo"); Path bar = new Path("/bar"); try { cluster = new MiniQJMHACluster.Builder(conf).Build(); MiniDFSCluster dfsCluster = cluster.GetDfsCluster(); dfsCluster.WaitActive(); // let NN1 tail editlog every 1s dfsCluster.GetConfiguration(1).SetInt(DFSConfigKeys.DfsHaTaileditsPeriodKey, 1); dfsCluster.RestartNameNode(1); dfsCluster.TransitionToActive(0); DistributedFileSystem dfs = dfsCluster.GetFileSystem(0); dfs.Mkdirs(foo); FSImage fsimage = dfsCluster.GetNamesystem(0).GetFSImage(); // start rolling upgrade RollingUpgradeInfo info = dfs.RollingUpgrade(HdfsConstants.RollingUpgradeAction.Prepare ); NUnit.Framework.Assert.IsTrue(info.IsStarted()); dfs.Mkdirs(bar); QueryForPreparation(dfs); // The NN should have a copy of the fsimage in case of rollbacks. NUnit.Framework.Assert.IsTrue(fsimage.HasRollbackFSImage()); info = dfs.RollingUpgrade(HdfsConstants.RollingUpgradeAction.Finalize); NUnit.Framework.Assert.IsTrue(info.IsFinalized()); NUnit.Framework.Assert.IsTrue(dfs.Exists(foo)); // Once finalized, there should be no more fsimage for rollbacks. NUnit.Framework.Assert.IsFalse(fsimage.HasRollbackFSImage()); // Should have no problem in restart and replaying edits that include // the FINALIZE op. dfsCluster.RestartNameNode(0); } finally { if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestNamespaceCommands() { Configuration conf = new HdfsConfiguration(); // Make it relinquish locks. When run serially, the result should // be identical. conf.SetInt(DFSConfigKeys.DfsContentSummaryLimitKey, 2); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build(); DistributedFileSystem dfs = cluster.GetFileSystem(); try { // 1: create directory /nqdir0/qdir1/qdir20/nqdir30 NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(new Path("/nqdir0/qdir1/qdir20/nqdir30") )); // 2: set the quota of /nqdir0/qdir1 to be 6 Path quotaDir1 = new Path("/nqdir0/qdir1"); dfs.SetQuota(quotaDir1, 6, HdfsConstants.QuotaDontSet); ContentSummary c = dfs.GetContentSummary(quotaDir1); NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount(), 3); NUnit.Framework.Assert.AreEqual(c.GetQuota(), 6); // 3: set the quota of /nqdir0/qdir1/qdir20 to be 7 Path quotaDir2 = new Path("/nqdir0/qdir1/qdir20"); dfs.SetQuota(quotaDir2, 7, HdfsConstants.QuotaDontSet); c = dfs.GetContentSummary(quotaDir2); NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount(), 2); NUnit.Framework.Assert.AreEqual(c.GetQuota(), 7); // 4: Create directory /nqdir0/qdir1/qdir21 and set its quota to 2 Path quotaDir3 = new Path("/nqdir0/qdir1/qdir21"); NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(quotaDir3)); dfs.SetQuota(quotaDir3, 2, HdfsConstants.QuotaDontSet); c = dfs.GetContentSummary(quotaDir3); NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount(), 1); NUnit.Framework.Assert.AreEqual(c.GetQuota(), 2); // 5: Create directory /nqdir0/qdir1/qdir21/nqdir32 Path tempPath = new Path(quotaDir3, "nqdir32"); NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(tempPath)); c = dfs.GetContentSummary(quotaDir3); NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount(), 2); NUnit.Framework.Assert.AreEqual(c.GetQuota(), 2); // 6: Create directory /nqdir0/qdir1/qdir21/nqdir33 tempPath = new Path(quotaDir3, "nqdir33"); bool hasException = false; try { NUnit.Framework.Assert.IsFalse(dfs.Mkdirs(tempPath)); } catch (NSQuotaExceededException) { hasException = true; } NUnit.Framework.Assert.IsTrue(hasException); c = dfs.GetContentSummary(quotaDir3); NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount(), 2); NUnit.Framework.Assert.AreEqual(c.GetQuota(), 2); // 7: Create directory /nqdir0/qdir1/qdir20/nqdir31 tempPath = new Path(quotaDir2, "nqdir31"); NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(tempPath)); c = dfs.GetContentSummary(quotaDir2); NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount(), 3); NUnit.Framework.Assert.AreEqual(c.GetQuota(), 7); c = dfs.GetContentSummary(quotaDir1); NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount(), 6); NUnit.Framework.Assert.AreEqual(c.GetQuota(), 6); // 8: Create directory /nqdir0/qdir1/qdir20/nqdir33 tempPath = new Path(quotaDir2, "nqdir33"); hasException = false; try { NUnit.Framework.Assert.IsFalse(dfs.Mkdirs(tempPath)); } catch (NSQuotaExceededException) { hasException = true; } NUnit.Framework.Assert.IsTrue(hasException); // 9: Move /nqdir0/qdir1/qdir21/nqdir32 /nqdir0/qdir1/qdir20/nqdir30 tempPath = new Path(quotaDir2, "nqdir30"); dfs.Rename(new Path(quotaDir3, "nqdir32"), tempPath); c = dfs.GetContentSummary(quotaDir2); NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount(), 4); NUnit.Framework.Assert.AreEqual(c.GetQuota(), 7); c = dfs.GetContentSummary(quotaDir1); NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount(), 6); NUnit.Framework.Assert.AreEqual(c.GetQuota(), 6); // 10: Move /nqdir0/qdir1/qdir20/nqdir30 to /nqdir0/qdir1/qdir21 hasException = false; try { NUnit.Framework.Assert.IsFalse(dfs.Rename(tempPath, quotaDir3)); } catch (NSQuotaExceededException) { hasException = true; } NUnit.Framework.Assert.IsTrue(hasException); NUnit.Framework.Assert.IsTrue(dfs.Exists(tempPath)); NUnit.Framework.Assert.IsFalse(dfs.Exists(new Path(quotaDir3, "nqdir30"))); // 10.a: Rename /nqdir0/qdir1/qdir20/nqdir30 to /nqdir0/qdir1/qdir21/nqdir32 hasException = false; try { NUnit.Framework.Assert.IsFalse(dfs.Rename(tempPath, new Path(quotaDir3, "nqdir32" ))); } catch (QuotaExceededException) { hasException = true; } NUnit.Framework.Assert.IsTrue(hasException); NUnit.Framework.Assert.IsTrue(dfs.Exists(tempPath)); NUnit.Framework.Assert.IsFalse(dfs.Exists(new Path(quotaDir3, "nqdir32"))); // 11: Move /nqdir0/qdir1/qdir20/nqdir30 to /nqdir0 NUnit.Framework.Assert.IsTrue(dfs.Rename(tempPath, new Path("/nqdir0"))); c = dfs.GetContentSummary(quotaDir2); NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount(), 2); NUnit.Framework.Assert.AreEqual(c.GetQuota(), 7); c = dfs.GetContentSummary(quotaDir1); NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount(), 4); NUnit.Framework.Assert.AreEqual(c.GetQuota(), 6); // 12: Create directory /nqdir0/nqdir30/nqdir33 NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(new Path("/nqdir0/nqdir30/nqdir33"))); // 13: Move /nqdir0/nqdir30 /nqdir0/qdir1/qdir20/qdir30 hasException = false; try { NUnit.Framework.Assert.IsFalse(dfs.Rename(new Path("/nqdir0/nqdir30"), tempPath)); } catch (NSQuotaExceededException) { hasException = true; } NUnit.Framework.Assert.IsTrue(hasException); // 14: Move /nqdir0/qdir1/qdir21 /nqdir0/qdir1/qdir20 NUnit.Framework.Assert.IsTrue(dfs.Rename(quotaDir3, quotaDir2)); c = dfs.GetContentSummary(quotaDir1); NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount(), 4); NUnit.Framework.Assert.AreEqual(c.GetQuota(), 6); c = dfs.GetContentSummary(quotaDir2); NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount(), 3); NUnit.Framework.Assert.AreEqual(c.GetQuota(), 7); tempPath = new Path(quotaDir2, "qdir21"); c = dfs.GetContentSummary(tempPath); NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount(), 1); NUnit.Framework.Assert.AreEqual(c.GetQuota(), 2); // 15: Delete /nqdir0/qdir1/qdir20/qdir21 dfs.Delete(tempPath, true); c = dfs.GetContentSummary(quotaDir2); NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount(), 2); NUnit.Framework.Assert.AreEqual(c.GetQuota(), 7); c = dfs.GetContentSummary(quotaDir1); NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount(), 3); NUnit.Framework.Assert.AreEqual(c.GetQuota(), 6); // 16: Move /nqdir0/qdir30 /nqdir0/qdir1/qdir20 NUnit.Framework.Assert.IsTrue(dfs.Rename(new Path("/nqdir0/nqdir30"), quotaDir2)); c = dfs.GetContentSummary(quotaDir2); NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount(), 5); NUnit.Framework.Assert.AreEqual(c.GetQuota(), 7); c = dfs.GetContentSummary(quotaDir1); NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount(), 6); NUnit.Framework.Assert.AreEqual(c.GetQuota(), 6); NUnit.Framework.Assert.AreEqual(14, cluster.GetNamesystem().GetFSDirectory().GetYieldCount ()); } finally { cluster.Shutdown(); } }
public virtual void TestBlockSynchronization() { int OrgFileSize = 3000; Configuration conf = new HdfsConfiguration(); conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, BlockSize); cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(5).Build(); cluster.WaitActive(); //create a file DistributedFileSystem dfs = cluster.GetFileSystem(); string filestr = "/foo"; Path filepath = new Path(filestr); DFSTestUtil.CreateFile(dfs, filepath, OrgFileSize, ReplicationNum, 0L); NUnit.Framework.Assert.IsTrue(dfs.Exists(filepath)); DFSTestUtil.WaitReplication(dfs, filepath, ReplicationNum); //get block info for the last block LocatedBlock locatedblock = TestInterDatanodeProtocol.GetLastLocatedBlock(dfs.dfs .GetNamenode(), filestr); DatanodeInfo[] datanodeinfos = locatedblock.GetLocations(); NUnit.Framework.Assert.AreEqual(ReplicationNum, datanodeinfos.Length); //connect to data nodes DataNode[] datanodes = new DataNode[ReplicationNum]; for (int i = 0; i < ReplicationNum; i++) { datanodes[i] = cluster.GetDataNode(datanodeinfos[i].GetIpcPort()); NUnit.Framework.Assert.IsTrue(datanodes[i] != null); } //verify Block Info ExtendedBlock lastblock = locatedblock.GetBlock(); DataNode.Log.Info("newblocks=" + lastblock); for (int i_1 = 0; i_1 < ReplicationNum; i_1++) { CheckMetaInfo(lastblock, datanodes[i_1]); } DataNode.Log.Info("dfs.dfs.clientName=" + dfs.dfs.clientName); cluster.GetNameNodeRpc().Append(filestr, dfs.dfs.clientName, new EnumSetWritable < CreateFlag>(EnumSet.Of(CreateFlag.Append))); // expire lease to trigger block recovery. WaitLeaseRecovery(cluster); Block[] updatedmetainfo = new Block[ReplicationNum]; long oldSize = lastblock.GetNumBytes(); lastblock = TestInterDatanodeProtocol.GetLastLocatedBlock(dfs.dfs.GetNamenode(), filestr).GetBlock(); long currentGS = lastblock.GetGenerationStamp(); for (int i_2 = 0; i_2 < ReplicationNum; i_2++) { updatedmetainfo[i_2] = DataNodeTestUtils.GetFSDataset(datanodes[i_2]).GetStoredBlock (lastblock.GetBlockPoolId(), lastblock.GetBlockId()); NUnit.Framework.Assert.AreEqual(lastblock.GetBlockId(), updatedmetainfo[i_2].GetBlockId ()); NUnit.Framework.Assert.AreEqual(oldSize, updatedmetainfo[i_2].GetNumBytes()); NUnit.Framework.Assert.AreEqual(currentGS, updatedmetainfo[i_2].GetGenerationStamp ()); } // verify that lease recovery does not occur when namenode is in safemode System.Console.Out.WriteLine("Testing that lease recovery cannot happen during safemode." ); filestr = "/foo.safemode"; filepath = new Path(filestr); dfs.Create(filepath, (short)1); cluster.GetNameNodeRpc().SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter, false); NUnit.Framework.Assert.IsTrue(dfs.dfs.Exists(filestr)); DFSTestUtil.WaitReplication(dfs, filepath, (short)1); WaitLeaseRecovery(cluster); // verify that we still cannot recover the lease LeaseManager lm = NameNodeAdapter.GetLeaseManager(cluster.GetNamesystem()); NUnit.Framework.Assert.IsTrue("Found " + lm.CountLease() + " lease, expected 1", lm.CountLease() == 1); cluster.GetNameNodeRpc().SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave, false); }
public virtual void TestDFSAdminRollingUpgradeCommands() { // start a cluster Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = null; try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build(); cluster.WaitActive(); Path foo = new Path("/foo"); Path bar = new Path("/bar"); Path baz = new Path("/baz"); { DistributedFileSystem dfs = cluster.GetFileSystem(); DFSAdmin dfsadmin = new DFSAdmin(conf); dfs.Mkdirs(foo); //illegal argument "abc" to rollingUpgrade option RunCmd(dfsadmin, false, "-rollingUpgrade", "abc"); CheckMxBeanIsNull(); //query rolling upgrade RunCmd(dfsadmin, true, "-rollingUpgrade"); //start rolling upgrade dfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter); RunCmd(dfsadmin, true, "-rollingUpgrade", "prepare"); dfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave); //query rolling upgrade RunCmd(dfsadmin, true, "-rollingUpgrade", "query"); CheckMxBean(); dfs.Mkdirs(bar); //finalize rolling upgrade RunCmd(dfsadmin, true, "-rollingUpgrade", "finalize"); // RollingUpgradeInfo should be null after finalization, both via // Java API and in JMX NUnit.Framework.Assert.IsNull(dfs.RollingUpgrade(HdfsConstants.RollingUpgradeAction .Query)); CheckMxBeanIsNull(); dfs.Mkdirs(baz); RunCmd(dfsadmin, true, "-rollingUpgrade"); // All directories created before upgrade, when upgrade in progress and // after upgrade finalize exists NUnit.Framework.Assert.IsTrue(dfs.Exists(foo)); NUnit.Framework.Assert.IsTrue(dfs.Exists(bar)); NUnit.Framework.Assert.IsTrue(dfs.Exists(baz)); dfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter); dfs.SaveNamespace(); dfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave); } // Ensure directories exist after restart cluster.RestartNameNode(); { DistributedFileSystem dfs = cluster.GetFileSystem(); NUnit.Framework.Assert.IsTrue(dfs.Exists(foo)); NUnit.Framework.Assert.IsTrue(dfs.Exists(bar)); NUnit.Framework.Assert.IsTrue(dfs.Exists(baz)); } } finally { if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestQuotaCommands() { Configuration conf = new HdfsConfiguration(); // set a smaller block size so that we can test with smaller // Space quotas int DefaultBlockSize = 512; conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, DefaultBlockSize); // Make it relinquish locks. When run serially, the result should // be identical. conf.SetInt(DFSConfigKeys.DfsContentSummaryLimitKey, 2); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build(); FileSystem fs = cluster.GetFileSystem(); NUnit.Framework.Assert.IsTrue("Not a HDFS: " + fs.GetUri(), fs is DistributedFileSystem ); DistributedFileSystem dfs = (DistributedFileSystem)fs; DFSAdmin admin = new DFSAdmin(conf); try { int fileLen = 1024; short replication = 5; long spaceQuota = fileLen * replication * 15 / 8; // 1: create a directory /test and set its quota to be 3 Path parent = new Path("/test"); NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(parent)); string[] args = new string[] { "-setQuota", "3", parent.ToString() }; RunCommand(admin, args, false); //try setting space quota with a 'binary prefix' RunCommand(admin, false, "-setSpaceQuota", "2t", parent.ToString()); NUnit.Framework.Assert.AreEqual(2L << 40, dfs.GetContentSummary(parent).GetSpaceQuota ()); // set diskspace quota to 10000 RunCommand(admin, false, "-setSpaceQuota", System.Convert.ToString(spaceQuota), parent .ToString()); // 2: create directory /test/data0 Path childDir0 = new Path(parent, "data0"); NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(childDir0)); // 3: create a file /test/datafile0 Path childFile0 = new Path(parent, "datafile0"); DFSTestUtil.CreateFile(fs, childFile0, fileLen, replication, 0); // 4: count -q /test ContentSummary c = dfs.GetContentSummary(parent); NUnit.Framework.Assert.AreEqual(c.GetFileCount() + c.GetDirectoryCount(), 3); NUnit.Framework.Assert.AreEqual(c.GetQuota(), 3); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), fileLen * replication); NUnit.Framework.Assert.AreEqual(c.GetSpaceQuota(), spaceQuota); // 5: count -q /test/data0 c = dfs.GetContentSummary(childDir0); NUnit.Framework.Assert.AreEqual(c.GetFileCount() + c.GetDirectoryCount(), 1); NUnit.Framework.Assert.AreEqual(c.GetQuota(), -1); // check disk space consumed c = dfs.GetContentSummary(parent); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), fileLen * replication); // 6: create a directory /test/data1 Path childDir1 = new Path(parent, "data1"); bool hasException = false; try { NUnit.Framework.Assert.IsFalse(dfs.Mkdirs(childDir1)); } catch (QuotaExceededException) { hasException = true; } NUnit.Framework.Assert.IsTrue(hasException); OutputStream fout; // 7: create a file /test/datafile1 Path childFile1 = new Path(parent, "datafile1"); hasException = false; try { fout = dfs.Create(childFile1); } catch (QuotaExceededException) { hasException = true; } NUnit.Framework.Assert.IsTrue(hasException); // 8: clear quota /test RunCommand(admin, new string[] { "-clrQuota", parent.ToString() }, false); c = dfs.GetContentSummary(parent); NUnit.Framework.Assert.AreEqual(c.GetQuota(), -1); NUnit.Framework.Assert.AreEqual(c.GetSpaceQuota(), spaceQuota); // 9: clear quota /test/data0 RunCommand(admin, new string[] { "-clrQuota", childDir0.ToString() }, false); c = dfs.GetContentSummary(childDir0); NUnit.Framework.Assert.AreEqual(c.GetQuota(), -1); // 10: create a file /test/datafile1 fout = dfs.Create(childFile1, replication); // 10.s: but writing fileLen bytes should result in an quota exception try { fout.Write(new byte[fileLen]); fout.Close(); NUnit.Framework.Assert.Fail(); } catch (QuotaExceededException) { IOUtils.CloseStream(fout); } //delete the file dfs.Delete(childFile1, false); // 9.s: clear diskspace quota RunCommand(admin, false, "-clrSpaceQuota", parent.ToString()); c = dfs.GetContentSummary(parent); NUnit.Framework.Assert.AreEqual(c.GetQuota(), -1); NUnit.Framework.Assert.AreEqual(c.GetSpaceQuota(), -1); // now creating childFile1 should succeed DFSTestUtil.CreateFile(dfs, childFile1, fileLen, replication, 0); // 11: set the quota of /test to be 1 // HADOOP-5872 - we can set quota even if it is immediately violated args = new string[] { "-setQuota", "1", parent.ToString() }; RunCommand(admin, args, false); RunCommand(admin, false, "-setSpaceQuota", Sharpen.Extensions.ToString(fileLen), args[2]); // for space quota // 12: set the quota of /test/data0 to be 1 args = new string[] { "-setQuota", "1", childDir0.ToString() }; RunCommand(admin, args, false); // 13: not able create a directory under data0 hasException = false; try { NUnit.Framework.Assert.IsFalse(dfs.Mkdirs(new Path(childDir0, "in"))); } catch (QuotaExceededException) { hasException = true; } NUnit.Framework.Assert.IsTrue(hasException); c = dfs.GetContentSummary(childDir0); NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount() + c.GetFileCount(), 1); NUnit.Framework.Assert.AreEqual(c.GetQuota(), 1); // 14a: set quota on a non-existent directory Path nonExistentPath = new Path("/test1"); NUnit.Framework.Assert.IsFalse(dfs.Exists(nonExistentPath)); args = new string[] { "-setQuota", "1", nonExistentPath.ToString() }; RunCommand(admin, args, true); RunCommand(admin, true, "-setSpaceQuota", "1g", nonExistentPath.ToString()); // for space quota // 14b: set quota on a file NUnit.Framework.Assert.IsTrue(dfs.IsFile(childFile0)); args[1] = childFile0.ToString(); RunCommand(admin, args, true); // same for space quota RunCommand(admin, true, "-setSpaceQuota", "1t", args[1]); // 15a: clear quota on a file args[0] = "-clrQuota"; RunCommand(admin, args, true); RunCommand(admin, true, "-clrSpaceQuota", args[1]); // 15b: clear quota on a non-existent directory args[1] = nonExistentPath.ToString(); RunCommand(admin, args, true); RunCommand(admin, true, "-clrSpaceQuota", args[1]); // 16a: set the quota of /test to be 0 args = new string[] { "-setQuota", "0", parent.ToString() }; RunCommand(admin, args, true); RunCommand(admin, true, "-setSpaceQuota", "0", args[2]); // 16b: set the quota of /test to be -1 args[1] = "-1"; RunCommand(admin, args, true); RunCommand(admin, true, "-setSpaceQuota", args[1], args[2]); // 16c: set the quota of /test to be Long.MAX_VALUE+1 args[1] = (long.MaxValue + 1L).ToString(); RunCommand(admin, args, true); RunCommand(admin, true, "-setSpaceQuota", args[1], args[2]); // 16d: set the quota of /test to be a non integer args[1] = "33aa1.5"; RunCommand(admin, args, true); RunCommand(admin, true, "-setSpaceQuota", args[1], args[2]); // 16e: set space quota with a value larger than Long.MAX_VALUE RunCommand(admin, true, "-setSpaceQuota", (long.MaxValue / 1024 / 1024 + 1024) + "m", args[2]); // 17: setQuota by a non-administrator string username = "******"; UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting(username, new string[] { "groupyy" }); string[] args2 = args.MemberwiseClone(); // need final ref for doAs block ugi.DoAs(new _PrivilegedExceptionAction_275(this, username, conf, args2, parent)); // 18: clrQuota by a non-administrator // 19: clrQuota on the root directory ("/") should fail RunCommand(admin, true, "-clrQuota", "/"); // 20: setQuota on the root directory ("/") should succeed RunCommand(admin, false, "-setQuota", "1000000", "/"); RunCommand(admin, true, "-clrQuota", "/"); RunCommand(admin, false, "-clrSpaceQuota", "/"); RunCommand(admin, new string[] { "-clrQuota", parent.ToString() }, false); RunCommand(admin, false, "-clrSpaceQuota", parent.ToString()); // 2: create directory /test/data2 Path childDir2 = new Path(parent, "data2"); NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(childDir2)); Path childFile2 = new Path(childDir2, "datafile2"); Path childFile3 = new Path(childDir2, "datafile3"); long spaceQuota2 = DefaultBlockSize * replication; long fileLen2 = DefaultBlockSize; // set space quota to a real low value RunCommand(admin, false, "-setSpaceQuota", System.Convert.ToString(spaceQuota2), childDir2.ToString()); // clear space quota RunCommand(admin, false, "-clrSpaceQuota", childDir2.ToString()); // create a file that is greater than the size of space quota DFSTestUtil.CreateFile(fs, childFile2, fileLen2, replication, 0); // now set space quota again. This should succeed RunCommand(admin, false, "-setSpaceQuota", System.Convert.ToString(spaceQuota2), childDir2.ToString()); hasException = false; try { DFSTestUtil.CreateFile(fs, childFile3, fileLen2, replication, 0); } catch (DSQuotaExceededException) { hasException = true; } NUnit.Framework.Assert.IsTrue(hasException); // now test the same for root Path childFile4 = new Path("/", "datafile2"); Path childFile5 = new Path("/", "datafile3"); RunCommand(admin, true, "-clrQuota", "/"); RunCommand(admin, false, "-clrSpaceQuota", "/"); // set space quota to a real low value RunCommand(admin, false, "-setSpaceQuota", System.Convert.ToString(spaceQuota2), "/"); RunCommand(admin, false, "-clrSpaceQuota", "/"); DFSTestUtil.CreateFile(fs, childFile4, fileLen2, replication, 0); RunCommand(admin, false, "-setSpaceQuota", System.Convert.ToString(spaceQuota2), "/"); hasException = false; try { DFSTestUtil.CreateFile(fs, childFile5, fileLen2, replication, 0); } catch (DSQuotaExceededException) { hasException = true; } NUnit.Framework.Assert.IsTrue(hasException); NUnit.Framework.Assert.AreEqual(4, cluster.GetNamesystem().GetFSDirectory().GetYieldCount ()); } finally { cluster.Shutdown(); } }
public virtual void TestSpaceCommands() { Configuration conf = new HdfsConfiguration(); // set a smaller block size so that we can test with smaller // diskspace quotas conf.Set(DFSConfigKeys.DfsBlockSizeKey, "512"); // Make it relinquish locks. When run serially, the result should // be identical. conf.SetInt(DFSConfigKeys.DfsContentSummaryLimitKey, 2); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build(); FileSystem fs = cluster.GetFileSystem(); NUnit.Framework.Assert.IsTrue("Not a HDFS: " + fs.GetUri(), fs is DistributedFileSystem ); DistributedFileSystem dfs = (DistributedFileSystem)fs; try { int fileLen = 1024; short replication = 3; int fileSpace = fileLen * replication; // create directory /nqdir0/qdir1/qdir20/nqdir30 NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(new Path("/nqdir0/qdir1/qdir20/nqdir30") )); // set the quota of /nqdir0/qdir1 to 4 * fileSpace Path quotaDir1 = new Path("/nqdir0/qdir1"); dfs.SetQuota(quotaDir1, HdfsConstants.QuotaDontSet, 4 * fileSpace); ContentSummary c = dfs.GetContentSummary(quotaDir1); NUnit.Framework.Assert.AreEqual(c.GetSpaceQuota(), 4 * fileSpace); // set the quota of /nqdir0/qdir1/qdir20 to 6 * fileSpace Path quotaDir20 = new Path("/nqdir0/qdir1/qdir20"); dfs.SetQuota(quotaDir20, HdfsConstants.QuotaDontSet, 6 * fileSpace); c = dfs.GetContentSummary(quotaDir20); NUnit.Framework.Assert.AreEqual(c.GetSpaceQuota(), 6 * fileSpace); // Create /nqdir0/qdir1/qdir21 and set its space quota to 2 * fileSpace Path quotaDir21 = new Path("/nqdir0/qdir1/qdir21"); NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(quotaDir21)); dfs.SetQuota(quotaDir21, HdfsConstants.QuotaDontSet, 2 * fileSpace); c = dfs.GetContentSummary(quotaDir21); NUnit.Framework.Assert.AreEqual(c.GetSpaceQuota(), 2 * fileSpace); // 5: Create directory /nqdir0/qdir1/qdir21/nqdir32 Path tempPath = new Path(quotaDir21, "nqdir32"); NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(tempPath)); // create a file under nqdir32/fileDir DFSTestUtil.CreateFile(dfs, new Path(tempPath, "fileDir/file1"), fileLen, replication , 0); c = dfs.GetContentSummary(quotaDir21); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), fileSpace); // Create a larger file /nqdir0/qdir1/qdir21/nqdir33/ bool hasException = false; try { DFSTestUtil.CreateFile(dfs, new Path(quotaDir21, "nqdir33/file2"), 2 * fileLen, replication , 0); } catch (DSQuotaExceededException) { hasException = true; } NUnit.Framework.Assert.IsTrue(hasException); // delete nqdir33 NUnit.Framework.Assert.IsTrue(dfs.Delete(new Path(quotaDir21, "nqdir33"), true)); c = dfs.GetContentSummary(quotaDir21); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), fileSpace); NUnit.Framework.Assert.AreEqual(c.GetSpaceQuota(), 2 * fileSpace); // Verify space before the move: c = dfs.GetContentSummary(quotaDir20); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 0); // Move /nqdir0/qdir1/qdir21/nqdir32 /nqdir0/qdir1/qdir20/nqdir30 Path dstPath = new Path(quotaDir20, "nqdir30"); Path srcPath = new Path(quotaDir21, "nqdir32"); NUnit.Framework.Assert.IsTrue(dfs.Rename(srcPath, dstPath)); // verify space after the move c = dfs.GetContentSummary(quotaDir20); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), fileSpace); // verify space for its parent c = dfs.GetContentSummary(quotaDir1); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), fileSpace); // verify space for source for the move c = dfs.GetContentSummary(quotaDir21); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 0); Path file2 = new Path(dstPath, "fileDir/file2"); int file2Len = 2 * fileLen; // create a larger file under /nqdir0/qdir1/qdir20/nqdir30 DFSTestUtil.CreateFile(dfs, file2, file2Len, replication, 0); c = dfs.GetContentSummary(quotaDir20); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 3 * fileSpace); c = dfs.GetContentSummary(quotaDir21); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 0); // Reverse: Move /nqdir0/qdir1/qdir20/nqdir30 to /nqdir0/qdir1/qdir21/ hasException = false; try { NUnit.Framework.Assert.IsFalse(dfs.Rename(dstPath, srcPath)); } catch (DSQuotaExceededException) { hasException = true; } NUnit.Framework.Assert.IsTrue(hasException); // make sure no intermediate directories left by failed rename NUnit.Framework.Assert.IsFalse(dfs.Exists(srcPath)); // directory should exist NUnit.Framework.Assert.IsTrue(dfs.Exists(dstPath)); // verify space after the failed move c = dfs.GetContentSummary(quotaDir20); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 3 * fileSpace); c = dfs.GetContentSummary(quotaDir21); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 0); // Test Append : // verify space quota c = dfs.GetContentSummary(quotaDir1); NUnit.Framework.Assert.AreEqual(c.GetSpaceQuota(), 4 * fileSpace); // verify space before append; c = dfs.GetContentSummary(dstPath); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 3 * fileSpace); OutputStream @out = dfs.Append(file2); // appending 1 fileLen should succeed @out.Write(new byte[fileLen]); @out.Close(); file2Len += fileLen; // after append // verify space after append; c = dfs.GetContentSummary(dstPath); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 4 * fileSpace); // now increase the quota for quotaDir1 dfs.SetQuota(quotaDir1, HdfsConstants.QuotaDontSet, 5 * fileSpace); // Now, appending more than 1 fileLen should result in an error @out = dfs.Append(file2); hasException = false; try { @out.Write(new byte[fileLen + 1024]); @out.Flush(); @out.Close(); } catch (DSQuotaExceededException) { hasException = true; IOUtils.CloseStream(@out); } NUnit.Framework.Assert.IsTrue(hasException); file2Len += fileLen; // after partial append // verify space after partial append c = dfs.GetContentSummary(dstPath); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 5 * fileSpace); // Test set replication : // first reduce the replication dfs.SetReplication(file2, (short)(replication - 1)); // verify that space is reduced by file2Len c = dfs.GetContentSummary(dstPath); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 5 * fileSpace - file2Len); // now try to increase the replication and and expect an error. hasException = false; try { dfs.SetReplication(file2, (short)(replication + 1)); } catch (DSQuotaExceededException) { hasException = true; } NUnit.Framework.Assert.IsTrue(hasException); // verify space consumed remains unchanged. c = dfs.GetContentSummary(dstPath); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 5 * fileSpace - file2Len); // now increase the quota for quotaDir1 and quotaDir20 dfs.SetQuota(quotaDir1, HdfsConstants.QuotaDontSet, 10 * fileSpace); dfs.SetQuota(quotaDir20, HdfsConstants.QuotaDontSet, 10 * fileSpace); // then increasing replication should be ok. dfs.SetReplication(file2, (short)(replication + 1)); // verify increase in space c = dfs.GetContentSummary(dstPath); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 5 * fileSpace + file2Len); // Test HDFS-2053 : // Create directory /hdfs-2053 Path quotaDir2053 = new Path("/hdfs-2053"); NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(quotaDir2053)); // Create subdirectories /hdfs-2053/{A,B,C} Path quotaDir2053_A = new Path(quotaDir2053, "A"); NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(quotaDir2053_A)); Path quotaDir2053_B = new Path(quotaDir2053, "B"); NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(quotaDir2053_B)); Path quotaDir2053_C = new Path(quotaDir2053, "C"); NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(quotaDir2053_C)); // Factors to vary the sizes of test files created in each subdir. // The actual factors are not really important but they allow us to create // identifiable file sizes per subdir, which helps during debugging. int sizeFactorA = 1; int sizeFactorB = 2; int sizeFactorC = 4; // Set space quota for subdirectory C dfs.SetQuota(quotaDir2053_C, HdfsConstants.QuotaDontSet, (sizeFactorC + 1) * fileSpace ); c = dfs.GetContentSummary(quotaDir2053_C); NUnit.Framework.Assert.AreEqual(c.GetSpaceQuota(), (sizeFactorC + 1) * fileSpace); // Create a file under subdirectory A DFSTestUtil.CreateFile(dfs, new Path(quotaDir2053_A, "fileA"), sizeFactorA * fileLen , replication, 0); c = dfs.GetContentSummary(quotaDir2053_A); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), sizeFactorA * fileSpace); // Create a file under subdirectory B DFSTestUtil.CreateFile(dfs, new Path(quotaDir2053_B, "fileB"), sizeFactorB * fileLen , replication, 0); c = dfs.GetContentSummary(quotaDir2053_B); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), sizeFactorB * fileSpace); // Create a file under subdirectory C (which has a space quota) DFSTestUtil.CreateFile(dfs, new Path(quotaDir2053_C, "fileC"), sizeFactorC * fileLen , replication, 0); c = dfs.GetContentSummary(quotaDir2053_C); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), sizeFactorC * fileSpace); // Check space consumed for /hdfs-2053 c = dfs.GetContentSummary(quotaDir2053); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), (sizeFactorA + sizeFactorB + sizeFactorC) * fileSpace); NUnit.Framework.Assert.AreEqual(20, cluster.GetNamesystem().GetFSDirectory().GetYieldCount ()); } finally { cluster.Shutdown(); } }
/// <summary>The following test first creates a file.</summary> /// <remarks> /// The following test first creates a file. /// It verifies the block information from a datanode. /// Then, it updates the block with new information and verifies again. /// </remarks> /// <param name="useDnHostname">whether DNs should connect to other DNs by hostname</param> /// <exception cref="System.Exception"/> private void CheckBlockMetaDataInfo(bool useDnHostname) { MiniDFSCluster cluster = null; conf.SetBoolean(DFSConfigKeys.DfsDatanodeUseDnHostname, useDnHostname); if (useDnHostname) { // Since the mini cluster only listens on the loopback we have to // ensure the hostname used to access DNs maps to the loopback. We // do this by telling the DN to advertise localhost as its hostname // instead of the default hostname. conf.Set(DFSConfigKeys.DfsDatanodeHostNameKey, "localhost"); } try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).CheckDataNodeHostConfig (true).Build(); cluster.WaitActive(); //create a file DistributedFileSystem dfs = cluster.GetFileSystem(); string filestr = "/foo"; Path filepath = new Path(filestr); DFSTestUtil.CreateFile(dfs, filepath, 1024L, (short)3, 0L); NUnit.Framework.Assert.IsTrue(dfs.Exists(filepath)); //get block info LocatedBlock locatedblock = GetLastLocatedBlock(DFSClientAdapter.GetDFSClient(dfs ).GetNamenode(), filestr); DatanodeInfo[] datanodeinfo = locatedblock.GetLocations(); NUnit.Framework.Assert.IsTrue(datanodeinfo.Length > 0); //connect to a data node DataNode datanode = cluster.GetDataNode(datanodeinfo[0].GetIpcPort()); InterDatanodeProtocol idp = DataNodeTestUtils.CreateInterDatanodeProtocolProxy(datanode , datanodeinfo[0], conf, useDnHostname); // Stop the block scanners. datanode.GetBlockScanner().RemoveAllVolumeScanners(); //verify BlockMetaDataInfo ExtendedBlock b = locatedblock.GetBlock(); InterDatanodeProtocol.Log.Info("b=" + b + ", " + b.GetType()); CheckMetaInfo(b, datanode); long recoveryId = b.GetGenerationStamp() + 1; idp.InitReplicaRecovery(new BlockRecoveryCommand.RecoveringBlock(b, locatedblock. GetLocations(), recoveryId)); //verify updateBlock ExtendedBlock newblock = new ExtendedBlock(b.GetBlockPoolId(), b.GetBlockId(), b. GetNumBytes() / 2, b.GetGenerationStamp() + 1); idp.UpdateReplicaUnderRecovery(b, recoveryId, b.GetBlockId(), newblock.GetNumBytes ()); CheckMetaInfo(newblock, datanode); // Verify correct null response trying to init recovery for a missing block ExtendedBlock badBlock = new ExtendedBlock("fake-pool", b.GetBlockId(), 0, 0); NUnit.Framework.Assert.IsNull(idp.InitReplicaRecovery(new BlockRecoveryCommand.RecoveringBlock (badBlock, locatedblock.GetLocations(), recoveryId))); } finally { if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestLeaseAfterRename() { MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build(); try { Path p = new Path("/test-file"); Path d = new Path("/test-d"); Path d2 = new Path("/test-d-other"); // open a file to get a lease FileSystem fs = cluster.GetFileSystem(); FSDataOutputStream @out = fs.Create(p); @out.WriteBytes("something"); //out.hsync(); NUnit.Framework.Assert.IsTrue(HasLease(cluster, p)); NUnit.Framework.Assert.AreEqual(1, LeaseCount(cluster)); // just to ensure first fs doesn't have any logic to twiddle leases DistributedFileSystem fs2 = (DistributedFileSystem)FileSystem.NewInstance(fs.GetUri (), fs.GetConf()); // rename the file into an existing dir Log.Info("DMS: rename file into dir"); Path pRenamed = new Path(d, p.GetName()); fs2.Mkdirs(d); fs2.Rename(p, pRenamed); NUnit.Framework.Assert.IsFalse(p + " exists", fs2.Exists(p)); NUnit.Framework.Assert.IsTrue(pRenamed + " not found", fs2.Exists(pRenamed)); NUnit.Framework.Assert.IsFalse("has lease for " + p, HasLease(cluster, p)); NUnit.Framework.Assert.IsTrue("no lease for " + pRenamed, HasLease(cluster, pRenamed )); NUnit.Framework.Assert.AreEqual(1, LeaseCount(cluster)); // rename the parent dir to a new non-existent dir Log.Info("DMS: rename parent dir"); Path pRenamedAgain = new Path(d2, pRenamed.GetName()); fs2.Rename(d, d2); // src gone NUnit.Framework.Assert.IsFalse(d + " exists", fs2.Exists(d)); NUnit.Framework.Assert.IsFalse("has lease for " + pRenamed, HasLease(cluster, pRenamed )); // dst checks NUnit.Framework.Assert.IsTrue(d2 + " not found", fs2.Exists(d2)); NUnit.Framework.Assert.IsTrue(pRenamedAgain + " not found", fs2.Exists(pRenamedAgain )); NUnit.Framework.Assert.IsTrue("no lease for " + pRenamedAgain, HasLease(cluster, pRenamedAgain)); NUnit.Framework.Assert.AreEqual(1, LeaseCount(cluster)); // rename the parent dir to existing dir // NOTE: rename w/o options moves paths into existing dir Log.Info("DMS: rename parent again"); pRenamed = pRenamedAgain; pRenamedAgain = new Path(new Path(d, d2.GetName()), p.GetName()); fs2.Mkdirs(d); fs2.Rename(d2, d); // src gone NUnit.Framework.Assert.IsFalse(d2 + " exists", fs2.Exists(d2)); NUnit.Framework.Assert.IsFalse("no lease for " + pRenamed, HasLease(cluster, pRenamed )); // dst checks NUnit.Framework.Assert.IsTrue(d + " not found", fs2.Exists(d)); NUnit.Framework.Assert.IsTrue(pRenamedAgain + " not found", fs2.Exists(pRenamedAgain )); NUnit.Framework.Assert.IsTrue("no lease for " + pRenamedAgain, HasLease(cluster, pRenamedAgain)); NUnit.Framework.Assert.AreEqual(1, LeaseCount(cluster)); // rename with opts to non-existent dir pRenamed = pRenamedAgain; pRenamedAgain = new Path(d2, p.GetName()); fs2.Rename(pRenamed.GetParent(), d2, Options.Rename.Overwrite); // src gone NUnit.Framework.Assert.IsFalse(pRenamed.GetParent() + " not found", fs2.Exists(pRenamed .GetParent())); NUnit.Framework.Assert.IsFalse("has lease for " + pRenamed, HasLease(cluster, pRenamed )); // dst checks NUnit.Framework.Assert.IsTrue(d2 + " not found", fs2.Exists(d2)); NUnit.Framework.Assert.IsTrue(pRenamedAgain + " not found", fs2.Exists(pRenamedAgain )); NUnit.Framework.Assert.IsTrue("no lease for " + pRenamedAgain, HasLease(cluster, pRenamedAgain)); NUnit.Framework.Assert.AreEqual(1, LeaseCount(cluster)); // rename with opts to existing dir // NOTE: rename with options will not move paths into the existing dir pRenamed = pRenamedAgain; pRenamedAgain = new Path(d, p.GetName()); fs2.Rename(pRenamed.GetParent(), d, Options.Rename.Overwrite); // src gone NUnit.Framework.Assert.IsFalse(pRenamed.GetParent() + " not found", fs2.Exists(pRenamed .GetParent())); NUnit.Framework.Assert.IsFalse("has lease for " + pRenamed, HasLease(cluster, pRenamed )); // dst checks NUnit.Framework.Assert.IsTrue(d + " not found", fs2.Exists(d)); NUnit.Framework.Assert.IsTrue(pRenamedAgain + " not found", fs2.Exists(pRenamedAgain )); NUnit.Framework.Assert.IsTrue("no lease for " + pRenamedAgain, HasLease(cluster, pRenamedAgain)); NUnit.Framework.Assert.AreEqual(1, LeaseCount(cluster)); @out.Close(); } finally { cluster.Shutdown(); } }
public virtual void TestRollbackWithHAQJM() { Configuration conf = new HdfsConfiguration(); MiniQJMHACluster cluster = null; Path foo = new Path("/foo"); Path bar = new Path("/bar"); try { cluster = new MiniQJMHACluster.Builder(conf).Build(); MiniDFSCluster dfsCluster = cluster.GetDfsCluster(); dfsCluster.WaitActive(); // let NN1 tail editlog every 1s dfsCluster.GetConfiguration(1).SetInt(DFSConfigKeys.DfsHaTaileditsPeriodKey, 1); dfsCluster.RestartNameNode(1); dfsCluster.TransitionToActive(0); DistributedFileSystem dfs = dfsCluster.GetFileSystem(0); dfs.Mkdirs(foo); // start rolling upgrade RollingUpgradeInfo info = dfs.RollingUpgrade(HdfsConstants.RollingUpgradeAction.Prepare ); NUnit.Framework.Assert.IsTrue(info.IsStarted()); // create new directory dfs.Mkdirs(bar); dfs.Close(); TestRollingUpgrade.QueryForPreparation(dfs); // If the query returns true, both active and the standby NN should have // rollback fsimage ready. NUnit.Framework.Assert.IsTrue(dfsCluster.GetNameNode(0).GetFSImage().HasRollbackFSImage ()); NUnit.Framework.Assert.IsTrue(dfsCluster.GetNameNode(1).GetFSImage().HasRollbackFSImage ()); // rollback NN0 dfsCluster.RestartNameNode(0, true, "-rollingUpgrade", "rollback"); // shutdown NN1 dfsCluster.ShutdownNameNode(1); dfsCluster.TransitionToActive(0); // make sure /foo is still there, but /bar is not dfs = dfsCluster.GetFileSystem(0); NUnit.Framework.Assert.IsTrue(dfs.Exists(foo)); NUnit.Framework.Assert.IsFalse(dfs.Exists(bar)); // check the details of NNStorage NNStorage storage = dfsCluster.GetNamesystem(0).GetFSImage().GetStorage(); // segments:(startSegment, mkdir, start upgrade endSegment), // (startSegment, mkdir, endSegment) CheckNNStorage(storage, 4, 7); // check storage in JNs for (int i = 0; i < NumJournalNodes; i++) { FilePath dir = cluster.GetJournalCluster().GetCurrentDir(0, MiniQJMHACluster.Nameservice ); CheckJNStorage(dir, 5, 7); } // restart NN0 again to make sure we can start using the new fsimage and // the corresponding md5 checksum dfsCluster.RestartNameNode(0); // start the rolling upgrade again to make sure we do not load upgrade // status after the rollback dfsCluster.TransitionToActive(0); dfs.RollingUpgrade(HdfsConstants.RollingUpgradeAction.Prepare); } finally { if (cluster != null) { cluster.Shutdown(); } } }
/// <exception cref="System.Exception"/> public virtual void TestRollingUpgradeWithQJM() { string nnDirPrefix = MiniDFSCluster.GetBaseDirectory() + "/nn/"; FilePath nn1Dir = new FilePath(nnDirPrefix + "image1"); FilePath nn2Dir = new FilePath(nnDirPrefix + "image2"); Log.Info("nn1Dir=" + nn1Dir); Log.Info("nn2Dir=" + nn2Dir); Configuration conf = new HdfsConfiguration(); MiniJournalCluster mjc = new MiniJournalCluster.Builder(conf).Build(); SetConf(conf, nn1Dir, mjc); { // Start the cluster once to generate the dfs dirs MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).ManageNameDfsDirs (false).CheckExitOnShutdown(false).Build(); // Shutdown the cluster before making a copy of the namenode dir to release // all file locks, otherwise, the copy will fail on some platforms. cluster.Shutdown(); } MiniDFSCluster cluster2 = null; try { // Start a second NN pointed to the same quorum. // We need to copy the image dir from the first NN -- or else // the new NN will just be rejected because of Namespace mismatch. FileUtil.FullyDelete(nn2Dir); FileUtil.Copy(nn1Dir, FileSystem.GetLocal(conf).GetRaw(), new Path(nn2Dir.GetAbsolutePath ()), false, conf); // Start the cluster again MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Format( false).ManageNameDfsDirs(false).CheckExitOnShutdown(false).Build(); Path foo = new Path("/foo"); Path bar = new Path("/bar"); Path baz = new Path("/baz"); RollingUpgradeInfo info1; { DistributedFileSystem dfs = cluster.GetFileSystem(); dfs.Mkdirs(foo); //start rolling upgrade dfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter); info1 = dfs.RollingUpgrade(HdfsConstants.RollingUpgradeAction.Prepare); dfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave); Log.Info("START\n" + info1); //query rolling upgrade NUnit.Framework.Assert.AreEqual(info1, dfs.RollingUpgrade(HdfsConstants.RollingUpgradeAction .Query)); dfs.Mkdirs(bar); cluster.Shutdown(); } // cluster2 takes over QJM Configuration conf2 = SetConf(new Configuration(), nn2Dir, mjc); cluster2 = new MiniDFSCluster.Builder(conf2).NumDataNodes(0).Format(false).ManageNameDfsDirs (false).Build(); DistributedFileSystem dfs2 = cluster2.GetFileSystem(); // Check that cluster2 sees the edits made on cluster1 NUnit.Framework.Assert.IsTrue(dfs2.Exists(foo)); NUnit.Framework.Assert.IsTrue(dfs2.Exists(bar)); NUnit.Framework.Assert.IsFalse(dfs2.Exists(baz)); //query rolling upgrade in cluster2 NUnit.Framework.Assert.AreEqual(info1, dfs2.RollingUpgrade(HdfsConstants.RollingUpgradeAction .Query)); dfs2.Mkdirs(baz); Log.Info("RESTART cluster 2"); cluster2.RestartNameNode(); NUnit.Framework.Assert.AreEqual(info1, dfs2.RollingUpgrade(HdfsConstants.RollingUpgradeAction .Query)); NUnit.Framework.Assert.IsTrue(dfs2.Exists(foo)); NUnit.Framework.Assert.IsTrue(dfs2.Exists(bar)); NUnit.Framework.Assert.IsTrue(dfs2.Exists(baz)); //restart cluster with -upgrade should fail. try { cluster2.RestartNameNode("-upgrade"); } catch (IOException e) { Log.Info("The exception is expected.", e); } Log.Info("RESTART cluster 2 again"); cluster2.RestartNameNode(); NUnit.Framework.Assert.AreEqual(info1, dfs2.RollingUpgrade(HdfsConstants.RollingUpgradeAction .Query)); NUnit.Framework.Assert.IsTrue(dfs2.Exists(foo)); NUnit.Framework.Assert.IsTrue(dfs2.Exists(bar)); NUnit.Framework.Assert.IsTrue(dfs2.Exists(baz)); //finalize rolling upgrade RollingUpgradeInfo finalize = dfs2.RollingUpgrade(HdfsConstants.RollingUpgradeAction .Finalize); NUnit.Framework.Assert.IsTrue(finalize.IsFinalized()); Log.Info("RESTART cluster 2 with regular startup option"); cluster2.GetNameNodeInfos()[0].SetStartOpt(HdfsServerConstants.StartupOption.Regular ); cluster2.RestartNameNode(); NUnit.Framework.Assert.IsTrue(dfs2.Exists(foo)); NUnit.Framework.Assert.IsTrue(dfs2.Exists(bar)); NUnit.Framework.Assert.IsTrue(dfs2.Exists(baz)); } finally { if (cluster2 != null) { cluster2.Shutdown(); } } }