private static void SetTimeout(HttpURLConnection connection) { if (timeout <= 0) { Configuration conf = new HdfsConfiguration(); timeout = conf.GetInt(DFSConfigKeys.DfsImageTransferTimeoutKey, DFSConfigKeys.DfsImageTransferTimeoutDefault ); Log.Info("Image Transfer timeout configured to " + timeout + " milliseconds"); } if (timeout > 0) { connection.SetConnectTimeout(timeout); connection.SetReadTimeout(timeout); } }
public virtual void TestCreate() { Configuration conf = new HdfsConfiguration(); conf.SetBoolean(DFSConfigKeys.DfsPermissionsEnabledKey, true); conf.Set(FsPermission.UmaskLabel, "000"); MiniDFSCluster cluster = null; FileSystem fs = null; try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build(); cluster.WaitActive(); fs = FileSystem.Get(conf); FsPermission rootPerm = CheckPermission(fs, "/", null); FsPermission inheritPerm = FsPermission.CreateImmutable((short)(rootPerm.ToShort( ) | 0xc0)); FsPermission dirPerm = new FsPermission((short)0x1ff); fs.Mkdirs(new Path("/a1/a2/a3"), dirPerm); CheckPermission(fs, "/a1", dirPerm); CheckPermission(fs, "/a1/a2", dirPerm); CheckPermission(fs, "/a1/a2/a3", dirPerm); dirPerm = new FsPermission((short)0x53); FsPermission permission = FsPermission.CreateImmutable((short)(dirPerm.ToShort() | 0xc0)); fs.Mkdirs(new Path("/aa/1/aa/2/aa/3"), dirPerm); CheckPermission(fs, "/aa/1", permission); CheckPermission(fs, "/aa/1/aa/2", permission); CheckPermission(fs, "/aa/1/aa/2/aa/3", dirPerm); FsPermission filePerm = new FsPermission((short)0x124); Path p = new Path("/b1/b2/b3.txt"); FSDataOutputStream @out = fs.Create(p, filePerm, true, conf.GetInt(CommonConfigurationKeys .IoFileBufferSizeKey, 4096), fs.GetDefaultReplication(p), fs.GetDefaultBlockSize (p), null); @out.Write(123); @out.Close(); CheckPermission(fs, "/b1", inheritPerm); CheckPermission(fs, "/b1/b2", inheritPerm); CheckPermission(fs, "/b1/b2/b3.txt", filePerm); conf.Set(FsPermission.UmaskLabel, "022"); permission = FsPermission.CreateImmutable((short)0x1b6); FileSystem.Mkdirs(fs, new Path("/c1"), new FsPermission(permission)); FileSystem.Create(fs, new Path("/c1/c2.txt"), new FsPermission(permission)); CheckPermission(fs, "/c1", permission); CheckPermission(fs, "/c1/c2.txt", permission); } finally { try { if (fs != null) { fs.Close(); } } catch (Exception e) { Log.Error(StringUtils.StringifyException(e)); } try { if (cluster != null) { cluster.Shutdown(); } } catch (Exception e) { Log.Error(StringUtils.StringifyException(e)); } } }
public virtual void TestNameEditsConfigsFailure() { Path file1 = new Path("TestNameEditsConfigs1"); Path file2 = new Path("TestNameEditsConfigs2"); Path file3 = new Path("TestNameEditsConfigs3"); MiniDFSCluster cluster = null; Configuration conf = null; FileSystem fileSys = null; FilePath nameOnlyDir = new FilePath(base_dir, "name"); FilePath editsOnlyDir = new FilePath(base_dir, "edits"); FilePath nameAndEditsDir = new FilePath(base_dir, "name_and_edits"); // 1 // Start namenode with same dfs.namenode.name.dir and dfs.namenode.edits.dir conf = new HdfsConfiguration(); conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, nameAndEditsDir.GetPath()); conf.Set(DFSConfigKeys.DfsNamenodeEditsDirKey, nameAndEditsDir.GetPath()); replication = (short)conf.GetInt(DFSConfigKeys.DfsReplicationKey, 3); try { // Manage our own dfs directories cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(NumDataNodes).ManageNameDfsDirs (false).Build(); cluster.WaitActive(); // Check that the dir has a VERSION file NUnit.Framework.Assert.IsTrue(new FilePath(nameAndEditsDir, "current/VERSION").Exists ()); fileSys = cluster.GetFileSystem(); NUnit.Framework.Assert.IsTrue(!fileSys.Exists(file1)); WriteFile(fileSys, file1, replication); CheckFile(fileSys, file1, replication); } finally { fileSys.Close(); cluster.Shutdown(); } // 2 // Start namenode with additional dfs.namenode.name.dir and dfs.namenode.edits.dir conf = new HdfsConfiguration(); NUnit.Framework.Assert.IsTrue(nameOnlyDir.Mkdir()); NUnit.Framework.Assert.IsTrue(editsOnlyDir.Mkdir()); conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, nameAndEditsDir.GetPath() + "," + nameOnlyDir .GetPath()); conf.Set(DFSConfigKeys.DfsNamenodeEditsDirKey, nameAndEditsDir.GetPath() + "," + editsOnlyDir.GetPath()); replication = (short)conf.GetInt(DFSConfigKeys.DfsReplicationKey, 3); try { // Manage our own dfs directories. Do not format. cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(NumDataNodes).Format(false ).ManageNameDfsDirs(false).Build(); cluster.WaitActive(); // Check that the dirs have a VERSION file NUnit.Framework.Assert.IsTrue(new FilePath(nameAndEditsDir, "current/VERSION").Exists ()); NUnit.Framework.Assert.IsTrue(new FilePath(nameOnlyDir, "current/VERSION").Exists ()); NUnit.Framework.Assert.IsTrue(new FilePath(editsOnlyDir, "current/VERSION").Exists ()); fileSys = cluster.GetFileSystem(); NUnit.Framework.Assert.IsTrue(fileSys.Exists(file1)); CheckFile(fileSys, file1, replication); CleanupFile(fileSys, file1); WriteFile(fileSys, file2, replication); CheckFile(fileSys, file2, replication); } finally { fileSys.Close(); cluster.Shutdown(); } // 3 // Now remove common directory both have and start namenode with // separate name and edits dirs try { conf = new HdfsConfiguration(); conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, nameOnlyDir.GetPath()); conf.Set(DFSConfigKeys.DfsNamenodeEditsDirKey, editsOnlyDir.GetPath()); replication = (short)conf.GetInt(DFSConfigKeys.DfsReplicationKey, 3); cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(NumDataNodes).Format(false ).ManageNameDfsDirs(false).Build(); cluster.WaitActive(); fileSys = cluster.GetFileSystem(); NUnit.Framework.Assert.IsFalse(fileSys.Exists(file1)); NUnit.Framework.Assert.IsTrue(fileSys.Exists(file2)); CheckFile(fileSys, file2, replication); CleanupFile(fileSys, file2); WriteFile(fileSys, file3, replication); CheckFile(fileSys, file3, replication); } finally { fileSys.Close(); cluster.Shutdown(); } // 4 // Add old shared directory for name and edits along with latest name conf = new HdfsConfiguration(); conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, nameOnlyDir.GetPath() + "," + nameAndEditsDir .GetPath()); conf.Set(DFSConfigKeys.DfsNamenodeEditsDirKey, nameAndEditsDir.GetPath()); replication = (short)conf.GetInt(DFSConfigKeys.DfsReplicationKey, 3); try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(NumDataNodes).Format(false ).ManageNameDfsDirs(false).Build(); NUnit.Framework.Assert.Fail("Successfully started cluster but should not have been able to." ); } catch (IOException e) { // expect to fail Log.Info("EXPECTED: cluster start failed due to missing " + "latest edits dir", e ); } finally { if (cluster != null) { cluster.Shutdown(); } cluster = null; } // 5 // Add old shared directory for name and edits along with latest edits. // This is OK, since the latest edits will have segments leading all // the way from the image in name_and_edits. conf = new HdfsConfiguration(); conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, nameAndEditsDir.GetPath()); conf.Set(DFSConfigKeys.DfsNamenodeEditsDirKey, editsOnlyDir.GetPath() + "," + nameAndEditsDir .GetPath()); replication = (short)conf.GetInt(DFSConfigKeys.DfsReplicationKey, 3); try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(NumDataNodes).Format(false ).ManageNameDfsDirs(false).Build(); fileSys = cluster.GetFileSystem(); NUnit.Framework.Assert.IsFalse(fileSys.Exists(file1)); NUnit.Framework.Assert.IsFalse(fileSys.Exists(file2)); NUnit.Framework.Assert.IsTrue(fileSys.Exists(file3)); CheckFile(fileSys, file3, replication); CleanupFile(fileSys, file3); WriteFile(fileSys, file3, replication); CheckFile(fileSys, file3, replication); } finally { fileSys.Close(); cluster.Shutdown(); } }
public virtual void TestNameEditsConfigs() { Path file1 = new Path("TestNameEditsConfigs1"); Path file2 = new Path("TestNameEditsConfigs2"); Path file3 = new Path("TestNameEditsConfigs3"); MiniDFSCluster cluster = null; SecondaryNameNode secondary = null; Configuration conf = null; FileSystem fileSys = null; FilePath newNameDir = new FilePath(base_dir, "name"); FilePath newEditsDir = new FilePath(base_dir, "edits"); FilePath nameAndEdits = new FilePath(base_dir, "name_and_edits"); FilePath checkpointNameDir = new FilePath(base_dir, "secondname"); FilePath checkpointEditsDir = new FilePath(base_dir, "secondedits"); FilePath checkpointNameAndEdits = new FilePath(base_dir, "second_name_and_edits"); ImmutableList <FilePath> allCurrentDirs = ImmutableList.Of(new FilePath(nameAndEdits , "current"), new FilePath(newNameDir, "current"), new FilePath(newEditsDir, "current" ), new FilePath(checkpointNameAndEdits, "current"), new FilePath(checkpointNameDir , "current"), new FilePath(checkpointEditsDir, "current")); ImmutableList <FilePath> imageCurrentDirs = ImmutableList.Of(new FilePath(nameAndEdits , "current"), new FilePath(newNameDir, "current"), new FilePath(checkpointNameAndEdits , "current"), new FilePath(checkpointNameDir, "current")); // Start namenode with same dfs.namenode.name.dir and dfs.namenode.edits.dir conf = new HdfsConfiguration(); conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, nameAndEdits.GetPath()); conf.Set(DFSConfigKeys.DfsNamenodeEditsDirKey, nameAndEdits.GetPath()); conf.Set(DFSConfigKeys.DfsNamenodeCheckpointDirKey, checkpointNameAndEdits.GetPath ()); conf.Set(DFSConfigKeys.DfsNamenodeCheckpointEditsDirKey, checkpointNameAndEdits.GetPath ()); replication = (short)conf.GetInt(DFSConfigKeys.DfsReplicationKey, 3); // Manage our own dfs directories cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(NumDataNodes).ManageNameDfsDirs (false).Build(); cluster.WaitActive(); secondary = StartSecondaryNameNode(conf); fileSys = cluster.GetFileSystem(); try { NUnit.Framework.Assert.IsTrue(!fileSys.Exists(file1)); WriteFile(fileSys, file1, replication); CheckFile(fileSys, file1, replication); secondary.DoCheckpoint(); } finally { fileSys.Close(); cluster.Shutdown(); secondary.Shutdown(); } // Start namenode with additional dfs.namenode.name.dir and dfs.namenode.edits.dir conf = new HdfsConfiguration(); NUnit.Framework.Assert.IsTrue(newNameDir.Mkdir()); NUnit.Framework.Assert.IsTrue(newEditsDir.Mkdir()); conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, nameAndEdits.GetPath() + "," + newNameDir .GetPath()); conf.Set(DFSConfigKeys.DfsNamenodeEditsDirKey, nameAndEdits.GetPath() + "," + newEditsDir .GetPath()); conf.Set(DFSConfigKeys.DfsNamenodeCheckpointDirKey, checkpointNameDir.GetPath() + "," + checkpointNameAndEdits.GetPath()); conf.Set(DFSConfigKeys.DfsNamenodeCheckpointEditsDirKey, checkpointEditsDir.GetPath () + "," + checkpointNameAndEdits.GetPath()); replication = (short)conf.GetInt(DFSConfigKeys.DfsReplicationKey, 3); // Manage our own dfs directories. Do not format. cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(NumDataNodes).Format(false ).ManageNameDfsDirs(false).Build(); cluster.WaitActive(); secondary = StartSecondaryNameNode(conf); fileSys = cluster.GetFileSystem(); try { NUnit.Framework.Assert.IsTrue(fileSys.Exists(file1)); CheckFile(fileSys, file1, replication); CleanupFile(fileSys, file1); WriteFile(fileSys, file2, replication); CheckFile(fileSys, file2, replication); secondary.DoCheckpoint(); } finally { fileSys.Close(); cluster.Shutdown(); secondary.Shutdown(); } FSImageTestUtil.AssertParallelFilesAreIdentical(allCurrentDirs, ImmutableSet.Of("VERSION" )); FSImageTestUtil.AssertSameNewestImage(imageCurrentDirs); // Now remove common directory both have and start namenode with // separate name and edits dirs conf = new HdfsConfiguration(); conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, newNameDir.GetPath()); conf.Set(DFSConfigKeys.DfsNamenodeEditsDirKey, newEditsDir.GetPath()); conf.Set(DFSConfigKeys.DfsNamenodeCheckpointDirKey, checkpointNameDir.GetPath()); conf.Set(DFSConfigKeys.DfsNamenodeCheckpointEditsDirKey, checkpointEditsDir.GetPath ()); replication = (short)conf.GetInt(DFSConfigKeys.DfsReplicationKey, 3); cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(NumDataNodes).Format(false ).ManageNameDfsDirs(false).Build(); cluster.WaitActive(); secondary = StartSecondaryNameNode(conf); fileSys = cluster.GetFileSystem(); try { NUnit.Framework.Assert.IsTrue(!fileSys.Exists(file1)); NUnit.Framework.Assert.IsTrue(fileSys.Exists(file2)); CheckFile(fileSys, file2, replication); CleanupFile(fileSys, file2); WriteFile(fileSys, file3, replication); CheckFile(fileSys, file3, replication); secondary.DoCheckpoint(); } finally { fileSys.Close(); cluster.Shutdown(); secondary.Shutdown(); } // No edit logs in new name dir CheckImageAndEditsFilesExistence(newNameDir, true, false); CheckImageAndEditsFilesExistence(newEditsDir, false, true); CheckImageAndEditsFilesExistence(checkpointNameDir, true, false); CheckImageAndEditsFilesExistence(checkpointEditsDir, false, true); // Add old name_and_edits dir. File system should not read image or edits // from old dir NUnit.Framework.Assert.IsTrue(FileUtil.FullyDelete(new FilePath(nameAndEdits, "current" ))); NUnit.Framework.Assert.IsTrue(FileUtil.FullyDelete(new FilePath(checkpointNameAndEdits , "current"))); conf = new HdfsConfiguration(); conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, nameAndEdits.GetPath() + "," + newNameDir .GetPath()); conf.Set(DFSConfigKeys.DfsNamenodeEditsDirKey, nameAndEdits + "," + newEditsDir.GetPath ()); conf.Set(DFSConfigKeys.DfsNamenodeCheckpointDirKey, checkpointNameDir.GetPath() + "," + checkpointNameAndEdits.GetPath()); conf.Set(DFSConfigKeys.DfsNamenodeCheckpointEditsDirKey, checkpointEditsDir.GetPath () + "," + checkpointNameAndEdits.GetPath()); replication = (short)conf.GetInt(DFSConfigKeys.DfsReplicationKey, 3); cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(NumDataNodes).Format(false ).ManageNameDfsDirs(false).Build(); cluster.WaitActive(); secondary = StartSecondaryNameNode(conf); fileSys = cluster.GetFileSystem(); try { NUnit.Framework.Assert.IsTrue(!fileSys.Exists(file1)); NUnit.Framework.Assert.IsTrue(!fileSys.Exists(file2)); NUnit.Framework.Assert.IsTrue(fileSys.Exists(file3)); CheckFile(fileSys, file3, replication); secondary.DoCheckpoint(); } finally { fileSys.Close(); cluster.Shutdown(); secondary.Shutdown(); } CheckImageAndEditsFilesExistence(nameAndEdits, true, true); CheckImageAndEditsFilesExistence(checkpointNameAndEdits, true, true); }
public virtual void TestHeartbeat() { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build(); try { cluster.WaitActive(); FSNamesystem namesystem = cluster.GetNamesystem(); HeartbeatManager hm = namesystem.GetBlockManager().GetDatanodeManager().GetHeartbeatManager (); string poolId = namesystem.GetBlockPoolId(); DatanodeRegistration nodeReg = DataNodeTestUtils.GetDNRegistrationForBP(cluster.GetDataNodes ()[0], poolId); DatanodeDescriptor dd = NameNodeAdapter.GetDatanode(namesystem, nodeReg); string storageID = DatanodeStorage.GenerateUuid(); dd.UpdateStorage(new DatanodeStorage(storageID)); int RemainingBlocks = 1; int MaxReplicateLimit = conf.GetInt(DFSConfigKeys.DfsNamenodeReplicationMaxStreamsKey , 2); int MaxInvalidateLimit = DFSConfigKeys.DfsBlockInvalidateLimitDefault; int MaxInvalidateBlocks = 2 * MaxInvalidateLimit + RemainingBlocks; int MaxReplicateBlocks = 2 * MaxReplicateLimit + RemainingBlocks; DatanodeStorageInfo[] OneTarget = new DatanodeStorageInfo[] { dd.GetStorageInfo(storageID ) }; try { namesystem.WriteLock(); lock (hm) { for (int i = 0; i < MaxReplicateBlocks; i++) { dd.AddBlockToBeReplicated(new Block(i, 0, GenerationStamp.LastReservedStamp), OneTarget ); } DatanodeCommand[] cmds = NameNodeAdapter.SendHeartBeat(nodeReg, dd, namesystem).GetCommands (); NUnit.Framework.Assert.AreEqual(1, cmds.Length); NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaTransfer, cmds[0].GetAction() ); NUnit.Framework.Assert.AreEqual(MaxReplicateLimit, ((BlockCommand)cmds[0]).GetBlocks ().Length); AList <Block> blockList = new AList <Block>(MaxInvalidateBlocks); for (int i_1 = 0; i_1 < MaxInvalidateBlocks; i_1++) { blockList.AddItem(new Block(i_1, 0, GenerationStamp.LastReservedStamp)); } dd.AddBlocksToBeInvalidated(blockList); cmds = NameNodeAdapter.SendHeartBeat(nodeReg, dd, namesystem).GetCommands(); NUnit.Framework.Assert.AreEqual(2, cmds.Length); NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaTransfer, cmds[0].GetAction() ); NUnit.Framework.Assert.AreEqual(MaxReplicateLimit, ((BlockCommand)cmds[0]).GetBlocks ().Length); NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaInvalidate, cmds[1].GetAction ()); NUnit.Framework.Assert.AreEqual(MaxInvalidateLimit, ((BlockCommand)cmds[1]).GetBlocks ().Length); cmds = NameNodeAdapter.SendHeartBeat(nodeReg, dd, namesystem).GetCommands(); NUnit.Framework.Assert.AreEqual(2, cmds.Length); NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaTransfer, cmds[0].GetAction() ); NUnit.Framework.Assert.AreEqual(RemainingBlocks, ((BlockCommand)cmds[0]).GetBlocks ().Length); NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaInvalidate, cmds[1].GetAction ()); NUnit.Framework.Assert.AreEqual(MaxInvalidateLimit, ((BlockCommand)cmds[1]).GetBlocks ().Length); cmds = NameNodeAdapter.SendHeartBeat(nodeReg, dd, namesystem).GetCommands(); NUnit.Framework.Assert.AreEqual(1, cmds.Length); NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaInvalidate, cmds[0].GetAction ()); NUnit.Framework.Assert.AreEqual(RemainingBlocks, ((BlockCommand)cmds[0]).GetBlocks ().Length); cmds = NameNodeAdapter.SendHeartBeat(nodeReg, dd, namesystem).GetCommands(); NUnit.Framework.Assert.AreEqual(0, cmds.Length); } } finally { namesystem.WriteUnlock(); } } finally { cluster.Shutdown(); } }
/// <exception cref="System.Exception"/> internal virtual void TestCheckpoint(HdfsServerConstants.StartupOption op) { Path file1 = new Path("/checkpoint.dat"); Path file2 = new Path("/checkpoint2.dat"); Path file3 = new Path("/backup.dat"); Configuration conf = new HdfsConfiguration(); HAUtil.SetAllowStandbyReads(conf, true); short replication = (short)conf.GetInt("dfs.replication", 3); int numDatanodes = Math.Max(3, replication); conf.Set(DFSConfigKeys.DfsNamenodeBackupHttpAddressKey, "localhost:0"); conf.Set(DFSConfigKeys.DfsBlockreportInitialDelayKey, "0"); conf.SetInt(DFSConfigKeys.DfsDatanodeScanPeriodHoursKey, -1); // disable block scanner conf.SetInt(DFSConfigKeys.DfsNamenodeCheckpointTxnsKey, 1); MiniDFSCluster cluster = null; FileSystem fileSys = null; BackupNode backup = null; try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build(); fileSys = cluster.GetFileSystem(); // // verify that 'format' really blew away all pre-existing files // NUnit.Framework.Assert.IsTrue(!fileSys.Exists(file1)); NUnit.Framework.Assert.IsTrue(!fileSys.Exists(file2)); // // Create file1 // NUnit.Framework.Assert.IsTrue(fileSys.Mkdirs(file1)); // // Take a checkpoint // long txid = cluster.GetNameNodeRpc().GetTransactionID(); backup = StartBackupNode(conf, op, 1); WaitCheckpointDone(cluster, txid); } catch (IOException e) { Log.Error("Error in TestBackupNode:", e); NUnit.Framework.Assert.IsTrue(e.GetLocalizedMessage(), false); } finally { if (backup != null) { backup.Stop(); } if (fileSys != null) { fileSys.Close(); } if (cluster != null) { cluster.Shutdown(); } } FilePath nnCurDir = new FilePath(BaseDir, "name1/current/"); FilePath bnCurDir = new FilePath(GetBackupNodeDir(op, 1), "/current/"); FSImageTestUtil.AssertParallelFilesAreIdentical(ImmutableList.Of(bnCurDir, nnCurDir ), ImmutableSet.Of <string>("VERSION")); try { // // Restart cluster and verify that file1 still exist. // cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDatanodes).Format(false ).Build(); fileSys = cluster.GetFileSystem(); // check that file1 still exists NUnit.Framework.Assert.IsTrue(fileSys.Exists(file1)); fileSys.Delete(file1, true); // create new file file2 fileSys.Mkdirs(file2); // // Take a checkpoint // long txid = cluster.GetNameNodeRpc().GetTransactionID(); backup = StartBackupNode(conf, op, 1); WaitCheckpointDone(cluster, txid); for (int i = 0; i < 10; i++) { fileSys.Mkdirs(new Path("file_" + i)); } txid = cluster.GetNameNodeRpc().GetTransactionID(); backup.DoCheckpoint(); WaitCheckpointDone(cluster, txid); txid = cluster.GetNameNodeRpc().GetTransactionID(); backup.DoCheckpoint(); WaitCheckpointDone(cluster, txid); // Try BackupNode operations IPEndPoint add = backup.GetNameNodeAddress(); // Write to BN FileSystem bnFS = FileSystem.Get(new Path("hdfs://" + NetUtils.GetHostPortString( add)).ToUri(), conf); bool canWrite = true; try { Org.Apache.Hadoop.Hdfs.Server.Namenode.TestCheckpoint.WriteFile(bnFS, file3, replication ); } catch (IOException eio) { Log.Info("Write to " + backup.GetRole() + " failed as expected: ", eio); canWrite = false; } NUnit.Framework.Assert.IsFalse("Write to BackupNode must be prohibited.", canWrite ); // Reads are allowed for BackupNode, but not for CheckpointNode bool canRead = true; try { bnFS.Exists(file2); } catch (IOException eio) { Log.Info("Read from " + backup.GetRole() + " failed: ", eio); canRead = false; } NUnit.Framework.Assert.AreEqual("Reads to BackupNode are allowed, but not CheckpointNode." , canRead, backup.IsRole(HdfsServerConstants.NamenodeRole.Backup)); Org.Apache.Hadoop.Hdfs.Server.Namenode.TestCheckpoint.WriteFile(fileSys, file3, replication ); Org.Apache.Hadoop.Hdfs.Server.Namenode.TestCheckpoint.CheckFile(fileSys, file3, replication ); // should also be on BN right away NUnit.Framework.Assert.IsTrue("file3 does not exist on BackupNode", op != HdfsServerConstants.StartupOption .Backup || backup.GetNamesystem().GetFileInfo(file3.ToUri().GetPath(), false) != null); } catch (IOException e) { Log.Error("Error in TestBackupNode:", e); throw new Exception(e); } finally { if (backup != null) { backup.Stop(); } if (fileSys != null) { fileSys.Close(); } if (cluster != null) { cluster.Shutdown(); } } FSImageTestUtil.AssertParallelFilesAreIdentical(ImmutableList.Of(bnCurDir, nnCurDir ), ImmutableSet.Of <string>("VERSION")); try { // // Restart cluster and verify that file2 exists and // file1 does not exist. // cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Format(false).Build(); fileSys = cluster.GetFileSystem(); NUnit.Framework.Assert.IsTrue(!fileSys.Exists(file1)); // verify that file2 exists NUnit.Framework.Assert.IsTrue(fileSys.Exists(file2)); } catch (IOException e) { Log.Error("Error in TestBackupNode: ", e); NUnit.Framework.Assert.IsTrue(e.GetLocalizedMessage(), false); } finally { fileSys.Close(); cluster.Shutdown(); } }