public virtual void TestBlockIdGeneration() { Configuration conf = new HdfsConfiguration(); conf.SetInt(DFSConfigKeys.DfsReplicationKey, 1); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); try { cluster.WaitActive(); FileSystem fs = cluster.GetFileSystem(); // Create a file that is 10 blocks long. Path path = new Path("testBlockIdGeneration.dat"); DFSTestUtil.CreateFile(fs, path, IoSize, BlockSize * 10, BlockSize, Replication, Seed); IList <LocatedBlock> blocks = DFSTestUtil.GetAllBlocks(fs, path); Log.Info("Block0 id is " + blocks[0].GetBlock().GetBlockId()); long nextBlockExpectedId = blocks[0].GetBlock().GetBlockId() + 1; // Ensure that the block IDs are sequentially increasing. for (int i = 1; i < blocks.Count; ++i) { long nextBlockId = blocks[i].GetBlock().GetBlockId(); Log.Info("Block" + i + " id is " + nextBlockId); Assert.AssertThat(nextBlockId, CoreMatchers.Is(nextBlockExpectedId)); ++nextBlockExpectedId; } } finally { cluster.Shutdown(); } }
public virtual void TestBlockTokenInLastLocatedBlock() { Configuration conf = new HdfsConfiguration(); conf.SetBoolean(DFSConfigKeys.DfsBlockAccessTokenEnableKey, true); conf.SetInt(DFSConfigKeys.DfsBlockSizeKey, 512); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); cluster.WaitActive(); try { FileSystem fs = cluster.GetFileSystem(); string fileName = "/testBlockTokenInLastLocatedBlock"; Path filePath = new Path(fileName); FSDataOutputStream @out = fs.Create(filePath, (short)1); @out.Write(new byte[1000]); // ensure that the first block is written out (see FSOutputSummer#flush) @out.Flush(); LocatedBlocks locatedBlocks = cluster.GetNameNodeRpc().GetBlockLocations(fileName , 0, 1000); while (locatedBlocks.GetLastLocatedBlock() == null) { Sharpen.Thread.Sleep(100); locatedBlocks = cluster.GetNameNodeRpc().GetBlockLocations(fileName, 0, 1000); } Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token = locatedBlocks .GetLastLocatedBlock().GetBlockToken(); NUnit.Framework.Assert.AreEqual(BlockTokenIdentifier.KindName, token.GetKind()); @out.Close(); } finally { cluster.Shutdown(); } }
public virtual void TestFinalizedReplicas() { // bring up a cluster of 3 Configuration conf = new HdfsConfiguration(); conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, 1024L); conf.SetInt(DFSConfigKeys.DfsClientWritePacketSizeKey, 512); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build(); cluster.WaitActive(); FileSystem fs = cluster.GetFileSystem(); try { // test finalized replicas string TopDir = "/test"; DFSTestUtil util = new DFSTestUtil.Builder().SetName("TestDatanodeRestart").SetNumFiles (2).Build(); util.CreateFiles(fs, TopDir, (short)3); util.WaitReplication(fs, TopDir, (short)3); util.CheckFiles(fs, TopDir); cluster.RestartDataNodes(); cluster.WaitActive(); util.CheckFiles(fs, TopDir); } finally { cluster.Shutdown(); } }
public virtual void TestServerSaslNoClientSasl() { HdfsConfiguration clusterConf = CreateSecureConfig("authentication,integrity,privacy" ); // Set short retry timeouts so this test runs faster clusterConf.SetInt(DFSConfigKeys.DfsClientRetryWindowBase, 10); StartCluster(clusterConf); HdfsConfiguration clientConf = new HdfsConfiguration(clusterConf); clientConf.Set(DFSConfigKeys.DfsDataTransferProtectionKey, string.Empty); GenericTestUtils.LogCapturer logs = GenericTestUtils.LogCapturer.CaptureLogs(LogFactory .GetLog(typeof(DataNode))); try { DoTest(clientConf); NUnit.Framework.Assert.Fail("Should fail if SASL data transfer protection is not " + "configured or not supported in client"); } catch (IOException e) { GenericTestUtils.AssertMatches(e.Message, "could only be replicated to 0 nodes"); } finally { logs.StopCapturing(); } GenericTestUtils.AssertMatches(logs.GetOutput(), "Failed to read expected SASL data transfer protection " + "handshake from client at"); }
/// <summary> /// The test verifies the number of outstanding replication requests for a /// given DN shouldn't exceed the limit set by configuration property /// dfs.namenode.replication.max-streams-hard-limit. /// </summary> /// <remarks> /// The test verifies the number of outstanding replication requests for a /// given DN shouldn't exceed the limit set by configuration property /// dfs.namenode.replication.max-streams-hard-limit. /// The test does the followings: /// 1. Create a mini cluster with 2 DNs. Set large heartbeat interval so that /// replication requests won't be picked by any DN right away. /// 2. Create a file with 10 blocks and replication factor 2. Thus each /// of the 2 DNs have one replica of each block. /// 3. Add a DN to the cluster for later replication. /// 4. Remove a DN that has data. /// 5. Ask BlockManager to compute the replication work. This will assign /// replication requests to the only DN that has data. /// 6. Make sure the number of pending replication requests of that DN don't /// exceed the limit. /// </remarks> /// <exception cref="System.Exception"/> public virtual void TestNumberOfBlocksToBeReplicated() { // 1 min timeout Configuration conf = new HdfsConfiguration(); conf.SetLong(DFSConfigKeys.DfsNamenodeMinBlockSizeKey, 0); conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, 1); conf.SetInt(DFSConfigKeys.DfsBytesPerChecksumKey, 1); // Large value to make sure the pending replication request can stay in // DatanodeDescriptor.replicateBlocks before test timeout. conf.SetInt(DFSConfigKeys.DfsHeartbeatIntervalKey, 100); // Make sure BlockManager can pull all blocks from UnderReplicatedBlocks via // chooseUnderReplicatedBlocks at once. conf.SetInt(DFSConfigKeys.DfsNamenodeReplicationWorkMultiplierPerIteration, 5); int NumOfBlocks = 10; short RepFactor = 2; string FileName = "/testFile"; Path FilePath = new Path(FileName); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(RepFactor) .Build(); try { // create a file with 10 blocks with a replication factor of 2 FileSystem fs = cluster.GetFileSystem(); DFSTestUtil.CreateFile(fs, FilePath, NumOfBlocks, RepFactor, 1L); DFSTestUtil.WaitReplication(fs, FilePath, RepFactor); cluster.StartDataNodes(conf, 1, true, null, null, null, null); BlockManager bm = cluster.GetNamesystem().GetBlockManager(); ExtendedBlock b = DFSTestUtil.GetFirstBlock(fs, FilePath); IEnumerator <DatanodeStorageInfo> storageInfos = bm.blocksMap.GetStorages(b.GetLocalBlock ()).GetEnumerator(); DatanodeDescriptor firstDn = storageInfos.Next().GetDatanodeDescriptor(); DatanodeDescriptor secondDn = storageInfos.Next().GetDatanodeDescriptor(); bm.GetDatanodeManager().RemoveDatanode(firstDn); NUnit.Framework.Assert.AreEqual(NumOfBlocks, bm.GetUnderReplicatedNotMissingBlocks ()); bm.ComputeDatanodeWork(); NUnit.Framework.Assert.IsTrue("The number of blocks to be replicated should be less than " + "or equal to " + bm.replicationStreamsHardLimit, secondDn.GetNumberOfBlocksToBeReplicated () <= bm.replicationStreamsHardLimit); } finally { cluster.Shutdown(); } }
public static void Setup() { Configuration conf = new HdfsConfiguration(); conf.SetInt(DFSConfigKeys.DfsListLimit, ListLimit); cluster = new MiniDFSCluster.Builder(conf).Build(); cluster.WaitClusterUp(); fs = cluster.GetFileSystem(); }
public virtual void TestDeadDatanode() { Configuration conf = new HdfsConfiguration(); conf.SetInt(DFSConfigKeys.DfsNamenodeHeartbeatRecheckIntervalKey, 500); conf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, 1L); cluster = new MiniDFSCluster.Builder(conf).Build(); cluster.WaitActive(); string poolId = cluster.GetNamesystem().GetBlockPoolId(); // wait for datanode to be marked live DataNode dn = cluster.GetDataNodes()[0]; DatanodeRegistration reg = DataNodeTestUtils.GetDNRegistrationForBP(cluster.GetDataNodes ()[0], poolId); DFSTestUtil.WaitForDatanodeState(cluster, reg.GetDatanodeUuid(), true, 20000); // Shutdown and wait for datanode to be marked dead dn.Shutdown(); DFSTestUtil.WaitForDatanodeState(cluster, reg.GetDatanodeUuid(), false, 20000); DatanodeProtocol dnp = cluster.GetNameNodeRpc(); ReceivedDeletedBlockInfo[] blocks = new ReceivedDeletedBlockInfo[] { new ReceivedDeletedBlockInfo (new Block(0), ReceivedDeletedBlockInfo.BlockStatus.ReceivedBlock, null) }; StorageReceivedDeletedBlocks[] storageBlocks = new StorageReceivedDeletedBlocks[] { new StorageReceivedDeletedBlocks(reg.GetDatanodeUuid(), blocks) }; // Ensure blockReceived call from dead datanode is rejected with IOException try { dnp.BlockReceivedAndDeleted(reg, poolId, storageBlocks); NUnit.Framework.Assert.Fail("Expected IOException is not thrown"); } catch (IOException) { } // Expected // Ensure blockReport from dead datanode is rejected with IOException StorageBlockReport[] report = new StorageBlockReport[] { new StorageBlockReport(new DatanodeStorage(reg.GetDatanodeUuid()), BlockListAsLongs.Empty) }; try { dnp.BlockReport(reg, poolId, report, new BlockReportContext(1, 0, Runtime.NanoTime ())); NUnit.Framework.Assert.Fail("Expected IOException is not thrown"); } catch (IOException) { } // Expected // Ensure heartbeat from dead datanode is rejected with a command // that asks datanode to register again StorageReport[] rep = new StorageReport[] { new StorageReport(new DatanodeStorage (reg.GetDatanodeUuid()), false, 0, 0, 0, 0) }; DatanodeCommand[] cmd = dnp.SendHeartbeat(reg, rep, 0L, 0L, 0, 0, 0, null).GetCommands (); NUnit.Framework.Assert.AreEqual(1, cmd.Length); NUnit.Framework.Assert.AreEqual(cmd[0].GetAction(), RegisterCommand.Register.GetAction ()); }
/// <exception cref="System.IO.IOException"/> public static void CreateCluster() { HdfsConfiguration conf = new HdfsConfiguration(); conf.AddResource(ContractHdfsXml); //hack in a 256 byte block size conf.SetInt(DFSConfigKeys.DfsBlockSizeKey, BlockSize); cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build(); cluster.WaitClusterUp(); }
public virtual void TestRecoverReplicas() { Configuration conf = new HdfsConfiguration(); conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, 1024L); conf.SetInt(DFSConfigKeys.DfsClientWritePacketSizeKey, 512); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build(); cluster.WaitActive(); try { FileSystem fs = cluster.GetFileSystem(); for (int i = 0; i < 4; i++) { Path fileName = new Path("/test" + i); DFSTestUtil.CreateFile(fs, fileName, 1, (short)1, 0L); DFSTestUtil.WaitReplication(fs, fileName, (short)1); } string bpid = cluster.GetNamesystem().GetBlockPoolId(); DataNode dn = cluster.GetDataNodes()[0]; IEnumerator <ReplicaInfo> replicasItor = Dataset(dn).volumeMap.Replicas(bpid).GetEnumerator (); ReplicaInfo replica = replicasItor.Next(); CreateUnlinkTmpFile(replica, true, true); // rename block file CreateUnlinkTmpFile(replica, false, true); // rename meta file replica = replicasItor.Next(); CreateUnlinkTmpFile(replica, true, false); // copy block file CreateUnlinkTmpFile(replica, false, false); // copy meta file replica = replicasItor.Next(); CreateUnlinkTmpFile(replica, true, true); // rename block file CreateUnlinkTmpFile(replica, false, false); // copy meta file cluster.RestartDataNodes(); cluster.WaitActive(); dn = cluster.GetDataNodes()[0]; // check volumeMap: 4 finalized replica ICollection <ReplicaInfo> replicas = Dataset(dn).volumeMap.Replicas(bpid); NUnit.Framework.Assert.AreEqual(4, replicas.Count); replicasItor = replicas.GetEnumerator(); while (replicasItor.HasNext()) { NUnit.Framework.Assert.AreEqual(HdfsServerConstants.ReplicaState.Finalized, replicasItor .Next().GetState()); } } finally { cluster.Shutdown(); } }
/* * Return a configuration object with low timeouts for testing and * a topology script set (which enables rack awareness). */ private Configuration GetConf() { Configuration conf = new HdfsConfiguration(); // Lower the heart beat interval so the NN quickly learns of dead // or decommissioned DNs and the NN issues replication and invalidation // commands quickly (as replies to heartbeats) conf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, 1L); // Have the NN ReplicationMonitor compute the replication and // invalidation commands to send DNs every second. conf.SetInt(DFSConfigKeys.DfsNamenodeReplicationIntervalKey, 1); // Have the NN check for pending replications every second so it // quickly schedules additional replicas as they are identified. conf.SetInt(DFSConfigKeys.DfsNamenodeReplicationPendingTimeoutSecKey, 1); // The DNs report blocks every second. conf.SetLong(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 1000L); // Indicates we have multiple racks conf.Set(DFSConfigKeys.NetTopologyScriptFileNameKey, "xyz"); return(conf); }
public static void BeforeClassSetup() { Configuration conf = new HdfsConfiguration(); conf.SetBoolean(DFSConfigKeys.DfsWebhdfsEnabledKey, true); conf.Set(FsPermission.UmaskLabel, "000"); conf.SetInt(DFSConfigKeys.DfsNamenodeMaxComponentLengthKey, 0); cluster = new MiniDFSCluster.Builder(conf).Build(); webhdfs = WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem.Scheme); dfs = cluster.GetFileSystem(); }
public virtual void TestSingleRequiredFailedEditsDirOnSetReadyToFlush() { // Set one of the edits dirs to be required. string[] editsDirs = cluster.GetConfiguration(0).GetTrimmedStrings(DFSConfigKeys. DfsNamenodeNameDirKey); ShutDownMiniCluster(); Configuration conf = new HdfsConfiguration(); conf.Set(DFSConfigKeys.DfsNamenodeEditsDirRequiredKey, editsDirs[0]); conf.SetInt(DFSConfigKeys.DfsNamenodeEditsDirMinimumKey, 0); conf.SetInt(DFSConfigKeys.DfsNamenodeCheckedVolumesMinimumKey, 0); SetUpMiniCluster(conf, true); NUnit.Framework.Assert.IsTrue(DoAnEdit()); // Invalidated the one required edits journal. InvalidateEditsDirAtIndex(0, false, false); JournalSet.JournalAndStream nonRequiredJas = GetJournalAndStream(1); EditLogFileOutputStream nonRequiredSpy = SpyOnStream(nonRequiredJas); // The NN has not terminated (no ExitException thrown) // ..and that the other stream is active. NUnit.Framework.Assert.IsTrue(nonRequiredJas.IsActive()); try { DoAnEdit(); NUnit.Framework.Assert.Fail("A single failure of a required journal should have halted the NN" ); } catch (RemoteException re) { NUnit.Framework.Assert.IsTrue(re.GetClassName().Contains("ExitException")); GenericTestUtils.AssertExceptionContains("setReadyToFlush failed for required journal" , re); } // Since the required directory failed setReadyToFlush, and that // directory was listed prior to the non-required directory, // we should not call setReadyToFlush on the non-required // directory. Regression test for HDFS-2874. Org.Mockito.Mockito.Verify(nonRequiredSpy, Org.Mockito.Mockito.Never()).SetReadyToFlush (); NUnit.Framework.Assert.IsFalse(nonRequiredJas.IsActive()); }
public virtual void TestReplicationAdjusted() { // start a cluster Configuration conf = new HdfsConfiguration(); // Replicate and heartbeat fast to shave a few seconds off test conf.SetInt(DFSConfigKeys.DfsNamenodeReplicationIntervalKey, 1); conf.SetInt(DFSConfigKeys.DfsHeartbeatIntervalKey, 1); MiniDFSCluster cluster = null; try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build(); cluster.WaitActive(); FileSystem fs = cluster.GetFileSystem(); // Create a file with replication count 1 Path p = new Path("/testfile"); DFSTestUtil.CreateFile(fs, p, 10, (short)1, 1); /*repl*/ DFSTestUtil.WaitReplication(fs, p, (short)1); // Shut down and restart cluster with new minimum replication of 2 cluster.Shutdown(); cluster = null; conf.SetInt(DFSConfigKeys.DfsNamenodeReplicationMinKey, 2); cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Format(false).Build(); cluster.WaitActive(); fs = cluster.GetFileSystem(); // The file should get adjusted to replication 2 when // the edit log is replayed. DFSTestUtil.WaitReplication(fs, p, (short)2); } finally { if (cluster != null) { cluster.Shutdown(); } } }
public static void Init() { baseDir = new FilePath(Runtime.GetProperty("test.build.dir", "target/test-dir"), typeof(TestSecureNNWithQJM).Name); FileUtil.FullyDelete(baseDir); NUnit.Framework.Assert.IsTrue(baseDir.Mkdirs()); Properties kdcConf = MiniKdc.CreateConf(); kdc = new MiniKdc(kdcConf, baseDir); kdc.Start(); baseConf = new HdfsConfiguration(); SecurityUtil.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Kerberos , baseConf); UserGroupInformation.SetConfiguration(baseConf); NUnit.Framework.Assert.IsTrue("Expected configuration to enable security", UserGroupInformation .IsSecurityEnabled()); string userName = UserGroupInformation.GetLoginUser().GetShortUserName(); FilePath keytabFile = new FilePath(baseDir, userName + ".keytab"); string keytab = keytabFile.GetAbsolutePath(); // Windows will not reverse name lookup "127.0.0.1" to "localhost". string krbInstance = Path.Windows ? "127.0.0.1" : "localhost"; kdc.CreatePrincipal(keytabFile, userName + "/" + krbInstance, "HTTP/" + krbInstance ); string hdfsPrincipal = userName + "/" + krbInstance + "@" + kdc.GetRealm(); string spnegoPrincipal = "HTTP/" + krbInstance + "@" + kdc.GetRealm(); baseConf.Set(DFSConfigKeys.DfsNamenodeKerberosPrincipalKey, hdfsPrincipal); baseConf.Set(DFSConfigKeys.DfsNamenodeKeytabFileKey, keytab); baseConf.Set(DFSConfigKeys.DfsDatanodeKerberosPrincipalKey, hdfsPrincipal); baseConf.Set(DFSConfigKeys.DfsDatanodeKeytabFileKey, keytab); baseConf.Set(DFSConfigKeys.DfsWebAuthenticationKerberosPrincipalKey, spnegoPrincipal ); baseConf.Set(DFSConfigKeys.DfsJournalnodeKeytabFileKey, keytab); baseConf.Set(DFSConfigKeys.DfsJournalnodeKerberosPrincipalKey, hdfsPrincipal); baseConf.Set(DFSConfigKeys.DfsJournalnodeKerberosInternalSpnegoPrincipalKey, spnegoPrincipal ); baseConf.SetBoolean(DFSConfigKeys.DfsBlockAccessTokenEnableKey, true); baseConf.Set(DFSConfigKeys.DfsDataTransferProtectionKey, "authentication"); baseConf.Set(DFSConfigKeys.DfsHttpPolicyKey, HttpConfig.Policy.HttpsOnly.ToString ()); baseConf.Set(DFSConfigKeys.DfsNamenodeHttpsAddressKey, "localhost:0"); baseConf.Set(DFSConfigKeys.DfsDatanodeHttpsAddressKey, "localhost:0"); baseConf.Set(DFSConfigKeys.DfsJournalnodeHttpsAddressKey, "localhost:0"); baseConf.SetInt(CommonConfigurationKeys.IpcClientConnectMaxRetriesOnSaslKey, 10); string keystoresDir = baseDir.GetAbsolutePath(); string sslConfDir = KeyStoreTestUtil.GetClasspathDir(typeof(TestSecureNNWithQJM)); KeyStoreTestUtil.SetupSSLConfig(keystoresDir, sslConfDir, baseConf, false); }
public static void SetUp() { // start a cluster Configuration conf = new HdfsConfiguration(); // High value of replication interval // so that blocks remain under-replicated conf.SetInt(DFSConfigKeys.DfsNamenodeReplicationIntervalKey, 1000); conf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, 1L); conf.SetLong(DFSConfigKeys.DfsNamenodeHeartbeatRecheckIntervalKey, 1L); cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(NumDataNodes).Build(); cluster.WaitActive(); fileSys = cluster.GetFileSystem(); nnRpc = cluster.GetNameNodeRpc(); }
public virtual void Setup() { StaticMapping.ResetMap(); Configuration conf = new HdfsConfiguration(); string[] racks = new string[] { "/RACK0", "/RACK0", "/RACK2", "/RACK3", "/RACK2" }; string[] hosts = new string[] { "/host0", "/host1", "/host2", "/host3", "/host4" }; conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, DefaultBlockSize); conf.SetInt(DFSConfigKeys.DfsBytesPerChecksumKey, DefaultBlockSize / 2); cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(5).Racks(racks).Hosts(hosts ).Build(); cluster.WaitActive(); nameNodeRpc = cluster.GetNameNodeRpc(); namesystem = cluster.GetNamesystem(); perm = new PermissionStatus("TestDefaultBlockPlacementPolicy", null, FsPermission .GetDefault()); }
public static HdfsConfiguration InitZeroCopyTest() { Assume.AssumeTrue(NativeIO.IsAvailable()); Assume.AssumeTrue(SystemUtils.IsOsUnix); HdfsConfiguration conf = new HdfsConfiguration(); conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, true); conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, BlockSize); conf.SetInt(DFSConfigKeys.DfsClientMmapCacheSize, 3); conf.SetLong(DFSConfigKeys.DfsClientMmapCacheTimeoutMs, 100); conf.Set(DFSConfigKeys.DfsDomainSocketPathKey, new FilePath(sockDir.GetDir(), "TestRequestMmapAccess._PORT.sock" ).GetAbsolutePath()); conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, true); conf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, 1); conf.SetLong(DFSConfigKeys.DfsCachereportIntervalMsecKey, 1000); conf.SetLong(DFSConfigKeys.DfsNamenodePathBasedCacheRefreshIntervalMs, 1000); return(conf); }
public virtual void TestMultipleRedundantFailedEditsDirOnSetReadyToFlush() { // Set up 4 name/edits dirs. ShutDownMiniCluster(); Configuration conf = new HdfsConfiguration(); string[] nameDirs = new string[4]; for (int i = 0; i < nameDirs.Length; i++) { FilePath nameDir = new FilePath(PathUtils.GetTestDir(GetType()), "name-dir" + i); nameDir.Mkdirs(); nameDirs[i] = nameDir.GetAbsolutePath(); } conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, StringUtils.Join(nameDirs, ",")); // Keep running unless there are less than 2 edits dirs remaining. conf.SetInt(DFSConfigKeys.DfsNamenodeEditsDirMinimumKey, 2); SetUpMiniCluster(conf, false); // All journals active. NUnit.Framework.Assert.IsTrue(DoAnEdit()); // The NN has not terminated (no ExitException thrown) // Invalidate 1/4 of the redundant journals. InvalidateEditsDirAtIndex(0, false, false); NUnit.Framework.Assert.IsTrue(DoAnEdit()); // The NN has not terminated (no ExitException thrown) // Invalidate 2/4 of the redundant journals. InvalidateEditsDirAtIndex(1, false, false); NUnit.Framework.Assert.IsTrue(DoAnEdit()); // The NN has not terminated (no ExitException thrown) // Invalidate 3/4 of the redundant journals. InvalidateEditsDirAtIndex(2, false, false); try { DoAnEdit(); NUnit.Framework.Assert.Fail("A failure of more than the minimum number of redundant journals " + "should have halted "); } catch (RemoteException re) { NUnit.Framework.Assert.IsTrue(re.GetClassName().Contains("ExitException")); GenericTestUtils.AssertExceptionContains("Could not sync enough journals to persistent storage due to " + "setReadyToFlush failed for too many journals. " + "Unsynced transactions: 1" , re); } }
public virtual void TestTailer() { Configuration conf = new HdfsConfiguration(); conf.SetInt(DFSConfigKeys.DfsHaTaileditsPeriodKey, 1); HAUtil.SetAllowStandbyReads(conf, true); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology .SimpleHATopology()).NumDataNodes(0).Build(); cluster.WaitActive(); cluster.TransitionToActive(0); NameNode nn1 = cluster.GetNameNode(0); NameNode nn2 = cluster.GetNameNode(1); try { for (int i = 0; i < DirsToMake / 2; i++) { NameNodeAdapter.Mkdirs(nn1, GetDirPath(i), new PermissionStatus("test", "test", new FsPermission((short)0x1ed)), true); } HATestUtil.WaitForStandbyToCatchUp(nn1, nn2); for (int i_1 = 0; i_1 < DirsToMake / 2; i_1++) { NUnit.Framework.Assert.IsTrue(NameNodeAdapter.GetFileInfo(nn2, GetDirPath(i_1), false ).IsDir()); } for (int i_2 = DirsToMake / 2; i_2 < DirsToMake; i_2++) { NameNodeAdapter.Mkdirs(nn1, GetDirPath(i_2), new PermissionStatus("test", "test", new FsPermission((short)0x1ed)), true); } HATestUtil.WaitForStandbyToCatchUp(nn1, nn2); for (int i_3 = DirsToMake / 2; i_3 < DirsToMake; i_3++) { NUnit.Framework.Assert.IsTrue(NameNodeAdapter.GetFileInfo(nn2, GetDirPath(i_3), false ).IsDir()); } } finally { cluster.Shutdown(); } }
// test rbw replicas persist across DataNode restarts /// <exception cref="System.IO.IOException"/> public virtual void TestRbwReplicas() { Configuration conf = new HdfsConfiguration(); conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, 1024L); conf.SetInt(DFSConfigKeys.DfsClientWritePacketSizeKey, 512); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build(); cluster.WaitActive(); try { TestRbwReplicas(cluster, false); TestRbwReplicas(cluster, true); } finally { cluster.Shutdown(); } }
public virtual void TestTriggerBlockIdCollision() { Configuration conf = new HdfsConfiguration(); conf.SetInt(DFSConfigKeys.DfsReplicationKey, 1); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); try { cluster.WaitActive(); FileSystem fs = cluster.GetFileSystem(); FSNamesystem fsn = cluster.GetNamesystem(); int blockCount = 10; // Create a file with a few blocks to rev up the global block ID // counter. Path path1 = new Path("testBlockIdCollisionDetection_file1.dat"); DFSTestUtil.CreateFile(fs, path1, IoSize, BlockSize * blockCount, BlockSize, Replication , Seed); IList <LocatedBlock> blocks1 = DFSTestUtil.GetAllBlocks(fs, path1); // Rewind the block ID counter in the name system object. This will result // in block ID collisions when we try to allocate new blocks. SequentialBlockIdGenerator blockIdGenerator = fsn.GetBlockIdManager().GetBlockIdGenerator (); blockIdGenerator.SetCurrentValue(blockIdGenerator.GetCurrentValue() - 5); // Trigger collisions by creating a new file. Path path2 = new Path("testBlockIdCollisionDetection_file2.dat"); DFSTestUtil.CreateFile(fs, path2, IoSize, BlockSize * blockCount, BlockSize, Replication , Seed); IList <LocatedBlock> blocks2 = DFSTestUtil.GetAllBlocks(fs, path2); Assert.AssertThat(blocks2.Count, CoreMatchers.Is(blockCount)); // Make sure that file2 block IDs start immediately after file1 Assert.AssertThat(blocks2[0].GetBlock().GetBlockId(), CoreMatchers.Is(blocks1[9]. GetBlock().GetBlockId() + 1)); } finally { cluster.Shutdown(); } }
public virtual void TestStartNNWithTrashEmptier() { MiniDFSCluster cluster = null; Configuration conf = new HdfsConfiguration(); // enable both trash emptier and dropping response conf.SetLong("fs.trash.interval", 360); conf.SetInt(DFSConfigKeys.DfsClientTestDropNamenodeResponseNumKey, 2); try { cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology.SimpleHATopology ()).NumDataNodes(0).Build(); cluster.WaitActive(); cluster.TransitionToActive(0); } finally { if (cluster != null) { cluster.Shutdown(); } } }
/// <summary>Creates configuration for starting a secure cluster.</summary> /// <param name="dataTransferProtection">supported QOPs</param> /// <returns>configuration for starting a secure cluster</returns> /// <exception cref="System.Exception">if there is any failure</exception> protected internal virtual HdfsConfiguration CreateSecureConfig(string dataTransferProtection ) { HdfsConfiguration conf = new HdfsConfiguration(); SecurityUtil.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Kerberos , conf); conf.Set(DFSConfigKeys.DfsNamenodeKerberosPrincipalKey, hdfsPrincipal); conf.Set(DFSConfigKeys.DfsNamenodeKeytabFileKey, keytab); conf.Set(DFSConfigKeys.DfsDatanodeKerberosPrincipalKey, hdfsPrincipal); conf.Set(DFSConfigKeys.DfsDatanodeKeytabFileKey, keytab); conf.Set(DFSConfigKeys.DfsWebAuthenticationKerberosPrincipalKey, spnegoPrincipal); conf.SetBoolean(DFSConfigKeys.DfsBlockAccessTokenEnableKey, true); conf.Set(DFSConfigKeys.DfsDataTransferProtectionKey, dataTransferProtection); conf.Set(DFSConfigKeys.DfsHttpPolicyKey, HttpConfig.Policy.HttpsOnly.ToString()); conf.Set(DFSConfigKeys.DfsNamenodeHttpsAddressKey, "localhost:0"); conf.Set(DFSConfigKeys.DfsDatanodeHttpsAddressKey, "localhost:0"); conf.SetInt(CommonConfigurationKeys.IpcClientConnectMaxRetriesOnSaslKey, 10); string keystoresDir = baseDir.GetAbsolutePath(); string sslConfDir = KeyStoreTestUtil.GetClasspathDir(this.GetType()); KeyStoreTestUtil.SetupSSLConfig(keystoresDir, sslConfDir, conf, false); return(conf); }
public virtual void TestChangedStorageId() { HdfsConfiguration conf = new HdfsConfiguration(); conf.SetInt(DFSConfigKeys.DfsHaTaileditsPeriodKey, 1); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).NnTopology (MiniDFSNNTopology.SimpleHATopology()).Build(); try { cluster.TransitionToActive(0); FileSystem fs = HATestUtil.ConfigureFailoverFs(cluster, conf); OutputStream @out = fs.Create(filePath); @out.Write(Sharpen.Runtime.GetBytesForString("foo bar baz")); @out.Close(); HATestUtil.WaitForStandbyToCatchUp(cluster.GetNameNode(0), cluster.GetNameNode(1) ); // Change the gen stamp of the block on datanode to go back in time (gen // stamps start at 1000) ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, filePath); NUnit.Framework.Assert.IsTrue(cluster.ChangeGenStampOfBlock(0, block, 900)); // Stop the DN so the replica with the changed gen stamp will be reported // when this DN starts up. MiniDFSCluster.DataNodeProperties dnProps = cluster.StopDataNode(0); // Restart the namenode so that when the DN comes up it will see an initial // block report. cluster.RestartNameNode(1, false); NUnit.Framework.Assert.IsTrue(cluster.RestartDataNode(dnProps, true)); // Wait until the standby NN queues up the corrupt block in the pending DN // message queue. while (cluster.GetNamesystem(1).GetBlockManager().GetPendingDataNodeMessageCount( ) < 1) { ThreadUtil.SleepAtLeastIgnoreInterrupts(1000); } NUnit.Framework.Assert.AreEqual(1, cluster.GetNamesystem(1).GetBlockManager().GetPendingDataNodeMessageCount ()); string oldStorageId = GetRegisteredDatanodeUid(cluster, 1); // Reformat/restart the DN. NUnit.Framework.Assert.IsTrue(WipeAndRestartDn(cluster, 0)); // Give the DN time to start up and register, which will cause the // DatanodeManager to dissociate the old storage ID from the DN xfer addr. string newStorageId = string.Empty; do { ThreadUtil.SleepAtLeastIgnoreInterrupts(1000); newStorageId = GetRegisteredDatanodeUid(cluster, 1); System.Console.Out.WriteLine("====> oldStorageId: " + oldStorageId + " newStorageId: " + newStorageId); }while (newStorageId.Equals(oldStorageId)); NUnit.Framework.Assert.AreEqual(0, cluster.GetNamesystem(1).GetBlockManager().GetPendingDataNodeMessageCount ()); // Now try to fail over. cluster.TransitionToStandby(0); cluster.TransitionToActive(1); } finally { cluster.Shutdown(); } }
/// <summary>Check that listCorruptFileBlocks works while the namenode is still in safemode. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestListCorruptFileBlocksInSafeMode() { MiniDFSCluster cluster = null; Random random = new Random(); try { Configuration conf = new HdfsConfiguration(); // datanode scans directories conf.SetInt(DFSConfigKeys.DfsDatanodeDirectoryscanIntervalKey, 1); // datanode sends block reports conf.SetInt(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 3 * 1000); // never leave safemode automatically conf.SetFloat(DFSConfigKeys.DfsNamenodeSafemodeThresholdPctKey, 1.5f); // start populating repl queues immediately conf.SetFloat(DFSConfigKeys.DfsNamenodeReplQueueThresholdPctKey, 0f); // Set short retry timeouts so this test runs faster conf.SetInt(DFSConfigKeys.DfsClientRetryWindowBase, 10); cluster = new MiniDFSCluster.Builder(conf).WaitSafeMode(false).Build(); cluster.GetNameNodeRpc().SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave, false); FileSystem fs = cluster.GetFileSystem(); // create two files with one block each DFSTestUtil util = new DFSTestUtil.Builder().SetName("testListCorruptFileBlocksInSafeMode" ).SetNumFiles(2).SetMaxLevels(1).SetMaxSize(512).Build(); util.CreateFiles(fs, "/srcdat10"); // fetch bad file list from namenode. There should be none. ICollection <FSNamesystem.CorruptFileBlockInfo> badFiles = cluster.GetNameNode().GetNamesystem ().ListCorruptFileBlocks("/", null); NUnit.Framework.Assert.IsTrue("Namenode has " + badFiles.Count + " corrupt files. Expecting None." , badFiles.Count == 0); // Now deliberately corrupt one block FilePath storageDir = cluster.GetInstanceStorageDir(0, 0); FilePath data_dir = MiniDFSCluster.GetFinalizedDir(storageDir, cluster.GetNamesystem ().GetBlockPoolId()); NUnit.Framework.Assert.IsTrue("data directory does not exist", data_dir.Exists()); IList <FilePath> metaFiles = MiniDFSCluster.GetAllBlockMetadataFiles(data_dir); NUnit.Framework.Assert.IsTrue("Data directory does not contain any blocks or there was an " + "IO error", metaFiles != null && !metaFiles.IsEmpty()); FilePath metaFile = metaFiles[0]; RandomAccessFile file = new RandomAccessFile(metaFile, "rw"); FileChannel channel = file.GetChannel(); long position = channel.Size() - 2; int length = 2; byte[] buffer = new byte[length]; random.NextBytes(buffer); channel.Write(ByteBuffer.Wrap(buffer), position); file.Close(); Log.Info("Deliberately corrupting file " + metaFile.GetName() + " at offset " + position + " length " + length); // read all files to trigger detection of corrupted replica try { util.CheckFiles(fs, "/srcdat10"); } catch (BlockMissingException) { System.Console.Out.WriteLine("Received BlockMissingException as expected."); } catch (IOException e) { NUnit.Framework.Assert.IsTrue("Corrupted replicas not handled properly. " + "Expecting BlockMissingException " + " but received IOException " + e, false); } // fetch bad file list from namenode. There should be one file. badFiles = cluster.GetNameNode().GetNamesystem().ListCorruptFileBlocks("/", null); Log.Info("Namenode has bad files. " + badFiles.Count); NUnit.Framework.Assert.IsTrue("Namenode has " + badFiles.Count + " bad files. Expecting 1." , badFiles.Count == 1); // restart namenode cluster.RestartNameNode(0); fs = cluster.GetFileSystem(); // wait until replication queues have been initialized while (!cluster.GetNameNode().namesystem.IsPopulatingReplQueues()) { try { Log.Info("waiting for replication queues"); Sharpen.Thread.Sleep(1000); } catch (Exception) { } } // read all files to trigger detection of corrupted replica try { util.CheckFiles(fs, "/srcdat10"); } catch (BlockMissingException) { System.Console.Out.WriteLine("Received BlockMissingException as expected."); } catch (IOException e) { NUnit.Framework.Assert.IsTrue("Corrupted replicas not handled properly. " + "Expecting BlockMissingException " + " but received IOException " + e, false); } // fetch bad file list from namenode. There should be one file. badFiles = cluster.GetNameNode().GetNamesystem().ListCorruptFileBlocks("/", null); Log.Info("Namenode has bad files. " + badFiles.Count); NUnit.Framework.Assert.IsTrue("Namenode has " + badFiles.Count + " bad files. Expecting 1." , badFiles.Count == 1); // check that we are still in safe mode NUnit.Framework.Assert.IsTrue("Namenode is not in safe mode", cluster.GetNameNode ().IsInSafeMode()); // now leave safe mode so that we can clean up cluster.GetNameNodeRpc().SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave, false); util.Cleanup(fs, "/srcdat10"); } catch (Exception e) { Log.Error(StringUtils.StringifyException(e)); throw; } finally { if (cluster != null) { cluster.Shutdown(); } } }
/// <summary>check if nn.getCorruptFiles() returns a file that has corrupted blocks</summary> /// <exception cref="System.Exception"/> public virtual void TestListCorruptFilesCorruptedBlock() { MiniDFSCluster cluster = null; Random random = new Random(); try { Configuration conf = new HdfsConfiguration(); conf.SetInt(DFSConfigKeys.DfsDatanodeDirectoryscanIntervalKey, 1); // datanode scans directories conf.SetInt(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 3 * 1000); // datanode sends block reports // Set short retry timeouts so this test runs faster conf.SetInt(DFSConfigKeys.DfsClientRetryWindowBase, 10); cluster = new MiniDFSCluster.Builder(conf).Build(); FileSystem fs = cluster.GetFileSystem(); // create two files with one block each DFSTestUtil util = new DFSTestUtil.Builder().SetName("testCorruptFilesCorruptedBlock" ).SetNumFiles(2).SetMaxLevels(1).SetMaxSize(512).Build(); util.CreateFiles(fs, "/srcdat10"); // fetch bad file list from namenode. There should be none. NameNode namenode = cluster.GetNameNode(); ICollection <FSNamesystem.CorruptFileBlockInfo> badFiles = namenode.GetNamesystem( ).ListCorruptFileBlocks("/", null); NUnit.Framework.Assert.IsTrue("Namenode has " + badFiles.Count + " corrupt files. Expecting None." , badFiles.Count == 0); // Now deliberately corrupt one block string bpid = cluster.GetNamesystem().GetBlockPoolId(); FilePath storageDir = cluster.GetInstanceStorageDir(0, 1); FilePath data_dir = MiniDFSCluster.GetFinalizedDir(storageDir, bpid); NUnit.Framework.Assert.IsTrue("data directory does not exist", data_dir.Exists()); IList <FilePath> metaFiles = MiniDFSCluster.GetAllBlockMetadataFiles(data_dir); NUnit.Framework.Assert.IsTrue("Data directory does not contain any blocks or there was an " + "IO error", metaFiles != null && !metaFiles.IsEmpty()); FilePath metaFile = metaFiles[0]; RandomAccessFile file = new RandomAccessFile(metaFile, "rw"); FileChannel channel = file.GetChannel(); long position = channel.Size() - 2; int length = 2; byte[] buffer = new byte[length]; random.NextBytes(buffer); channel.Write(ByteBuffer.Wrap(buffer), position); file.Close(); Log.Info("Deliberately corrupting file " + metaFile.GetName() + " at offset " + position + " length " + length); // read all files to trigger detection of corrupted replica try { util.CheckFiles(fs, "/srcdat10"); } catch (BlockMissingException) { System.Console.Out.WriteLine("Received BlockMissingException as expected."); } catch (IOException e) { NUnit.Framework.Assert.IsTrue("Corrupted replicas not handled properly. Expecting BlockMissingException " + " but received IOException " + e, false); } // fetch bad file list from namenode. There should be one file. badFiles = namenode.GetNamesystem().ListCorruptFileBlocks("/", null); Log.Info("Namenode has bad files. " + badFiles.Count); NUnit.Framework.Assert.IsTrue("Namenode has " + badFiles.Count + " bad files. Expecting 1." , badFiles.Count == 1); util.Cleanup(fs, "/srcdat10"); } finally { if (cluster != null) { cluster.Shutdown(); } } }
/// <summary> /// Regression test for HDFS-7960.<p/> /// Shutting down a datanode, removing a storage directory, and restarting /// the DataNode should not produce zombie storages. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestRemovingStorageDoesNotProduceZombies() { Configuration conf = new HdfsConfiguration(); conf.SetInt(DFSConfigKeys.DfsDatanodeFailedVolumesToleratedKey, 1); int NumStoragesPerDn = 2; MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).StoragesPerDatanode (NumStoragesPerDn).Build(); try { cluster.WaitActive(); foreach (DataNode dn in cluster.GetDataNodes()) { NUnit.Framework.Assert.AreEqual(NumStoragesPerDn, cluster.GetNamesystem().GetBlockManager ().GetDatanodeManager().GetDatanode(dn.GetDatanodeId()).GetStorageInfos().Length ); } // Create a file which will end up on all 3 datanodes. Path TestPath = new Path("/foo1"); DistributedFileSystem fs = cluster.GetFileSystem(); DFSTestUtil.CreateFile(fs, TestPath, 1024, (short)3, unchecked ((int)(0xcafecafe)) ); foreach (DataNode dn_1 in cluster.GetDataNodes()) { DataNodeTestUtils.TriggerBlockReport(dn_1); } ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, new Path("/foo1")); cluster.GetNamesystem().WriteLock(); string storageIdToRemove; string datanodeUuid; // Find the first storage which this block is in. try { IEnumerator <DatanodeStorageInfo> storageInfoIter = cluster.GetNamesystem().GetBlockManager ().GetStorages(block.GetLocalBlock()).GetEnumerator(); NUnit.Framework.Assert.IsTrue(storageInfoIter.HasNext()); DatanodeStorageInfo info = storageInfoIter.Next(); storageIdToRemove = info.GetStorageID(); datanodeUuid = info.GetDatanodeDescriptor().GetDatanodeUuid(); } finally { cluster.GetNamesystem().WriteUnlock(); } // Find the DataNode which holds that first storage. DataNode datanodeToRemoveStorageFrom; int datanodeToRemoveStorageFromIdx = 0; while (true) { if (datanodeToRemoveStorageFromIdx >= cluster.GetDataNodes().Count) { NUnit.Framework.Assert.Fail("failed to find datanode with uuid " + datanodeUuid); datanodeToRemoveStorageFrom = null; break; } DataNode dn_2 = cluster.GetDataNodes()[datanodeToRemoveStorageFromIdx]; if (dn_2.GetDatanodeUuid().Equals(datanodeUuid)) { datanodeToRemoveStorageFrom = dn_2; break; } datanodeToRemoveStorageFromIdx++; } // Find the volume within the datanode which holds that first storage. IList <FsVolumeSpi> volumes = datanodeToRemoveStorageFrom.GetFSDataset().GetVolumes (); NUnit.Framework.Assert.AreEqual(NumStoragesPerDn, volumes.Count); string volumeDirectoryToRemove = null; foreach (FsVolumeSpi volume in volumes) { if (volume.GetStorageID().Equals(storageIdToRemove)) { volumeDirectoryToRemove = volume.GetBasePath(); } } // Shut down the datanode and remove the volume. // Replace the volume directory with a regular file, which will // cause a volume failure. (If we merely removed the directory, // it would be re-initialized with a new storage ID.) NUnit.Framework.Assert.IsNotNull(volumeDirectoryToRemove); datanodeToRemoveStorageFrom.Shutdown(); FileUtil.FullyDelete(new FilePath(volumeDirectoryToRemove)); FileOutputStream fos = new FileOutputStream(volumeDirectoryToRemove); try { fos.Write(1); } finally { fos.Close(); } cluster.RestartDataNode(datanodeToRemoveStorageFromIdx); // Wait for the NameNode to remove the storage. Log.Info("waiting for the datanode to remove " + storageIdToRemove); GenericTestUtils.WaitFor(new _Supplier_227(cluster, datanodeToRemoveStorageFrom, storageIdToRemove, NumStoragesPerDn), 10, 30000); } finally { if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestPendingAndInvalidate() { Configuration Conf = new HdfsConfiguration(); Conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, 1024); Conf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, DfsReplicationInterval); Conf.SetInt(DFSConfigKeys.DfsNamenodeReplicationIntervalKey, DfsReplicationInterval ); MiniDFSCluster cluster = new MiniDFSCluster.Builder(Conf).NumDataNodes(DatanodeCount ).Build(); cluster.WaitActive(); FSNamesystem namesystem = cluster.GetNamesystem(); BlockManager bm = namesystem.GetBlockManager(); DistributedFileSystem fs = cluster.GetFileSystem(); try { // 1. create a file Path filePath = new Path("/tmp.txt"); DFSTestUtil.CreateFile(fs, filePath, 1024, (short)3, 0L); // 2. disable the heartbeats foreach (DataNode dn in cluster.GetDataNodes()) { DataNodeTestUtils.SetHeartbeatsDisabledForTests(dn, true); } // 3. mark a couple of blocks as corrupt LocatedBlock block = NameNodeAdapter.GetBlockLocations(cluster.GetNameNode(), filePath .ToString(), 0, 1).Get(0); cluster.GetNamesystem().WriteLock(); try { bm.FindAndMarkBlockAsCorrupt(block.GetBlock(), block.GetLocations()[0], "STORAGE_ID" , "TEST"); bm.FindAndMarkBlockAsCorrupt(block.GetBlock(), block.GetLocations()[1], "STORAGE_ID" , "TEST"); } finally { cluster.GetNamesystem().WriteUnlock(); } BlockManagerTestUtil.ComputeAllPendingWork(bm); BlockManagerTestUtil.UpdateState(bm); NUnit.Framework.Assert.AreEqual(bm.GetPendingReplicationBlocksCount(), 1L); NUnit.Framework.Assert.AreEqual(bm.pendingReplications.GetNumReplicas(block.GetBlock ().GetLocalBlock()), 2); // 4. delete the file fs.Delete(filePath, true); // retry at most 10 times, each time sleep for 1s. Note that 10s is much // less than the default pending record timeout (5~10min) int retries = 10; long pendingNum = bm.GetPendingReplicationBlocksCount(); while (pendingNum != 0 && retries-- > 0) { Sharpen.Thread.Sleep(1000); // let NN do the deletion BlockManagerTestUtil.UpdateState(bm); pendingNum = bm.GetPendingReplicationBlocksCount(); } NUnit.Framework.Assert.AreEqual(pendingNum, 0L); } finally { cluster.Shutdown(); } }
public virtual void TestFileLimit() { Configuration conf = new HdfsConfiguration(); int maxObjects = 5; conf.SetLong(DFSConfigKeys.DfsNamenodeMaxObjectsKey, maxObjects); conf.SetLong(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 1000L); conf.SetInt(DFSConfigKeys.DfsHeartbeatIntervalKey, 1); int currentNodes = 0; if (simulatedStorage) { SimulatedFSDataset.SetFactory(conf); } MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build(); FileSystem fs = cluster.GetFileSystem(); FSNamesystem namesys = cluster.GetNamesystem(); try { // // check that / exists // Path path = new Path("/"); NUnit.Framework.Assert.IsTrue("/ should be a directory", fs.GetFileStatus(path).IsDirectory ()); currentNodes = 1; // root inode // verify that we can create the specified number of files. We leave // one for the "/". Each file takes an inode and a block. // for (int i = 0; i < maxObjects / 2; i++) { Path file = new Path("/filestatus" + i); CreateFile(fs, file); System.Console.Out.WriteLine("Created file " + file); currentNodes += 2; } // two more objects for this creation. // verify that creating another file fails bool hitException = false; try { Path file = new Path("/filestatus"); CreateFile(fs, file); System.Console.Out.WriteLine("Created file " + file); } catch (IOException) { hitException = true; } NUnit.Framework.Assert.IsTrue("Was able to exceed file limit", hitException); // delete one file Path file0 = new Path("/filestatus0"); fs.Delete(file0, true); System.Console.Out.WriteLine("Deleted file " + file0); currentNodes -= 2; // wait for number of blocks to decrease WaitForLimit(namesys, currentNodes); // now, we shud be able to create a new file CreateFile(fs, file0); System.Console.Out.WriteLine("Created file " + file0 + " again."); currentNodes += 2; // delete the file again file0 = new Path("/filestatus0"); fs.Delete(file0, true); System.Console.Out.WriteLine("Deleted file " + file0 + " again."); currentNodes -= 2; // wait for number of blocks to decrease WaitForLimit(namesys, currentNodes); // create two directories in place of the file that we deleted Path dir = new Path("/dir0/dir1"); fs.Mkdirs(dir); System.Console.Out.WriteLine("Created directories " + dir); currentNodes += 2; WaitForLimit(namesys, currentNodes); // verify that creating another directory fails hitException = false; try { fs.Mkdirs(new Path("dir.fail")); System.Console.Out.WriteLine("Created directory should not have succeeded."); } catch (IOException) { hitException = true; } NUnit.Framework.Assert.IsTrue("Was able to exceed dir limit", hitException); } finally { fs.Close(); cluster.Shutdown(); } }
/// <summary>Test if NN.listCorruptFiles() returns the right number of results.</summary> /// <remarks> /// Test if NN.listCorruptFiles() returns the right number of results. /// The corrupt blocks are detected by the BlockPoolSliceScanner. /// Also, test that DFS.listCorruptFileBlocks can make multiple successive /// calls. /// </remarks> /// <exception cref="System.Exception"/> public virtual void TestMaxCorruptFiles() { MiniDFSCluster cluster = null; try { Configuration conf = new HdfsConfiguration(); conf.SetInt(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 3 * 1000); // datanode sends block reports cluster = new MiniDFSCluster.Builder(conf).Build(); FileSystem fs = cluster.GetFileSystem(); int maxCorruptFileBlocks = FSNamesystem.DefaultMaxCorruptFileblocksReturned; // create 110 files with one block each DFSTestUtil util = new DFSTestUtil.Builder().SetName("testMaxCorruptFiles").SetNumFiles (maxCorruptFileBlocks * 3).SetMaxLevels(1).SetMaxSize(512).Build(); util.CreateFiles(fs, "/srcdat2", (short)1); util.WaitReplication(fs, "/srcdat2", (short)1); // verify that there are no bad blocks. NameNode namenode = cluster.GetNameNode(); ICollection <FSNamesystem.CorruptFileBlockInfo> badFiles = namenode.GetNamesystem( ).ListCorruptFileBlocks("/srcdat2", null); NUnit.Framework.Assert.IsTrue("Namenode has " + badFiles.Count + " corrupt files. Expecting none." , badFiles.Count == 0); // Now deliberately blocks from all files string bpid = cluster.GetNamesystem().GetBlockPoolId(); for (int i = 0; i < 4; i++) { for (int j = 0; j <= 1; j++) { FilePath storageDir = cluster.GetInstanceStorageDir(i, j); FilePath data_dir = MiniDFSCluster.GetFinalizedDir(storageDir, bpid); Log.Info("Removing files from " + data_dir); IList <FilePath> metadataFiles = MiniDFSCluster.GetAllBlockMetadataFiles(data_dir); if (metadataFiles == null) { continue; } foreach (FilePath metadataFile in metadataFiles) { FilePath blockFile = Block.MetaToBlockFile(metadataFile); NUnit.Framework.Assert.IsTrue("Cannot remove file.", blockFile.Delete()); NUnit.Framework.Assert.IsTrue("Cannot remove file.", metadataFile.Delete()); } } } // Occasionally the BlockPoolSliceScanner can run before we have removed // the blocks. Restart the Datanode to trigger the scanner into running // once more. Log.Info("Restarting Datanode to trigger BlockPoolSliceScanner"); cluster.RestartDataNodes(); cluster.WaitActive(); badFiles = namenode.GetNamesystem().ListCorruptFileBlocks("/srcdat2", null); while (badFiles.Count < maxCorruptFileBlocks) { Log.Info("# of corrupt files is: " + badFiles.Count); Sharpen.Thread.Sleep(10000); badFiles = namenode.GetNamesystem().ListCorruptFileBlocks("/srcdat2", null); } badFiles = namenode.GetNamesystem().ListCorruptFileBlocks("/srcdat2", null); Log.Info("Namenode has bad files. " + badFiles.Count); NUnit.Framework.Assert.IsTrue("Namenode has " + badFiles.Count + " bad files. Expecting " + maxCorruptFileBlocks + ".", badFiles.Count == maxCorruptFileBlocks); CorruptFileBlockIterator iter = (CorruptFileBlockIterator)fs.ListCorruptFileBlocks (new Path("/srcdat2")); int corruptPaths = CountPaths(iter); NUnit.Framework.Assert.IsTrue("Expected more than " + maxCorruptFileBlocks + " corrupt file blocks but got " + corruptPaths, corruptPaths > maxCorruptFileBlocks); NUnit.Framework.Assert.IsTrue("Iterator should have made more than 1 call but made " + iter.GetCallsMade(), iter.GetCallsMade() > 1); util.Cleanup(fs, "/srcdat2"); } finally { if (cluster != null) { cluster.Shutdown(); } } }