/// <exception cref="System.IO.IOException"/> private void ValidateNumberReplicas(int expectedReplicas) { NumberReplicas numberReplicas = blockManager.CountNodes(block); Assert.AssertThat(numberReplicas.LiveReplicas(), CoreMatchers.Is(expectedReplicas )); Assert.AssertThat(numberReplicas.ExcessReplicas(), CoreMatchers.Is(0)); Assert.AssertThat(numberReplicas.CorruptReplicas(), CoreMatchers.Is(0)); Assert.AssertThat(numberReplicas.DecommissionedReplicas(), CoreMatchers.Is(0)); Assert.AssertThat(numberReplicas.ReplicasOnStaleNodes(), CoreMatchers.Is(0)); BlockManagerTestUtil.UpdateState(blockManager); Assert.AssertThat(blockManager.GetUnderReplicatedBlocksCount(), CoreMatchers.Is(0L )); Assert.AssertThat(blockManager.GetExcessBlocksCount(), CoreMatchers.Is(0L)); }
public virtual void TestInvalidateOverReplicatedBlock() { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build(); try { FSNamesystem namesystem = cluster.GetNamesystem(); BlockManager bm = namesystem.GetBlockManager(); FileSystem fs = cluster.GetFileSystem(); Path p = new Path(MiniDFSCluster.GetBaseDirectory(), "/foo1"); FSDataOutputStream @out = fs.Create(p, (short)2); @out.WriteBytes("HDFS-3119: " + p); @out.Hsync(); fs.SetReplication(p, (short)1); @out.Close(); ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, p); NUnit.Framework.Assert.AreEqual("Expected only one live replica for the block", 1 , bm.CountNodes(block.GetLocalBlock()).LiveReplicas()); } finally { cluster.Shutdown(); } }
/// <returns> /// a tuple of the replica state (number racks, number live /// replicas, and number needed replicas) for the given block. /// </returns> public static int[] GetReplicaInfo(FSNamesystem namesystem, Block b) { BlockManager bm = namesystem.GetBlockManager(); namesystem.ReadLock(); try { return(new int[] { GetNumberOfRacks(bm, b), bm.CountNodes(b).LiveReplicas(), bm.neededReplications .Contains(b) ? 1 : 0 }); } finally { namesystem.ReadUnlock(); } }
public virtual void TestProcesOverReplicateBlock() { Configuration conf = new HdfsConfiguration(); conf.SetLong(DFSConfigKeys.DfsDatanodeScanPeriodHoursKey, 100L); conf.SetLong(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 1000L); conf.Set(DFSConfigKeys.DfsNamenodeReplicationPendingTimeoutSecKey, Sharpen.Extensions.ToString (2)); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build(); FileSystem fs = cluster.GetFileSystem(); try { Path fileName = new Path("/foo1"); DFSTestUtil.CreateFile(fs, fileName, 2, (short)3, 0L); DFSTestUtil.WaitReplication(fs, fileName, (short)3); // corrupt the block on datanode 0 ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, fileName); NUnit.Framework.Assert.IsTrue(cluster.CorruptReplica(0, block)); MiniDFSCluster.DataNodeProperties dnProps = cluster.StopDataNode(0); // remove block scanner log to trigger block scanning FilePath scanCursor = new FilePath(new FilePath(MiniDFSCluster.GetFinalizedDir(cluster .GetInstanceStorageDir(0, 0), cluster.GetNamesystem().GetBlockPoolId()).GetParent ()).GetParent(), "scanner.cursor"); //wait for one minute for deletion to succeed; for (int i = 0; !scanCursor.Delete(); i++) { NUnit.Framework.Assert.IsTrue("Could not delete " + scanCursor.GetAbsolutePath() + " in one minute", i < 60); try { Sharpen.Thread.Sleep(1000); } catch (Exception) { } } // restart the datanode so the corrupt replica will be detected cluster.RestartDataNode(dnProps); DFSTestUtil.WaitReplication(fs, fileName, (short)2); string blockPoolId = cluster.GetNamesystem().GetBlockPoolId(); DatanodeID corruptDataNode = DataNodeTestUtils.GetDNRegistrationForBP(cluster.GetDataNodes ()[2], blockPoolId); FSNamesystem namesystem = cluster.GetNamesystem(); BlockManager bm = namesystem.GetBlockManager(); HeartbeatManager hm = bm.GetDatanodeManager().GetHeartbeatManager(); try { namesystem.WriteLock(); lock (hm) { // set live datanode's remaining space to be 0 // so they will be chosen to be deleted when over-replication occurs string corruptMachineName = corruptDataNode.GetXferAddr(); foreach (DatanodeDescriptor datanode in hm.GetDatanodes()) { if (!corruptMachineName.Equals(datanode.GetXferAddr())) { datanode.GetStorageInfos()[0].SetUtilizationForTesting(100L, 100L, 0, 100L); datanode.UpdateHeartbeat(BlockManagerTestUtil.GetStorageReportsForDatanode(datanode ), 0L, 0L, 0, 0, null); } } // decrease the replication factor to 1; NameNodeAdapter.SetReplication(namesystem, fileName.ToString(), (short)1); // corrupt one won't be chosen to be excess one // without 4910 the number of live replicas would be 0: block gets lost NUnit.Framework.Assert.AreEqual(1, bm.CountNodes(block.GetLocalBlock()).LiveReplicas ()); } } finally { namesystem.WriteUnlock(); } } finally { cluster.Shutdown(); } }