Beispiel #1
0
        public virtual void TestBlocksCounter()
        {
            DatanodeDescriptor dd = BlockManagerTestUtil.GetLocalDatanodeDescriptor(true);

            NUnit.Framework.Assert.AreEqual(0, dd.NumBlocks());
            BlockInfoContiguous blk  = new BlockInfoContiguous(new Block(1L), (short)1);
            BlockInfoContiguous blk1 = new BlockInfoContiguous(new Block(2L), (short)2);

            DatanodeStorageInfo[] storages = dd.GetStorageInfos();
            NUnit.Framework.Assert.IsTrue(storages.Length > 0);
            // add first block
            NUnit.Framework.Assert.IsTrue(storages[0].AddBlock(blk) == DatanodeStorageInfo.AddBlockResult
                                          .Added);
            NUnit.Framework.Assert.AreEqual(1, dd.NumBlocks());
            // remove a non-existent block
            NUnit.Framework.Assert.IsFalse(dd.RemoveBlock(blk1));
            NUnit.Framework.Assert.AreEqual(1, dd.NumBlocks());
            // add an existent block
            NUnit.Framework.Assert.IsFalse(storages[0].AddBlock(blk) == DatanodeStorageInfo.AddBlockResult
                                           .Added);
            NUnit.Framework.Assert.AreEqual(1, dd.NumBlocks());
            // add second block
            NUnit.Framework.Assert.IsTrue(storages[0].AddBlock(blk1) == DatanodeStorageInfo.AddBlockResult
                                          .Added);
            NUnit.Framework.Assert.AreEqual(2, dd.NumBlocks());
            // remove first block
            NUnit.Framework.Assert.IsTrue(dd.RemoveBlock(blk));
            NUnit.Framework.Assert.AreEqual(1, dd.NumBlocks());
            // remove second block
            NUnit.Framework.Assert.IsTrue(dd.RemoveBlock(blk1));
            NUnit.Framework.Assert.AreEqual(0, dd.NumBlocks());
        }
Beispiel #2
0
 public virtual void TestChooseTargetWithDecomNodes()
 {
     namenode.GetNamesystem().WriteLock();
     try
     {
         string blockPoolId = namenode.GetNamesystem().GetBlockPoolId();
         dnManager.HandleHeartbeat(dnrList[3], BlockManagerTestUtil.GetStorageReportsForDatanode
                                       (dataNodes[3]), blockPoolId, dataNodes[3].GetCacheCapacity(), dataNodes[3].GetCacheRemaining
                                       (), 2, 0, 0, null);
         dnManager.HandleHeartbeat(dnrList[4], BlockManagerTestUtil.GetStorageReportsForDatanode
                                       (dataNodes[4]), blockPoolId, dataNodes[4].GetCacheCapacity(), dataNodes[4].GetCacheRemaining
                                       (), 4, 0, 0, null);
         dnManager.HandleHeartbeat(dnrList[5], BlockManagerTestUtil.GetStorageReportsForDatanode
                                       (dataNodes[5]), blockPoolId, dataNodes[5].GetCacheCapacity(), dataNodes[5].GetCacheRemaining
                                       (), 4, 0, 0, null);
         // value in the above heartbeats
         int          load = 2 + 4 + 4;
         FSNamesystem fsn  = namenode.GetNamesystem();
         NUnit.Framework.Assert.AreEqual((double)load / 6, dnManager.GetFSClusterStats().GetInServiceXceiverAverage
                                             (), Epsilon);
         // Decommission DNs so BlockPlacementPolicyDefault.isGoodTarget()
         // returns false
         for (int i = 0; i < 3; i++)
         {
             DatanodeDescriptor d = dnManager.GetDatanode(dnrList[i]);
             dnManager.GetDecomManager().StartDecommission(d);
             d.SetDecommissioned();
         }
         NUnit.Framework.Assert.AreEqual((double)load / 3, dnManager.GetFSClusterStats().GetInServiceXceiverAverage
                                             (), Epsilon);
         // update references of writer DN to update the de-commissioned state
         IList <DatanodeDescriptor> liveNodes = new AList <DatanodeDescriptor>();
         dnManager.FetchDatanodes(liveNodes, null, false);
         DatanodeDescriptor writerDn = null;
         if (liveNodes.Contains(dataNodes[0]))
         {
             writerDn = liveNodes[liveNodes.IndexOf(dataNodes[0])];
         }
         // Call chooseTarget()
         DatanodeStorageInfo[] targets = namenode.GetNamesystem().GetBlockManager().GetBlockPlacementPolicy
                                             ().ChooseTarget("testFile.txt", 3, writerDn, new AList <DatanodeStorageInfo>(), false
                                                             , null, 1024, TestBlockStoragePolicy.DefaultStoragePolicy);
         NUnit.Framework.Assert.AreEqual(3, targets.Length);
         ICollection <DatanodeStorageInfo> targetSet = new HashSet <DatanodeStorageInfo>(Arrays
                                                                                         .AsList(targets));
         for (int i_1 = 3; i_1 < storages.Length; i_1++)
         {
             NUnit.Framework.Assert.IsTrue(targetSet.Contains(storages[i_1]));
         }
     }
     finally
     {
         dataNodes[0].StopDecommission();
         dataNodes[1].StopDecommission();
         dataNodes[2].StopDecommission();
         namenode.GetNamesystem().WriteUnlock();
     }
 }
 private static void UpdateHeartbeatWithUsage(DatanodeDescriptor dn, long capacity
                                              , long dfsUsed, long remaining, long blockPoolUsed, long dnCacheCapacity, long dnCacheUsed
                                              , int xceiverCount, int volFailures)
 {
     dn.GetStorageInfos()[0].SetUtilizationForTesting(capacity, dfsUsed, remaining, blockPoolUsed
                                                      );
     dn.UpdateHeartbeat(BlockManagerTestUtil.GetStorageReportsForDatanode(dn), dnCacheCapacity
                        , dnCacheUsed, xceiverCount, volFailures, null);
 }
        private void AddNodes(IEnumerable <DatanodeDescriptor> nodesToAdd)
        {
            NetworkTopology cluster = bm.GetDatanodeManager().GetNetworkTopology();

            // construct network topology
            foreach (DatanodeDescriptor dn in nodesToAdd)
            {
                cluster.Add(dn);
                dn.GetStorageInfos()[0].SetUtilizationForTesting(2 * HdfsConstants.MinBlocksForWrite
                                                                 * BlockSize, 0L, 2 * HdfsConstants.MinBlocksForWrite * BlockSize, 0L);
                dn.UpdateHeartbeat(BlockManagerTestUtil.GetStorageReportsForDatanode(dn), 0L, 0L,
                                   0, 0, null);
                bm.GetDatanodeManager().CheckIfClusterIsNowMultiRack(dn);
            }
        }
        public virtual void TestBlocksAreNotUnderreplicatedInSingleRack()
        {
            IList <DatanodeDescriptor> nodes = ImmutableList.Of(BlockManagerTestUtil.GetDatanodeDescriptor
                                                                    ("1.1.1.1", "/rackA", true), BlockManagerTestUtil.GetDatanodeDescriptor("2.2.2.2"
                                                                                                                                            , "/rackA", true), BlockManagerTestUtil.GetDatanodeDescriptor("3.3.3.3", "/rackA"
                                                                                                                                                                                                          , true), BlockManagerTestUtil.GetDatanodeDescriptor("4.4.4.4", "/rackA", true),
                                                                BlockManagerTestUtil.GetDatanodeDescriptor("5.5.5.5", "/rackA", true), BlockManagerTestUtil
                                                                .GetDatanodeDescriptor("6.6.6.6", "/rackA", true));

            AddNodes(nodes);
            IList <DatanodeDescriptor> origNodes = nodes.SubList(0, 3);

            for (int i = 0; i < NumTestIters; i++)
            {
                DoTestSingleRackClusterIsSufficientlyReplicated(i, origNodes);
            }
        }
Beispiel #6
0
        public static void SetupCluster()
        {
            Configuration conf = new HdfsConfiguration();

            string[] racks = new string[] { "/rack1", "/rack1", "/rack1", "/rack2", "/rack2",
                                            "/rack2" };
            storages  = DFSTestUtil.CreateDatanodeStorageInfos(racks);
            dataNodes = DFSTestUtil.ToDatanodeDescriptor(storages);
            FileSystem.SetDefaultUri(conf, "hdfs://localhost:0");
            conf.Set(DFSConfigKeys.DfsNamenodeHttpAddressKey, "0.0.0.0:0");
            FilePath baseDir = PathUtils.GetTestDir(typeof(TestReplicationPolicy));

            conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, new FilePath(baseDir, "name").GetPath
                         ());
            conf.SetBoolean(DFSConfigKeys.DfsNamenodeAvoidStaleDatanodeForReadKey, true);
            conf.SetBoolean(DFSConfigKeys.DfsNamenodeAvoidStaleDatanodeForWriteKey, true);
            conf.SetBoolean(DFSConfigKeys.DfsNamenodeReplicationConsiderloadKey, true);
            DFSTestUtil.FormatNameNode(conf);
            namenode = new NameNode(conf);
            int blockSize = 1024;

            dnrList   = new AList <DatanodeRegistration>();
            dnManager = namenode.GetNamesystem().GetBlockManager().GetDatanodeManager();
            // Register DNs
            for (int i = 0; i < 6; i++)
            {
                DatanodeRegistration dnr = new DatanodeRegistration(dataNodes[i], new StorageInfo
                                                                        (HdfsServerConstants.NodeType.DataNode), new ExportedBlockKeys(), VersionInfo.GetVersion
                                                                        ());
                dnrList.AddItem(dnr);
                dnManager.RegisterDatanode(dnr);
                dataNodes[i].GetStorageInfos()[0].SetUtilizationForTesting(2 * HdfsConstants.MinBlocksForWrite
                                                                           * blockSize, 0L, 2 * HdfsConstants.MinBlocksForWrite * blockSize, 0L);
                dataNodes[i].UpdateHeartbeat(BlockManagerTestUtil.GetStorageReportsForDatanode(dataNodes
                                                                                               [i]), 0L, 0L, 0, 0, null);
            }
        }
        public virtual void TestPendingAndInvalidate()
        {
            Configuration Conf = new HdfsConfiguration();

            Conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, 1024);
            Conf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, DfsReplicationInterval);
            Conf.SetInt(DFSConfigKeys.DfsNamenodeReplicationIntervalKey, DfsReplicationInterval
                        );
            MiniDFSCluster cluster = new MiniDFSCluster.Builder(Conf).NumDataNodes(DatanodeCount
                                                                                   ).Build();

            cluster.WaitActive();
            FSNamesystem          namesystem = cluster.GetNamesystem();
            BlockManager          bm         = namesystem.GetBlockManager();
            DistributedFileSystem fs         = cluster.GetFileSystem();

            try
            {
                // 1. create a file
                Path filePath = new Path("/tmp.txt");
                DFSTestUtil.CreateFile(fs, filePath, 1024, (short)3, 0L);
                // 2. disable the heartbeats
                foreach (DataNode dn in cluster.GetDataNodes())
                {
                    DataNodeTestUtils.SetHeartbeatsDisabledForTests(dn, true);
                }
                // 3. mark a couple of blocks as corrupt
                LocatedBlock block = NameNodeAdapter.GetBlockLocations(cluster.GetNameNode(), filePath
                                                                       .ToString(), 0, 1).Get(0);
                cluster.GetNamesystem().WriteLock();
                try
                {
                    bm.FindAndMarkBlockAsCorrupt(block.GetBlock(), block.GetLocations()[0], "STORAGE_ID"
                                                 , "TEST");
                    bm.FindAndMarkBlockAsCorrupt(block.GetBlock(), block.GetLocations()[1], "STORAGE_ID"
                                                 , "TEST");
                }
                finally
                {
                    cluster.GetNamesystem().WriteUnlock();
                }
                BlockManagerTestUtil.ComputeAllPendingWork(bm);
                BlockManagerTestUtil.UpdateState(bm);
                NUnit.Framework.Assert.AreEqual(bm.GetPendingReplicationBlocksCount(), 1L);
                NUnit.Framework.Assert.AreEqual(bm.pendingReplications.GetNumReplicas(block.GetBlock
                                                                                          ().GetLocalBlock()), 2);
                // 4. delete the file
                fs.Delete(filePath, true);
                // retry at most 10 times, each time sleep for 1s. Note that 10s is much
                // less than the default pending record timeout (5~10min)
                int  retries    = 10;
                long pendingNum = bm.GetPendingReplicationBlocksCount();
                while (pendingNum != 0 && retries-- > 0)
                {
                    Sharpen.Thread.Sleep(1000);
                    // let NN do the deletion
                    BlockManagerTestUtil.UpdateState(bm);
                    pendingNum = bm.GetPendingReplicationBlocksCount();
                }
                NUnit.Framework.Assert.AreEqual(pendingNum, 0L);
            }
            finally
            {
                cluster.Shutdown();
            }
        }
        public virtual void TestBlockReceived()
        {
            Configuration conf = new HdfsConfiguration();

            conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, 1024);
            MiniDFSCluster cluster = null;

            try
            {
                cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(DatanodeCount).Build();
                cluster.WaitActive();
                DistributedFileSystem hdfs       = cluster.GetFileSystem();
                FSNamesystem          fsn        = cluster.GetNamesystem();
                BlockManager          blkManager = fsn.GetBlockManager();
                string file       = "/tmp.txt";
                Path   filePath   = new Path(file);
                short  replFactor = 1;
                DFSTestUtil.CreateFile(hdfs, filePath, 1024L, replFactor, 0);
                // temporarily stop the heartbeat
                AList <DataNode> datanodes = cluster.GetDataNodes();
                for (int i = 0; i < DatanodeCount; i++)
                {
                    DataNodeTestUtils.SetHeartbeatsDisabledForTests(datanodes[i], true);
                }
                hdfs.SetReplication(filePath, (short)DatanodeCount);
                BlockManagerTestUtil.ComputeAllPendingWork(blkManager);
                NUnit.Framework.Assert.AreEqual(1, blkManager.pendingReplications.Size());
                INodeFile fileNode = fsn.GetFSDirectory().GetINode4Write(file).AsFile();
                Block[]   blocks   = fileNode.GetBlocks();
                NUnit.Framework.Assert.AreEqual(DatanodeCount - 1, blkManager.pendingReplications
                                                .GetNumReplicas(blocks[0]));
                LocatedBlock locatedBlock = hdfs.GetClient().GetLocatedBlocks(file, 0).Get(0);
                DatanodeInfo existingDn   = (locatedBlock.GetLocations())[0];
                int          reportDnNum  = 0;
                string       poolId       = cluster.GetNamesystem().GetBlockPoolId();
                // let two datanodes (other than the one that already has the data) to
                // report to NN
                for (int i_1 = 0; i_1 < DatanodeCount && reportDnNum < 2; i_1++)
                {
                    if (!datanodes[i_1].GetDatanodeId().Equals(existingDn))
                    {
                        DatanodeRegistration           dnR    = datanodes[i_1].GetDNRegistrationForBP(poolId);
                        StorageReceivedDeletedBlocks[] report = new StorageReceivedDeletedBlocks[] { new
                                                                                                     StorageReceivedDeletedBlocks("Fake-storage-ID-Ignored", new ReceivedDeletedBlockInfo
                                                                                                                                  [] { new ReceivedDeletedBlockInfo(blocks[0], ReceivedDeletedBlockInfo.BlockStatus
                                                                                                                                                                    .ReceivedBlock, string.Empty) }) };
                        cluster.GetNameNodeRpc().BlockReceivedAndDeleted(dnR, poolId, report);
                        reportDnNum++;
                    }
                }
                NUnit.Framework.Assert.AreEqual(DatanodeCount - 3, blkManager.pendingReplications
                                                .GetNumReplicas(blocks[0]));
                // let the same datanodes report again
                for (int i_2 = 0; i_2 < DatanodeCount && reportDnNum < 2; i_2++)
                {
                    if (!datanodes[i_2].GetDatanodeId().Equals(existingDn))
                    {
                        DatanodeRegistration           dnR    = datanodes[i_2].GetDNRegistrationForBP(poolId);
                        StorageReceivedDeletedBlocks[] report = new StorageReceivedDeletedBlocks[] { new
                                                                                                     StorageReceivedDeletedBlocks("Fake-storage-ID-Ignored", new ReceivedDeletedBlockInfo
                                                                                                                                  [] { new ReceivedDeletedBlockInfo(blocks[0], ReceivedDeletedBlockInfo.BlockStatus
                                                                                                                                                                    .ReceivedBlock, string.Empty) }) };
                        cluster.GetNameNodeRpc().BlockReceivedAndDeleted(dnR, poolId, report);
                        reportDnNum++;
                    }
                }
                NUnit.Framework.Assert.AreEqual(DatanodeCount - 3, blkManager.pendingReplications
                                                .GetNumReplicas(blocks[0]));
                // re-enable heartbeat for the datanode that has data
                for (int i_3 = 0; i_3 < DatanodeCount; i_3++)
                {
                    DataNodeTestUtils.SetHeartbeatsDisabledForTests(datanodes[i_3], false);
                    DataNodeTestUtils.TriggerHeartbeat(datanodes[i_3]);
                }
                Sharpen.Thread.Sleep(5000);
                NUnit.Framework.Assert.AreEqual(0, blkManager.pendingReplications.Size());
            }
            finally
            {
                if (cluster != null)
                {
                    cluster.Shutdown();
                }
            }
        }
        public virtual void TestProcesOverReplicateBlock()
        {
            Configuration conf = new HdfsConfiguration();

            conf.SetLong(DFSConfigKeys.DfsDatanodeScanPeriodHoursKey, 100L);
            conf.SetLong(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 1000L);
            conf.Set(DFSConfigKeys.DfsNamenodeReplicationPendingTimeoutSecKey, Sharpen.Extensions.ToString
                         (2));
            MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build();
            FileSystem     fs      = cluster.GetFileSystem();

            try
            {
                Path fileName = new Path("/foo1");
                DFSTestUtil.CreateFile(fs, fileName, 2, (short)3, 0L);
                DFSTestUtil.WaitReplication(fs, fileName, (short)3);
                // corrupt the block on datanode 0
                ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, fileName);
                NUnit.Framework.Assert.IsTrue(cluster.CorruptReplica(0, block));
                MiniDFSCluster.DataNodeProperties dnProps = cluster.StopDataNode(0);
                // remove block scanner log to trigger block scanning
                FilePath scanCursor = new FilePath(new FilePath(MiniDFSCluster.GetFinalizedDir(cluster
                                                                                               .GetInstanceStorageDir(0, 0), cluster.GetNamesystem().GetBlockPoolId()).GetParent
                                                                    ()).GetParent(), "scanner.cursor");
                //wait for one minute for deletion to succeed;
                for (int i = 0; !scanCursor.Delete(); i++)
                {
                    NUnit.Framework.Assert.IsTrue("Could not delete " + scanCursor.GetAbsolutePath()
                                                  + " in one minute", i < 60);
                    try
                    {
                        Sharpen.Thread.Sleep(1000);
                    }
                    catch (Exception)
                    {
                    }
                }
                // restart the datanode so the corrupt replica will be detected
                cluster.RestartDataNode(dnProps);
                DFSTestUtil.WaitReplication(fs, fileName, (short)2);
                string     blockPoolId     = cluster.GetNamesystem().GetBlockPoolId();
                DatanodeID corruptDataNode = DataNodeTestUtils.GetDNRegistrationForBP(cluster.GetDataNodes
                                                                                          ()[2], blockPoolId);
                FSNamesystem     namesystem = cluster.GetNamesystem();
                BlockManager     bm         = namesystem.GetBlockManager();
                HeartbeatManager hm         = bm.GetDatanodeManager().GetHeartbeatManager();
                try
                {
                    namesystem.WriteLock();
                    lock (hm)
                    {
                        // set live datanode's remaining space to be 0
                        // so they will be chosen to be deleted when over-replication occurs
                        string corruptMachineName = corruptDataNode.GetXferAddr();
                        foreach (DatanodeDescriptor datanode in hm.GetDatanodes())
                        {
                            if (!corruptMachineName.Equals(datanode.GetXferAddr()))
                            {
                                datanode.GetStorageInfos()[0].SetUtilizationForTesting(100L, 100L, 0, 100L);
                                datanode.UpdateHeartbeat(BlockManagerTestUtil.GetStorageReportsForDatanode(datanode
                                                                                                           ), 0L, 0L, 0, 0, null);
                            }
                        }
                        // decrease the replication factor to 1;
                        NameNodeAdapter.SetReplication(namesystem, fileName.ToString(), (short)1);
                        // corrupt one won't be chosen to be excess one
                        // without 4910 the number of live replicas would be 0: block gets lost
                        NUnit.Framework.Assert.AreEqual(1, bm.CountNodes(block.GetLocalBlock()).LiveReplicas
                                                            ());
                    }
                }
                finally
                {
                    namesystem.WriteUnlock();
                }
            }
            finally
            {
                cluster.Shutdown();
            }
        }
Beispiel #10
0
        public virtual void TestNodeCount()
        {
            // start a mini dfs cluster of 2 nodes
            Configuration  conf    = new HdfsConfiguration();
            MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(ReplicationFactor
                                                                                   ).Build();

            try
            {
                FSNamesystem     namesystem = cluster.GetNamesystem();
                BlockManager     bm         = namesystem.GetBlockManager();
                HeartbeatManager hm         = bm.GetDatanodeManager().GetHeartbeatManager();
                FileSystem       fs         = cluster.GetFileSystem();
                // populate the cluster with a one block file
                Path FilePath = new Path("/testfile");
                DFSTestUtil.CreateFile(fs, FilePath, 1L, ReplicationFactor, 1L);
                DFSTestUtil.WaitReplication(fs, FilePath, ReplicationFactor);
                ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, FilePath);
                // keep a copy of all datanode descriptor
                DatanodeDescriptor[] datanodes = hm.GetDatanodes();
                // start two new nodes
                cluster.StartDataNodes(conf, 2, true, null, null);
                cluster.WaitActive();
                // bring down first datanode
                DatanodeDescriptor datanode = datanodes[0];
                MiniDFSCluster.DataNodeProperties dnprop = cluster.StopDataNode(datanode.GetXferAddr
                                                                                    ());
                // make sure that NN detects that the datanode is down
                BlockManagerTestUtil.NoticeDeadDatanode(cluster.GetNameNode(), datanode.GetXferAddr
                                                            ());
                // the block will be replicated
                DFSTestUtil.WaitReplication(fs, FilePath, ReplicationFactor);
                // restart the first datanode
                cluster.RestartDataNode(dnprop);
                cluster.WaitActive();
                // check if excessive replica is detected (transient)
                InitializeTimeout(Timeout);
                while (CountNodes(block.GetLocalBlock(), namesystem).ExcessReplicas() == 0)
                {
                    CheckTimeout("excess replicas not detected");
                }
                // find out a non-excess node
                DatanodeDescriptor nonExcessDN = null;
                foreach (DatanodeStorageInfo storage in bm.blocksMap.GetStorages(block.GetLocalBlock
                                                                                     ()))
                {
                    DatanodeDescriptor  dn     = storage.GetDatanodeDescriptor();
                    ICollection <Block> blocks = bm.excessReplicateMap[dn.GetDatanodeUuid()];
                    if (blocks == null || !blocks.Contains(block.GetLocalBlock()))
                    {
                        nonExcessDN = dn;
                        break;
                    }
                }
                NUnit.Framework.Assert.IsTrue(nonExcessDN != null);
                // bring down non excessive datanode
                dnprop = cluster.StopDataNode(nonExcessDN.GetXferAddr());
                // make sure that NN detects that the datanode is down
                BlockManagerTestUtil.NoticeDeadDatanode(cluster.GetNameNode(), nonExcessDN.GetXferAddr
                                                            ());
                // The block should be replicated
                InitializeTimeout(Timeout);
                while (CountNodes(block.GetLocalBlock(), namesystem).LiveReplicas() != ReplicationFactor
                       )
                {
                    CheckTimeout("live replica count not correct", 1000);
                }
                // restart the first datanode
                cluster.RestartDataNode(dnprop);
                cluster.WaitActive();
                // check if excessive replica is detected (transient)
                InitializeTimeout(Timeout);
                while (CountNodes(block.GetLocalBlock(), namesystem).ExcessReplicas() != 2)
                {
                    CheckTimeout("excess replica count not equal to 2");
                }
            }
            finally
            {
                cluster.Shutdown();
            }
        }