예제 #1
0
        public virtual void TestProcessPendingReplications()
        {
            Configuration conf = new HdfsConfiguration();

            conf.SetLong(DFSConfigKeys.DfsNamenodeReplicationPendingTimeoutSecKey, Timeout);
            MiniDFSCluster      cluster = null;
            Block               block;
            BlockInfoContiguous blockInfo;

            try
            {
                cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(DatanodeCount).Build();
                cluster.WaitActive();
                FSNamesystem             fsn                 = cluster.GetNamesystem();
                BlockManager             blkManager          = fsn.GetBlockManager();
                PendingReplicationBlocks pendingReplications = blkManager.pendingReplications;
                UnderReplicatedBlocks    neededReplications  = blkManager.neededReplications;
                BlocksMap blocksMap = blkManager.blocksMap;
                //
                // Add 1 block to pendingReplications with GenerationStamp = 0.
                //
                block     = new Block(1, 1, 0);
                blockInfo = new BlockInfoContiguous(block, (short)3);
                pendingReplications.Increment(block, DatanodeStorageInfo.ToDatanodeDescriptors(DFSTestUtil
                                                                                               .CreateDatanodeStorageInfos(1)));
                BlockCollection bc = Org.Mockito.Mockito.Mock <BlockCollection>();
                Org.Mockito.Mockito.DoReturn((short)3).When(bc).GetBlockReplication();
                // Place into blocksmap with GenerationStamp = 1
                blockInfo.SetGenerationStamp(1);
                blocksMap.AddBlockCollection(blockInfo, bc);
                NUnit.Framework.Assert.AreEqual("Size of pendingReplications ", 1, pendingReplications
                                                .Size());
                // Add a second block to pendingReplications that has no
                // corresponding entry in blocksmap
                block = new Block(2, 2, 0);
                pendingReplications.Increment(block, DatanodeStorageInfo.ToDatanodeDescriptors(DFSTestUtil
                                                                                               .CreateDatanodeStorageInfos(1)));
                // verify 2 blocks in pendingReplications
                NUnit.Framework.Assert.AreEqual("Size of pendingReplications ", 2, pendingReplications
                                                .Size());
                //
                // Wait for everything to timeout.
                //
                while (pendingReplications.Size() > 0)
                {
                    try
                    {
                        Sharpen.Thread.Sleep(100);
                    }
                    catch (Exception)
                    {
                    }
                }
                //
                // Verify that block moves to neededReplications
                //
                while (neededReplications.Size() == 0)
                {
                    try
                    {
                        Sharpen.Thread.Sleep(100);
                    }
                    catch (Exception)
                    {
                    }
                }
                // Verify that the generation stamp we will try to replicate
                // is now 1
                foreach (Block b in neededReplications)
                {
                    NUnit.Framework.Assert.AreEqual("Generation stamp is 1 ", 1, b.GetGenerationStamp
                                                        ());
                }
                // Verify size of neededReplications is exactly 1.
                NUnit.Framework.Assert.AreEqual("size of neededReplications is 1 ", 1, neededReplications
                                                .Size());
            }
            finally
            {
                if (cluster != null)
                {
                    cluster.Shutdown();
                }
            }
        }
예제 #2
0
        public virtual void TestPendingReplication()
        {
            PendingReplicationBlocks pendingReplications;

            pendingReplications = new PendingReplicationBlocks(Timeout * 1000);
            pendingReplications.Start();
            //
            // Add 10 blocks to pendingReplications.
            //
            DatanodeStorageInfo[] storages = DFSTestUtil.CreateDatanodeStorageInfos(10);
            for (int i = 0; i < storages.Length; i++)
            {
                Block block = new Block(i, i, 0);
                DatanodeStorageInfo[] targets = new DatanodeStorageInfo[i];
                System.Array.Copy(storages, 0, targets, 0, i);
                pendingReplications.Increment(block, DatanodeStorageInfo.ToDatanodeDescriptors(targets
                                                                                               ));
            }
            NUnit.Framework.Assert.AreEqual("Size of pendingReplications ", 10, pendingReplications
                                            .Size());
            //
            // remove one item and reinsert it
            //
            Block blk = new Block(8, 8, 0);

            pendingReplications.Decrement(blk, storages[7].GetDatanodeDescriptor());
            // removes one replica
            NUnit.Framework.Assert.AreEqual("pendingReplications.getNumReplicas ", 7, pendingReplications
                                            .GetNumReplicas(blk));
            for (int i_1 = 0; i_1 < 7; i_1++)
            {
                // removes all replicas
                pendingReplications.Decrement(blk, storages[i_1].GetDatanodeDescriptor());
            }
            NUnit.Framework.Assert.IsTrue(pendingReplications.Size() == 9);
            pendingReplications.Increment(blk, DatanodeStorageInfo.ToDatanodeDescriptors(DFSTestUtil
                                                                                         .CreateDatanodeStorageInfos(8)));
            NUnit.Framework.Assert.IsTrue(pendingReplications.Size() == 10);
            //
            // verify that the number of replicas returned
            // are sane.
            //
            for (int i_2 = 0; i_2 < 10; i_2++)
            {
                Block block       = new Block(i_2, i_2, 0);
                int   numReplicas = pendingReplications.GetNumReplicas(block);
                NUnit.Framework.Assert.IsTrue(numReplicas == i_2);
            }
            //
            // verify that nothing has timed out so far
            //
            NUnit.Framework.Assert.IsTrue(pendingReplications.GetTimedOutBlocks() == null);
            //
            // Wait for one second and then insert some more items.
            //
            try
            {
                Sharpen.Thread.Sleep(1000);
            }
            catch (Exception)
            {
            }
            for (int i_3 = 10; i_3 < 15; i_3++)
            {
                Block block = new Block(i_3, i_3, 0);
                pendingReplications.Increment(block, DatanodeStorageInfo.ToDatanodeDescriptors(DFSTestUtil
                                                                                               .CreateDatanodeStorageInfos(i_3)));
            }
            NUnit.Framework.Assert.IsTrue(pendingReplications.Size() == 15);
            //
            // Wait for everything to timeout.
            //
            int loop = 0;

            while (pendingReplications.Size() > 0)
            {
                try
                {
                    Sharpen.Thread.Sleep(1000);
                }
                catch (Exception)
                {
                }
                loop++;
            }
            System.Console.Out.WriteLine("Had to wait for " + loop + " seconds for the lot to timeout"
                                         );
            //
            // Verify that everything has timed out.
            //
            NUnit.Framework.Assert.AreEqual("Size of pendingReplications ", 0, pendingReplications
                                            .Size());
            Block[] timedOut = pendingReplications.GetTimedOutBlocks();
            NUnit.Framework.Assert.IsTrue(timedOut != null && timedOut.Length == 15);
            for (int i_4 = 0; i_4 < timedOut.Length; i_4++)
            {
                NUnit.Framework.Assert.IsTrue(timedOut[i_4].GetBlockId() < 15);
            }
            pendingReplications.Stop();
        }
 internal PendingReplicationMonitor(PendingReplicationBlocks _enclosing)
 {
     this._enclosing = _enclosing;
 }