/// <summary>Constrict an iterator for a single queue level</summary> /// <param name="l">the priority level to iterate over</param> private BlockIterator(UnderReplicatedBlocks _enclosing, int l) { this._enclosing = _enclosing; this.level = l; this.isIteratorForLevel = true; this.iterators.AddItem(this._enclosing.priorityQueues[this.level].GetEnumerator() ); }
/// <summary>Construct an iterator over all queues.</summary> private BlockIterator(UnderReplicatedBlocks _enclosing) { this._enclosing = _enclosing; this.level = 0; for (int i = 0; i < UnderReplicatedBlocks.Level; i++) { this.iterators.AddItem(this._enclosing.priorityQueues[i].GetEnumerator()); } }
/// <summary>Determine whether or not a block is in a level without changing the API. /// </summary> /// <remarks> /// Determine whether or not a block is in a level without changing the API. /// Instead get the per-level iterator and run though it looking for a match. /// If the block is not found, an assertion is thrown. /// This is inefficient, but this is only a test case. /// </remarks> /// <param name="queues">queues to scan</param> /// <param name="block">block to look for</param> /// <param name="level">level to select</param> private void AssertInLevel(UnderReplicatedBlocks queues, Block block, int level) { UnderReplicatedBlocks.BlockIterator bi = queues.Iterator(level); while (bi.HasNext()) { Block next = bi.Next(); if (block.Equals(next)) { return; } } NUnit.Framework.Assert.Fail("Block " + block + " not found in level " + level); }
public virtual void TestBlockPriorities() { UnderReplicatedBlocks queues = new UnderReplicatedBlocks(); Block block1 = new Block(1); Block block2 = new Block(2); Block block_very_under_replicated = new Block(3); Block block_corrupt = new Block(4); Block block_corrupt_repl_one = new Block(5); //add a block with a single entry AssertAdded(queues, block1, 1, 0, 3); NUnit.Framework.Assert.AreEqual(1, queues.GetUnderReplicatedBlockCount()); NUnit.Framework.Assert.AreEqual(1, queues.Size()); AssertInLevel(queues, block1, UnderReplicatedBlocks.QueueHighestPriority); //repeated additions fail NUnit.Framework.Assert.IsFalse(queues.Add(block1, 1, 0, 3)); //add a second block with two replicas AssertAdded(queues, block2, 2, 0, 3); NUnit.Framework.Assert.AreEqual(2, queues.GetUnderReplicatedBlockCount()); NUnit.Framework.Assert.AreEqual(2, queues.Size()); AssertInLevel(queues, block2, UnderReplicatedBlocks.QueueUnderReplicated); //now try to add a block that is corrupt AssertAdded(queues, block_corrupt, 0, 0, 3); NUnit.Framework.Assert.AreEqual(3, queues.Size()); NUnit.Framework.Assert.AreEqual(2, queues.GetUnderReplicatedBlockCount()); NUnit.Framework.Assert.AreEqual(1, queues.GetCorruptBlockSize()); AssertInLevel(queues, block_corrupt, UnderReplicatedBlocks.QueueWithCorruptBlocks ); //insert a very under-replicated block AssertAdded(queues, block_very_under_replicated, 4, 0, 25); AssertInLevel(queues, block_very_under_replicated, UnderReplicatedBlocks.QueueVeryUnderReplicated ); //insert a corrupt block with replication factor 1 AssertAdded(queues, block_corrupt_repl_one, 0, 0, 1); NUnit.Framework.Assert.AreEqual(2, queues.GetCorruptBlockSize()); NUnit.Framework.Assert.AreEqual(1, queues.GetCorruptReplOneBlockSize()); queues.Update(block_corrupt_repl_one, 0, 0, 3, 0, 2); NUnit.Framework.Assert.AreEqual(0, queues.GetCorruptReplOneBlockSize()); queues.Update(block_corrupt, 0, 0, 1, 0, -2); NUnit.Framework.Assert.AreEqual(1, queues.GetCorruptReplOneBlockSize()); queues.Update(block_very_under_replicated, 0, 0, 1, -4, -24); NUnit.Framework.Assert.AreEqual(2, queues.GetCorruptReplOneBlockSize()); }
public virtual void TestProcessPendingReplications() { Configuration conf = new HdfsConfiguration(); conf.SetLong(DFSConfigKeys.DfsNamenodeReplicationPendingTimeoutSecKey, Timeout); MiniDFSCluster cluster = null; Block block; BlockInfoContiguous blockInfo; try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(DatanodeCount).Build(); cluster.WaitActive(); FSNamesystem fsn = cluster.GetNamesystem(); BlockManager blkManager = fsn.GetBlockManager(); PendingReplicationBlocks pendingReplications = blkManager.pendingReplications; UnderReplicatedBlocks neededReplications = blkManager.neededReplications; BlocksMap blocksMap = blkManager.blocksMap; // // Add 1 block to pendingReplications with GenerationStamp = 0. // block = new Block(1, 1, 0); blockInfo = new BlockInfoContiguous(block, (short)3); pendingReplications.Increment(block, DatanodeStorageInfo.ToDatanodeDescriptors(DFSTestUtil .CreateDatanodeStorageInfos(1))); BlockCollection bc = Org.Mockito.Mockito.Mock <BlockCollection>(); Org.Mockito.Mockito.DoReturn((short)3).When(bc).GetBlockReplication(); // Place into blocksmap with GenerationStamp = 1 blockInfo.SetGenerationStamp(1); blocksMap.AddBlockCollection(blockInfo, bc); NUnit.Framework.Assert.AreEqual("Size of pendingReplications ", 1, pendingReplications .Size()); // Add a second block to pendingReplications that has no // corresponding entry in blocksmap block = new Block(2, 2, 0); pendingReplications.Increment(block, DatanodeStorageInfo.ToDatanodeDescriptors(DFSTestUtil .CreateDatanodeStorageInfos(1))); // verify 2 blocks in pendingReplications NUnit.Framework.Assert.AreEqual("Size of pendingReplications ", 2, pendingReplications .Size()); // // Wait for everything to timeout. // while (pendingReplications.Size() > 0) { try { Sharpen.Thread.Sleep(100); } catch (Exception) { } } // // Verify that block moves to neededReplications // while (neededReplications.Size() == 0) { try { Sharpen.Thread.Sleep(100); } catch (Exception) { } } // Verify that the generation stamp we will try to replicate // is now 1 foreach (Block b in neededReplications) { NUnit.Framework.Assert.AreEqual("Generation stamp is 1 ", 1, b.GetGenerationStamp ()); } // Verify size of neededReplications is exactly 1. NUnit.Framework.Assert.AreEqual("size of neededReplications is 1 ", 1, neededReplications .Size()); } finally { if (cluster != null) { cluster.Shutdown(); } } }
private void AssertAdded(UnderReplicatedBlocks queues, Block block, int curReplicas , int decomissionedReplicas, int expectedReplicas) { NUnit.Framework.Assert.IsTrue("Failed to add " + block, queues.Add(block, curReplicas , decomissionedReplicas, expectedReplicas)); }