/// <summary>Constrict an iterator for a single queue level</summary>
/// <param name="l">the priority level to iterate over</param>
private BlockIterator(UnderReplicatedBlocks _enclosing, int l)
{
this._enclosing = _enclosing;
this.level = l;
this.isIteratorForLevel = true;
this.iterators.AddItem(this._enclosing.priorityQueues[this.level].GetEnumerator()
);
}
/// <summary>Construct an iterator over all queues.</summary>
private BlockIterator(UnderReplicatedBlocks _enclosing)
{
this._enclosing = _enclosing;
this.level = 0;
for (int i = 0; i < UnderReplicatedBlocks.Level; i++)
{
this.iterators.AddItem(this._enclosing.priorityQueues[i].GetEnumerator());
}
}
/// <summary>Determine whether or not a block is in a level without changing the API.
/// </summary>
/// <remarks>
/// Determine whether or not a block is in a level without changing the API.
/// Instead get the per-level iterator and run though it looking for a match.
/// If the block is not found, an assertion is thrown.
/// This is inefficient, but this is only a test case.
/// </remarks>
/// <param name="queues">queues to scan</param>
/// <param name="block">block to look for</param>
/// <param name="level">level to select</param>
private void AssertInLevel(UnderReplicatedBlocks queues, Block block, int level)
{
UnderReplicatedBlocks.BlockIterator bi = queues.Iterator(level);
while (bi.HasNext())
{
Block next = bi.Next();
if (block.Equals(next))
{
return;
}
}
NUnit.Framework.Assert.Fail("Block " + block + " not found in level " + level);
}
public virtual void TestBlockPriorities()
{
UnderReplicatedBlocks queues = new UnderReplicatedBlocks();
Block block1 = new Block(1);
Block block2 = new Block(2);
Block block_very_under_replicated = new Block(3);
Block block_corrupt = new Block(4);
Block block_corrupt_repl_one = new Block(5);
//add a block with a single entry
AssertAdded(queues, block1, 1, 0, 3);
NUnit.Framework.Assert.AreEqual(1, queues.GetUnderReplicatedBlockCount());
NUnit.Framework.Assert.AreEqual(1, queues.Size());
AssertInLevel(queues, block1, UnderReplicatedBlocks.QueueHighestPriority);
//repeated additions fail
NUnit.Framework.Assert.IsFalse(queues.Add(block1, 1, 0, 3));
//add a second block with two replicas
AssertAdded(queues, block2, 2, 0, 3);
NUnit.Framework.Assert.AreEqual(2, queues.GetUnderReplicatedBlockCount());
NUnit.Framework.Assert.AreEqual(2, queues.Size());
AssertInLevel(queues, block2, UnderReplicatedBlocks.QueueUnderReplicated);
//now try to add a block that is corrupt
AssertAdded(queues, block_corrupt, 0, 0, 3);
NUnit.Framework.Assert.AreEqual(3, queues.Size());
NUnit.Framework.Assert.AreEqual(2, queues.GetUnderReplicatedBlockCount());
NUnit.Framework.Assert.AreEqual(1, queues.GetCorruptBlockSize());
AssertInLevel(queues, block_corrupt, UnderReplicatedBlocks.QueueWithCorruptBlocks
);
//insert a very under-replicated block
AssertAdded(queues, block_very_under_replicated, 4, 0, 25);
AssertInLevel(queues, block_very_under_replicated, UnderReplicatedBlocks.QueueVeryUnderReplicated
);
//insert a corrupt block with replication factor 1
AssertAdded(queues, block_corrupt_repl_one, 0, 0, 1);
NUnit.Framework.Assert.AreEqual(2, queues.GetCorruptBlockSize());
NUnit.Framework.Assert.AreEqual(1, queues.GetCorruptReplOneBlockSize());
queues.Update(block_corrupt_repl_one, 0, 0, 3, 0, 2);
NUnit.Framework.Assert.AreEqual(0, queues.GetCorruptReplOneBlockSize());
queues.Update(block_corrupt, 0, 0, 1, 0, -2);
NUnit.Framework.Assert.AreEqual(1, queues.GetCorruptReplOneBlockSize());
queues.Update(block_very_under_replicated, 0, 0, 1, -4, -24);
NUnit.Framework.Assert.AreEqual(2, queues.GetCorruptReplOneBlockSize());
}
public virtual void TestProcessPendingReplications()
{
Configuration conf = new HdfsConfiguration();
conf.SetLong(DFSConfigKeys.DfsNamenodeReplicationPendingTimeoutSecKey, Timeout);
MiniDFSCluster cluster = null;
Block block;
BlockInfoContiguous blockInfo;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(DatanodeCount).Build();
cluster.WaitActive();
FSNamesystem fsn = cluster.GetNamesystem();
BlockManager blkManager = fsn.GetBlockManager();
PendingReplicationBlocks pendingReplications = blkManager.pendingReplications;
UnderReplicatedBlocks neededReplications = blkManager.neededReplications;
BlocksMap blocksMap = blkManager.blocksMap;
//
// Add 1 block to pendingReplications with GenerationStamp = 0.
//
block = new Block(1, 1, 0);
blockInfo = new BlockInfoContiguous(block, (short)3);
pendingReplications.Increment(block, DatanodeStorageInfo.ToDatanodeDescriptors(DFSTestUtil
.CreateDatanodeStorageInfos(1)));
BlockCollection bc = Org.Mockito.Mockito.Mock <BlockCollection>();
Org.Mockito.Mockito.DoReturn((short)3).When(bc).GetBlockReplication();
// Place into blocksmap with GenerationStamp = 1
blockInfo.SetGenerationStamp(1);
blocksMap.AddBlockCollection(blockInfo, bc);
NUnit.Framework.Assert.AreEqual("Size of pendingReplications ", 1, pendingReplications
.Size());
// Add a second block to pendingReplications that has no
// corresponding entry in blocksmap
block = new Block(2, 2, 0);
pendingReplications.Increment(block, DatanodeStorageInfo.ToDatanodeDescriptors(DFSTestUtil
.CreateDatanodeStorageInfos(1)));
// verify 2 blocks in pendingReplications
NUnit.Framework.Assert.AreEqual("Size of pendingReplications ", 2, pendingReplications
.Size());
//
// Wait for everything to timeout.
//
while (pendingReplications.Size() > 0)
{
try
{
Sharpen.Thread.Sleep(100);
}
catch (Exception)
{
}
}
//
// Verify that block moves to neededReplications
//
while (neededReplications.Size() == 0)
{
try
{
Sharpen.Thread.Sleep(100);
}
catch (Exception)
{
}
}
// Verify that the generation stamp we will try to replicate
// is now 1
foreach (Block b in neededReplications)
{
NUnit.Framework.Assert.AreEqual("Generation stamp is 1 ", 1, b.GetGenerationStamp
());
}
// Verify size of neededReplications is exactly 1.
NUnit.Framework.Assert.AreEqual("size of neededReplications is 1 ", 1, neededReplications
.Size());
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
private void AssertAdded(UnderReplicatedBlocks queues, Block block, int curReplicas
, int decomissionedReplicas, int expectedReplicas)
{
NUnit.Framework.Assert.IsTrue("Failed to add " + block, queues.Add(block, curReplicas
, decomissionedReplicas, expectedReplicas));
}