public virtual void TestProcessPendingReplications()
{
Configuration conf = new HdfsConfiguration();
conf.SetLong(DFSConfigKeys.DfsNamenodeReplicationPendingTimeoutSecKey, Timeout);
MiniDFSCluster cluster = null;
Block block;
BlockInfoContiguous blockInfo;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(DatanodeCount).Build();
cluster.WaitActive();
FSNamesystem fsn = cluster.GetNamesystem();
BlockManager blkManager = fsn.GetBlockManager();
PendingReplicationBlocks pendingReplications = blkManager.pendingReplications;
UnderReplicatedBlocks neededReplications = blkManager.neededReplications;
BlocksMap blocksMap = blkManager.blocksMap;
//
// Add 1 block to pendingReplications with GenerationStamp = 0.
//
block = new Block(1, 1, 0);
blockInfo = new BlockInfoContiguous(block, (short)3);
pendingReplications.Increment(block, DatanodeStorageInfo.ToDatanodeDescriptors(DFSTestUtil
.CreateDatanodeStorageInfos(1)));
BlockCollection bc = Org.Mockito.Mockito.Mock <BlockCollection>();
Org.Mockito.Mockito.DoReturn((short)3).When(bc).GetBlockReplication();
// Place into blocksmap with GenerationStamp = 1
blockInfo.SetGenerationStamp(1);
blocksMap.AddBlockCollection(blockInfo, bc);
NUnit.Framework.Assert.AreEqual("Size of pendingReplications ", 1, pendingReplications
.Size());
// Add a second block to pendingReplications that has no
// corresponding entry in blocksmap
block = new Block(2, 2, 0);
pendingReplications.Increment(block, DatanodeStorageInfo.ToDatanodeDescriptors(DFSTestUtil
.CreateDatanodeStorageInfos(1)));
// verify 2 blocks in pendingReplications
NUnit.Framework.Assert.AreEqual("Size of pendingReplications ", 2, pendingReplications
.Size());
//
// Wait for everything to timeout.
//
while (pendingReplications.Size() > 0)
{
try
{
Sharpen.Thread.Sleep(100);
}
catch (Exception)
{
}
}
//
// Verify that block moves to neededReplications
//
while (neededReplications.Size() == 0)
{
try
{
Sharpen.Thread.Sleep(100);
}
catch (Exception)
{
}
}
// Verify that the generation stamp we will try to replicate
// is now 1
foreach (Block b in neededReplications)
{
NUnit.Framework.Assert.AreEqual("Generation stamp is 1 ", 1, b.GetGenerationStamp
());
}
// Verify size of neededReplications is exactly 1.
NUnit.Framework.Assert.AreEqual("size of neededReplications is 1 ", 1, neededReplications
.Size());
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestPendingReplication()
{
PendingReplicationBlocks pendingReplications;
pendingReplications = new PendingReplicationBlocks(Timeout * 1000);
pendingReplications.Start();
//
// Add 10 blocks to pendingReplications.
//
DatanodeStorageInfo[] storages = DFSTestUtil.CreateDatanodeStorageInfos(10);
for (int i = 0; i < storages.Length; i++)
{
Block block = new Block(i, i, 0);
DatanodeStorageInfo[] targets = new DatanodeStorageInfo[i];
System.Array.Copy(storages, 0, targets, 0, i);
pendingReplications.Increment(block, DatanodeStorageInfo.ToDatanodeDescriptors(targets
));
}
NUnit.Framework.Assert.AreEqual("Size of pendingReplications ", 10, pendingReplications
.Size());
//
// remove one item and reinsert it
//
Block blk = new Block(8, 8, 0);
pendingReplications.Decrement(blk, storages[7].GetDatanodeDescriptor());
// removes one replica
NUnit.Framework.Assert.AreEqual("pendingReplications.getNumReplicas ", 7, pendingReplications
.GetNumReplicas(blk));
for (int i_1 = 0; i_1 < 7; i_1++)
{
// removes all replicas
pendingReplications.Decrement(blk, storages[i_1].GetDatanodeDescriptor());
}
NUnit.Framework.Assert.IsTrue(pendingReplications.Size() == 9);
pendingReplications.Increment(blk, DatanodeStorageInfo.ToDatanodeDescriptors(DFSTestUtil
.CreateDatanodeStorageInfos(8)));
NUnit.Framework.Assert.IsTrue(pendingReplications.Size() == 10);
//
// verify that the number of replicas returned
// are sane.
//
for (int i_2 = 0; i_2 < 10; i_2++)
{
Block block = new Block(i_2, i_2, 0);
int numReplicas = pendingReplications.GetNumReplicas(block);
NUnit.Framework.Assert.IsTrue(numReplicas == i_2);
}
//
// verify that nothing has timed out so far
//
NUnit.Framework.Assert.IsTrue(pendingReplications.GetTimedOutBlocks() == null);
//
// Wait for one second and then insert some more items.
//
try
{
Sharpen.Thread.Sleep(1000);
}
catch (Exception)
{
}
for (int i_3 = 10; i_3 < 15; i_3++)
{
Block block = new Block(i_3, i_3, 0);
pendingReplications.Increment(block, DatanodeStorageInfo.ToDatanodeDescriptors(DFSTestUtil
.CreateDatanodeStorageInfos(i_3)));
}
NUnit.Framework.Assert.IsTrue(pendingReplications.Size() == 15);
//
// Wait for everything to timeout.
//
int loop = 0;
while (pendingReplications.Size() > 0)
{
try
{
Sharpen.Thread.Sleep(1000);
}
catch (Exception)
{
}
loop++;
}
System.Console.Out.WriteLine("Had to wait for " + loop + " seconds for the lot to timeout"
);
//
// Verify that everything has timed out.
//
NUnit.Framework.Assert.AreEqual("Size of pendingReplications ", 0, pendingReplications
.Size());
Block[] timedOut = pendingReplications.GetTimedOutBlocks();
NUnit.Framework.Assert.IsTrue(timedOut != null && timedOut.Length == 15);
for (int i_4 = 0; i_4 < timedOut.Length; i_4++)
{
NUnit.Framework.Assert.IsTrue(timedOut[i_4].GetBlockId() < 15);
}
pendingReplications.Stop();
}
internal PendingReplicationMonitor(PendingReplicationBlocks _enclosing)
{
this._enclosing = _enclosing;
}