public virtual void TestPendingAndInvalidate()
{
Configuration Conf = new HdfsConfiguration();
Conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, 1024);
Conf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, DfsReplicationInterval);
Conf.SetInt(DFSConfigKeys.DfsNamenodeReplicationIntervalKey, DfsReplicationInterval
);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(Conf).NumDataNodes(DatanodeCount
).Build();
cluster.WaitActive();
FSNamesystem namesystem = cluster.GetNamesystem();
BlockManager bm = namesystem.GetBlockManager();
DistributedFileSystem fs = cluster.GetFileSystem();
try
{
// 1. create a file
Path filePath = new Path("/tmp.txt");
DFSTestUtil.CreateFile(fs, filePath, 1024, (short)3, 0L);
// 2. disable the heartbeats
foreach (DataNode dn in cluster.GetDataNodes())
{
DataNodeTestUtils.SetHeartbeatsDisabledForTests(dn, true);
}
// 3. mark a couple of blocks as corrupt
LocatedBlock block = NameNodeAdapter.GetBlockLocations(cluster.GetNameNode(), filePath
.ToString(), 0, 1).Get(0);
cluster.GetNamesystem().WriteLock();
try
{
bm.FindAndMarkBlockAsCorrupt(block.GetBlock(), block.GetLocations()[0], "STORAGE_ID"
, "TEST");
bm.FindAndMarkBlockAsCorrupt(block.GetBlock(), block.GetLocations()[1], "STORAGE_ID"
, "TEST");
}
finally
{
cluster.GetNamesystem().WriteUnlock();
}
BlockManagerTestUtil.ComputeAllPendingWork(bm);
BlockManagerTestUtil.UpdateState(bm);
NUnit.Framework.Assert.AreEqual(bm.GetPendingReplicationBlocksCount(), 1L);
NUnit.Framework.Assert.AreEqual(bm.pendingReplications.GetNumReplicas(block.GetBlock
().GetLocalBlock()), 2);
// 4. delete the file
fs.Delete(filePath, true);
// retry at most 10 times, each time sleep for 1s. Note that 10s is much
// less than the default pending record timeout (5~10min)
int retries = 10;
long pendingNum = bm.GetPendingReplicationBlocksCount();
while (pendingNum != 0 && retries-- > 0)
{
Sharpen.Thread.Sleep(1000);
// let NN do the deletion
BlockManagerTestUtil.UpdateState(bm);
pendingNum = bm.GetPendingReplicationBlocksCount();
}
NUnit.Framework.Assert.AreEqual(pendingNum, 0L);
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestBlockReceived()
{
Configuration conf = new HdfsConfiguration();
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, 1024);
MiniDFSCluster cluster = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(DatanodeCount).Build();
cluster.WaitActive();
DistributedFileSystem hdfs = cluster.GetFileSystem();
FSNamesystem fsn = cluster.GetNamesystem();
BlockManager blkManager = fsn.GetBlockManager();
string file = "/tmp.txt";
Path filePath = new Path(file);
short replFactor = 1;
DFSTestUtil.CreateFile(hdfs, filePath, 1024L, replFactor, 0);
// temporarily stop the heartbeat
AList <DataNode> datanodes = cluster.GetDataNodes();
for (int i = 0; i < DatanodeCount; i++)
{
DataNodeTestUtils.SetHeartbeatsDisabledForTests(datanodes[i], true);
}
hdfs.SetReplication(filePath, (short)DatanodeCount);
BlockManagerTestUtil.ComputeAllPendingWork(blkManager);
NUnit.Framework.Assert.AreEqual(1, blkManager.pendingReplications.Size());
INodeFile fileNode = fsn.GetFSDirectory().GetINode4Write(file).AsFile();
Block[] blocks = fileNode.GetBlocks();
NUnit.Framework.Assert.AreEqual(DatanodeCount - 1, blkManager.pendingReplications
.GetNumReplicas(blocks[0]));
LocatedBlock locatedBlock = hdfs.GetClient().GetLocatedBlocks(file, 0).Get(0);
DatanodeInfo existingDn = (locatedBlock.GetLocations())[0];
int reportDnNum = 0;
string poolId = cluster.GetNamesystem().GetBlockPoolId();
// let two datanodes (other than the one that already has the data) to
// report to NN
for (int i_1 = 0; i_1 < DatanodeCount && reportDnNum < 2; i_1++)
{
if (!datanodes[i_1].GetDatanodeId().Equals(existingDn))
{
DatanodeRegistration dnR = datanodes[i_1].GetDNRegistrationForBP(poolId);
StorageReceivedDeletedBlocks[] report = new StorageReceivedDeletedBlocks[] { new
StorageReceivedDeletedBlocks("Fake-storage-ID-Ignored", new ReceivedDeletedBlockInfo
[] { new ReceivedDeletedBlockInfo(blocks[0], ReceivedDeletedBlockInfo.BlockStatus
.ReceivedBlock, string.Empty) }) };
cluster.GetNameNodeRpc().BlockReceivedAndDeleted(dnR, poolId, report);
reportDnNum++;
}
}
NUnit.Framework.Assert.AreEqual(DatanodeCount - 3, blkManager.pendingReplications
.GetNumReplicas(blocks[0]));
// let the same datanodes report again
for (int i_2 = 0; i_2 < DatanodeCount && reportDnNum < 2; i_2++)
{
if (!datanodes[i_2].GetDatanodeId().Equals(existingDn))
{
DatanodeRegistration dnR = datanodes[i_2].GetDNRegistrationForBP(poolId);
StorageReceivedDeletedBlocks[] report = new StorageReceivedDeletedBlocks[] { new
StorageReceivedDeletedBlocks("Fake-storage-ID-Ignored", new ReceivedDeletedBlockInfo
[] { new ReceivedDeletedBlockInfo(blocks[0], ReceivedDeletedBlockInfo.BlockStatus
.ReceivedBlock, string.Empty) }) };
cluster.GetNameNodeRpc().BlockReceivedAndDeleted(dnR, poolId, report);
reportDnNum++;
}
}
NUnit.Framework.Assert.AreEqual(DatanodeCount - 3, blkManager.pendingReplications
.GetNumReplicas(blocks[0]));
// re-enable heartbeat for the datanode that has data
for (int i_3 = 0; i_3 < DatanodeCount; i_3++)
{
DataNodeTestUtils.SetHeartbeatsDisabledForTests(datanodes[i_3], false);
DataNodeTestUtils.TriggerHeartbeat(datanodes[i_3]);
}
Sharpen.Thread.Sleep(5000);
NUnit.Framework.Assert.AreEqual(0, blkManager.pendingReplications.Size());
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}