public virtual void TestBlocksCounter()
{
DatanodeDescriptor dd = BlockManagerTestUtil.GetLocalDatanodeDescriptor(true);
NUnit.Framework.Assert.AreEqual(0, dd.NumBlocks());
BlockInfoContiguous blk = new BlockInfoContiguous(new Block(1L), (short)1);
BlockInfoContiguous blk1 = new BlockInfoContiguous(new Block(2L), (short)2);
DatanodeStorageInfo[] storages = dd.GetStorageInfos();
NUnit.Framework.Assert.IsTrue(storages.Length > 0);
// add first block
NUnit.Framework.Assert.IsTrue(storages[0].AddBlock(blk) == DatanodeStorageInfo.AddBlockResult
.Added);
NUnit.Framework.Assert.AreEqual(1, dd.NumBlocks());
// remove a non-existent block
NUnit.Framework.Assert.IsFalse(dd.RemoveBlock(blk1));
NUnit.Framework.Assert.AreEqual(1, dd.NumBlocks());
// add an existent block
NUnit.Framework.Assert.IsFalse(storages[0].AddBlock(blk) == DatanodeStorageInfo.AddBlockResult
.Added);
NUnit.Framework.Assert.AreEqual(1, dd.NumBlocks());
// add second block
NUnit.Framework.Assert.IsTrue(storages[0].AddBlock(blk1) == DatanodeStorageInfo.AddBlockResult
.Added);
NUnit.Framework.Assert.AreEqual(2, dd.NumBlocks());
// remove first block
NUnit.Framework.Assert.IsTrue(dd.RemoveBlock(blk));
NUnit.Framework.Assert.AreEqual(1, dd.NumBlocks());
// remove second block
NUnit.Framework.Assert.IsTrue(dd.RemoveBlock(blk1));
NUnit.Framework.Assert.AreEqual(0, dd.NumBlocks());
}
public virtual void TestChooseTargetWithDecomNodes()
{
namenode.GetNamesystem().WriteLock();
try
{
string blockPoolId = namenode.GetNamesystem().GetBlockPoolId();
dnManager.HandleHeartbeat(dnrList[3], BlockManagerTestUtil.GetStorageReportsForDatanode
(dataNodes[3]), blockPoolId, dataNodes[3].GetCacheCapacity(), dataNodes[3].GetCacheRemaining
(), 2, 0, 0, null);
dnManager.HandleHeartbeat(dnrList[4], BlockManagerTestUtil.GetStorageReportsForDatanode
(dataNodes[4]), blockPoolId, dataNodes[4].GetCacheCapacity(), dataNodes[4].GetCacheRemaining
(), 4, 0, 0, null);
dnManager.HandleHeartbeat(dnrList[5], BlockManagerTestUtil.GetStorageReportsForDatanode
(dataNodes[5]), blockPoolId, dataNodes[5].GetCacheCapacity(), dataNodes[5].GetCacheRemaining
(), 4, 0, 0, null);
// value in the above heartbeats
int load = 2 + 4 + 4;
FSNamesystem fsn = namenode.GetNamesystem();
NUnit.Framework.Assert.AreEqual((double)load / 6, dnManager.GetFSClusterStats().GetInServiceXceiverAverage
(), Epsilon);
// Decommission DNs so BlockPlacementPolicyDefault.isGoodTarget()
// returns false
for (int i = 0; i < 3; i++)
{
DatanodeDescriptor d = dnManager.GetDatanode(dnrList[i]);
dnManager.GetDecomManager().StartDecommission(d);
d.SetDecommissioned();
}
NUnit.Framework.Assert.AreEqual((double)load / 3, dnManager.GetFSClusterStats().GetInServiceXceiverAverage
(), Epsilon);
// update references of writer DN to update the de-commissioned state
IList <DatanodeDescriptor> liveNodes = new AList <DatanodeDescriptor>();
dnManager.FetchDatanodes(liveNodes, null, false);
DatanodeDescriptor writerDn = null;
if (liveNodes.Contains(dataNodes[0]))
{
writerDn = liveNodes[liveNodes.IndexOf(dataNodes[0])];
}
// Call chooseTarget()
DatanodeStorageInfo[] targets = namenode.GetNamesystem().GetBlockManager().GetBlockPlacementPolicy
().ChooseTarget("testFile.txt", 3, writerDn, new AList <DatanodeStorageInfo>(), false
, null, 1024, TestBlockStoragePolicy.DefaultStoragePolicy);
NUnit.Framework.Assert.AreEqual(3, targets.Length);
ICollection <DatanodeStorageInfo> targetSet = new HashSet <DatanodeStorageInfo>(Arrays
.AsList(targets));
for (int i_1 = 3; i_1 < storages.Length; i_1++)
{
NUnit.Framework.Assert.IsTrue(targetSet.Contains(storages[i_1]));
}
}
finally
{
dataNodes[0].StopDecommission();
dataNodes[1].StopDecommission();
dataNodes[2].StopDecommission();
namenode.GetNamesystem().WriteUnlock();
}
}
private static void UpdateHeartbeatWithUsage(DatanodeDescriptor dn, long capacity
, long dfsUsed, long remaining, long blockPoolUsed, long dnCacheCapacity, long dnCacheUsed
, int xceiverCount, int volFailures)
{
dn.GetStorageInfos()[0].SetUtilizationForTesting(capacity, dfsUsed, remaining, blockPoolUsed
);
dn.UpdateHeartbeat(BlockManagerTestUtil.GetStorageReportsForDatanode(dn), dnCacheCapacity
, dnCacheUsed, xceiverCount, volFailures, null);
}
private void AddNodes(IEnumerable <DatanodeDescriptor> nodesToAdd)
{
NetworkTopology cluster = bm.GetDatanodeManager().GetNetworkTopology();
// construct network topology
foreach (DatanodeDescriptor dn in nodesToAdd)
{
cluster.Add(dn);
dn.GetStorageInfos()[0].SetUtilizationForTesting(2 * HdfsConstants.MinBlocksForWrite
* BlockSize, 0L, 2 * HdfsConstants.MinBlocksForWrite * BlockSize, 0L);
dn.UpdateHeartbeat(BlockManagerTestUtil.GetStorageReportsForDatanode(dn), 0L, 0L,
0, 0, null);
bm.GetDatanodeManager().CheckIfClusterIsNowMultiRack(dn);
}
}
public virtual void TestBlocksAreNotUnderreplicatedInSingleRack()
{
IList <DatanodeDescriptor> nodes = ImmutableList.Of(BlockManagerTestUtil.GetDatanodeDescriptor
("1.1.1.1", "/rackA", true), BlockManagerTestUtil.GetDatanodeDescriptor("2.2.2.2"
, "/rackA", true), BlockManagerTestUtil.GetDatanodeDescriptor("3.3.3.3", "/rackA"
, true), BlockManagerTestUtil.GetDatanodeDescriptor("4.4.4.4", "/rackA", true),
BlockManagerTestUtil.GetDatanodeDescriptor("5.5.5.5", "/rackA", true), BlockManagerTestUtil
.GetDatanodeDescriptor("6.6.6.6", "/rackA", true));
AddNodes(nodes);
IList <DatanodeDescriptor> origNodes = nodes.SubList(0, 3);
for (int i = 0; i < NumTestIters; i++)
{
DoTestSingleRackClusterIsSufficientlyReplicated(i, origNodes);
}
}
public static void SetupCluster()
{
Configuration conf = new HdfsConfiguration();
string[] racks = new string[] { "/rack1", "/rack1", "/rack1", "/rack2", "/rack2",
"/rack2" };
storages = DFSTestUtil.CreateDatanodeStorageInfos(racks);
dataNodes = DFSTestUtil.ToDatanodeDescriptor(storages);
FileSystem.SetDefaultUri(conf, "hdfs://localhost:0");
conf.Set(DFSConfigKeys.DfsNamenodeHttpAddressKey, "0.0.0.0:0");
FilePath baseDir = PathUtils.GetTestDir(typeof(TestReplicationPolicy));
conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, new FilePath(baseDir, "name").GetPath
());
conf.SetBoolean(DFSConfigKeys.DfsNamenodeAvoidStaleDatanodeForReadKey, true);
conf.SetBoolean(DFSConfigKeys.DfsNamenodeAvoidStaleDatanodeForWriteKey, true);
conf.SetBoolean(DFSConfigKeys.DfsNamenodeReplicationConsiderloadKey, true);
DFSTestUtil.FormatNameNode(conf);
namenode = new NameNode(conf);
int blockSize = 1024;
dnrList = new AList <DatanodeRegistration>();
dnManager = namenode.GetNamesystem().GetBlockManager().GetDatanodeManager();
// Register DNs
for (int i = 0; i < 6; i++)
{
DatanodeRegistration dnr = new DatanodeRegistration(dataNodes[i], new StorageInfo
(HdfsServerConstants.NodeType.DataNode), new ExportedBlockKeys(), VersionInfo.GetVersion
());
dnrList.AddItem(dnr);
dnManager.RegisterDatanode(dnr);
dataNodes[i].GetStorageInfos()[0].SetUtilizationForTesting(2 * HdfsConstants.MinBlocksForWrite
* blockSize, 0L, 2 * HdfsConstants.MinBlocksForWrite * blockSize, 0L);
dataNodes[i].UpdateHeartbeat(BlockManagerTestUtil.GetStorageReportsForDatanode(dataNodes
[i]), 0L, 0L, 0, 0, null);
}
}
public virtual void TestPendingAndInvalidate()
{
Configuration Conf = new HdfsConfiguration();
Conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, 1024);
Conf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, DfsReplicationInterval);
Conf.SetInt(DFSConfigKeys.DfsNamenodeReplicationIntervalKey, DfsReplicationInterval
);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(Conf).NumDataNodes(DatanodeCount
).Build();
cluster.WaitActive();
FSNamesystem namesystem = cluster.GetNamesystem();
BlockManager bm = namesystem.GetBlockManager();
DistributedFileSystem fs = cluster.GetFileSystem();
try
{
// 1. create a file
Path filePath = new Path("/tmp.txt");
DFSTestUtil.CreateFile(fs, filePath, 1024, (short)3, 0L);
// 2. disable the heartbeats
foreach (DataNode dn in cluster.GetDataNodes())
{
DataNodeTestUtils.SetHeartbeatsDisabledForTests(dn, true);
}
// 3. mark a couple of blocks as corrupt
LocatedBlock block = NameNodeAdapter.GetBlockLocations(cluster.GetNameNode(), filePath
.ToString(), 0, 1).Get(0);
cluster.GetNamesystem().WriteLock();
try
{
bm.FindAndMarkBlockAsCorrupt(block.GetBlock(), block.GetLocations()[0], "STORAGE_ID"
, "TEST");
bm.FindAndMarkBlockAsCorrupt(block.GetBlock(), block.GetLocations()[1], "STORAGE_ID"
, "TEST");
}
finally
{
cluster.GetNamesystem().WriteUnlock();
}
BlockManagerTestUtil.ComputeAllPendingWork(bm);
BlockManagerTestUtil.UpdateState(bm);
NUnit.Framework.Assert.AreEqual(bm.GetPendingReplicationBlocksCount(), 1L);
NUnit.Framework.Assert.AreEqual(bm.pendingReplications.GetNumReplicas(block.GetBlock
().GetLocalBlock()), 2);
// 4. delete the file
fs.Delete(filePath, true);
// retry at most 10 times, each time sleep for 1s. Note that 10s is much
// less than the default pending record timeout (5~10min)
int retries = 10;
long pendingNum = bm.GetPendingReplicationBlocksCount();
while (pendingNum != 0 && retries-- > 0)
{
Sharpen.Thread.Sleep(1000);
// let NN do the deletion
BlockManagerTestUtil.UpdateState(bm);
pendingNum = bm.GetPendingReplicationBlocksCount();
}
NUnit.Framework.Assert.AreEqual(pendingNum, 0L);
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestBlockReceived()
{
Configuration conf = new HdfsConfiguration();
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, 1024);
MiniDFSCluster cluster = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(DatanodeCount).Build();
cluster.WaitActive();
DistributedFileSystem hdfs = cluster.GetFileSystem();
FSNamesystem fsn = cluster.GetNamesystem();
BlockManager blkManager = fsn.GetBlockManager();
string file = "/tmp.txt";
Path filePath = new Path(file);
short replFactor = 1;
DFSTestUtil.CreateFile(hdfs, filePath, 1024L, replFactor, 0);
// temporarily stop the heartbeat
AList <DataNode> datanodes = cluster.GetDataNodes();
for (int i = 0; i < DatanodeCount; i++)
{
DataNodeTestUtils.SetHeartbeatsDisabledForTests(datanodes[i], true);
}
hdfs.SetReplication(filePath, (short)DatanodeCount);
BlockManagerTestUtil.ComputeAllPendingWork(blkManager);
NUnit.Framework.Assert.AreEqual(1, blkManager.pendingReplications.Size());
INodeFile fileNode = fsn.GetFSDirectory().GetINode4Write(file).AsFile();
Block[] blocks = fileNode.GetBlocks();
NUnit.Framework.Assert.AreEqual(DatanodeCount - 1, blkManager.pendingReplications
.GetNumReplicas(blocks[0]));
LocatedBlock locatedBlock = hdfs.GetClient().GetLocatedBlocks(file, 0).Get(0);
DatanodeInfo existingDn = (locatedBlock.GetLocations())[0];
int reportDnNum = 0;
string poolId = cluster.GetNamesystem().GetBlockPoolId();
// let two datanodes (other than the one that already has the data) to
// report to NN
for (int i_1 = 0; i_1 < DatanodeCount && reportDnNum < 2; i_1++)
{
if (!datanodes[i_1].GetDatanodeId().Equals(existingDn))
{
DatanodeRegistration dnR = datanodes[i_1].GetDNRegistrationForBP(poolId);
StorageReceivedDeletedBlocks[] report = new StorageReceivedDeletedBlocks[] { new
StorageReceivedDeletedBlocks("Fake-storage-ID-Ignored", new ReceivedDeletedBlockInfo
[] { new ReceivedDeletedBlockInfo(blocks[0], ReceivedDeletedBlockInfo.BlockStatus
.ReceivedBlock, string.Empty) }) };
cluster.GetNameNodeRpc().BlockReceivedAndDeleted(dnR, poolId, report);
reportDnNum++;
}
}
NUnit.Framework.Assert.AreEqual(DatanodeCount - 3, blkManager.pendingReplications
.GetNumReplicas(blocks[0]));
// let the same datanodes report again
for (int i_2 = 0; i_2 < DatanodeCount && reportDnNum < 2; i_2++)
{
if (!datanodes[i_2].GetDatanodeId().Equals(existingDn))
{
DatanodeRegistration dnR = datanodes[i_2].GetDNRegistrationForBP(poolId);
StorageReceivedDeletedBlocks[] report = new StorageReceivedDeletedBlocks[] { new
StorageReceivedDeletedBlocks("Fake-storage-ID-Ignored", new ReceivedDeletedBlockInfo
[] { new ReceivedDeletedBlockInfo(blocks[0], ReceivedDeletedBlockInfo.BlockStatus
.ReceivedBlock, string.Empty) }) };
cluster.GetNameNodeRpc().BlockReceivedAndDeleted(dnR, poolId, report);
reportDnNum++;
}
}
NUnit.Framework.Assert.AreEqual(DatanodeCount - 3, blkManager.pendingReplications
.GetNumReplicas(blocks[0]));
// re-enable heartbeat for the datanode that has data
for (int i_3 = 0; i_3 < DatanodeCount; i_3++)
{
DataNodeTestUtils.SetHeartbeatsDisabledForTests(datanodes[i_3], false);
DataNodeTestUtils.TriggerHeartbeat(datanodes[i_3]);
}
Sharpen.Thread.Sleep(5000);
NUnit.Framework.Assert.AreEqual(0, blkManager.pendingReplications.Size());
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestProcesOverReplicateBlock()
{
Configuration conf = new HdfsConfiguration();
conf.SetLong(DFSConfigKeys.DfsDatanodeScanPeriodHoursKey, 100L);
conf.SetLong(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 1000L);
conf.Set(DFSConfigKeys.DfsNamenodeReplicationPendingTimeoutSecKey, Sharpen.Extensions.ToString
(2));
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build();
FileSystem fs = cluster.GetFileSystem();
try
{
Path fileName = new Path("/foo1");
DFSTestUtil.CreateFile(fs, fileName, 2, (short)3, 0L);
DFSTestUtil.WaitReplication(fs, fileName, (short)3);
// corrupt the block on datanode 0
ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, fileName);
NUnit.Framework.Assert.IsTrue(cluster.CorruptReplica(0, block));
MiniDFSCluster.DataNodeProperties dnProps = cluster.StopDataNode(0);
// remove block scanner log to trigger block scanning
FilePath scanCursor = new FilePath(new FilePath(MiniDFSCluster.GetFinalizedDir(cluster
.GetInstanceStorageDir(0, 0), cluster.GetNamesystem().GetBlockPoolId()).GetParent
()).GetParent(), "scanner.cursor");
//wait for one minute for deletion to succeed;
for (int i = 0; !scanCursor.Delete(); i++)
{
NUnit.Framework.Assert.IsTrue("Could not delete " + scanCursor.GetAbsolutePath()
+ " in one minute", i < 60);
try
{
Sharpen.Thread.Sleep(1000);
}
catch (Exception)
{
}
}
// restart the datanode so the corrupt replica will be detected
cluster.RestartDataNode(dnProps);
DFSTestUtil.WaitReplication(fs, fileName, (short)2);
string blockPoolId = cluster.GetNamesystem().GetBlockPoolId();
DatanodeID corruptDataNode = DataNodeTestUtils.GetDNRegistrationForBP(cluster.GetDataNodes
()[2], blockPoolId);
FSNamesystem namesystem = cluster.GetNamesystem();
BlockManager bm = namesystem.GetBlockManager();
HeartbeatManager hm = bm.GetDatanodeManager().GetHeartbeatManager();
try
{
namesystem.WriteLock();
lock (hm)
{
// set live datanode's remaining space to be 0
// so they will be chosen to be deleted when over-replication occurs
string corruptMachineName = corruptDataNode.GetXferAddr();
foreach (DatanodeDescriptor datanode in hm.GetDatanodes())
{
if (!corruptMachineName.Equals(datanode.GetXferAddr()))
{
datanode.GetStorageInfos()[0].SetUtilizationForTesting(100L, 100L, 0, 100L);
datanode.UpdateHeartbeat(BlockManagerTestUtil.GetStorageReportsForDatanode(datanode
), 0L, 0L, 0, 0, null);
}
}
// decrease the replication factor to 1;
NameNodeAdapter.SetReplication(namesystem, fileName.ToString(), (short)1);
// corrupt one won't be chosen to be excess one
// without 4910 the number of live replicas would be 0: block gets lost
NUnit.Framework.Assert.AreEqual(1, bm.CountNodes(block.GetLocalBlock()).LiveReplicas
());
}
}
finally
{
namesystem.WriteUnlock();
}
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestNodeCount()
{
// start a mini dfs cluster of 2 nodes
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(ReplicationFactor
).Build();
try
{
FSNamesystem namesystem = cluster.GetNamesystem();
BlockManager bm = namesystem.GetBlockManager();
HeartbeatManager hm = bm.GetDatanodeManager().GetHeartbeatManager();
FileSystem fs = cluster.GetFileSystem();
// populate the cluster with a one block file
Path FilePath = new Path("/testfile");
DFSTestUtil.CreateFile(fs, FilePath, 1L, ReplicationFactor, 1L);
DFSTestUtil.WaitReplication(fs, FilePath, ReplicationFactor);
ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, FilePath);
// keep a copy of all datanode descriptor
DatanodeDescriptor[] datanodes = hm.GetDatanodes();
// start two new nodes
cluster.StartDataNodes(conf, 2, true, null, null);
cluster.WaitActive();
// bring down first datanode
DatanodeDescriptor datanode = datanodes[0];
MiniDFSCluster.DataNodeProperties dnprop = cluster.StopDataNode(datanode.GetXferAddr
());
// make sure that NN detects that the datanode is down
BlockManagerTestUtil.NoticeDeadDatanode(cluster.GetNameNode(), datanode.GetXferAddr
());
// the block will be replicated
DFSTestUtil.WaitReplication(fs, FilePath, ReplicationFactor);
// restart the first datanode
cluster.RestartDataNode(dnprop);
cluster.WaitActive();
// check if excessive replica is detected (transient)
InitializeTimeout(Timeout);
while (CountNodes(block.GetLocalBlock(), namesystem).ExcessReplicas() == 0)
{
CheckTimeout("excess replicas not detected");
}
// find out a non-excess node
DatanodeDescriptor nonExcessDN = null;
foreach (DatanodeStorageInfo storage in bm.blocksMap.GetStorages(block.GetLocalBlock
()))
{
DatanodeDescriptor dn = storage.GetDatanodeDescriptor();
ICollection <Block> blocks = bm.excessReplicateMap[dn.GetDatanodeUuid()];
if (blocks == null || !blocks.Contains(block.GetLocalBlock()))
{
nonExcessDN = dn;
break;
}
}
NUnit.Framework.Assert.IsTrue(nonExcessDN != null);
// bring down non excessive datanode
dnprop = cluster.StopDataNode(nonExcessDN.GetXferAddr());
// make sure that NN detects that the datanode is down
BlockManagerTestUtil.NoticeDeadDatanode(cluster.GetNameNode(), nonExcessDN.GetXferAddr
());
// The block should be replicated
InitializeTimeout(Timeout);
while (CountNodes(block.GetLocalBlock(), namesystem).LiveReplicas() != ReplicationFactor
)
{
CheckTimeout("live replica count not correct", 1000);
}
// restart the first datanode
cluster.RestartDataNode(dnprop);
cluster.WaitActive();
// check if excessive replica is detected (transient)
InitializeTimeout(Timeout);
while (CountNodes(block.GetLocalBlock(), namesystem).ExcessReplicas() != 2)
{
CheckTimeout("excess replica count not equal to 2");
}
}
finally
{
cluster.Shutdown();
}
}