public virtual void TestDeadDatanode()
{
Configuration conf = new HdfsConfiguration();
conf.SetInt(DFSConfigKeys.DfsNamenodeHeartbeatRecheckIntervalKey, 500);
conf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, 1L);
cluster = new MiniDFSCluster.Builder(conf).Build();
cluster.WaitActive();
string poolId = cluster.GetNamesystem().GetBlockPoolId();
// wait for datanode to be marked live
DataNode dn = cluster.GetDataNodes()[0];
DatanodeRegistration reg = DataNodeTestUtils.GetDNRegistrationForBP(cluster.GetDataNodes
()[0], poolId);
DFSTestUtil.WaitForDatanodeState(cluster, reg.GetDatanodeUuid(), true, 20000);
// Shutdown and wait for datanode to be marked dead
dn.Shutdown();
DFSTestUtil.WaitForDatanodeState(cluster, reg.GetDatanodeUuid(), false, 20000);
DatanodeProtocol dnp = cluster.GetNameNodeRpc();
ReceivedDeletedBlockInfo[] blocks = new ReceivedDeletedBlockInfo[] { new ReceivedDeletedBlockInfo
(new Block(0), ReceivedDeletedBlockInfo.BlockStatus.ReceivedBlock, null) };
StorageReceivedDeletedBlocks[] storageBlocks = new StorageReceivedDeletedBlocks[]
{ new StorageReceivedDeletedBlocks(reg.GetDatanodeUuid(), blocks) };
// Ensure blockReceived call from dead datanode is rejected with IOException
try
{
dnp.BlockReceivedAndDeleted(reg, poolId, storageBlocks);
NUnit.Framework.Assert.Fail("Expected IOException is not thrown");
}
catch (IOException)
{
}
// Expected
// Ensure blockReport from dead datanode is rejected with IOException
StorageBlockReport[] report = new StorageBlockReport[] { new StorageBlockReport(new
DatanodeStorage(reg.GetDatanodeUuid()), BlockListAsLongs.Empty) };
try
{
dnp.BlockReport(reg, poolId, report, new BlockReportContext(1, 0, Runtime.NanoTime
()));
NUnit.Framework.Assert.Fail("Expected IOException is not thrown");
}
catch (IOException)
{
}
// Expected
// Ensure heartbeat from dead datanode is rejected with a command
// that asks datanode to register again
StorageReport[] rep = new StorageReport[] { new StorageReport(new DatanodeStorage
(reg.GetDatanodeUuid()), false, 0, 0, 0, 0) };
DatanodeCommand[] cmd = dnp.SendHeartbeat(reg, rep, 0L, 0L, 0, 0, 0, null).GetCommands
();
NUnit.Framework.Assert.AreEqual(1, cmd.Length);
NUnit.Framework.Assert.AreEqual(cmd[0].GetAction(), RegisterCommand.Register.GetAction
());
}
public virtual void TestChooseReplicaToDelete()
{
MiniDFSCluster cluster = null;
FileSystem fs = null;
try
{
Configuration conf = new HdfsConfiguration();
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, SmallBlockSize);
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build();
fs = cluster.GetFileSystem();
FSNamesystem namesystem = cluster.GetNamesystem();
conf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, 300);
cluster.StartDataNodes(conf, 1, true, null, null, null);
DataNode lastDN = cluster.GetDataNodes()[3];
DatanodeRegistration dnReg = DataNodeTestUtils.GetDNRegistrationForBP(lastDN, namesystem
.GetBlockPoolId());
string lastDNid = dnReg.GetDatanodeUuid();
Path fileName = new Path("/foo2");
DFSTestUtil.CreateFile(fs, fileName, SmallFileLength, (short)4, 0L);
DFSTestUtil.WaitReplication(fs, fileName, (short)4);
// Wait for tolerable number of heartbeats plus one
DatanodeDescriptor nodeInfo = null;
long lastHeartbeat = 0;
long waitTime = DFSConfigKeys.DfsHeartbeatIntervalDefault * 1000 * (DFSConfigKeys
.DfsNamenodeTolerateHeartbeatMultiplierDefault + 1);
do
{
nodeInfo = namesystem.GetBlockManager().GetDatanodeManager().GetDatanode(dnReg);
lastHeartbeat = nodeInfo.GetLastUpdateMonotonic();
}while (Time.MonotonicNow() - lastHeartbeat < waitTime);
fs.SetReplication(fileName, (short)3);
BlockLocation[] locs = fs.GetFileBlockLocations(fs.GetFileStatus(fileName), 0, long.MaxValue
);
// All replicas for deletion should be scheduled on lastDN.
// And should not actually be deleted, because lastDN does not heartbeat.
namesystem.ReadLock();
ICollection <Block> dnBlocks = namesystem.GetBlockManager().excessReplicateMap[lastDNid
];
NUnit.Framework.Assert.AreEqual("Replicas on node " + lastDNid + " should have been deleted"
, SmallFileLength / SmallBlockSize, dnBlocks.Count);
namesystem.ReadUnlock();
foreach (BlockLocation location in locs)
{
NUnit.Framework.Assert.AreEqual("Block should still have 4 replicas", 4, location
.GetNames().Length);
}
}
finally
{
if (fs != null)
{
fs.Close();
}
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestRejectUnresolvedDatanodes()
{
//Create the DatanodeManager which will be tested
FSNamesystem fsn = Org.Mockito.Mockito.Mock <FSNamesystem>();
Org.Mockito.Mockito.When(fsn.HasWriteLock()).ThenReturn(true);
Configuration conf = new Configuration();
//Set configuration property for rejecting unresolved topology mapping
conf.SetBoolean(DFSConfigKeys.DfsRejectUnresolvedDnTopologyMappingKey, true);
//set TestDatanodeManager.MyResolver to be used for topology resolving
conf.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey,
typeof(TestDatanodeManager.MyResolver), typeof(DNSToSwitchMapping));
//create DatanodeManager
DatanodeManager dm = new DatanodeManager(Org.Mockito.Mockito.Mock <BlockManager>()
, fsn, conf);
//storageID to register.
string storageID = "someStorageID-123";
DatanodeRegistration dr = Org.Mockito.Mockito.Mock <DatanodeRegistration>();
Org.Mockito.Mockito.When(dr.GetDatanodeUuid()).ThenReturn(storageID);
try
{
//Register this node
dm.RegisterDatanode(dr);
NUnit.Framework.Assert.Fail("Expected an UnresolvedTopologyException");
}
catch (UnresolvedTopologyException)
{
Log.Info("Expected - topology is not resolved and " + "registration is rejected."
);
}
catch (Exception)
{
NUnit.Framework.Assert.Fail("Expected an UnresolvedTopologyException");
}
}
public virtual void TestNumVersionsReportedCorrect()
{
//Create the DatanodeManager which will be tested
FSNamesystem fsn = Org.Mockito.Mockito.Mock <FSNamesystem>();
Org.Mockito.Mockito.When(fsn.HasWriteLock()).ThenReturn(true);
DatanodeManager dm = new DatanodeManager(Org.Mockito.Mockito.Mock <BlockManager>()
, fsn, new Configuration());
//Seed the RNG with a known value so test failures are easier to reproduce
Random rng = new Random();
int seed = rng.Next();
rng = new Random(seed);
Log.Info("Using seed " + seed + " for testing");
//A map of the Storage IDs to the DN registration it was registered with
Dictionary <string, DatanodeRegistration> sIdToDnReg = new Dictionary <string, DatanodeRegistration
>();
for (int i = 0; i < NumIterations; ++i)
{
//If true, remove a node for every 3rd time (if there's one)
if (rng.NextBoolean() && i % 3 == 0 && sIdToDnReg.Count != 0)
{
//Pick a random node.
int randomIndex = rng.Next() % sIdToDnReg.Count;
//Iterate to that random position
IEnumerator <KeyValuePair <string, DatanodeRegistration> > it = sIdToDnReg.GetEnumerator
();
for (int j = 0; j < randomIndex - 1; ++j)
{
it.Next();
}
DatanodeRegistration toRemove = it.Next().Value;
Log.Info("Removing node " + toRemove.GetDatanodeUuid() + " ip " + toRemove.GetXferAddr
() + " version : " + toRemove.GetSoftwareVersion());
//Remove that random node
dm.RemoveDatanode(toRemove);
it.Remove();
}
else
{
// Otherwise register a node. This node may be a new / an old one
//Pick a random storageID to register.
string storageID = "someStorageID" + rng.Next(5000);
DatanodeRegistration dr = Org.Mockito.Mockito.Mock <DatanodeRegistration>();
Org.Mockito.Mockito.When(dr.GetDatanodeUuid()).ThenReturn(storageID);
//If this storageID had already been registered before
if (sIdToDnReg.Contains(storageID))
{
dr = sIdToDnReg[storageID];
//Half of the times, change the IP address
if (rng.NextBoolean())
{
dr.SetIpAddr(dr.GetIpAddr() + "newIP");
}
}
else
{
//This storageID has never been registered
//Ensure IP address is unique to storageID
string ip = "someIP" + storageID;
Org.Mockito.Mockito.When(dr.GetIpAddr()).ThenReturn(ip);
Org.Mockito.Mockito.When(dr.GetXferAddr()).ThenReturn(ip + ":9000");
Org.Mockito.Mockito.When(dr.GetXferPort()).ThenReturn(9000);
}
//Pick a random version to register with
Org.Mockito.Mockito.When(dr.GetSoftwareVersion()).ThenReturn("version" + rng.Next
(5));
Log.Info("Registering node storageID: " + dr.GetDatanodeUuid() + ", version: " +
dr.GetSoftwareVersion() + ", IP address: " + dr.GetXferAddr());
//Register this random node
dm.RegisterDatanode(dr);
sIdToDnReg[storageID] = dr;
}
//Verify DatanodeManager still has the right count
IDictionary <string, int> mapToCheck = dm.GetDatanodesSoftwareVersions();
//Remove counts from versions and make sure that after removing all nodes
//mapToCheck is empty
foreach (KeyValuePair <string, DatanodeRegistration> it_1 in sIdToDnReg)
{
string ver = it_1.Value.GetSoftwareVersion();
if (!mapToCheck.Contains(ver))
{
throw new Exception("The correct number of datanodes of a " + "version was not found on iteration "
+ i);
}
mapToCheck[ver] = mapToCheck[ver] - 1;
if (mapToCheck[ver] == 0)
{
Sharpen.Collections.Remove(mapToCheck, ver);
}
}
foreach (KeyValuePair <string, int> entry in mapToCheck)
{
Log.Info("Still in map: " + entry.Key + " has " + entry.Value);
}
NUnit.Framework.Assert.AreEqual("The map of version counts returned by DatanodeManager was"
+ " not what it was expected to be on iteration " + i, 0, mapToCheck.Count);
}
}
public virtual void TestSortLocatedBlocks()
{
// create the DatanodeManager which will be tested
FSNamesystem fsn = Org.Mockito.Mockito.Mock <FSNamesystem>();
Org.Mockito.Mockito.When(fsn.HasWriteLock()).ThenReturn(true);
DatanodeManager dm = new DatanodeManager(Org.Mockito.Mockito.Mock <BlockManager>()
, fsn, new Configuration());
// register 5 datanodes, each with different storage ID and type
DatanodeInfo[] locs = new DatanodeInfo[5];
string[] storageIDs = new string[5];
StorageType[] storageTypes = new StorageType[] { StorageType.Archive, StorageType
.Default, StorageType.Disk, StorageType.RamDisk, StorageType.Ssd };
for (int i = 0; i < 5; i++)
{
// register new datanode
string uuid = "UUID-" + i;
string ip = "IP-" + i;
DatanodeRegistration dr = Org.Mockito.Mockito.Mock <DatanodeRegistration>();
Org.Mockito.Mockito.When(dr.GetDatanodeUuid()).ThenReturn(uuid);
Org.Mockito.Mockito.When(dr.GetIpAddr()).ThenReturn(ip);
Org.Mockito.Mockito.When(dr.GetXferAddr()).ThenReturn(ip + ":9000");
Org.Mockito.Mockito.When(dr.GetXferPort()).ThenReturn(9000);
Org.Mockito.Mockito.When(dr.GetSoftwareVersion()).ThenReturn("version1");
dm.RegisterDatanode(dr);
// get location and storage information
locs[i] = dm.GetDatanode(uuid);
storageIDs[i] = "storageID-" + i;
}
// set first 2 locations as decomissioned
locs[0].SetDecommissioned();
locs[1].SetDecommissioned();
// create LocatedBlock with above locations
ExtendedBlock b = new ExtendedBlock("somePoolID", 1234);
LocatedBlock block = new LocatedBlock(b, locs, storageIDs, storageTypes);
IList <LocatedBlock> blocks = new AList <LocatedBlock>();
blocks.AddItem(block);
string targetIp = locs[4].GetIpAddr();
// sort block locations
dm.SortLocatedBlocks(targetIp, blocks);
// check that storage IDs/types are aligned with datanode locs
DatanodeInfo[] sortedLocs = block.GetLocations();
storageIDs = block.GetStorageIDs();
storageTypes = block.GetStorageTypes();
Assert.AssertThat(sortedLocs.Length, IS.Is(5));
Assert.AssertThat(storageIDs.Length, IS.Is(5));
Assert.AssertThat(storageTypes.Length, IS.Is(5));
for (int i_1 = 0; i_1 < sortedLocs.Length; i_1++)
{
Assert.AssertThat(((DatanodeInfoWithStorage)sortedLocs[i_1]).GetStorageID(), IS.Is
(storageIDs[i_1]));
Assert.AssertThat(((DatanodeInfoWithStorage)sortedLocs[i_1]).GetStorageType(), IS.Is
(storageTypes[i_1]));
}
// Ensure the local node is first.
Assert.AssertThat(sortedLocs[0].GetIpAddr(), IS.Is(targetIp));
// Ensure the two decommissioned DNs were moved to the end.
Assert.AssertThat(sortedLocs[sortedLocs.Length - 1].GetAdminState(), IS.Is(DatanodeInfo.AdminStates
.Decommissioned));
Assert.AssertThat(sortedLocs[sortedLocs.Length - 2].GetAdminState(), IS.Is(DatanodeInfo.AdminStates
.Decommissioned));
}