public virtual void TestDNSLookups()
{
TestDatanodeRegistration.MonitorDNS sm = new TestDatanodeRegistration.MonitorDNS(
);
Runtime.SetSecurityManager(sm);
MiniDFSCluster cluster = null;
try
{
HdfsConfiguration conf = new HdfsConfiguration();
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(8).Build();
cluster.WaitActive();
int initialLookups = sm.lookups;
NUnit.Framework.Assert.IsTrue("dns security manager is active", initialLookups !=
0);
DatanodeManager dm = cluster.GetNamesystem().GetBlockManager().GetDatanodeManager
();
// make sure no lookups occur
dm.RefreshNodes(conf);
NUnit.Framework.Assert.AreEqual(initialLookups, sm.lookups);
dm.RefreshNodes(conf);
NUnit.Framework.Assert.AreEqual(initialLookups, sm.lookups);
// ensure none of the reports trigger lookups
dm.GetDatanodeListForReport(HdfsConstants.DatanodeReportType.All);
NUnit.Framework.Assert.AreEqual(initialLookups, sm.lookups);
dm.GetDatanodeListForReport(HdfsConstants.DatanodeReportType.Live);
NUnit.Framework.Assert.AreEqual(initialLookups, sm.lookups);
dm.GetDatanodeListForReport(HdfsConstants.DatanodeReportType.Dead);
NUnit.Framework.Assert.AreEqual(initialLookups, sm.lookups);
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
Runtime.SetSecurityManager(null);
}
}
public virtual void TestIncludeExcludeLists()
{
BlockManager bm = Org.Mockito.Mockito.Mock <BlockManager>();
FSNamesystem fsn = Org.Mockito.Mockito.Mock <FSNamesystem>();
Configuration conf = new Configuration();
HostFileManager hm = new HostFileManager();
HostFileManager.HostSet includedNodes = new HostFileManager.HostSet();
HostFileManager.HostSet excludedNodes = new HostFileManager.HostSet();
includedNodes.Add(Entry("127.0.0.1:12345"));
includedNodes.Add(Entry("localhost:12345"));
includedNodes.Add(Entry("127.0.0.1:12345"));
includedNodes.Add(Entry("127.0.0.2"));
excludedNodes.Add(Entry("127.0.0.1:12346"));
excludedNodes.Add(Entry("127.0.30.1:12346"));
NUnit.Framework.Assert.AreEqual(2, includedNodes.Size());
NUnit.Framework.Assert.AreEqual(2, excludedNodes.Size());
hm.Refresh(includedNodes, excludedNodes);
DatanodeManager dm = new DatanodeManager(bm, fsn, conf);
Whitebox.SetInternalState(dm, "hostFileManager", hm);
IDictionary <string, DatanodeDescriptor> dnMap = (IDictionary <string, DatanodeDescriptor
>)Whitebox.GetInternalState(dm, "datanodeMap");
// After the de-duplication, there should be only one DN from the included
// nodes declared as dead.
NUnit.Framework.Assert.AreEqual(2, dm.GetDatanodeListForReport(HdfsConstants.DatanodeReportType
.All).Count);
NUnit.Framework.Assert.AreEqual(2, dm.GetDatanodeListForReport(HdfsConstants.DatanodeReportType
.Dead).Count);
dnMap["uuid-foo"] = new DatanodeDescriptor(new DatanodeID("127.0.0.1", "localhost"
, "uuid-foo", 12345, 1020, 1021, 1022));
NUnit.Framework.Assert.AreEqual(1, dm.GetDatanodeListForReport(HdfsConstants.DatanodeReportType
.Dead).Count);
dnMap["uuid-bar"] = new DatanodeDescriptor(new DatanodeID("127.0.0.2", "127.0.0.2"
, "uuid-bar", 12345, 1020, 1021, 1022));
NUnit.Framework.Assert.AreEqual(0, dm.GetDatanodeListForReport(HdfsConstants.DatanodeReportType
.Dead).Count);
DatanodeDescriptor spam = new DatanodeDescriptor(new DatanodeID("127.0.0" + ".3",
"127.0.0.3", "uuid-spam", 12345, 1020, 1021, 1022));
DFSTestUtil.SetDatanodeDead(spam);
includedNodes.Add(Entry("127.0.0.3:12345"));
dnMap["uuid-spam"] = spam;
NUnit.Framework.Assert.AreEqual(1, dm.GetDatanodeListForReport(HdfsConstants.DatanodeReportType
.Dead).Count);
Sharpen.Collections.Remove(dnMap, "uuid-spam");
NUnit.Framework.Assert.AreEqual(1, dm.GetDatanodeListForReport(HdfsConstants.DatanodeReportType
.Dead).Count);
excludedNodes.Add(Entry("127.0.0.3"));
NUnit.Framework.Assert.AreEqual(0, dm.GetDatanodeListForReport(HdfsConstants.DatanodeReportType
.Dead).Count);
}
public virtual void TestRemoveIncludedNode()
{
FSNamesystem fsn = Org.Mockito.Mockito.Mock <FSNamesystem>();
// Set the write lock so that the DatanodeManager can start
Org.Mockito.Mockito.When(fsn.HasWriteLock()).ThenReturn(true);
DatanodeManager dm = MockDatanodeManager(fsn, new Configuration());
HostFileManager hm = new HostFileManager();
HostFileManager.HostSet noNodes = new HostFileManager.HostSet();
HostFileManager.HostSet oneNode = new HostFileManager.HostSet();
HostFileManager.HostSet twoNodes = new HostFileManager.HostSet();
DatanodeRegistration dr1 = new DatanodeRegistration(new DatanodeID("127.0.0.1", "127.0.0.1"
, "someStorageID-123", 12345, 12345, 12345, 12345), new StorageInfo(HdfsServerConstants.NodeType
.DataNode), new ExportedBlockKeys(), "test");
DatanodeRegistration dr2 = new DatanodeRegistration(new DatanodeID("127.0.0.1", "127.0.0.1"
, "someStorageID-234", 23456, 23456, 23456, 23456), new StorageInfo(HdfsServerConstants.NodeType
.DataNode), new ExportedBlockKeys(), "test");
twoNodes.Add(Entry("127.0.0.1:12345"));
twoNodes.Add(Entry("127.0.0.1:23456"));
oneNode.Add(Entry("127.0.0.1:23456"));
hm.Refresh(twoNodes, noNodes);
Whitebox.SetInternalState(dm, "hostFileManager", hm);
// Register two data nodes to simulate them coming up.
// We need to add two nodes, because if we have only one node, removing it
// will cause the includes list to be empty, which means all hosts will be
// allowed.
dm.RegisterDatanode(dr1);
dm.RegisterDatanode(dr2);
// Make sure that both nodes are reported
IList <DatanodeDescriptor> both = dm.GetDatanodeListForReport(HdfsConstants.DatanodeReportType
.All);
// Sort the list so that we know which one is which
both.Sort();
NUnit.Framework.Assert.AreEqual("Incorrect number of hosts reported", 2, both.Count
);
NUnit.Framework.Assert.AreEqual("Unexpected host or host in unexpected position",
"127.0.0.1:12345", both[0].GetInfoAddr());
NUnit.Framework.Assert.AreEqual("Unexpected host or host in unexpected position",
"127.0.0.1:23456", both[1].GetInfoAddr());
// Remove one node from includes, but do not add it to excludes.
hm.Refresh(oneNode, noNodes);
// Make sure that only one node is still reported
IList <DatanodeDescriptor> onlyOne = dm.GetDatanodeListForReport(HdfsConstants.DatanodeReportType
.All);
NUnit.Framework.Assert.AreEqual("Incorrect number of hosts reported", 1, onlyOne.
Count);
NUnit.Framework.Assert.AreEqual("Unexpected host reported", "127.0.0.1:23456", onlyOne
[0].GetInfoAddr());
// Remove all nodes from includes
hm.Refresh(noNodes, noNodes);
// Check that both nodes are reported again
IList <DatanodeDescriptor> bothAgain = dm.GetDatanodeListForReport(HdfsConstants.DatanodeReportType
.All);
// Sort the list so that we know which one is which
bothAgain.Sort();
NUnit.Framework.Assert.AreEqual("Incorrect number of hosts reported", 2, bothAgain
.Count);
NUnit.Framework.Assert.AreEqual("Unexpected host or host in unexpected position",
"127.0.0.1:12345", bothAgain[0].GetInfoAddr());
NUnit.Framework.Assert.AreEqual("Unexpected host or host in unexpected position",
"127.0.0.1:23456", bothAgain[1].GetInfoAddr());
}