internal virtual void Refresh(HostFileManager.HostSet newIncludes, HostFileManager.HostSet
newExcludes)
{
lock (this)
{
includes = newIncludes;
excludes = newExcludes;
}
}
public virtual void TestIncludeExcludeLists()
{
BlockManager bm = Org.Mockito.Mockito.Mock <BlockManager>();
FSNamesystem fsn = Org.Mockito.Mockito.Mock <FSNamesystem>();
Configuration conf = new Configuration();
HostFileManager hm = new HostFileManager();
HostFileManager.HostSet includedNodes = new HostFileManager.HostSet();
HostFileManager.HostSet excludedNodes = new HostFileManager.HostSet();
includedNodes.Add(Entry("127.0.0.1:12345"));
includedNodes.Add(Entry("localhost:12345"));
includedNodes.Add(Entry("127.0.0.1:12345"));
includedNodes.Add(Entry("127.0.0.2"));
excludedNodes.Add(Entry("127.0.0.1:12346"));
excludedNodes.Add(Entry("127.0.30.1:12346"));
NUnit.Framework.Assert.AreEqual(2, includedNodes.Size());
NUnit.Framework.Assert.AreEqual(2, excludedNodes.Size());
hm.Refresh(includedNodes, excludedNodes);
DatanodeManager dm = new DatanodeManager(bm, fsn, conf);
Whitebox.SetInternalState(dm, "hostFileManager", hm);
IDictionary <string, DatanodeDescriptor> dnMap = (IDictionary <string, DatanodeDescriptor
>)Whitebox.GetInternalState(dm, "datanodeMap");
// After the de-duplication, there should be only one DN from the included
// nodes declared as dead.
NUnit.Framework.Assert.AreEqual(2, dm.GetDatanodeListForReport(HdfsConstants.DatanodeReportType
.All).Count);
NUnit.Framework.Assert.AreEqual(2, dm.GetDatanodeListForReport(HdfsConstants.DatanodeReportType
.Dead).Count);
dnMap["uuid-foo"] = new DatanodeDescriptor(new DatanodeID("127.0.0.1", "localhost"
, "uuid-foo", 12345, 1020, 1021, 1022));
NUnit.Framework.Assert.AreEqual(1, dm.GetDatanodeListForReport(HdfsConstants.DatanodeReportType
.Dead).Count);
dnMap["uuid-bar"] = new DatanodeDescriptor(new DatanodeID("127.0.0.2", "127.0.0.2"
, "uuid-bar", 12345, 1020, 1021, 1022));
NUnit.Framework.Assert.AreEqual(0, dm.GetDatanodeListForReport(HdfsConstants.DatanodeReportType
.Dead).Count);
DatanodeDescriptor spam = new DatanodeDescriptor(new DatanodeID("127.0.0" + ".3",
"127.0.0.3", "uuid-spam", 12345, 1020, 1021, 1022));
DFSTestUtil.SetDatanodeDead(spam);
includedNodes.Add(Entry("127.0.0.3:12345"));
dnMap["uuid-spam"] = spam;
NUnit.Framework.Assert.AreEqual(1, dm.GetDatanodeListForReport(HdfsConstants.DatanodeReportType
.Dead).Count);
Sharpen.Collections.Remove(dnMap, "uuid-spam");
NUnit.Framework.Assert.AreEqual(1, dm.GetDatanodeListForReport(HdfsConstants.DatanodeReportType
.Dead).Count);
excludedNodes.Add(Entry("127.0.0.3"));
NUnit.Framework.Assert.AreEqual(0, dm.GetDatanodeListForReport(HdfsConstants.DatanodeReportType
.Dead).Count);
}
public virtual void TestDeduplication()
{
HostFileManager.HostSet s = new HostFileManager.HostSet();
// These entries will be de-duped, since they refer to the same IP
// address + port combo.
s.Add(Entry("127.0.0.1:12345"));
s.Add(Entry("localhost:12345"));
NUnit.Framework.Assert.AreEqual(1, s.Size());
s.Add(Entry("127.0.0.1:12345"));
NUnit.Framework.Assert.AreEqual(1, s.Size());
// The following entries should not be de-duped.
s.Add(Entry("127.0.0.1:12346"));
NUnit.Framework.Assert.AreEqual(2, s.Size());
s.Add(Entry("127.0.0.1"));
NUnit.Framework.Assert.AreEqual(3, s.Size());
s.Add(Entry("127.0.0.10"));
NUnit.Framework.Assert.AreEqual(4, s.Size());
}
/// <exception cref="System.IO.IOException"/>
private static HostFileManager.HostSet ReadFile(string type, string filename)
{
HostFileManager.HostSet res = new HostFileManager.HostSet();
if (!filename.IsEmpty())
{
HashSet <string> entrySet = new HashSet <string>();
HostsFileReader.ReadFileToSet(type, filename, entrySet);
foreach (string str in entrySet)
{
IPEndPoint addr = ParseEntry(type, filename, str);
if (addr != null)
{
res.Add(addr);
}
}
}
return(res);
}
public virtual void TestRelation()
{
HostFileManager.HostSet s = new HostFileManager.HostSet();
s.Add(Entry("127.0.0.1:123"));
NUnit.Framework.Assert.IsTrue(s.Match(Entry("127.0.0.1:123")));
NUnit.Framework.Assert.IsFalse(s.Match(Entry("127.0.0.1:12")));
NUnit.Framework.Assert.IsFalse(s.Match(Entry("127.0.0.1")));
NUnit.Framework.Assert.IsFalse(s.MatchedBy(Entry("127.0.0.1:12")));
NUnit.Framework.Assert.IsTrue(s.MatchedBy(Entry("127.0.0.1")));
NUnit.Framework.Assert.IsTrue(s.MatchedBy(Entry("127.0.0.1:123")));
NUnit.Framework.Assert.IsFalse(s.Match(Entry("127.0.0.2")));
NUnit.Framework.Assert.IsFalse(s.Match(Entry("127.0.0.2:123")));
NUnit.Framework.Assert.IsFalse(s.MatchedBy(Entry("127.0.0.2")));
NUnit.Framework.Assert.IsFalse(s.MatchedBy(Entry("127.0.0.2:123")));
s.Add(Entry("127.0.0.1"));
NUnit.Framework.Assert.IsTrue(s.Match(Entry("127.0.0.1:123")));
NUnit.Framework.Assert.IsTrue(s.Match(Entry("127.0.0.1:12")));
NUnit.Framework.Assert.IsTrue(s.Match(Entry("127.0.0.1")));
NUnit.Framework.Assert.IsFalse(s.MatchedBy(Entry("127.0.0.1:12")));
NUnit.Framework.Assert.IsTrue(s.MatchedBy(Entry("127.0.0.1")));
NUnit.Framework.Assert.IsTrue(s.MatchedBy(Entry("127.0.0.1:123")));
NUnit.Framework.Assert.IsFalse(s.Match(Entry("127.0.0.2")));
NUnit.Framework.Assert.IsFalse(s.Match(Entry("127.0.0.2:123")));
NUnit.Framework.Assert.IsFalse(s.MatchedBy(Entry("127.0.0.2")));
NUnit.Framework.Assert.IsFalse(s.MatchedBy(Entry("127.0.0.2:123")));
s.Add(Entry("127.0.0.2:123"));
NUnit.Framework.Assert.IsTrue(s.Match(Entry("127.0.0.1:123")));
NUnit.Framework.Assert.IsTrue(s.Match(Entry("127.0.0.1:12")));
NUnit.Framework.Assert.IsTrue(s.Match(Entry("127.0.0.1")));
NUnit.Framework.Assert.IsFalse(s.MatchedBy(Entry("127.0.0.1:12")));
NUnit.Framework.Assert.IsTrue(s.MatchedBy(Entry("127.0.0.1")));
NUnit.Framework.Assert.IsTrue(s.MatchedBy(Entry("127.0.0.1:123")));
NUnit.Framework.Assert.IsFalse(s.Match(Entry("127.0.0.2")));
NUnit.Framework.Assert.IsTrue(s.Match(Entry("127.0.0.2:123")));
NUnit.Framework.Assert.IsTrue(s.MatchedBy(Entry("127.0.0.2")));
NUnit.Framework.Assert.IsTrue(s.MatchedBy(Entry("127.0.0.2:123")));
}
public virtual void TestRemoveIncludedNode()
{
FSNamesystem fsn = Org.Mockito.Mockito.Mock <FSNamesystem>();
// Set the write lock so that the DatanodeManager can start
Org.Mockito.Mockito.When(fsn.HasWriteLock()).ThenReturn(true);
DatanodeManager dm = MockDatanodeManager(fsn, new Configuration());
HostFileManager hm = new HostFileManager();
HostFileManager.HostSet noNodes = new HostFileManager.HostSet();
HostFileManager.HostSet oneNode = new HostFileManager.HostSet();
HostFileManager.HostSet twoNodes = new HostFileManager.HostSet();
DatanodeRegistration dr1 = new DatanodeRegistration(new DatanodeID("127.0.0.1", "127.0.0.1"
, "someStorageID-123", 12345, 12345, 12345, 12345), new StorageInfo(HdfsServerConstants.NodeType
.DataNode), new ExportedBlockKeys(), "test");
DatanodeRegistration dr2 = new DatanodeRegistration(new DatanodeID("127.0.0.1", "127.0.0.1"
, "someStorageID-234", 23456, 23456, 23456, 23456), new StorageInfo(HdfsServerConstants.NodeType
.DataNode), new ExportedBlockKeys(), "test");
twoNodes.Add(Entry("127.0.0.1:12345"));
twoNodes.Add(Entry("127.0.0.1:23456"));
oneNode.Add(Entry("127.0.0.1:23456"));
hm.Refresh(twoNodes, noNodes);
Whitebox.SetInternalState(dm, "hostFileManager", hm);
// Register two data nodes to simulate them coming up.
// We need to add two nodes, because if we have only one node, removing it
// will cause the includes list to be empty, which means all hosts will be
// allowed.
dm.RegisterDatanode(dr1);
dm.RegisterDatanode(dr2);
// Make sure that both nodes are reported
IList <DatanodeDescriptor> both = dm.GetDatanodeListForReport(HdfsConstants.DatanodeReportType
.All);
// Sort the list so that we know which one is which
both.Sort();
NUnit.Framework.Assert.AreEqual("Incorrect number of hosts reported", 2, both.Count
);
NUnit.Framework.Assert.AreEqual("Unexpected host or host in unexpected position",
"127.0.0.1:12345", both[0].GetInfoAddr());
NUnit.Framework.Assert.AreEqual("Unexpected host or host in unexpected position",
"127.0.0.1:23456", both[1].GetInfoAddr());
// Remove one node from includes, but do not add it to excludes.
hm.Refresh(oneNode, noNodes);
// Make sure that only one node is still reported
IList <DatanodeDescriptor> onlyOne = dm.GetDatanodeListForReport(HdfsConstants.DatanodeReportType
.All);
NUnit.Framework.Assert.AreEqual("Incorrect number of hosts reported", 1, onlyOne.
Count);
NUnit.Framework.Assert.AreEqual("Unexpected host reported", "127.0.0.1:23456", onlyOne
[0].GetInfoAddr());
// Remove all nodes from includes
hm.Refresh(noNodes, noNodes);
// Check that both nodes are reported again
IList <DatanodeDescriptor> bothAgain = dm.GetDatanodeListForReport(HdfsConstants.DatanodeReportType
.All);
// Sort the list so that we know which one is which
bothAgain.Sort();
NUnit.Framework.Assert.AreEqual("Incorrect number of hosts reported", 2, bothAgain
.Count);
NUnit.Framework.Assert.AreEqual("Unexpected host or host in unexpected position",
"127.0.0.1:12345", bothAgain[0].GetInfoAddr());
NUnit.Framework.Assert.AreEqual("Unexpected host or host in unexpected position",
"127.0.0.1:23456", bothAgain[1].GetInfoAddr());
}
/// <summary>Read the includes and excludes lists from the named files.</summary>
/// <remarks>
/// Read the includes and excludes lists from the named files. Any previous
/// includes and excludes lists are discarded.
/// </remarks>
/// <param name="includeFile">the path to the new includes list</param>
/// <param name="excludeFile">the path to the new excludes list</param>
/// <exception cref="System.IO.IOException">thrown if there is a problem reading one of the files
/// </exception>
internal virtual void Refresh(string includeFile, string excludeFile)
{
HostFileManager.HostSet newIncludes = ReadFile("included", includeFile);
HostFileManager.HostSet newExcludes = ReadFile("excluded", excludeFile);
Refresh(newIncludes, newExcludes);
}