public virtual void TestStaticIsSingleSwitchOnNullScript()
{
StaticMapping mapping = NewInstance(null);
mapping.SetConf(CreateConf(null));
AssertSingleSwitch(mapping);
}
public virtual void TestReadNodesFromConfig()
{
StaticMapping mapping = NewInstance();
Configuration conf = new Configuration();
conf.Set(StaticMapping.KeyHadoopConfiguredNodeMapping, "n1=/r1,n2=/r2");
mapping.SetConf(conf);
//even though we have inserted elements into the list, because
//it is driven by the script key in the configuration, it still
//thinks that it is single rack
AssertSingleSwitch(mapping);
IList <string> l1 = new AList <string>(3);
l1.AddItem("n1");
l1.AddItem("unknown");
l1.AddItem("n2");
IList <string> resolved = mapping.Resolve(l1);
Assert.Equal(3, resolved.Count);
Assert.Equal("/r1", resolved[0]);
Assert.Equal(NetworkTopology.DefaultRack, resolved[1]);
Assert.Equal("/r2", resolved[2]);
IDictionary <string, string> switchMap = mapping.GetSwitchMap();
string topology = mapping.DumpTopology();
Log.Info(topology);
Assert.Equal(topology, 2, switchMap.Count);
Assert.Equal(topology, "/r1", switchMap["n1"]);
NUnit.Framework.Assert.IsNull(topology, switchMap["unknown"]);
}
/// <summary>
/// Reset the map then create a new instance of the
/// <see cref="StaticMapping"/>
/// class with the topology script in the configuration set to
/// the parameter
/// </summary>
/// <param name="script">a (never executed) script, can be null</param>
/// <returns>a new instance</returns>
private StaticMapping NewInstance(string script)
{
StaticMapping mapping = NewInstance();
mapping.SetConf(CreateConf(script));
return(mapping);
}
public virtual void TestCachingRelaysMultiSwitchQueries()
{
StaticMapping staticMapping = NewInstance("top");
AssertMultiSwitch(staticMapping);
CachedDNSToSwitchMapping cachedMap = new CachedDNSToSwitchMapping(staticMapping);
Log.Info("Mapping: " + cachedMap + "\n" + cachedMap.DumpTopology());
AssertMultiSwitch(cachedMap);
}
public virtual void TestCachingRelaysSingleSwitchQueries()
{
//create a single switch map
StaticMapping staticMapping = NewInstance(null);
AssertSingleSwitch(staticMapping);
CachedDNSToSwitchMapping cachedMap = new CachedDNSToSwitchMapping(staticMapping);
Log.Info("Mapping: " + cachedMap + "\n" + cachedMap.DumpTopology());
AssertSingleSwitch(cachedMap);
}
public virtual void TestCachingCachesNegativeEntries()
{
StaticMapping staticMapping = NewInstance();
CachedDNSToSwitchMapping cachedMap = new CachedDNSToSwitchMapping(staticMapping);
AssertMapSize(cachedMap, 0);
AssertMapSize(staticMapping, 0);
IList <string> resolved = cachedMap.Resolve(CreateQueryList());
//and verify the cache is no longer empty while the static map is
AssertMapSize(staticMapping, 0);
AssertMapSize(cachedMap, 2);
}
public virtual void TestAddResolveNodes()
{
StaticMapping mapping = NewInstance();
StaticMapping.AddNodeToRack("n1", "/r1");
IList <string> queryList = CreateQueryList();
IList <string> resolved = mapping.Resolve(queryList);
Assert.Equal(2, resolved.Count);
Assert.Equal("/r1", resolved[0]);
Assert.Equal(NetworkTopology.DefaultRack, resolved[1]);
// get the switch map and examine it
IDictionary <string, string> switchMap = mapping.GetSwitchMap();
string topology = mapping.DumpTopology();
Log.Info(topology);
Assert.Equal(topology, 1, switchMap.Count);
Assert.Equal(topology, "/r1", switchMap["n1"]);
}
public virtual void TestCachingRelaysResolveQueries()
{
StaticMapping mapping = NewInstance();
mapping.SetConf(CreateConf("top"));
StaticMapping staticMapping = mapping;
CachedDNSToSwitchMapping cachedMap = new CachedDNSToSwitchMapping(staticMapping);
AssertMapSize(cachedMap, 0);
//add a node to the static map
StaticMapping.AddNodeToRack("n1", "/r1");
//verify it is there
AssertMapSize(staticMapping, 1);
//verify that the cache hasn't picked it up yet
AssertMapSize(cachedMap, 0);
//now relay the query
cachedMap.Resolve(CreateQueryList());
//and verify the cache is no longer empty
AssertMapSize(cachedMap, 2);
}
/// <exception cref="System.Exception"/>
public virtual void TestInvalidNetworkTopologiesNotCachedInHdfs()
{
// start a cluster
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = null;
try
{
// bad rack topology
string[] racks = new string[] { "/a/b", "/c" };
string[] hosts = new string[] { "foo1.example.com", "foo2.example.com" };
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Racks(racks).Hosts(hosts
).Build();
cluster.WaitActive();
NamenodeProtocols nn = cluster.GetNameNodeRpc();
NUnit.Framework.Assert.IsNotNull(nn);
// Wait for one DataNode to register.
// The other DataNode will not be able to register up because of the rack mismatch.
DatanodeInfo[] info;
while (true)
{
info = nn.GetDatanodeReport(HdfsConstants.DatanodeReportType.Live);
NUnit.Framework.Assert.IsFalse(info.Length == 2);
if (info.Length == 1)
{
break;
}
Sharpen.Thread.Sleep(1000);
}
// Set the network topology of the other node to the match the network
// topology of the node that came up.
int validIdx = info[0].GetHostName().Equals(hosts[0]) ? 0 : 1;
int invalidIdx = validIdx == 1 ? 0 : 1;
StaticMapping.AddNodeToRack(hosts[invalidIdx], racks[validIdx]);
Log.Info("datanode " + validIdx + " came up with network location " + info[0].GetNetworkLocation
());
// Restart the DN with the invalid topology and wait for it to register.
cluster.RestartDataNode(invalidIdx);
Sharpen.Thread.Sleep(5000);
while (true)
{
info = nn.GetDatanodeReport(HdfsConstants.DatanodeReportType.Live);
if (info.Length == 2)
{
break;
}
if (info.Length == 0)
{
Log.Info("got no valid DNs");
}
else
{
if (info.Length == 1)
{
Log.Info("got one valid DN: " + info[0].GetHostName() + " (at " + info[0].GetNetworkLocation
() + ")");
}
}
Sharpen.Thread.Sleep(1000);
}
NUnit.Framework.Assert.AreEqual(info[0].GetNetworkLocation(), info[1].GetNetworkLocation
());
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <summary>
/// Reset the map then create a new instance of the
/// <see cref="StaticMapping"/>
/// class with a null configuration
/// </summary>
/// <returns>a new instance</returns>
private StaticMapping NewInstance()
{
StaticMapping.ResetMap();
return(new StaticMapping());
}
public virtual void TestStaticIsMultiSwitchOnScript()
{
StaticMapping mapping = NewInstance("ls");
AssertMultiSwitch(mapping);
}