public virtual void TestGetNNServiceRpcAddressesForNsIds()
{
Configuration conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsNameservices, "nn1,nn2");
conf.Set(DFSConfigKeys.DfsInternalNameservicesKey, "nn1");
// Test - configured list of namenodes are returned
string Nn1Address = "localhost:9000";
string Nn2Address = "localhost:9001";
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "nn1"), Nn1Address
);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "nn2"), Nn2Address
);
IDictionary <string, IDictionary <string, IPEndPoint> > nnMap = DFSUtil.GetNNServiceRpcAddressesForCluster
(conf);
NUnit.Framework.Assert.AreEqual(1, nnMap.Count);
NUnit.Framework.Assert.IsTrue(nnMap.Contains("nn1"));
conf.Set(DFSConfigKeys.DfsInternalNameservicesKey, "nn3");
try
{
DFSUtil.GetNNServiceRpcAddressesForCluster(conf);
NUnit.Framework.Assert.Fail("Should fail for misconfiguration");
}
catch (IOException)
{
}
}
/// <summary>
/// Create a test configuration that will exercise the initializeGenericKeys
/// code path.
/// </summary>
/// <remarks>
/// Create a test configuration that will exercise the initializeGenericKeys
/// code path. This is a regression test for HDFS-4279.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
internal static void SetupRecoveryTestConf(Configuration conf)
{
conf.Set(DFSConfigKeys.DfsNameservices, "ns1");
conf.Set(DFSConfigKeys.DfsHaNamenodeIdKey, "nn1");
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsHaNamenodesKeyPrefix, "ns1"), "nn1,nn2"
);
string baseDir = Runtime.GetProperty(MiniDFSCluster.PropTestBuildData, "build/test/data"
) + "/dfs/";
FilePath nameDir = new FilePath(baseDir, "nameR");
FilePath secondaryDir = new FilePath(baseDir, "namesecondaryR");
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeNameDirKey, "ns1", "nn1"
), nameDir.GetCanonicalPath());
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeCheckpointDirKey, "ns1",
"nn1"), secondaryDir.GetCanonicalPath());
conf.Unset(DFSConfigKeys.DfsNamenodeNameDirKey);
conf.Unset(DFSConfigKeys.DfsNamenodeCheckpointDirKey);
FileUtils.DeleteQuietly(nameDir);
if (!nameDir.Mkdirs())
{
throw new RuntimeException("failed to make directory " + nameDir.GetAbsolutePath(
));
}
FileUtils.DeleteQuietly(secondaryDir);
if (!secondaryDir.Mkdirs())
{
throw new RuntimeException("failed to make directory " + secondaryDir.GetAbsolutePath
());
}
}
public virtual void TestGetNNUris()
{
HdfsConfiguration conf = new HdfsConfiguration();
string Ns1Nn1Addr = "ns1-nn1.example.com:8020";
string Ns1Nn2Addr = "ns1-nn2.example.com:8020";
string Ns2NnAddr = "ns2-nn.example.com:8020";
string Nn1Addr = "nn.example.com:8020";
string Nn1SrvcAddr = "nn.example.com:8021";
string Nn2Addr = "nn2.example.com:8020";
conf.Set(DFSConfigKeys.DfsNameservices, "ns1,ns2");
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsHaNamenodesKeyPrefix, "ns1"), "nn1,nn2"
);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "ns1", "nn1"
), Ns1Nn1Addr);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "ns1", "nn2"
), Ns1Nn2Addr);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeServiceRpcAddressKey, "ns2"
), Ns2NnAddr);
conf.Set(DFSConfigKeys.DfsNamenodeRpcAddressKey, "hdfs://" + Nn1Addr);
conf.Set(CommonConfigurationKeys.FsDefaultNameKey, "hdfs://" + Nn2Addr);
ICollection <URI> uris = DFSUtil.GetNameServiceUris(conf, DFSConfigKeys.DfsNamenodeServiceRpcAddressKey
, DFSConfigKeys.DfsNamenodeRpcAddressKey);
NUnit.Framework.Assert.AreEqual(4, uris.Count);
NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://ns1")));
NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://" + Ns2NnAddr)));
NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://" + Nn1Addr)));
NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://" + Nn2Addr)));
// Make sure that non-HDFS URIs in fs.defaultFS don't get included.
conf.Set(CommonConfigurationKeys.FsDefaultNameKey, "viewfs://vfs-name.example.com"
);
uris = DFSUtil.GetNameServiceUris(conf, DFSConfigKeys.DfsNamenodeServiceRpcAddressKey
, DFSConfigKeys.DfsNamenodeRpcAddressKey);
NUnit.Framework.Assert.AreEqual(3, uris.Count);
NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://ns1")));
NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://" + Ns2NnAddr)));
NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://" + Nn1Addr)));
// Make sure that an HA URI being the default URI doesn't result in multiple
// entries being returned.
conf.Set(CommonConfigurationKeys.FsDefaultNameKey, "hdfs://ns1");
uris = DFSUtil.GetNameServiceUris(conf, DFSConfigKeys.DfsNamenodeServiceRpcAddressKey
, DFSConfigKeys.DfsNamenodeRpcAddressKey);
NUnit.Framework.Assert.AreEqual(3, uris.Count);
NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://ns1")));
NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://" + Ns2NnAddr)));
NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://" + Nn1Addr)));
// Make sure that when a service RPC address is used that is distinct from
// the client RPC address, and that client RPC address is also used as the
// default URI, that the client URI does not end up in the set of URIs
// returned.
conf = new HdfsConfiguration();
conf.Set(CommonConfigurationKeys.FsDefaultNameKey, "hdfs://" + Nn1Addr);
conf.Set(DFSConfigKeys.DfsNamenodeRpcAddressKey, Nn1Addr);
conf.Set(DFSConfigKeys.DfsNamenodeServiceRpcAddressKey, Nn1SrvcAddr);
uris = DFSUtil.GetNameServiceUris(conf, DFSConfigKeys.DfsNamenodeServiceRpcAddressKey
, DFSConfigKeys.DfsNamenodeRpcAddressKey);
NUnit.Framework.Assert.AreEqual(1, uris.Count);
NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://" + Nn1SrvcAddr)));
}
private Configuration SetupAddress(string key)
{
HdfsConfiguration conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsNameservices, "nn1");
conf.Set(DFSUtil.AddKeySuffixes(key, "nn1"), "localhost:9000");
return(conf);
}
public virtual void TestHANameNodesWithFederation()
{
HdfsConfiguration conf = new HdfsConfiguration();
string Ns1Nn1Host = "ns1-nn1.example.com:8020";
string Ns1Nn2Host = "ns1-nn2.example.com:8020";
string Ns2Nn1Host = "ns2-nn1.example.com:8020";
string Ns2Nn2Host = "ns2-nn2.example.com:8020";
conf.Set(CommonConfigurationKeys.FsDefaultNameKey, "hdfs://ns1");
// Two nameservices, each with two NNs.
conf.Set(DFSConfigKeys.DfsNameservices, "ns1,ns2");
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsHaNamenodesKeyPrefix, "ns1"), "ns1-nn1,ns1-nn2"
);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsHaNamenodesKeyPrefix, "ns2"), "ns2-nn1,ns2-nn2"
);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "ns1", "ns1-nn1"
), Ns1Nn1Host);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "ns1", "ns1-nn2"
), Ns1Nn2Host);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "ns2", "ns2-nn1"
), Ns2Nn1Host);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "ns2", "ns2-nn2"
), Ns2Nn2Host);
IDictionary <string, IDictionary <string, IPEndPoint> > map = DFSUtil.GetHaNnRpcAddresses
(conf);
NUnit.Framework.Assert.IsTrue(HAUtil.IsHAEnabled(conf, "ns1"));
NUnit.Framework.Assert.IsTrue(HAUtil.IsHAEnabled(conf, "ns2"));
NUnit.Framework.Assert.IsFalse(HAUtil.IsHAEnabled(conf, "ns3"));
NUnit.Framework.Assert.AreEqual(Ns1Nn1Host, map["ns1"]["ns1-nn1"].ToString());
NUnit.Framework.Assert.AreEqual(Ns1Nn2Host, map["ns1"]["ns1-nn2"].ToString());
NUnit.Framework.Assert.AreEqual(Ns2Nn1Host, map["ns2"]["ns2-nn1"].ToString());
NUnit.Framework.Assert.AreEqual(Ns2Nn2Host, map["ns2"]["ns2-nn2"].ToString());
NUnit.Framework.Assert.AreEqual(Ns1Nn1Host, DFSUtil.GetNamenodeServiceAddr(conf,
"ns1", "ns1-nn1"));
NUnit.Framework.Assert.AreEqual(Ns1Nn2Host, DFSUtil.GetNamenodeServiceAddr(conf,
"ns1", "ns1-nn2"));
NUnit.Framework.Assert.AreEqual(Ns2Nn1Host, DFSUtil.GetNamenodeServiceAddr(conf,
"ns2", "ns2-nn1"));
// No nameservice was given and we can't determine which service addr
// to use as two nameservices could share a namenode ID.
NUnit.Framework.Assert.AreEqual(null, DFSUtil.GetNamenodeServiceAddr(conf, null,
"ns1-nn1"));
// Ditto for nameservice IDs, if multiple are defined
NUnit.Framework.Assert.AreEqual(null, DFSUtil.GetNamenodeNameServiceId(conf));
NUnit.Framework.Assert.AreEqual(null, DFSUtil.GetSecondaryNameServiceId(conf));
ICollection <URI> uris = DFSUtil.GetNameServiceUris(conf, DFSConfigKeys.DfsNamenodeRpcAddressKey
);
NUnit.Framework.Assert.AreEqual(2, uris.Count);
NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://ns1")));
NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://ns2")));
}
/// <summary>
/// Test
/// <see cref="DFSUtil.GetNamenodeNameServiceId(Org.Apache.Hadoop.Conf.Configuration)
/// "/>
/// to ensure
/// exception is thrown when multiple rpc addresses match the local node's
/// address
/// </summary>
public virtual void TestGetNameServiceIdException()
{
HdfsConfiguration conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsNameservices, "nn1,nn2");
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "nn1"), "localhost:9000"
);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "nn2"), "localhost:9001"
);
DFSUtil.GetNamenodeNameServiceId(conf);
NUnit.Framework.Assert.Fail("Expected exception is not thrown");
}
private void SetHAConf(Configuration conf, string nn1Addr, string nn2Addr)
{
conf.Set(CommonConfigurationKeysPublic.FsDefaultNameKey, "hdfs://" + Nsid);
conf.Set(DFSConfigKeys.DfsNameservices, Nsid);
conf.Set(DFSConfigKeys.DfsNameserviceId, Nsid);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsHaNamenodesKeyPrefix, Nsid), "nn1,nn2"
);
conf.Set(DFSConfigKeys.DfsHaNamenodeIdKey, "nn1");
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, Nsid, "nn1"
), nn1Addr);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, Nsid, "nn2"
), nn2Addr);
}
/// <summary>Set a given key with value as address, for all the nameServiceIds.</summary>
/// <param name="conf">configuration to set the addresses in</param>
/// <param name="key">configuration key</param>
/// <param name="nameServiceIdCount">Number of nameServices for which the key is set</param>
/// <param name="portOffset">starting port offset</param>
/// <returns>list of addresses that are set in the configuration</returns>
private string[] SetupAddress(HdfsConfiguration conf, string key, int nameServiceIdCount
, int portOffset)
{
string[] values = new string[nameServiceIdCount];
for (int i = 0; i < nameServiceIdCount; i++, portOffset++)
{
string nsID = GetNameServiceId(i);
string specificKey = DFSUtil.AddKeySuffixes(key, nsID);
values[i] = "nn" + i + ":" + portOffset;
conf.Set(specificKey, values[i]);
}
return(values);
}
public virtual void TestInitializeSharedEditsConfiguresGenericConfKeys()
{
Configuration conf = new Configuration();
conf.Set(DFSConfigKeys.DfsNameservices, "ns1");
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsHaNamenodesKeyPrefix, "ns1"), "nn1,nn2"
);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "ns1", "nn1"
), "localhost:1234");
NUnit.Framework.Assert.IsNull(conf.Get(DFSConfigKeys.DfsNamenodeRpcAddressKey));
NameNode.InitializeSharedEdits(conf);
NUnit.Framework.Assert.IsNotNull(conf.Get(DFSConfigKeys.DfsNamenodeRpcAddressKey)
);
}
private Configuration GetHAConf(string nsId, string host1, string host2)
{
Configuration conf = new Configuration();
conf.Set(DFSConfigKeys.DfsNameservices, nsId);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsHaNamenodesKeyPrefix, nsId), "nn1,nn2"
);
conf.Set(DFSConfigKeys.DfsHaNamenodeIdKey, "nn1");
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, nsId, "nn1"
), host1 + ":12345");
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, nsId, "nn2"
), host2 + ":12345");
return(conf);
}
// Fencer shell commands that always return true and false respectively
// on Unix.
// Fencer shell commands that always return true and false respectively
// on Windows. Lacking POSIX 'true' and 'false' commands we use the DOS
// commands 'rem' and 'help.exe'.
private HdfsConfiguration GetHAConf()
{
HdfsConfiguration conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsNameservices, Nsid);
conf.Set(DFSConfigKeys.DfsNameserviceId, Nsid);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsHaNamenodesKeyPrefix, Nsid), "nn1,nn2"
);
conf.Set(DFSConfigKeys.DfsHaNamenodeIdKey, "nn1");
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, Nsid, "nn1"
), HostA + ":12345");
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, Nsid, "nn2"
), HostB + ":12345");
return(conf);
}
private static Configuration CreateWebHDFSHAConfiguration(string logicalHostName,
string nnaddr1, string nnaddr2)
{
HdfsConfiguration conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsNameservices, "ns1");
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsHaNamenodesKeyPrefix, "ns1"), "nn1,nn2"
);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeHttpAddressKey, "ns1", "nn1"
), nnaddr1);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeHttpAddressKey, "ns1", "nn2"
), nnaddr2);
conf.Set(DFSConfigKeys.DfsClientFailoverProxyProviderKeyPrefix + "." + logicalHostName
, typeof(ConfiguredFailoverProxyProvider).FullName);
return(conf);
}
public virtual void TestIsValidRequestor()
{
Configuration conf = new HdfsConfiguration();
KerberosName.SetRules("RULE:[1:$1]\nRULE:[2:$1]");
// Set up generic HA configs.
conf.Set(DFSConfigKeys.DfsNameservices, "ns1");
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsHaNamenodesKeyPrefix, "ns1"), "nn1,nn2"
);
// Set up NN1 HA configs.
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "ns1", "nn1"
), "host1:1234");
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeKerberosPrincipalKey, "ns1"
, "nn1"), "hdfs/[email protected]");
// Set up NN2 HA configs.
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "ns1", "nn2"
), "host2:1234");
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeKerberosPrincipalKey, "ns1"
, "nn2"), "hdfs/[email protected]");
// Initialize this conf object as though we're running on NN1.
NameNode.InitializeGenericKeys(conf, "ns1", "nn1");
AccessControlList acls = Org.Mockito.Mockito.Mock <AccessControlList>();
Org.Mockito.Mockito.When(acls.IsUserAllowed(Org.Mockito.Mockito.Any <UserGroupInformation
>())).ThenReturn(false);
ServletContext context = Org.Mockito.Mockito.Mock <ServletContext>();
Org.Mockito.Mockito.When(context.GetAttribute(HttpServer2.AdminsAcl)).ThenReturn(
acls);
// Make sure that NN2 is considered a valid fsimage/edits requestor.
NUnit.Framework.Assert.IsTrue(ImageServlet.IsValidRequestor(context, "hdfs/[email protected]"
, conf));
// Mark atm as an admin.
Org.Mockito.Mockito.When(acls.IsUserAllowed(Org.Mockito.Mockito.ArgThat(new _ArgumentMatcher_76
()))).ThenReturn(true);
// Make sure that NN2 is still considered a valid requestor.
NUnit.Framework.Assert.IsTrue(ImageServlet.IsValidRequestor(context, "hdfs/[email protected]"
, conf));
// Make sure an admin is considered a valid requestor.
NUnit.Framework.Assert.IsTrue(ImageServlet.IsValidRequestor(context, "*****@*****.**"
, conf));
// Make sure other users are *not* considered valid requestors.
NUnit.Framework.Assert.IsFalse(ImageServlet.IsValidRequestor(context, "*****@*****.**"
, conf));
}
private Configuration InitHAConf(URI journalURI, Configuration conf)
{
conf.Set(DFSConfigKeys.DfsNamenodeSharedEditsDirKey, journalURI.ToString());
string address1 = "127.0.0.1:" + basePort;
string address2 = "127.0.0.1:" + (basePort + 2);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, Nameservice
, Nn1), address1);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, Nameservice
, Nn2), address2);
conf.Set(DFSConfigKeys.DfsNameservices, Nameservice);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsHaNamenodesKeyPrefix, Nameservice
), Nn1 + "," + Nn2);
conf.Set(DFSConfigKeys.DfsClientFailoverProxyProviderKeyPrefix + "." + Nameservice
, typeof(ConfiguredFailoverProxyProvider).FullName);
conf.Set("fs.defaultFS", "hdfs://" + Nameservice);
return(conf);
}
/// <summary>Sets the required configurations for performing failover</summary>
public static void SetFailoverConfigurations(Configuration conf, string logicalName
, IPEndPoint nnAddr1, IPEndPoint nnAddr2)
{
string nameNodeId1 = "nn1";
string nameNodeId2 = "nn2";
string address1 = "hdfs://" + nnAddr1.GetHostName() + ":" + nnAddr1.Port;
string address2 = "hdfs://" + nnAddr2.GetHostName() + ":" + nnAddr2.Port;
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, logicalName
, nameNodeId1), address1);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, logicalName
, nameNodeId2), address2);
conf.Set(DFSConfigKeys.DfsNameservices, logicalName);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsHaNamenodesKeyPrefix, logicalName
), nameNodeId1 + "," + nameNodeId2);
conf.Set(DFSConfigKeys.DfsClientFailoverProxyProviderKeyPrefix + "." + logicalName
, typeof(ConfiguredFailoverProxyProvider).FullName);
conf.Set("fs.defaultFS", "hdfs://" + logicalName);
}
public virtual void TestMultipleNamenodes()
{
HdfsConfiguration conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsNameservices, "nn1,nn2");
// Test - configured list of namenodes are returned
string Nn1Address = "localhost:9000";
string Nn2Address = "localhost:9001";
string Nn3Address = "localhost:9002";
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "nn1"), Nn1Address
);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "nn2"), Nn2Address
);
IDictionary <string, IDictionary <string, IPEndPoint> > nnMap = DFSUtil.GetNNServiceRpcAddresses
(conf);
NUnit.Framework.Assert.AreEqual(2, nnMap.Count);
IDictionary <string, IPEndPoint> nn1Map = nnMap["nn1"];
NUnit.Framework.Assert.AreEqual(1, nn1Map.Count);
IPEndPoint addr = nn1Map[null];
NUnit.Framework.Assert.AreEqual("localhost", addr.GetHostName());
NUnit.Framework.Assert.AreEqual(9000, addr.Port);
IDictionary <string, IPEndPoint> nn2Map = nnMap["nn2"];
NUnit.Framework.Assert.AreEqual(1, nn2Map.Count);
addr = nn2Map[null];
NUnit.Framework.Assert.AreEqual("localhost", addr.GetHostName());
NUnit.Framework.Assert.AreEqual(9001, addr.Port);
// Test - can look up nameservice ID from service address
CheckNameServiceId(conf, Nn1Address, "nn1");
CheckNameServiceId(conf, Nn2Address, "nn2");
CheckNameServiceId(conf, Nn3Address, null);
// HA is not enabled in a purely federated config
NUnit.Framework.Assert.IsFalse(HAUtil.IsHAEnabled(conf, "nn1"));
NUnit.Framework.Assert.IsFalse(HAUtil.IsHAEnabled(conf, "nn2"));
}
public virtual void TestConfModificationFederationOnly()
{
HdfsConfiguration conf = new HdfsConfiguration();
string nsId = "ns1";
conf.Set(DFSConfigKeys.DfsNameservices, nsId);
conf.Set(DFSConfigKeys.DfsNameserviceId, nsId);
// Set the nameservice specific keys with nameserviceId in the config key
foreach (string key in NameNode.NamenodeSpecificKeys)
{
// Note: value is same as the key
conf.Set(DFSUtil.AddKeySuffixes(key, nsId), key);
}
// Initialize generic keys from specific keys
NameNode.InitializeGenericKeys(conf, nsId, null);
// Retrieve the keys without nameserviceId and Ensure generic keys are set
// to the correct value
foreach (string key_1 in NameNode.NamenodeSpecificKeys)
{
NUnit.Framework.Assert.AreEqual(key_1, conf.Get(key_1));
}
}
public virtual void GetNameNodeServiceAddr()
{
HdfsConfiguration conf = new HdfsConfiguration();
// One nameservice with two NNs
string Ns1Nn1Host = "ns1-nn1.example.com:8020";
string Ns1Nn1HostSvc = "ns1-nn2.example.com:8021";
string Ns1Nn2Host = "ns1-nn1.example.com:8020";
string Ns1Nn2HostSvc = "ns1-nn2.example.com:8021";
conf.Set(DFSConfigKeys.DfsNameservices, "ns1");
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsHaNamenodesKeyPrefix, "ns1"), "nn1,nn2"
);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "ns1", "nn1"
), Ns1Nn1Host);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "ns1", "nn2"
), Ns1Nn2Host);
// The rpc address is used if no service address is defined
NUnit.Framework.Assert.AreEqual(Ns1Nn1Host, DFSUtil.GetNamenodeServiceAddr(conf,
null, "nn1"));
NUnit.Framework.Assert.AreEqual(Ns1Nn2Host, DFSUtil.GetNamenodeServiceAddr(conf,
null, "nn2"));
// A nameservice is specified explicitly
NUnit.Framework.Assert.AreEqual(Ns1Nn1Host, DFSUtil.GetNamenodeServiceAddr(conf,
"ns1", "nn1"));
NUnit.Framework.Assert.AreEqual(null, DFSUtil.GetNamenodeServiceAddr(conf, "invalid"
, "nn1"));
// The service addrs are used when they are defined
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeServiceRpcAddressKey, "ns1"
, "nn1"), Ns1Nn1HostSvc);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeServiceRpcAddressKey, "ns1"
, "nn2"), Ns1Nn2HostSvc);
NUnit.Framework.Assert.AreEqual(Ns1Nn1HostSvc, DFSUtil.GetNamenodeServiceAddr(conf
, null, "nn1"));
NUnit.Framework.Assert.AreEqual(Ns1Nn2HostSvc, DFSUtil.GetNamenodeServiceAddr(conf
, null, "nn2"));
// We can determine the nameservice ID, there's only one listed
NUnit.Framework.Assert.AreEqual("ns1", DFSUtil.GetNamenodeNameServiceId(conf));
NUnit.Framework.Assert.AreEqual("ns1", DFSUtil.GetSecondaryNameServiceId(conf));
}
/// <summary>Get the NN ID of the other node in an HA setup.</summary>
/// <param name="conf">the configuration of this node</param>
/// <returns>the NN ID of the other node in this nameservice</returns>
public static string GetNameNodeIdOfOtherNode(Configuration conf, string nsId)
{
Preconditions.CheckArgument(nsId != null, "Could not determine namespace id. Please ensure that this "
+ "machine is one of the machines listed as a NN RPC address, " + "or configure "
+ DFSConfigKeys.DfsNameserviceId);
ICollection <string> nnIds = DFSUtil.GetNameNodeIds(conf, nsId);
string myNNId = conf.Get(DFSConfigKeys.DfsHaNamenodeIdKey);
Preconditions.CheckArgument(nnIds != null, "Could not determine namenode ids in namespace '%s'. "
+ "Please configure " + DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsHaNamenodesKeyPrefix
, nsId), nsId);
Preconditions.CheckArgument(nnIds.Count == 2, "Expected exactly 2 NameNodes in namespace '%s'. "
+ "Instead, got only %s (NN ids were '%s'", nsId, nnIds.Count, Joiner.On("','")
.Join(nnIds));
Preconditions.CheckState(myNNId != null && !myNNId.IsEmpty(), "Could not determine own NN ID in namespace '%s'. Please "
+ "ensure that this node is one of the machines listed as an " + "NN RPC address, or configure "
+ DFSConfigKeys.DfsHaNamenodeIdKey, nsId);
AList <string> nnSet = Lists.NewArrayList(nnIds);
nnSet.Remove(myNNId);
System.Diagnostics.Debug.Assert(nnSet.Count == 1);
return(nnSet[0]);
}
public virtual void TestFormatShouldBeIgnoredForNonFileBasedDirs()
{
Configuration conf = new HdfsConfiguration();
string logicalName = "mycluster";
// DFS_NAMENODE_RPC_ADDRESS_KEY are required to identify the NameNode
// is configured in HA, then only DFS_NAMENODE_SHARED_EDITS_DIR_KEY
// is considered.
string localhost = "127.0.0.1";
IPEndPoint nnAddr1 = new IPEndPoint(localhost, 8020);
IPEndPoint nnAddr2 = new IPEndPoint(localhost, 9020);
HATestUtil.SetFailoverConfigurations(conf, logicalName, nnAddr1, nnAddr2);
conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, new FilePath(DfsBaseDir, "name").GetAbsolutePath
());
conf.SetBoolean(DFSConfigKeys.DfsNamenodeSupportAllowFormatKey, true);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeEditsPluginPrefix, "dummy"
), typeof(TestGenericJournalConf.DummyJournalManager).FullName);
conf.Set(DFSConfigKeys.DfsNamenodeSharedEditsDirKey, "dummy://" + localhost + ":2181/ledgers"
);
conf.Set(DFSConfigKeys.DfsHaNamenodeIdKey, "nn1");
// An internal assert is added to verify the working of test
NameNode.Format(conf);
}
private static void AddNN(Configuration conf, string ns, string addr)
{
string key = DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, ns);
conf.Set(key, addr);
}
