public virtual void TestHANameNodesWithFederation()
{
HdfsConfiguration conf = new HdfsConfiguration();
string Ns1Nn1Host = "ns1-nn1.example.com:8020";
string Ns1Nn2Host = "ns1-nn2.example.com:8020";
string Ns2Nn1Host = "ns2-nn1.example.com:8020";
string Ns2Nn2Host = "ns2-nn2.example.com:8020";
conf.Set(CommonConfigurationKeys.FsDefaultNameKey, "hdfs://ns1");
// Two nameservices, each with two NNs.
conf.Set(DFSConfigKeys.DfsNameservices, "ns1,ns2");
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsHaNamenodesKeyPrefix, "ns1"), "ns1-nn1,ns1-nn2"
);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsHaNamenodesKeyPrefix, "ns2"), "ns2-nn1,ns2-nn2"
);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "ns1", "ns1-nn1"
), Ns1Nn1Host);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "ns1", "ns1-nn2"
), Ns1Nn2Host);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "ns2", "ns2-nn1"
), Ns2Nn1Host);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "ns2", "ns2-nn2"
), Ns2Nn2Host);
IDictionary <string, IDictionary <string, IPEndPoint> > map = DFSUtil.GetHaNnRpcAddresses
(conf);
NUnit.Framework.Assert.IsTrue(HAUtil.IsHAEnabled(conf, "ns1"));
NUnit.Framework.Assert.IsTrue(HAUtil.IsHAEnabled(conf, "ns2"));
NUnit.Framework.Assert.IsFalse(HAUtil.IsHAEnabled(conf, "ns3"));
NUnit.Framework.Assert.AreEqual(Ns1Nn1Host, map["ns1"]["ns1-nn1"].ToString());
NUnit.Framework.Assert.AreEqual(Ns1Nn2Host, map["ns1"]["ns1-nn2"].ToString());
NUnit.Framework.Assert.AreEqual(Ns2Nn1Host, map["ns2"]["ns2-nn1"].ToString());
NUnit.Framework.Assert.AreEqual(Ns2Nn2Host, map["ns2"]["ns2-nn2"].ToString());
NUnit.Framework.Assert.AreEqual(Ns1Nn1Host, DFSUtil.GetNamenodeServiceAddr(conf,
"ns1", "ns1-nn1"));
NUnit.Framework.Assert.AreEqual(Ns1Nn2Host, DFSUtil.GetNamenodeServiceAddr(conf,
"ns1", "ns1-nn2"));
NUnit.Framework.Assert.AreEqual(Ns2Nn1Host, DFSUtil.GetNamenodeServiceAddr(conf,
"ns2", "ns2-nn1"));
// No nameservice was given and we can't determine which service addr
// to use as two nameservices could share a namenode ID.
NUnit.Framework.Assert.AreEqual(null, DFSUtil.GetNamenodeServiceAddr(conf, null,
"ns1-nn1"));
// Ditto for nameservice IDs, if multiple are defined
NUnit.Framework.Assert.AreEqual(null, DFSUtil.GetNamenodeNameServiceId(conf));
NUnit.Framework.Assert.AreEqual(null, DFSUtil.GetSecondaryNameServiceId(conf));
ICollection <URI> uris = DFSUtil.GetNameServiceUris(conf, DFSConfigKeys.DfsNamenodeRpcAddressKey
);
NUnit.Framework.Assert.AreEqual(2, uris.Count);
NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://ns1")));
NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://ns2")));
}
/* Hidden constructor */
/// <summary>Returns true if HA for namenode is configured for the given nameservice</summary>
/// <param name="conf">Configuration</param>
/// <param name="nsId">nameservice, or null if no federated NS is configured</param>
/// <returns>true if HA is configured in the configuration; else false.</returns>
public static bool IsHAEnabled(Configuration conf, string nsId)
{
IDictionary <string, IDictionary <string, IPEndPoint> > addresses = DFSUtil.GetHaNnRpcAddresses
(conf);
if (addresses == null)
{
return(false);
}
IDictionary <string, IPEndPoint> nnMap = addresses[nsId];
return(nnMap != null && nnMap.Count > 1);
}
public ConfiguredFailoverProxyProvider(Configuration conf, URI uri, Type xface)
{
Preconditions.CheckArgument(xface.IsAssignableFrom(typeof(NamenodeProtocols)), "Interface class %s is not a valid NameNode protocol!"
);
this.xface = xface;
this.conf = new Configuration(conf);
int maxRetries = this.conf.GetInt(DFSConfigKeys.DfsClientFailoverConnectionRetriesKey
, DFSConfigKeys.DfsClientFailoverConnectionRetriesDefault);
this.conf.SetInt(CommonConfigurationKeysPublic.IpcClientConnectMaxRetriesKey, maxRetries
);
int maxRetriesOnSocketTimeouts = this.conf.GetInt(DFSConfigKeys.DfsClientFailoverConnectionRetriesOnSocketTimeoutsKey
, DFSConfigKeys.DfsClientFailoverConnectionRetriesOnSocketTimeoutsDefault);
this.conf.SetInt(CommonConfigurationKeysPublic.IpcClientConnectMaxRetriesOnSocketTimeoutsKey
, maxRetriesOnSocketTimeouts);
try
{
ugi = UserGroupInformation.GetCurrentUser();
IDictionary <string, IDictionary <string, IPEndPoint> > map = DFSUtil.GetHaNnRpcAddresses
(conf);
IDictionary <string, IPEndPoint> addressesInNN = map[uri.GetHost()];
if (addressesInNN == null || addressesInNN.Count == 0)
{
throw new RuntimeException("Could not find any configured addresses " + "for URI "
+ uri);
}
ICollection <IPEndPoint> addressesOfNns = addressesInNN.Values;
foreach (IPEndPoint address in addressesOfNns)
{
proxies.AddItem(new ConfiguredFailoverProxyProvider.AddressRpcProxyPair <T>(address
));
}
// The client may have a delegation token set for the logical
// URI of the cluster. Clone this token to apply to each of the
// underlying IPC addresses so that the IPC code can find it.
HAUtil.CloneDelegationTokenForLogicalUri(ugi, uri, addressesOfNns);
}
catch (IOException e)
{
throw new RuntimeException(e);
}
}
