/// <exception cref="System.IO.IOException"/>
private void ParseConfAndFindOtherNN()
{
Configuration conf = GetConf();
nsId = DFSUtil.GetNamenodeNameServiceId(conf);
if (!HAUtil.IsHAEnabled(conf, nsId))
{
throw new HadoopIllegalArgumentException("HA is not enabled for this namenode.");
}
nnId = HAUtil.GetNameNodeId(conf, nsId);
NameNode.InitializeGenericKeys(conf, nsId, nnId);
if (!HAUtil.UsesSharedEditsDir(conf))
{
throw new HadoopIllegalArgumentException("Shared edits storage is not enabled for this namenode."
);
}
Configuration otherNode = HAUtil.GetConfForOtherNode(conf);
otherNNId = HAUtil.GetNameNodeId(otherNode, nsId);
otherIpcAddr = NameNode.GetServiceAddress(otherNode, true);
Preconditions.CheckArgument(otherIpcAddr.Port != 0 && !otherIpcAddr.Address.IsAnyLocalAddress
(), "Could not determine valid IPC address for other NameNode (%s)" + ", got: %s"
, otherNNId, otherIpcAddr);
string scheme = DFSUtil.GetHttpClientScheme(conf);
otherHttpAddr = DFSUtil.GetInfoServerWithDefaultHost(otherIpcAddr.GetHostName(),
otherNode, scheme).ToURL();
dirsToFormat = FSNamesystem.GetNamespaceDirs(conf);
editUrisToFormat = FSNamesystem.GetNamespaceEditsDirs(conf, false);
sharedEditsUris = FSNamesystem.GetSharedEditsDirs(conf);
}
/// <exception cref="System.IO.IOException"/>
public SecondaryNameNode(Configuration conf, SecondaryNameNode.CommandLineOpts commandLineOpts
)
{
try
{
string nsId = DFSUtil.GetSecondaryNameServiceId(conf);
if (HAUtil.IsHAEnabled(conf, nsId))
{
throw new IOException("Cannot use SecondaryNameNode in an HA cluster." + " The Standby Namenode will perform checkpointing."
);
}
NameNode.InitializeGenericKeys(conf, nsId, null);
Initialize(conf, commandLineOpts);
}
catch (IOException e)
{
Shutdown();
throw;
}
catch (HadoopIllegalArgumentException e)
{
Shutdown();
throw;
}
}
public static string GetResolvedRemoteRMWebAppURLWithoutScheme(Configuration conf
, HttpConfig.Policy httpPolicy)
{
IPEndPoint address = null;
string rmId = null;
if (HAUtil.IsHAEnabled(conf))
{
// If HA enabled, pick one of the RM-IDs and rely on redirect to go to
// the Active RM
rmId = (string)Sharpen.Collections.ToArray(HAUtil.GetRMHAIds(conf))[0];
}
if (httpPolicy == HttpConfig.Policy.HttpsOnly)
{
address = conf.GetSocketAddr(rmId == null ? YarnConfiguration.RmWebappHttpsAddress
: HAUtil.AddSuffix(YarnConfiguration.RmWebappHttpsAddress, rmId), YarnConfiguration
.DefaultRmWebappHttpsAddress, YarnConfiguration.DefaultRmWebappHttpsPort);
}
else
{
address = conf.GetSocketAddr(rmId == null ? YarnConfiguration.RmWebappAddress : HAUtil
.AddSuffix(YarnConfiguration.RmWebappAddress, rmId), YarnConfiguration.DefaultRmWebappAddress
, YarnConfiguration.DefaultRmWebappPort);
}
return(GetResolvedAddress(address));
}
internal static bool IsValidRequestor(ServletContext context, string remoteUser,
Configuration conf)
{
if (remoteUser == null)
{
// This really shouldn't happen...
Log.Warn("Received null remoteUser while authorizing access to getImage servlet");
return(false);
}
ICollection <string> validRequestors = new HashSet <string>();
validRequestors.AddItem(SecurityUtil.GetServerPrincipal(conf.Get(DFSConfigKeys.DfsNamenodeKerberosPrincipalKey
), NameNode.GetAddress(conf).GetHostName()));
try
{
validRequestors.AddItem(SecurityUtil.GetServerPrincipal(conf.Get(DFSConfigKeys.DfsSecondaryNamenodeKerberosPrincipalKey
), SecondaryNameNode.GetHttpAddress(conf).GetHostName()));
}
catch (Exception e)
{
// Don't halt if SecondaryNameNode principal could not be added.
Log.Debug("SecondaryNameNode principal could not be added", e);
string msg = string.Format("SecondaryNameNode principal not considered, %s = %s, %s = %s"
, DFSConfigKeys.DfsSecondaryNamenodeKerberosPrincipalKey, conf.Get(DFSConfigKeys
.DfsSecondaryNamenodeKerberosPrincipalKey), DFSConfigKeys.DfsNamenodeSecondaryHttpAddressKey
, conf.GetTrimmed(DFSConfigKeys.DfsNamenodeSecondaryHttpAddressKey, DFSConfigKeys
.DfsNamenodeSecondaryHttpAddressDefault));
Log.Warn(msg);
}
if (HAUtil.IsHAEnabled(conf, DFSUtil.GetNamenodeNameServiceId(conf)))
{
Configuration otherNnConf = HAUtil.GetConfForOtherNode(conf);
validRequestors.AddItem(SecurityUtil.GetServerPrincipal(otherNnConf.Get(DFSConfigKeys
.DfsNamenodeKerberosPrincipalKey), NameNode.GetAddress(otherNnConf).GetHostName(
)));
}
foreach (string v in validRequestors)
{
if (v != null && v.Equals(remoteUser))
{
Log.Info("ImageServlet allowing checkpointer: " + remoteUser);
return(true);
}
}
if (HttpServer2.UserHasAdministratorAccess(context, remoteUser))
{
Log.Info("ImageServlet allowing administrator: " + remoteUser);
return(true);
}
Log.Info("ImageServlet rejecting: " + remoteUser);
return(false);
}
private void InitResourceManager(int index, Configuration conf)
{
lock (this)
{
if (HAUtil.IsHAEnabled(conf))
{
conf.Set(YarnConfiguration.RmHaId, rmIds[index]);
}
resourceManagers[index].Init(conf);
resourceManagers[index].GetRMContext().GetDispatcher().Register(typeof(RMAppAttemptEventType
), new _EventHandler_296(this));
}
}
/// <exception cref="System.Exception"/>
private void SetUpHaCluster(bool security)
{
conf = new Configuration();
conf.SetBoolean(CommonConfigurationKeys.HadoopSecurityAuthorization, security);
cluster = new MiniQJMHACluster.Builder(conf).Build();
SetHAConf(conf, cluster.GetDfsCluster().GetNameNode(0).GetHostAndPort(), cluster.
GetDfsCluster().GetNameNode(1).GetHostAndPort());
cluster.GetDfsCluster().GetNameNode(0).GetHostAndPort();
admin = new DFSAdmin();
admin.SetConf(conf);
NUnit.Framework.Assert.IsTrue(HAUtil.IsHAEnabled(conf, "ns1"));
originOut = System.Console.Out;
originErr = System.Console.Error;
Runtime.SetOut(new TextWriter(@out));
Runtime.SetErr(new TextWriter(err));
}
public static Text GetTokenService(Configuration conf, string address, string defaultAddr
, int defaultPort)
{
if (HAUtil.IsHAEnabled(conf))
{
// Build a list of service addresses to form the service name
AList <string> services = new AList <string>();
YarnConfiguration yarnConf = new YarnConfiguration(conf);
foreach (string rmId in HAUtil.GetRMHAIds(conf))
{
// Set RM_ID to get the corresponding RM_ADDRESS
yarnConf.Set(YarnConfiguration.RmHaId, rmId);
services.AddItem(SecurityUtil.BuildTokenService(yarnConf.GetSocketAddr(address, defaultAddr
, defaultPort)).ToString());
}
return(new Text(Joiner.On(',').Join(services)));
}
// Non-HA case - no need to set RM_ID
return(SecurityUtil.BuildTokenService(conf.GetSocketAddr(address, defaultAddr, defaultPort
)));
}
public static IList <string> GetProxyHostsAndPortsForAmFilter(Configuration conf)
{
IList <string> addrs = new AList <string>();
string proxyAddr = conf.Get(YarnConfiguration.ProxyAddress);
// If PROXY_ADDRESS isn't set, fallback to RM_WEBAPP(_HTTPS)_ADDRESS
// There could be multiple if using RM HA
if (proxyAddr == null || proxyAddr.IsEmpty())
{
// If RM HA is enabled, try getting those addresses
if (HAUtil.IsHAEnabled(conf))
{
IList <string> haAddrs = RMHAUtils.GetRMHAWebappAddresses(new YarnConfiguration(conf
));
foreach (string addr in haAddrs)
{
try
{
IPEndPoint socketAddr = NetUtils.CreateSocketAddr(addr);
addrs.AddItem(GetResolvedAddress(socketAddr));
}
catch (ArgumentException)
{
}
}
}
// skip if can't resolve
// If couldn't resolve any of the addresses or not using RM HA, fallback
if (addrs.IsEmpty())
{
addrs.AddItem(GetResolvedRMWebAppURLWithoutScheme(conf));
}
}
else
{
addrs.AddItem(proxyAddr);
}
return(addrs);
}
protected internal static T CreateRMProxy <T>(Configuration configuration, Org.Apache.Hadoop.Yarn.Client.RMProxy
instance)
{
System.Type protocol = typeof(T);
YarnConfiguration conf = (configuration is YarnConfiguration) ? (YarnConfiguration
)configuration : new YarnConfiguration(configuration);
RetryPolicy retryPolicy = CreateRetryPolicy(conf);
if (HAUtil.IsHAEnabled(conf))
{
RMFailoverProxyProvider <T> provider = instance.CreateRMFailoverProxyProvider(conf
, protocol);
return((T)RetryProxy.Create(protocol, provider, retryPolicy));
}
else
{
IPEndPoint rmAddress = instance.GetRMAddress(conf, protocol);
Log.Info("Connecting to ResourceManager at " + rmAddress);
T proxy = Org.Apache.Hadoop.Yarn.Client.RMProxy.GetProxy <T>(conf, protocol, rmAddress
);
return((T)RetryProxy.Create(protocol, proxy, retryPolicy));
}
}
public static Org.Apache.Hadoop.Hdfs.Tools.DFSZKFailoverController Create(Configuration
conf)
{
Configuration localNNConf = DFSHAAdmin.AddSecurityConfiguration(conf);
string nsId = DFSUtil.GetNamenodeNameServiceId(conf);
if (!HAUtil.IsHAEnabled(localNNConf, nsId))
{
throw new HadoopIllegalArgumentException("HA is not enabled for this namenode.");
}
string nnId = HAUtil.GetNameNodeId(localNNConf, nsId);
if (nnId == null)
{
string msg = "Could not get the namenode ID of this node. " + "You may run zkfc on the node other than namenode.";
throw new HadoopIllegalArgumentException(msg);
}
NameNode.InitializeGenericKeys(localNNConf, nsId, nnId);
DFSUtil.SetGenericConf(localNNConf, nsId, nnId, ZkfcConfKeys);
NNHAServiceTarget localTarget = new NNHAServiceTarget(localNNConf, nsId, nnId);
return(new Org.Apache.Hadoop.Hdfs.Tools.DFSZKFailoverController(localNNConf, localTarget
));
}
public static RetryPolicy CreateRetryPolicy(Configuration conf)
{
long rmConnectWaitMS = conf.GetLong(YarnConfiguration.ResourcemanagerConnectMaxWaitMs
, YarnConfiguration.DefaultResourcemanagerConnectMaxWaitMs);
long rmConnectionRetryIntervalMS = conf.GetLong(YarnConfiguration.ResourcemanagerConnectRetryIntervalMs
, YarnConfiguration.DefaultResourcemanagerConnectRetryIntervalMs);
bool waitForEver = (rmConnectWaitMS == -1);
if (!waitForEver)
{
if (rmConnectWaitMS < 0)
{
throw new YarnRuntimeException("Invalid Configuration. " + YarnConfiguration.ResourcemanagerConnectMaxWaitMs
+ " can be -1, but can not be other negative numbers");
}
// try connect once
if (rmConnectWaitMS < rmConnectionRetryIntervalMS)
{
Log.Warn(YarnConfiguration.ResourcemanagerConnectMaxWaitMs + " is smaller than "
+ YarnConfiguration.ResourcemanagerConnectRetryIntervalMs + ". Only try connect once."
);
rmConnectWaitMS = 0;
}
}
// Handle HA case first
if (HAUtil.IsHAEnabled(conf))
{
long failoverSleepBaseMs = conf.GetLong(YarnConfiguration.ClientFailoverSleeptimeBaseMs
, rmConnectionRetryIntervalMS);
long failoverSleepMaxMs = conf.GetLong(YarnConfiguration.ClientFailoverSleeptimeMaxMs
, rmConnectionRetryIntervalMS);
int maxFailoverAttempts = conf.GetInt(YarnConfiguration.ClientFailoverMaxAttempts
, -1);
if (maxFailoverAttempts == -1)
{
if (waitForEver)
{
maxFailoverAttempts = int.MaxValue;
}
else
{
maxFailoverAttempts = (int)(rmConnectWaitMS / failoverSleepBaseMs);
}
}
return(RetryPolicies.FailoverOnNetworkException(RetryPolicies.TryOnceThenFail, maxFailoverAttempts
, failoverSleepBaseMs, failoverSleepMaxMs));
}
if (rmConnectionRetryIntervalMS < 0)
{
throw new YarnRuntimeException("Invalid Configuration. " + YarnConfiguration.ResourcemanagerConnectRetryIntervalMs
+ " should not be negative.");
}
RetryPolicy retryPolicy = null;
if (waitForEver)
{
retryPolicy = RetryPolicies.RetryForever;
}
else
{
retryPolicy = RetryPolicies.RetryUpToMaximumTimeWithFixedSleep(rmConnectWaitMS, rmConnectionRetryIntervalMS
, TimeUnit.Milliseconds);
}
IDictionary <Type, RetryPolicy> exceptionToPolicyMap = new Dictionary <Type, RetryPolicy
>();
exceptionToPolicyMap[typeof(EOFException)] = retryPolicy;
exceptionToPolicyMap[typeof(ConnectException)] = retryPolicy;
exceptionToPolicyMap[typeof(NoRouteToHostException)] = retryPolicy;
exceptionToPolicyMap[typeof(UnknownHostException)] = retryPolicy;
exceptionToPolicyMap[typeof(ConnectTimeoutException)] = retryPolicy;
exceptionToPolicyMap[typeof(RetriableException)] = retryPolicy;
exceptionToPolicyMap[typeof(SocketException)] = retryPolicy;
return(RetryPolicies.RetryByException(RetryPolicies.TryOnceThenFail, exceptionToPolicyMap
));
}
/// <exception cref="System.Exception"/>
protected override void ServiceInit(Configuration conf)
{
useFixedPorts = conf.GetBoolean(YarnConfiguration.YarnMiniclusterFixedPorts, YarnConfiguration
.DefaultYarnMiniclusterFixedPorts);
useRpc = conf.GetBoolean(YarnConfiguration.YarnMiniclusterUseRpc, YarnConfiguration
.DefaultYarnMiniclusterUseRpc);
failoverTimeout = conf.GetInt(YarnConfiguration.RmZkTimeoutMs, YarnConfiguration.
DefaultRmZkTimeoutMs);
if (useRpc && !useFixedPorts)
{
throw new YarnRuntimeException("Invalid configuration!" + " Minicluster can use rpc only when configured to use fixed ports"
);
}
conf.SetBoolean(YarnConfiguration.IsMiniYarnCluster, true);
if (resourceManagers.Length > 1)
{
conf.SetBoolean(YarnConfiguration.RmHaEnabled, true);
if (conf.Get(YarnConfiguration.RmHaIds) == null)
{
StringBuilder rmIds = new StringBuilder();
for (int i = 0; i < resourceManagers.Length; i++)
{
if (i != 0)
{
rmIds.Append(",");
}
rmIds.Append("rm" + i);
}
conf.Set(YarnConfiguration.RmHaIds, rmIds.ToString());
}
ICollection <string> rmIdsCollection = HAUtil.GetRMHAIds(conf);
rmIds = Sharpen.Collections.ToArray(rmIdsCollection, new string[rmIdsCollection.Count
]);
}
for (int i_1 = 0; i_1 < resourceManagers.Length; i_1++)
{
resourceManagers[i_1] = CreateResourceManager();
if (!useFixedPorts)
{
if (HAUtil.IsHAEnabled(conf))
{
SetHARMConfigurationWithEphemeralPorts(i_1, conf);
}
else
{
SetNonHARMConfigurationWithEphemeralPorts(conf);
}
}
AddService(new MiniYARNCluster.ResourceManagerWrapper(this, i_1));
}
for (int index = 0; index < nodeManagers.Length; index++)
{
nodeManagers[index] = useRpc ? new MiniYARNCluster.CustomNodeManager(this) : new
MiniYARNCluster.ShortCircuitedNodeManager(this);
AddService(new MiniYARNCluster.NodeManagerWrapper(this, index));
}
if (conf.GetBoolean(YarnConfiguration.TimelineServiceEnabled, YarnConfiguration.DefaultTimelineServiceEnabled
) || enableAHS)
{
AddService(new MiniYARNCluster.ApplicationHistoryServerWrapper(this));
}
base.ServiceInit(conf is YarnConfiguration ? conf : new YarnConfiguration(conf));
}