public static string GetResolvedRemoteRMWebAppURLWithoutScheme(Configuration conf
, HttpConfig.Policy httpPolicy)
{
IPEndPoint address = null;
string rmId = null;
if (HAUtil.IsHAEnabled(conf))
{
// If HA enabled, pick one of the RM-IDs and rely on redirect to go to
// the Active RM
rmId = (string)Sharpen.Collections.ToArray(HAUtil.GetRMHAIds(conf))[0];
}
if (httpPolicy == HttpConfig.Policy.HttpsOnly)
{
address = conf.GetSocketAddr(rmId == null ? YarnConfiguration.RmWebappHttpsAddress
: HAUtil.AddSuffix(YarnConfiguration.RmWebappHttpsAddress, rmId), YarnConfiguration
.DefaultRmWebappHttpsAddress, YarnConfiguration.DefaultRmWebappHttpsPort);
}
else
{
address = conf.GetSocketAddr(rmId == null ? YarnConfiguration.RmWebappAddress : HAUtil
.AddSuffix(YarnConfiguration.RmWebappAddress, rmId), YarnConfiguration.DefaultRmWebappAddress
, YarnConfiguration.DefaultRmWebappPort);
}
return(GetResolvedAddress(address));
}
protected override HAServiceTarget ResolveTarget(string rmId)
{
ICollection <string> rmIds = HAUtil.GetRMHAIds(GetConf());
if (!rmIds.Contains(rmId))
{
StringBuilder msg = new StringBuilder();
msg.Append(rmId + " is not a valid serviceId. It should be one of ");
foreach (string id in rmIds)
{
msg.Append(id + " ");
}
throw new ArgumentException(msg.ToString());
}
try
{
YarnConfiguration conf = new YarnConfiguration(GetConf());
conf.Set(YarnConfiguration.RmHaId, rmId);
return(new RMHAServiceTarget(conf));
}
catch (ArgumentException)
{
throw new YarnRuntimeException("Could not connect to " + rmId + "; the configuration for it might be missing"
);
}
catch (IOException)
{
throw new YarnRuntimeException("Could not connect to RM HA Admin for node " + rmId
);
}
}
public virtual void Init(Configuration configuration, RMProxy <T> rmProxy, Type protocol
)
{
this.rmProxy = rmProxy;
this.protocol = protocol;
this.rmProxy.CheckAllowedProtocols(this.protocol);
this.conf = new YarnConfiguration(configuration);
ICollection <string> rmIds = HAUtil.GetRMHAIds(conf);
this.rmServiceIds = Sharpen.Collections.ToArray(rmIds, new string[rmIds.Count]);
conf.Set(YarnConfiguration.RmHaId, rmServiceIds[currentProxyIndex]);
conf.SetInt(CommonConfigurationKeysPublic.IpcClientConnectMaxRetriesKey, conf.GetInt
(YarnConfiguration.ClientFailoverRetries, YarnConfiguration.DefaultClientFailoverRetries
));
conf.SetInt(CommonConfigurationKeysPublic.IpcClientConnectMaxRetriesOnSocketTimeoutsKey
, conf.GetInt(YarnConfiguration.ClientFailoverRetriesOnSocketTimeouts, YarnConfiguration
.DefaultClientFailoverRetriesOnSocketTimeouts));
}
public static Text GetTokenService(Configuration conf, string address, string defaultAddr
, int defaultPort)
{
if (HAUtil.IsHAEnabled(conf))
{
// Build a list of service addresses to form the service name
AList <string> services = new AList <string>();
YarnConfiguration yarnConf = new YarnConfiguration(conf);
foreach (string rmId in HAUtil.GetRMHAIds(conf))
{
// Set RM_ID to get the corresponding RM_ADDRESS
yarnConf.Set(YarnConfiguration.RmHaId, rmId);
services.AddItem(SecurityUtil.BuildTokenService(yarnConf.GetSocketAddr(address, defaultAddr
, defaultPort)).ToString());
}
return(new Text(Joiner.On(',').Join(services)));
}
// Non-HA case - no need to set RM_ID
return(SecurityUtil.BuildTokenService(conf.GetSocketAddr(address, defaultAddr, defaultPort
)));
}
private Configuration CreateHARMConf(string rmIds, string rmId, int adminPort)
{
Configuration conf = new YarnConfiguration();
conf.SetBoolean(YarnConfiguration.RmHaEnabled, true);
conf.Set(YarnConfiguration.RmHaIds, rmIds);
conf.SetBoolean(YarnConfiguration.RecoveryEnabled, true);
conf.Set(YarnConfiguration.RmStore, typeof(ZKRMStateStore).FullName);
conf.Set(YarnConfiguration.RmZkAddress, hostPort);
conf.SetInt(YarnConfiguration.RmZkTimeoutMs, ZkTimeoutMs);
conf.Set(YarnConfiguration.RmHaId, rmId);
conf.Set(YarnConfiguration.RmWebappAddress, "localhost:0");
foreach (string rpcAddress in YarnConfiguration.GetServiceAddressConfKeys(conf))
{
foreach (string id in HAUtil.GetRMHAIds(conf))
{
conf.Set(HAUtil.AddSuffix(rpcAddress, id), "localhost:0");
}
}
conf.Set(HAUtil.AddSuffix(YarnConfiguration.RmAdminAddress, rmId), "localhost:" +
adminPort);
return(conf);
}
/// <exception cref="System.Exception"/>
protected override void ServiceInit(Configuration conf)
{
useFixedPorts = conf.GetBoolean(YarnConfiguration.YarnMiniclusterFixedPorts, YarnConfiguration
.DefaultYarnMiniclusterFixedPorts);
useRpc = conf.GetBoolean(YarnConfiguration.YarnMiniclusterUseRpc, YarnConfiguration
.DefaultYarnMiniclusterUseRpc);
failoverTimeout = conf.GetInt(YarnConfiguration.RmZkTimeoutMs, YarnConfiguration.
DefaultRmZkTimeoutMs);
if (useRpc && !useFixedPorts)
{
throw new YarnRuntimeException("Invalid configuration!" + " Minicluster can use rpc only when configured to use fixed ports"
);
}
conf.SetBoolean(YarnConfiguration.IsMiniYarnCluster, true);
if (resourceManagers.Length > 1)
{
conf.SetBoolean(YarnConfiguration.RmHaEnabled, true);
if (conf.Get(YarnConfiguration.RmHaIds) == null)
{
StringBuilder rmIds = new StringBuilder();
for (int i = 0; i < resourceManagers.Length; i++)
{
if (i != 0)
{
rmIds.Append(",");
}
rmIds.Append("rm" + i);
}
conf.Set(YarnConfiguration.RmHaIds, rmIds.ToString());
}
ICollection <string> rmIdsCollection = HAUtil.GetRMHAIds(conf);
rmIds = Sharpen.Collections.ToArray(rmIdsCollection, new string[rmIdsCollection.Count
]);
}
for (int i_1 = 0; i_1 < resourceManagers.Length; i_1++)
{
resourceManagers[i_1] = CreateResourceManager();
if (!useFixedPorts)
{
if (HAUtil.IsHAEnabled(conf))
{
SetHARMConfigurationWithEphemeralPorts(i_1, conf);
}
else
{
SetNonHARMConfigurationWithEphemeralPorts(conf);
}
}
AddService(new MiniYARNCluster.ResourceManagerWrapper(this, i_1));
}
for (int index = 0; index < nodeManagers.Length; index++)
{
nodeManagers[index] = useRpc ? new MiniYARNCluster.CustomNodeManager(this) : new
MiniYARNCluster.ShortCircuitedNodeManager(this);
AddService(new MiniYARNCluster.NodeManagerWrapper(this, index));
}
if (conf.GetBoolean(YarnConfiguration.TimelineServiceEnabled, YarnConfiguration.DefaultTimelineServiceEnabled
) || enableAHS)
{
AddService(new MiniYARNCluster.ApplicationHistoryServerWrapper(this));
}
base.ServiceInit(conf is YarnConfiguration ? conf : new YarnConfiguration(conf));
}