protected internal override IPEndPoint GetRMAddress(YarnConfiguration conf, Type
protocol)
{
if (protocol == typeof(ApplicationClientProtocol))
{
return(conf.GetSocketAddr(YarnConfiguration.RmAddress, YarnConfiguration.DefaultRmAddress
, YarnConfiguration.DefaultRmPort));
}
else
{
if (protocol == typeof(ResourceManagerAdministrationProtocol))
{
return(conf.GetSocketAddr(YarnConfiguration.RmAdminAddress, YarnConfiguration.DefaultRmAdminAddress
, YarnConfiguration.DefaultRmAdminPort));
}
else
{
if (protocol == typeof(ApplicationMasterProtocol))
{
SetAMRMTokenService(conf);
return(conf.GetSocketAddr(YarnConfiguration.RmSchedulerAddress, YarnConfiguration
.DefaultRmSchedulerAddress, YarnConfiguration.DefaultRmSchedulerPort));
}
else
{
string message = "Unsupported protocol found when creating the proxy " + "connection to ResourceManager: "
+ ((protocol != null) ? protocol.GetType().FullName : "null");
Log.Error(message);
throw new InvalidOperationException(message);
}
}
}
}
public virtual void TestGetSocketAddr()
{
YarnConfiguration conf;
IPEndPoint resourceTrackerAddress;
//all default
conf = new YarnConfiguration();
resourceTrackerAddress = conf.GetSocketAddr(YarnConfiguration.RmBindHost, YarnConfiguration
.RmResourceTrackerAddress, YarnConfiguration.DefaultRmResourceTrackerAddress, YarnConfiguration
.DefaultRmResourceTrackerPort);
NUnit.Framework.Assert.AreEqual(new IPEndPoint(YarnConfiguration.DefaultRmResourceTrackerAddress
.Split(":")[0], YarnConfiguration.DefaultRmResourceTrackerPort), resourceTrackerAddress
);
//with address
conf.Set(YarnConfiguration.RmResourceTrackerAddress, "10.0.0.1");
resourceTrackerAddress = conf.GetSocketAddr(YarnConfiguration.RmBindHost, YarnConfiguration
.RmResourceTrackerAddress, YarnConfiguration.DefaultRmResourceTrackerAddress, YarnConfiguration
.DefaultRmResourceTrackerPort);
NUnit.Framework.Assert.AreEqual(new IPEndPoint("10.0.0.1", YarnConfiguration.DefaultRmResourceTrackerPort
), resourceTrackerAddress);
//address and socket
conf.Set(YarnConfiguration.RmResourceTrackerAddress, "10.0.0.2:5001");
resourceTrackerAddress = conf.GetSocketAddr(YarnConfiguration.RmBindHost, YarnConfiguration
.RmResourceTrackerAddress, YarnConfiguration.DefaultRmResourceTrackerAddress, YarnConfiguration
.DefaultRmResourceTrackerPort);
NUnit.Framework.Assert.AreEqual(new IPEndPoint("10.0.0.2", 5001), resourceTrackerAddress
);
//bind host only
conf = new YarnConfiguration();
conf.Set(YarnConfiguration.RmBindHost, "10.0.0.3");
resourceTrackerAddress = conf.GetSocketAddr(YarnConfiguration.RmBindHost, YarnConfiguration
.RmResourceTrackerAddress, YarnConfiguration.DefaultRmResourceTrackerAddress, YarnConfiguration
.DefaultRmResourceTrackerPort);
NUnit.Framework.Assert.AreEqual(new IPEndPoint("10.0.0.3", YarnConfiguration.DefaultRmResourceTrackerPort
), resourceTrackerAddress);
//bind host and address no port
conf.Set(YarnConfiguration.RmBindHost, "0.0.0.0");
conf.Set(YarnConfiguration.RmResourceTrackerAddress, "10.0.0.2");
resourceTrackerAddress = conf.GetSocketAddr(YarnConfiguration.RmBindHost, YarnConfiguration
.RmResourceTrackerAddress, YarnConfiguration.DefaultRmResourceTrackerAddress, YarnConfiguration
.DefaultRmResourceTrackerPort);
NUnit.Framework.Assert.AreEqual(new IPEndPoint("0.0.0.0", YarnConfiguration.DefaultRmResourceTrackerPort
), resourceTrackerAddress);
//bind host and address with port
conf.Set(YarnConfiguration.RmBindHost, "0.0.0.0");
conf.Set(YarnConfiguration.RmResourceTrackerAddress, "10.0.0.2:5003");
resourceTrackerAddress = conf.GetSocketAddr(YarnConfiguration.RmBindHost, YarnConfiguration
.RmResourceTrackerAddress, YarnConfiguration.DefaultRmResourceTrackerAddress, YarnConfiguration
.DefaultRmResourceTrackerPort);
NUnit.Framework.Assert.AreEqual(new IPEndPoint("0.0.0.0", 5003), resourceTrackerAddress
);
}
/// <exception cref="System.IO.IOException"/>
protected internal virtual SCMAdminProtocol CreateSCMAdminProtocol()
{
// Get the current configuration
YarnConfiguration conf = new YarnConfiguration(GetConf());
// Create the admin client
IPEndPoint addr = conf.GetSocketAddr(YarnConfiguration.ScmAdminAddress, YarnConfiguration
.DefaultScmAdminAddress, YarnConfiguration.DefaultScmAdminPort);
YarnRPC rpc = YarnRPC.Create(conf);
SCMAdminProtocol scmAdminProtocol = (SCMAdminProtocol)rpc.GetProxy(typeof(SCMAdminProtocol
), addr, conf);
return(scmAdminProtocol);
}
public virtual void TestGetSocketAddressForNMWithHA()
{
YarnConfiguration conf = new YarnConfiguration();
// Set NM address
conf.Set(YarnConfiguration.NmAddress, "0.0.0.0:1234");
// Set HA
conf.SetBoolean(YarnConfiguration.RmHaEnabled, true);
conf.Set(YarnConfiguration.RmHaId, "rm1");
NUnit.Framework.Assert.IsTrue(HAUtil.IsHAEnabled(conf));
IPEndPoint addr = conf.GetSocketAddr(YarnConfiguration.NmAddress, YarnConfiguration
.DefaultNmAddress, YarnConfiguration.DefaultNmPort);
NUnit.Framework.Assert.AreEqual(1234, addr.Port);
}
protected override IPEndPoint GetRMAddress(YarnConfiguration conf, Type protocol)
{
if (protocol == typeof(ResourceTracker))
{
return(conf.GetSocketAddr(YarnConfiguration.RmResourceTrackerAddress, YarnConfiguration
.DefaultRmResourceTrackerAddress, YarnConfiguration.DefaultRmResourceTrackerPort
));
}
else
{
string message = "Unsupported protocol found when creating the proxy " + "connection to ResourceManager: "
+ ((protocol != null) ? protocol.GetType().FullName : "null");
Log.Error(message);
throw new InvalidOperationException(message);
}
}
public virtual void TestContainerLocalizer()
{
if (!ShouldRun())
{
return;
}
IList <string> localDirs = dirsHandler.GetLocalDirs();
IList <string> logDirs = dirsHandler.GetLogDirs();
foreach (string localDir in localDirs)
{
Path userDir = new Path(localDir, ContainerLocalizer.Usercache);
files.Mkdir(userDir, new FsPermission("777"), false);
// $local/filecache
Path fileDir = new Path(localDir, ContainerLocalizer.Filecache);
files.Mkdir(fileDir, new FsPermission("777"), false);
}
string locId = "container_01_01";
Path nmPrivateContainerTokensPath = dirsHandler.GetLocalPathForWrite(ResourceLocalizationService
.NmPrivateDir + Path.Separator + string.Format(ContainerLocalizer.TokenFileNameFmt
, locId));
files.Create(nmPrivateContainerTokensPath, EnumSet.Of(CreateFlag.Create, CreateFlag
.Overwrite));
Configuration config = new YarnConfiguration(conf);
IPEndPoint nmAddr = config.GetSocketAddr(YarnConfiguration.NmBindHost, YarnConfiguration
.NmLocalizerAddress, YarnConfiguration.DefaultNmLocalizerAddress, YarnConfiguration
.DefaultNmLocalizerPort);
string appId = "application_01_01";
exec = new _LinuxContainerExecutor_258();
exec.SetConf(conf);
exec.StartLocalizer(nmPrivateContainerTokensPath, nmAddr, appSubmitter, appId, locId
, dirsHandler);
string locId2 = "container_01_02";
Path nmPrivateContainerTokensPath2 = dirsHandler.GetLocalPathForWrite(ResourceLocalizationService
.NmPrivateDir + Path.Separator + string.Format(ContainerLocalizer.TokenFileNameFmt
, locId2));
files.Create(nmPrivateContainerTokensPath2, EnumSet.Of(CreateFlag.Create, CreateFlag
.Overwrite));
exec.StartLocalizer(nmPrivateContainerTokensPath2, nmAddr, appSubmitter, appId, locId2
, dirsHandler);
}
private string BuildRedirectPath()
{
// make a copy of the original configuration so not to mutate it. Also use
// an YarnConfiguration to force loading of yarn-site.xml.
YarnConfiguration yarnConf = new YarnConfiguration(rm.GetConfig());
string activeRMHAId = RMHAUtils.FindActiveRMHAId(yarnConf);
string path = string.Empty;
if (activeRMHAId != null)
{
yarnConf.Set(YarnConfiguration.RmHaId, activeRMHAId);
IPEndPoint sock = YarnConfiguration.UseHttps(yarnConf) ? yarnConf.GetSocketAddr(YarnConfiguration
.RmWebappHttpsAddress, YarnConfiguration.DefaultRmWebappHttpsAddress, YarnConfiguration
.DefaultRmWebappHttpsPort) : yarnConf.GetSocketAddr(YarnConfiguration.RmWebappAddress
, YarnConfiguration.DefaultRmWebappAddress, YarnConfiguration.DefaultRmWebappPort
);
path = sock.GetHostName() + ":" + Sharpen.Extensions.ToString(sock.Port);
path = YarnConfiguration.UseHttps(yarnConf) ? "https://" + path : "http://" + path;
}
return(path);
}
public static Text GetTokenService(Configuration conf, string address, string defaultAddr
, int defaultPort)
{
if (HAUtil.IsHAEnabled(conf))
{
// Build a list of service addresses to form the service name
AList <string> services = new AList <string>();
YarnConfiguration yarnConf = new YarnConfiguration(conf);
foreach (string rmId in HAUtil.GetRMHAIds(conf))
{
// Set RM_ID to get the corresponding RM_ADDRESS
yarnConf.Set(YarnConfiguration.RmHaId, rmId);
services.AddItem(SecurityUtil.BuildTokenService(yarnConf.GetSocketAddr(address, defaultAddr
, defaultPort)).ToString());
}
return(new Text(Joiner.On(',').Join(services)));
}
// Non-HA case - no need to set RM_ID
return(SecurityUtil.BuildTokenService(conf.GetSocketAddr(address, defaultAddr, defaultPort
)));
}
/// <exception cref="System.IO.IOException"/>
public RMHAServiceTarget(YarnConfiguration conf)
{
autoFailoverEnabled = HAUtil.IsAutomaticFailoverEnabled(conf);
haAdminServiceAddress = conf.GetSocketAddr(YarnConfiguration.RmAdminAddress, YarnConfiguration
.DefaultRmAdminAddress, YarnConfiguration.DefaultRmAdminPort);
}