public static IList <string> GetRMHAWebappAddresses(YarnConfiguration conf)
{
ICollection <string> rmIds = conf.GetStringCollection(YarnConfiguration.RmHaIds);
IList <string> addrs = new AList <string>();
if (YarnConfiguration.UseHttps(conf))
{
foreach (string id in rmIds)
{
string addr = conf.Get(YarnConfiguration.RmWebappHttpsAddress + "." + id);
if (addr != null)
{
addrs.AddItem(addr);
}
}
}
else
{
foreach (string id in rmIds)
{
string addr = conf.Get(YarnConfiguration.RmWebappAddress + "." + id);
if (addr != null)
{
addrs.AddItem(addr);
}
}
}
return(addrs);
}
public static void SetUpResourceManager()
{
Configuration.AddDefaultResource("config-with-security.xml");
Configuration configuration = new YarnConfiguration();
resourceManager = new _ResourceManager_74();
resourceManager.Init(configuration);
new _Thread_80().Start();
int waitCount = 0;
while (resourceManager.GetServiceState() == Service.STATE.Inited && waitCount++ <
10)
{
Log.Info("Waiting for RM to start...");
Sharpen.Thread.Sleep(1000);
}
if (resourceManager.GetServiceState() != Service.STATE.Started)
{
throw new IOException("ResourceManager failed to start. Final state is " + resourceManager
.GetServiceState());
}
Log.Info("ResourceManager RMAdmin address: " + configuration.Get(YarnConfiguration
.RmAdminAddress));
client = new ResourceManagerAdministrationProtocolPBClientImpl(1L, GetProtocolAddress
(configuration), configuration);
}
public virtual void TestHAIDLookup()
{
//test implicitly lookup HA-ID
Configuration conf = new YarnConfiguration(configuration);
rm = new MockRM(conf);
rm.Init(conf);
NUnit.Framework.Assert.AreEqual(conf.Get(YarnConfiguration.RmHaId), Rm2NodeId);
//test explicitly lookup HA-ID
configuration.Set(YarnConfiguration.RmHaId, Rm1NodeId);
conf = new YarnConfiguration(configuration);
rm = new MockRM(conf);
rm.Init(conf);
NUnit.Framework.Assert.AreEqual(conf.Get(YarnConfiguration.RmHaId), Rm1NodeId);
//test if RM_HA_ID can not be found
configuration.Set(YarnConfiguration.RmHaIds, Rm1NodeId + "," + Rm3NodeId);
configuration.Unset(YarnConfiguration.RmHaId);
conf = new YarnConfiguration(configuration);
try
{
rm = new MockRM(conf);
rm.Init(conf);
NUnit.Framework.Assert.Fail("Should get an exception here.");
}
catch (Exception ex)
{
NUnit.Framework.Assert.IsTrue(ex.Message.Contains("Invalid configuration! Can not find valid RM_HA_ID."
));
}
}
/// <exception cref="Javax.Servlet.ServletException"/>
protected internal virtual string FindRedirectUrl()
{
string addr;
if (proxyUriBases.Count == 1)
{
// external proxy or not RM HA
addr = proxyUriBases.Values.GetEnumerator().Next();
}
else
{
// RM HA
YarnConfiguration conf = new YarnConfiguration();
string activeRMId = RMHAUtils.FindActiveRMHAId(conf);
string addressPropertyPrefix = YarnConfiguration.UseHttps(conf) ? YarnConfiguration
.RmWebappHttpsAddress : YarnConfiguration.RmWebappAddress;
string host = conf.Get(HAUtil.AddSuffix(addressPropertyPrefix, activeRMId));
addr = proxyUriBases[host];
}
if (addr == null)
{
throw new ServletException("Could not determine the proxy server for redirection"
);
}
return(addr);
}
/// <summary>
/// Add the requisite security principal settings to the given Configuration,
/// returning a copy.
/// </summary>
/// <param name="conf">the original config</param>
/// <returns>a copy with the security settings added</returns>
private static Configuration AddSecurityConfiguration(Configuration conf)
{
// Make a copy so we don't mutate it. Also use an YarnConfiguration to
// force loading of yarn-site.xml.
conf = new YarnConfiguration(conf);
conf.Set(CommonConfigurationKeys.HadoopSecurityServiceUserNameKey, conf.Get(YarnConfiguration
.RmPrincipal, string.Empty));
return(conf);
}
/// <exception cref="System.Exception"/>
protected internal virtual void SetupInternal(int numNodeManager)
{
Log.Info("Starting up YARN cluster");
conf = new YarnConfiguration();
conf.SetInt(YarnConfiguration.RmSchedulerMinimumAllocationMb, 128);
conf.Set("yarn.log.dir", "target");
conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, true);
conf.Set(YarnConfiguration.RmScheduler, typeof(CapacityScheduler).FullName);
conf.SetBoolean(YarnConfiguration.NodeLabelsEnabled, true);
if (yarnCluster == null)
{
yarnCluster = new MiniYARNCluster(typeof(TestDistributedShell).Name, 1, numNodeManager
, 1, 1);
yarnCluster.Init(conf);
yarnCluster.Start();
conf.Set(YarnConfiguration.TimelineServiceWebappAddress, MiniYARNCluster.GetHostname
() + ":" + yarnCluster.GetApplicationHistoryServer().GetPort());
WaitForNMsToRegister();
Uri url = Sharpen.Thread.CurrentThread().GetContextClassLoader().GetResource("yarn-site.xml"
);
if (url == null)
{
throw new RuntimeException("Could not find 'yarn-site.xml' dummy file in classpath"
);
}
Configuration yarnClusterConfig = yarnCluster.GetConfig();
yarnClusterConfig.Set("yarn.application.classpath", new FilePath(url.AbsolutePath
).GetParent());
//write the document to a buffer (not directly to the file, as that
//can cause the file being written to get read -which will then fail.
ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
yarnClusterConfig.WriteXml(bytesOut);
bytesOut.Close();
//write the bytes to the file in the classpath
OutputStream os = new FileOutputStream(new FilePath(url.AbsolutePath));
os.Write(bytesOut.ToByteArray());
os.Close();
}
FileContext fsContext = FileContext.GetLocalFSFileContext();
fsContext.Delete(new Path(conf.Get("yarn.timeline-service.leveldb-timeline-store.path"
)), true);
try
{
Sharpen.Thread.Sleep(2000);
}
catch (Exception e)
{
Log.Info("setup thread sleep interrupted. message=" + e.Message);
}
}
// Couldn't find an Active RM
/// <exception cref="System.Exception"/>
private static HAServiceProtocol.HAServiceState GetHAState(YarnConfiguration yarnConf
)
{
HAServiceTarget haServiceTarget;
int rpcTimeoutForChecks = yarnConf.GetInt(CommonConfigurationKeys.HaFcCliCheckTimeoutKey
, CommonConfigurationKeys.HaFcCliCheckTimeoutDefault);
yarnConf.Set(CommonConfigurationKeys.HadoopSecurityServiceUserNameKey, yarnConf.Get
(YarnConfiguration.RmPrincipal, string.Empty));
haServiceTarget = new RMHAServiceTarget(yarnConf);
HAServiceProtocol proto = haServiceTarget.GetProxy(yarnConf, rpcTimeoutForChecks);
HAServiceProtocol.HAServiceState haState = proto.GetServiceStatus().GetState();
return(haState);
}
public virtual void TearDown()
{
if (yarnCluster != null)
{
try
{
yarnCluster.Stop();
}
finally
{
yarnCluster = null;
}
}
FileContext fsContext = FileContext.GetLocalFSFileContext();
fsContext.Delete(new Path(conf.Get("yarn.timeline-service.leveldb-timeline-store.path"
)), true);
}
public virtual void TestTimelineServiceStartInMiniCluster()
{
Configuration conf = new YarnConfiguration();
int numNodeManagers = 1;
int numLocalDirs = 1;
int numLogDirs = 1;
bool enableAHS;
/*
* Timeline service should not start if TIMELINE_SERVICE_ENABLED == false
* and enableAHS flag == false
*/
conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, false);
enableAHS = false;
MiniYARNCluster cluster = null;
try
{
cluster = new MiniYARNCluster(typeof(TestMiniYarnCluster).Name, numNodeManagers,
numLocalDirs, numLogDirs, numLogDirs, enableAHS);
cluster.Init(conf);
cluster.Start();
//verify that the timeline service is not started.
NUnit.Framework.Assert.IsNull("Timeline Service should not have been started", cluster
.GetApplicationHistoryServer());
}
finally
{
if (cluster != null)
{
cluster.Stop();
}
}
/*
* Timeline service should start if TIMELINE_SERVICE_ENABLED == true
* and enableAHS == false
*/
conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, true);
enableAHS = false;
cluster = null;
try
{
cluster = new MiniYARNCluster(typeof(TestMiniYarnCluster).Name, numNodeManagers,
numLocalDirs, numLogDirs, numLogDirs, enableAHS);
cluster.Init(conf);
// Verify that the timeline-service starts on ephemeral ports by default
string hostname = MiniYARNCluster.GetHostname();
NUnit.Framework.Assert.AreEqual(hostname + ":0", conf.Get(YarnConfiguration.TimelineServiceAddress
));
NUnit.Framework.Assert.AreEqual(hostname + ":0", conf.Get(YarnConfiguration.TimelineServiceWebappAddress
));
cluster.Start();
//Timeline service may sometime take a while to get started
int wait = 0;
while (cluster.GetApplicationHistoryServer() == null && wait < 20)
{
Sharpen.Thread.Sleep(500);
wait++;
}
//verify that the timeline service is started.
NUnit.Framework.Assert.IsNotNull("Timeline Service should have been started", cluster
.GetApplicationHistoryServer());
}
finally
{
if (cluster != null)
{
cluster.Stop();
}
}
/*
* Timeline service should start if TIMELINE_SERVICE_ENABLED == false
* and enableAHS == true
*/
conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, false);
enableAHS = true;
cluster = null;
try
{
cluster = new MiniYARNCluster(typeof(TestMiniYarnCluster).Name, numNodeManagers,
numLocalDirs, numLogDirs, numLogDirs, enableAHS);
cluster.Init(conf);
cluster.Start();
//Timeline service may sometime take a while to get started
int wait = 0;
while (cluster.GetApplicationHistoryServer() == null && wait < 20)
{
Sharpen.Thread.Sleep(500);
wait++;
}
//verify that the timeline service is started.
NUnit.Framework.Assert.IsNotNull("Timeline Service should have been started", cluster
.GetApplicationHistoryServer());
}
finally
{
if (cluster != null)
{
cluster.Stop();
}
}
}
public virtual void InnerTestHAWithRMHostName(bool includeBindHost)
{
//this is run two times, with and without a bind host configured
if (includeBindHost)
{
configuration.Set(YarnConfiguration.RmBindHost, "9.9.9.9");
}
//test if both RM_HOSTBANE_{rm_id} and RM_RPCADDRESS_{rm_id} are set
//We should only read rpc addresses from RM_RPCADDRESS_{rm_id} configuration
configuration.Set(HAUtil.AddSuffix(YarnConfiguration.RmHostname, Rm1NodeId), "1.1.1.1"
);
configuration.Set(HAUtil.AddSuffix(YarnConfiguration.RmHostname, Rm2NodeId), "0.0.0.0"
);
configuration.Set(HAUtil.AddSuffix(YarnConfiguration.RmHostname, Rm3NodeId), "2.2.2.2"
);
try
{
Configuration conf = new YarnConfiguration(configuration);
rm = new MockRM(conf);
rm.Init(conf);
foreach (string confKey in YarnConfiguration.GetServiceAddressConfKeys(conf))
{
NUnit.Framework.Assert.AreEqual("RPC address not set for " + confKey, Rm1Address,
conf.Get(HAUtil.AddSuffix(confKey, Rm1NodeId)));
NUnit.Framework.Assert.AreEqual("RPC address not set for " + confKey, Rm2Address,
conf.Get(HAUtil.AddSuffix(confKey, Rm2NodeId)));
NUnit.Framework.Assert.AreEqual("RPC address not set for " + confKey, Rm3Address,
conf.Get(HAUtil.AddSuffix(confKey, Rm3NodeId)));
if (includeBindHost)
{
NUnit.Framework.Assert.AreEqual("Web address misconfigured WITH bind-host", Sharpen.Runtime.Substring
(rm.webAppAddress, 0, 7), "9.9.9.9");
}
else
{
//YarnConfiguration tries to figure out which rm host it's on by binding to it,
//which doesn't happen for any of these fake addresses, so we end up with 0.0.0.0
NUnit.Framework.Assert.AreEqual("Web address misconfigured WITHOUT bind-host", Sharpen.Runtime.Substring
(rm.webAppAddress, 0, 7), "0.0.0.0");
}
}
}
catch (YarnRuntimeException)
{
NUnit.Framework.Assert.Fail("Should not throw any exceptions.");
}
//test if only RM_HOSTBANE_{rm_id} is set
configuration.Clear();
configuration.SetBoolean(YarnConfiguration.RmHaEnabled, true);
configuration.Set(YarnConfiguration.RmHaIds, Rm1NodeId + "," + Rm2NodeId);
configuration.Set(HAUtil.AddSuffix(YarnConfiguration.RmHostname, Rm1NodeId), "1.1.1.1"
);
configuration.Set(HAUtil.AddSuffix(YarnConfiguration.RmHostname, Rm2NodeId), "0.0.0.0"
);
try
{
Configuration conf = new YarnConfiguration(configuration);
rm = new MockRM(conf);
rm.Init(conf);
NUnit.Framework.Assert.AreEqual("RPC address not set for " + YarnConfiguration.RmAddress
, "1.1.1.1:8032", conf.Get(HAUtil.AddSuffix(YarnConfiguration.RmAddress, Rm1NodeId
)));
NUnit.Framework.Assert.AreEqual("RPC address not set for " + YarnConfiguration.RmAddress
, "0.0.0.0:8032", conf.Get(HAUtil.AddSuffix(YarnConfiguration.RmAddress, Rm2NodeId
)));
}
catch (YarnRuntimeException)
{
NUnit.Framework.Assert.Fail("Should not throw any exceptions.");
}
}