/// <exception cref="System.Exception"/>
public virtual void TestContainerManager()
{
try
{
yarnCluster = new MiniYARNCluster(typeof(Org.Apache.Hadoop.Yarn.Server.TestContainerManagerSecurity
).FullName, 1, 1, 1);
yarnCluster.Init(conf);
yarnCluster.Start();
// TestNMTokens.
TestNMTokens(conf);
// Testing for container token tampering
TestContainerToken(conf);
}
catch (Exception e)
{
Sharpen.Runtime.PrintStackTrace(e);
throw;
}
finally
{
if (yarnCluster != null)
{
yarnCluster.Stop();
yarnCluster = null;
}
}
}
/// <exception cref="System.Exception"/>
protected override void ServiceInit(Configuration conf)
{
lock (this)
{
Configuration config = new YarnConfiguration(conf);
// create nm-local-dirs and configure them for the nodemanager
string localDirsString = this.PrepareDirs("local", this._enclosing.numLocalDirs);
config.Set(YarnConfiguration.NmLocalDirs, localDirsString);
// create nm-log-dirs and configure them for the nodemanager
string logDirsString = this.PrepareDirs("log", this._enclosing.numLogDirs);
config.Set(YarnConfiguration.NmLogDirs, logDirsString);
config.SetInt(YarnConfiguration.NmPmemMb, config.GetInt(YarnConfiguration.YarnMiniclusterNmPmemMb
, YarnConfiguration.DefaultYarnMiniclusterNmPmemMb));
config.Set(YarnConfiguration.NmAddress, MiniYARNCluster.GetHostname() + ":0");
config.Set(YarnConfiguration.NmLocalizerAddress, MiniYARNCluster.GetHostname() +
":0");
WebAppUtils.SetNMWebAppHostNameAndPort(config, MiniYARNCluster.GetHostname(), 0);
// Disable resource checks by default
if (!config.GetBoolean(YarnConfiguration.YarnMiniclusterControlResourceMonitoring
, YarnConfiguration.DefaultYarnMiniclusterControlResourceMonitoring))
{
config.SetBoolean(YarnConfiguration.NmPmemCheckEnabled, false);
config.SetBoolean(YarnConfiguration.NmVmemCheckEnabled, false);
}
MiniYARNCluster.Log.Info("Starting NM: " + this.index);
this._enclosing.nodeManagers[this.index].Init(config);
base.ServiceInit(config);
}
}
/// <exception cref="System.IO.IOException"/>
private void TestDirsFailures(bool localORLogDirs)
{
string dirType = localORLogDirs ? "local" : "log";
string dirsProperty = localORLogDirs ? YarnConfiguration.NmLocalDirs : YarnConfiguration
.NmLogDirs;
Configuration conf = new Configuration();
// set disk health check interval to a small value (say 1 sec).
conf.SetLong(YarnConfiguration.NmDiskHealthCheckIntervalMs, DiskHealthCheckInterval
);
// If 2 out of the total 4 local-dirs fail OR if 2 Out of the total 4
// log-dirs fail, then the node's health status should become unhealthy.
conf.SetFloat(YarnConfiguration.NmMinHealthyDisksFraction, 0.60F);
if (yarnCluster != null)
{
yarnCluster.Stop();
FileUtil.FullyDelete(localFSDirBase);
localFSDirBase.Mkdirs();
}
Log.Info("Starting up YARN cluster");
yarnCluster = new MiniYARNCluster(typeof(TestDiskFailures).FullName, 1, numLocalDirs
, numLogDirs);
yarnCluster.Init(conf);
yarnCluster.Start();
NodeManager nm = yarnCluster.GetNodeManager(0);
Log.Info("Configured nm-" + dirType + "-dirs=" + nm.GetConfig().Get(dirsProperty)
);
dirsHandler = nm.GetNodeHealthChecker().GetDiskHandler();
IList <string> list = localORLogDirs ? dirsHandler.GetLocalDirs() : dirsHandler.GetLogDirs
();
string[] dirs = Sharpen.Collections.ToArray(list, new string[list.Count]);
NUnit.Framework.Assert.AreEqual("Number of nm-" + dirType + "-dirs is wrong.", numLocalDirs
, dirs.Length);
string expectedDirs = StringUtils.Join(",", list);
// validate the health of disks initially
VerifyDisksHealth(localORLogDirs, expectedDirs, true);
// Make 1 nm-local-dir fail and verify if "the nodemanager can identify
// the disk failure(s) and can update the list of good nm-local-dirs.
PrepareDirToFail(dirs[2]);
expectedDirs = dirs[0] + "," + dirs[1] + "," + dirs[3];
VerifyDisksHealth(localORLogDirs, expectedDirs, true);
// Now, make 1 more nm-local-dir/nm-log-dir fail and verify if "the
// nodemanager can identify the disk failures and can update the list of
// good nm-local-dirs/nm-log-dirs and can update the overall health status
// of the node to unhealthy".
PrepareDirToFail(dirs[0]);
expectedDirs = dirs[1] + "," + dirs[3];
VerifyDisksHealth(localORLogDirs, expectedDirs, false);
// Fail the remaining 2 local-dirs/log-dirs and verify if NM remains with
// empty list of local-dirs/log-dirs and the overall health status is
// unhealthy.
PrepareDirToFail(dirs[1]);
PrepareDirToFail(dirs[3]);
expectedDirs = string.Empty;
VerifyDisksHealth(localORLogDirs, expectedDirs, false);
}
public static void Teardown()
{
if (yarnCluster != null)
{
yarnCluster.Stop();
yarnCluster = null;
}
FileUtil.FullyDelete(localFSDirBase);
}
public virtual void Setup()
{
Configuration conf = new YarnConfiguration();
conf.SetBoolean(YarnConfiguration.AutoFailoverEnabled, false);
conf.Set(YarnConfiguration.RmWebappAddress, "localhost:0");
cluster = new MiniYARNCluster(typeof(TestMiniYARNClusterForHA).FullName, 2, 1, 1,
1);
cluster.Init(conf);
cluster.Start();
cluster.GetResourceManager(0).GetRMContext().GetRMAdminService().TransitionToActive
(new HAServiceProtocol.StateChangeRequestInfo(HAServiceProtocol.RequestSource.RequestByUser
));
NUnit.Framework.Assert.IsFalse("RM never turned active", -1 == cluster.GetActiveRMIndex
());
}
/// <exception cref="System.Exception"/>
public virtual void TestContainerManagerWithEpoch()
{
try
{
yarnCluster = new MiniYARNCluster(typeof(Org.Apache.Hadoop.Yarn.Server.TestContainerManagerSecurity
).FullName, 1, 1, 1);
yarnCluster.Init(conf);
yarnCluster.Start();
// Testing for container token tampering
TestContainerTokenWithEpoch(conf);
}
finally
{
if (yarnCluster != null)
{
yarnCluster.Stop();
yarnCluster = null;
}
}
}
/// <exception cref="System.Exception"/>
protected override void ServiceInit(Configuration conf)
{
lock (this)
{
this._enclosing.appHistoryServer = new ApplicationHistoryServer();
conf.SetClass(YarnConfiguration.ApplicationHistoryStore, typeof(MemoryApplicationHistoryStore
), typeof(ApplicationHistoryStore));
conf.SetClass(YarnConfiguration.TimelineServiceStore, typeof(MemoryTimelineStore)
, typeof(TimelineStore));
conf.SetClass(YarnConfiguration.TimelineServiceStateStoreClass, typeof(MemoryTimelineStateStore
), typeof(TimelineStateStore));
if (!this._enclosing.useFixedPorts)
{
string hostname = MiniYARNCluster.GetHostname();
conf.Set(YarnConfiguration.TimelineServiceAddress, hostname + ":0");
conf.Set(YarnConfiguration.TimelineServiceWebappAddress, hostname + ":0");
}
this._enclosing.appHistoryServer.Init(conf);
base.ServiceInit(conf);
}
}
public virtual void TestTimelineServiceStartInMiniCluster()
{
Configuration conf = new YarnConfiguration();
int numNodeManagers = 1;
int numLocalDirs = 1;
int numLogDirs = 1;
bool enableAHS;
/*
* Timeline service should not start if TIMELINE_SERVICE_ENABLED == false
* and enableAHS flag == false
*/
conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, false);
enableAHS = false;
MiniYARNCluster cluster = null;
try
{
cluster = new MiniYARNCluster(typeof(TestMiniYarnCluster).Name, numNodeManagers,
numLocalDirs, numLogDirs, numLogDirs, enableAHS);
cluster.Init(conf);
cluster.Start();
//verify that the timeline service is not started.
NUnit.Framework.Assert.IsNull("Timeline Service should not have been started", cluster
.GetApplicationHistoryServer());
}
finally
{
if (cluster != null)
{
cluster.Stop();
}
}
/*
* Timeline service should start if TIMELINE_SERVICE_ENABLED == true
* and enableAHS == false
*/
conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, true);
enableAHS = false;
cluster = null;
try
{
cluster = new MiniYARNCluster(typeof(TestMiniYarnCluster).Name, numNodeManagers,
numLocalDirs, numLogDirs, numLogDirs, enableAHS);
cluster.Init(conf);
// Verify that the timeline-service starts on ephemeral ports by default
string hostname = MiniYARNCluster.GetHostname();
NUnit.Framework.Assert.AreEqual(hostname + ":0", conf.Get(YarnConfiguration.TimelineServiceAddress
));
NUnit.Framework.Assert.AreEqual(hostname + ":0", conf.Get(YarnConfiguration.TimelineServiceWebappAddress
));
cluster.Start();
//Timeline service may sometime take a while to get started
int wait = 0;
while (cluster.GetApplicationHistoryServer() == null && wait < 20)
{
Sharpen.Thread.Sleep(500);
wait++;
}
//verify that the timeline service is started.
NUnit.Framework.Assert.IsNotNull("Timeline Service should have been started", cluster
.GetApplicationHistoryServer());
}
finally
{
if (cluster != null)
{
cluster.Stop();
}
}
/*
* Timeline service should start if TIMELINE_SERVICE_ENABLED == false
* and enableAHS == true
*/
conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, false);
enableAHS = true;
cluster = null;
try
{
cluster = new MiniYARNCluster(typeof(TestMiniYarnCluster).Name, numNodeManagers,
numLocalDirs, numLogDirs, numLogDirs, enableAHS);
cluster.Init(conf);
cluster.Start();
//Timeline service may sometime take a while to get started
int wait = 0;
while (cluster.GetApplicationHistoryServer() == null && wait < 20)
{
Sharpen.Thread.Sleep(500);
wait++;
}
//verify that the timeline service is started.
NUnit.Framework.Assert.IsNotNull("Timeline Service should have been started", cluster
.GetApplicationHistoryServer());
}
finally
{
if (cluster != null)
{
cluster.Stop();
}
}
}
public ApplicationHistoryServerWrapper(MiniYARNCluster _enclosing)
: base(typeof(MiniYARNCluster.ApplicationHistoryServerWrapper).FullName)
{
this._enclosing = _enclosing;
}
internal ShortCircuitedNodeManager(MiniYARNCluster _enclosing)
: base(_enclosing)
{
this._enclosing = _enclosing;
}
internal CustomNodeManager(MiniYARNCluster _enclosing)
{
this._enclosing = _enclosing;
}
public NodeManagerWrapper(MiniYARNCluster _enclosing, int i)
: base(typeof(MiniYARNCluster.NodeManagerWrapper).FullName + "_" + i)
{
this._enclosing = _enclosing;
this.index = i;
}
public _Thread_310(MiniYARNCluster _enclosing, int index)
{
this._enclosing = _enclosing;
this.index = index;
}
public _EventHandler_296(MiniYARNCluster _enclosing)
{
this._enclosing = _enclosing;
}