public virtual void TestExplicitFailover()
{
conf.SetBoolean(YarnConfiguration.AutoFailoverEnabled, false);
cluster.Init(conf);
cluster.Start();
GetAdminService(0).TransitionToActive(req);
NUnit.Framework.Assert.IsFalse("RM never turned active", -1 == cluster.GetActiveRMIndex
());
VerifyConnections();
ExplicitFailover();
VerifyConnections();
ExplicitFailover();
VerifyConnections();
}
/// <exception cref="System.Exception"/>
protected internal virtual void SetupInternal(int numNodeManager)
{
Log.Info("Starting up YARN cluster");
conf = new YarnConfiguration();
conf.SetInt(YarnConfiguration.RmSchedulerMinimumAllocationMb, 128);
conf.Set("yarn.log.dir", "target");
conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, true);
conf.Set(YarnConfiguration.RmScheduler, typeof(CapacityScheduler).FullName);
conf.SetBoolean(YarnConfiguration.NodeLabelsEnabled, true);
if (yarnCluster == null)
{
yarnCluster = new MiniYARNCluster(typeof(TestDistributedShell).Name, 1, numNodeManager
, 1, 1);
yarnCluster.Init(conf);
yarnCluster.Start();
conf.Set(YarnConfiguration.TimelineServiceWebappAddress, MiniYARNCluster.GetHostname
() + ":" + yarnCluster.GetApplicationHistoryServer().GetPort());
WaitForNMsToRegister();
Uri url = Sharpen.Thread.CurrentThread().GetContextClassLoader().GetResource("yarn-site.xml"
);
if (url == null)
{
throw new RuntimeException("Could not find 'yarn-site.xml' dummy file in classpath"
);
}
Configuration yarnClusterConfig = yarnCluster.GetConfig();
yarnClusterConfig.Set("yarn.application.classpath", new FilePath(url.AbsolutePath
).GetParent());
//write the document to a buffer (not directly to the file, as that
//can cause the file being written to get read -which will then fail.
ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
yarnClusterConfig.WriteXml(bytesOut);
bytesOut.Close();
//write the bytes to the file in the classpath
OutputStream os = new FileOutputStream(new FilePath(url.AbsolutePath));
os.Write(bytesOut.ToByteArray());
os.Close();
}
FileContext fsContext = FileContext.GetLocalFSFileContext();
fsContext.Delete(new Path(conf.Get("yarn.timeline-service.leveldb-timeline-store.path"
)), true);
try
{
Sharpen.Thread.Sleep(2000);
}
catch (Exception e)
{
Log.Info("setup thread sleep interrupted. message=" + e.Message);
}
}
public virtual void TestRMWebAppRedirect()
{
cluster = new MiniYARNCluster(typeof(TestRMFailover).FullName, 2, 0, 1, 1);
conf.SetBoolean(YarnConfiguration.AutoFailoverEnabled, false);
cluster.Init(conf);
cluster.Start();
GetAdminService(0).TransitionToActive(req);
string rm1Url = "http://0.0.0.0:18088";
string rm2Url = "http://0.0.0.0:28088";
string redirectURL = GetRedirectURL(rm2Url);
// if uri is null, RMWebAppFilter will append a slash at the trail of the redirection url
NUnit.Framework.Assert.AreEqual(redirectURL, rm1Url + "/");
redirectURL = GetRedirectURL(rm2Url + "/metrics");
NUnit.Framework.Assert.AreEqual(redirectURL, rm1Url + "/metrics");
redirectURL = GetRedirectURL(rm2Url + "/jmx");
NUnit.Framework.Assert.AreEqual(redirectURL, rm1Url + "/jmx");
// standby RM links /conf, /stacks, /logLevel, /static, /logs,
// /cluster/cluster as well as webService
// /ws/v1/cluster/info should not be redirected to active RM
redirectURL = GetRedirectURL(rm2Url + "/cluster/cluster");
NUnit.Framework.Assert.IsNull(redirectURL);
redirectURL = GetRedirectURL(rm2Url + "/conf");
NUnit.Framework.Assert.IsNull(redirectURL);
redirectURL = GetRedirectURL(rm2Url + "/stacks");
NUnit.Framework.Assert.IsNull(redirectURL);
redirectURL = GetRedirectURL(rm2Url + "/logLevel");
NUnit.Framework.Assert.IsNull(redirectURL);
redirectURL = GetRedirectURL(rm2Url + "/static");
NUnit.Framework.Assert.IsNull(redirectURL);
redirectURL = GetRedirectURL(rm2Url + "/logs");
NUnit.Framework.Assert.IsNull(redirectURL);
redirectURL = GetRedirectURL(rm2Url + "/ws/v1/cluster/info");
NUnit.Framework.Assert.IsNull(redirectURL);
redirectURL = GetRedirectURL(rm2Url + "/ws/v1/cluster/apps");
NUnit.Framework.Assert.AreEqual(redirectURL, rm1Url + "/ws/v1/cluster/apps");
redirectURL = GetRedirectURL(rm2Url + "/proxy/" + fakeAppId);
NUnit.Framework.Assert.IsNull(redirectURL);
// transit the active RM to standby
// Both of RMs are in standby mode
GetAdminService(0).TransitionToStandby(req);
// RM2 is expected to send the httpRequest to itself.
// The Header Field: Refresh is expected to be set.
redirectURL = GetRefreshURL(rm2Url);
NUnit.Framework.Assert.IsTrue(redirectURL != null && redirectURL.Contains(YarnWebParams
.NextRefreshInterval) && redirectURL.Contains(rm2Url));
}
public static void Setup()
{
Log.Info("Starting up YARN cluster");
conf.SetInt(YarnConfiguration.RmSchedulerMinimumAllocationMb, 128);
if (yarnCluster == null)
{
yarnCluster = new MiniYARNCluster(typeof(TestUnmanagedAMLauncher).Name, 1, 1, 1);
yarnCluster.Init(conf);
yarnCluster.Start();
//get the address
Configuration yarnClusterConfig = yarnCluster.GetConfig();
Log.Info("MiniYARN ResourceManager published address: " + yarnClusterConfig.Get(YarnConfiguration
.RmAddress));
Log.Info("MiniYARN ResourceManager published web address: " + yarnClusterConfig.Get
(YarnConfiguration.RmWebappAddress));
string webapp = yarnClusterConfig.Get(YarnConfiguration.RmWebappAddress);
NUnit.Framework.Assert.IsTrue("Web app address still unbound to a host at " + webapp
, !webapp.StartsWith("0.0.0.0"));
Log.Info("Yarn webapp is at " + webapp);
Uri url = Sharpen.Thread.CurrentThread().GetContextClassLoader().GetResource("yarn-site.xml"
);
if (url == null)
{
throw new RuntimeException("Could not find 'yarn-site.xml' dummy file in classpath"
);
}
//write the document to a buffer (not directly to the file, as that
//can cause the file being written to get read -which will then fail.
ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
yarnClusterConfig.WriteXml(bytesOut);
bytesOut.Close();
//write the bytes to the file in the classpath
OutputStream os = new FileOutputStream(new FilePath(url.AbsolutePath));
os.Write(bytesOut.ToByteArray());
os.Close();
}
try
{
Sharpen.Thread.Sleep(2000);
}
catch (Exception e)
{
Log.Info("setup thread sleep interrupted. message=" + e.Message);
}
}
public virtual void Setup()
{
// start minicluster
conf = new YarnConfiguration();
yarnCluster = new MiniYARNCluster(typeof(TestAMRMClient).FullName, nodeCount, 1,
1);
yarnCluster.Init(conf);
yarnCluster.Start();
NUnit.Framework.Assert.IsNotNull(yarnCluster);
NUnit.Framework.Assert.AreEqual(Service.STATE.Started, yarnCluster.GetServiceState
());
// start rm client
yarnClient = (YarnClientImpl)YarnClient.CreateYarnClient();
yarnClient.Init(conf);
yarnClient.Start();
NUnit.Framework.Assert.IsNotNull(yarnClient);
NUnit.Framework.Assert.AreEqual(Service.STATE.Started, yarnClient.GetServiceState
());
// get node info
nodeReports = yarnClient.GetNodeReports(NodeState.Running);
// submit new app
ApplicationSubmissionContext appContext = yarnClient.CreateApplication().GetApplicationSubmissionContext
();
ApplicationId appId = appContext.GetApplicationId();
// set the application name
appContext.SetApplicationName("Test");
// Set the priority for the application master
Priority pri = Priority.NewInstance(0);
appContext.SetPriority(pri);
// Set the queue to which this application is to be submitted in the RM
appContext.SetQueue("default");
// Set up the container launch context for the application master
ContainerLaunchContext amContainer = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<ContainerLaunchContext>();
appContext.SetAMContainerSpec(amContainer);
// unmanaged AM
appContext.SetUnmanagedAM(true);
// Create the request to send to the applications manager
SubmitApplicationRequest appRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<SubmitApplicationRequest>();
appRequest.SetApplicationSubmissionContext(appContext);
// Submit the application to the applications manager
yarnClient.SubmitApplication(appContext);
// wait for app to start
int iterationsLeft = 30;
RMAppAttempt appAttempt = null;
while (iterationsLeft > 0)
{
ApplicationReport appReport = yarnClient.GetApplicationReport(appId);
if (appReport.GetYarnApplicationState() == YarnApplicationState.Accepted)
{
attemptId = appReport.GetCurrentApplicationAttemptId();
appAttempt = yarnCluster.GetResourceManager().GetRMContext().GetRMApps()[attemptId
.GetApplicationId()].GetCurrentAppAttempt();
while (true)
{
if (appAttempt.GetAppAttemptState() == RMAppAttemptState.Launched)
{
break;
}
}
break;
}
Sleep(1000);
--iterationsLeft;
}
if (iterationsLeft == 0)
{
NUnit.Framework.Assert.Fail("Application hasn't bee started");
}
// Just dig into the ResourceManager and get the AMRMToken just for the sake
// of testing.
UserGroupInformation.SetLoginUser(UserGroupInformation.CreateRemoteUser(UserGroupInformation
.GetCurrentUser().GetUserName()));
UserGroupInformation.GetCurrentUser().AddToken(appAttempt.GetAMRMToken());
//creating an instance NMTokenCase
nmTokenCache = new NMTokenCache();
// start am rm client
rmClient = (AMRMClientImpl <AMRMClient.ContainerRequest>)AMRMClient.CreateAMRMClient
<AMRMClient.ContainerRequest>();
//setting an instance NMTokenCase
rmClient.SetNMTokenCache(nmTokenCache);
rmClient.Init(conf);
rmClient.Start();
NUnit.Framework.Assert.IsNotNull(rmClient);
NUnit.Framework.Assert.AreEqual(Service.STATE.Started, rmClient.GetServiceState()
);
// start am nm client
nmClient = (NMClientImpl)NMClient.CreateNMClient();
//propagating the AMRMClient NMTokenCache instance
nmClient.SetNMTokenCache(rmClient.GetNMTokenCache());
nmClient.Init(conf);
nmClient.Start();
NUnit.Framework.Assert.IsNotNull(nmClient);
NUnit.Framework.Assert.AreEqual(Service.STATE.Started, nmClient.GetServiceState()
);
}