// nothing
/// <summary>Starts DFS and MR clusters, as specified in member-variable options.</summary>
/// <remarks>
/// Starts DFS and MR clusters, as specified in member-variable options. Also
/// writes out configuration and details, if requested.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.IO.FileNotFoundException"/>
/// <exception cref="Sharpen.URISyntaxException"/>
public virtual void Start()
{
if (!noDFS)
{
dfs = new MiniDFSCluster.Builder(conf).NameNodePort(nnPort).NumDataNodes(numDataNodes
).StartupOption(dfsOpts).Build();
Log.Info("Started MiniDFSCluster -- namenode on port " + dfs.GetNameNodePort());
}
if (!noMR)
{
if (fs == null && dfs != null)
{
fs = dfs.GetFileSystem().GetUri().ToString();
}
else
{
if (fs == null)
{
fs = "file:///tmp/minimr-" + Runtime.NanoTime();
}
}
FileSystem.SetDefaultUri(conf, new URI(fs));
// Instruct the minicluster to use fixed ports, so user will know which
// ports to use when communicating with the cluster.
conf.SetBoolean(YarnConfiguration.YarnMiniclusterFixedPorts, true);
conf.SetBoolean(JHAdminConfig.MrHistoryMiniclusterFixedPorts, true);
conf.Set(YarnConfiguration.RmAddress, MiniYARNCluster.GetHostname() + ":" + this.
rmPort);
conf.Set(JHAdminConfig.MrHistoryAddress, MiniYARNCluster.GetHostname() + ":" + this
.jhsPort);
mr = MiniMRClientClusterFactory.Create(this.GetType(), numNodeManagers, conf);
Log.Info("Started MiniMRCluster");
}
if (writeConfig != null)
{
FileOutputStream fos = new FileOutputStream(new FilePath(writeConfig));
conf.WriteXml(fos);
fos.Close();
}
if (writeDetails != null)
{
IDictionary <string, object> map = new SortedDictionary <string, object>();
if (dfs != null)
{
map["namenode_port"] = dfs.GetNameNodePort();
}
if (mr != null)
{
map["resourcemanager_port"] = mr.GetConfig().Get(YarnConfiguration.RmAddress).Split
(":")[1];
}
FileWriter fw = new FileWriter(new FilePath(writeDetails));
fw.Write(new JSON().ToJSON(map));
fw.Close();
}
}
public virtual void TestMRTimelineEventHandling()
{
Configuration conf = new YarnConfiguration();
conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, true);
conf.SetBoolean(MRJobConfig.MapreduceJobEmitTimelineData, true);
MiniMRYarnCluster cluster = null;
try
{
cluster = new MiniMRYarnCluster(typeof(TestJobHistoryEventHandler).Name, 1);
cluster.Init(conf);
cluster.Start();
conf.Set(YarnConfiguration.TimelineServiceWebappAddress, MiniYARNCluster.GetHostname
() + ":" + cluster.GetApplicationHistoryServer().GetPort());
TimelineStore ts = cluster.GetApplicationHistoryServer().GetTimelineStore();
Path inDir = new Path("input");
Path outDir = new Path("output");
RunningJob job = UtilsForTests.RunJobSucceed(new JobConf(conf), inDir, outDir);
NUnit.Framework.Assert.AreEqual(JobStatus.Succeeded, job.GetJobStatus().GetState(
).GetValue());
TimelineEntities entities = ts.GetEntities("MAPREDUCE_JOB", null, null, null, null
, null, null, null, null, null);
NUnit.Framework.Assert.AreEqual(1, entities.GetEntities().Count);
TimelineEntity tEntity = entities.GetEntities()[0];
NUnit.Framework.Assert.AreEqual(job.GetID().ToString(), tEntity.GetEntityId());
NUnit.Framework.Assert.AreEqual("MAPREDUCE_JOB", tEntity.GetEntityType());
NUnit.Framework.Assert.AreEqual(EventType.AmStarted.ToString(), tEntity.GetEvents
()[tEntity.GetEvents().Count - 1].GetEventType());
NUnit.Framework.Assert.AreEqual(EventType.JobFinished.ToString(), tEntity.GetEvents
()[0].GetEventType());
job = UtilsForTests.RunJobFail(new JobConf(conf), inDir, outDir);
NUnit.Framework.Assert.AreEqual(JobStatus.Failed, job.GetJobStatus().GetState().GetValue
());
entities = ts.GetEntities("MAPREDUCE_JOB", null, null, null, null, null, null, null
, null, null);
NUnit.Framework.Assert.AreEqual(2, entities.GetEntities().Count);
tEntity = entities.GetEntities()[0];
NUnit.Framework.Assert.AreEqual(job.GetID().ToString(), tEntity.GetEntityId());
NUnit.Framework.Assert.AreEqual("MAPREDUCE_JOB", tEntity.GetEntityType());
NUnit.Framework.Assert.AreEqual(EventType.AmStarted.ToString(), tEntity.GetEvents
()[tEntity.GetEvents().Count - 1].GetEventType());
NUnit.Framework.Assert.AreEqual(EventType.JobFailed.ToString(), tEntity.GetEvents
()[0].GetEventType());
}
finally
{
if (cluster != null)
{
cluster.Stop();
}
}
}
public virtual void Setup()
{
fakeAppId = ApplicationId.NewInstance(Runtime.CurrentTimeMillis(), 0);
conf = new YarnConfiguration();
conf.SetBoolean(YarnConfiguration.RmHaEnabled, true);
conf.Set(YarnConfiguration.RmHaIds, Rm1NodeId + "," + Rm2NodeId);
SetRpcAddressForRM(Rm1NodeId, Rm1PortBase);
SetRpcAddressForRM(Rm2NodeId, Rm2PortBase);
conf.SetLong(YarnConfiguration.ClientFailoverSleeptimeBaseMs, 100L);
conf.SetBoolean(YarnConfiguration.YarnMiniclusterFixedPorts, true);
conf.SetBoolean(YarnConfiguration.YarnMiniclusterUseRpc, true);
cluster = new MiniYARNCluster(typeof(TestRMFailover).FullName, 2, 1, 1, 1);
}
/// <exception cref="System.Exception"/>
protected internal virtual void SetupInternal(int numNodeManager)
{
Log.Info("Starting up YARN cluster");
conf = new YarnConfiguration();
conf.SetInt(YarnConfiguration.RmSchedulerMinimumAllocationMb, 128);
conf.Set("yarn.log.dir", "target");
conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, true);
conf.Set(YarnConfiguration.RmScheduler, typeof(CapacityScheduler).FullName);
conf.SetBoolean(YarnConfiguration.NodeLabelsEnabled, true);
if (yarnCluster == null)
{
yarnCluster = new MiniYARNCluster(typeof(TestDistributedShell).Name, 1, numNodeManager
, 1, 1);
yarnCluster.Init(conf);
yarnCluster.Start();
conf.Set(YarnConfiguration.TimelineServiceWebappAddress, MiniYARNCluster.GetHostname
() + ":" + yarnCluster.GetApplicationHistoryServer().GetPort());
WaitForNMsToRegister();
Uri url = Sharpen.Thread.CurrentThread().GetContextClassLoader().GetResource("yarn-site.xml"
);
if (url == null)
{
throw new RuntimeException("Could not find 'yarn-site.xml' dummy file in classpath"
);
}
Configuration yarnClusterConfig = yarnCluster.GetConfig();
yarnClusterConfig.Set("yarn.application.classpath", new FilePath(url.AbsolutePath
).GetParent());
//write the document to a buffer (not directly to the file, as that
//can cause the file being written to get read -which will then fail.
ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
yarnClusterConfig.WriteXml(bytesOut);
bytesOut.Close();
//write the bytes to the file in the classpath
OutputStream os = new FileOutputStream(new FilePath(url.AbsolutePath));
os.Write(bytesOut.ToByteArray());
os.Close();
}
FileContext fsContext = FileContext.GetLocalFSFileContext();
fsContext.Delete(new Path(conf.Get("yarn.timeline-service.leveldb-timeline-store.path"
)), true);
try
{
Sharpen.Thread.Sleep(2000);
}
catch (Exception e)
{
Log.Info("setup thread sleep interrupted. message=" + e.Message);
}
}
public static void TearDown()
{
if (yarnCluster != null)
{
try
{
yarnCluster.Stop();
}
finally
{
yarnCluster = null;
}
}
}
/// <exception cref="System.Exception"/>
protected override void ServiceStart()
{
lock (this)
{
try
{
if (!this.GetConfig().GetBoolean(JHAdminConfig.MrHistoryMiniclusterFixedPorts, JHAdminConfig
.DefaultMrHistoryMiniclusterFixedPorts))
{
string hostname = MiniYARNCluster.GetHostname();
// pick free random ports.
this.GetConfig().Set(JHAdminConfig.MrHistoryAddress, hostname + ":0");
MRWebAppUtil.SetJHSWebappURLWithoutScheme(this.GetConfig(), hostname + ":0");
this.GetConfig().Set(JHAdminConfig.JhsAdminAddress, hostname + ":0");
}
this._enclosing.historyServer = new JobHistoryServer();
this._enclosing.historyServer.Init(this.GetConfig());
new _Thread_212(this).Start();
while (!this.jhsStarted)
{
MiniMRYarnCluster.Log.Info("Waiting for HistoryServer to start...");
Sharpen.Thread.Sleep(1500);
}
//TODO Add a timeout. State.STOPPED check ?
if (this._enclosing.historyServer.GetServiceState() != Service.STATE.Started)
{
throw new IOException("HistoryServer failed to start");
}
base.ServiceStart();
}
catch (Exception t)
{
throw new YarnRuntimeException(t);
}
//need to do this because historyServer.init creates a new Configuration
this.GetConfig().Set(JHAdminConfig.MrHistoryAddress, this._enclosing.historyServer
.GetConfig().Get(JHAdminConfig.MrHistoryAddress));
MRWebAppUtil.SetJHSWebappURLWithoutScheme(this.GetConfig(), MRWebAppUtil.GetJHSWebappURLWithoutScheme
(this._enclosing.historyServer.GetConfig()));
MiniMRYarnCluster.Log.Info("MiniMRYARN ResourceManager address: " + this.GetConfig
().Get(YarnConfiguration.RmAddress));
MiniMRYarnCluster.Log.Info("MiniMRYARN ResourceManager web address: " + WebAppUtils
.GetRMWebAppURLWithoutScheme(this.GetConfig()));
MiniMRYarnCluster.Log.Info("MiniMRYARN HistoryServer address: " + this.GetConfig(
).Get(JHAdminConfig.MrHistoryAddress));
MiniMRYarnCluster.Log.Info("MiniMRYARN HistoryServer web address: " + MiniMRYarnCluster
.GetResolvedMRHistoryWebAppURLWithoutScheme(this.GetConfig(), MRWebAppUtil.GetJHSHttpPolicy
() == HttpConfig.Policy.HttpsOnly));
}
}
public virtual void TestRMWebAppRedirect()
{
cluster = new MiniYARNCluster(typeof(TestRMFailover).FullName, 2, 0, 1, 1);
conf.SetBoolean(YarnConfiguration.AutoFailoverEnabled, false);
cluster.Init(conf);
cluster.Start();
GetAdminService(0).TransitionToActive(req);
string rm1Url = "http://0.0.0.0:18088";
string rm2Url = "http://0.0.0.0:28088";
string redirectURL = GetRedirectURL(rm2Url);
// if uri is null, RMWebAppFilter will append a slash at the trail of the redirection url
NUnit.Framework.Assert.AreEqual(redirectURL, rm1Url + "/");
redirectURL = GetRedirectURL(rm2Url + "/metrics");
NUnit.Framework.Assert.AreEqual(redirectURL, rm1Url + "/metrics");
redirectURL = GetRedirectURL(rm2Url + "/jmx");
NUnit.Framework.Assert.AreEqual(redirectURL, rm1Url + "/jmx");
// standby RM links /conf, /stacks, /logLevel, /static, /logs,
// /cluster/cluster as well as webService
// /ws/v1/cluster/info should not be redirected to active RM
redirectURL = GetRedirectURL(rm2Url + "/cluster/cluster");
NUnit.Framework.Assert.IsNull(redirectURL);
redirectURL = GetRedirectURL(rm2Url + "/conf");
NUnit.Framework.Assert.IsNull(redirectURL);
redirectURL = GetRedirectURL(rm2Url + "/stacks");
NUnit.Framework.Assert.IsNull(redirectURL);
redirectURL = GetRedirectURL(rm2Url + "/logLevel");
NUnit.Framework.Assert.IsNull(redirectURL);
redirectURL = GetRedirectURL(rm2Url + "/static");
NUnit.Framework.Assert.IsNull(redirectURL);
redirectURL = GetRedirectURL(rm2Url + "/logs");
NUnit.Framework.Assert.IsNull(redirectURL);
redirectURL = GetRedirectURL(rm2Url + "/ws/v1/cluster/info");
NUnit.Framework.Assert.IsNull(redirectURL);
redirectURL = GetRedirectURL(rm2Url + "/ws/v1/cluster/apps");
NUnit.Framework.Assert.AreEqual(redirectURL, rm1Url + "/ws/v1/cluster/apps");
redirectURL = GetRedirectURL(rm2Url + "/proxy/" + fakeAppId);
NUnit.Framework.Assert.IsNull(redirectURL);
// transit the active RM to standby
// Both of RMs are in standby mode
GetAdminService(0).TransitionToStandby(req);
// RM2 is expected to send the httpRequest to itself.
// The Header Field: Refresh is expected to be set.
redirectURL = GetRefreshURL(rm2Url);
NUnit.Framework.Assert.IsTrue(redirectURL != null && redirectURL.Contains(YarnWebParams
.NextRefreshInterval) && redirectURL.Contains(rm2Url));
}
public static void Setup()
{
Log.Info("Starting up YARN cluster");
conf.SetInt(YarnConfiguration.RmSchedulerMinimumAllocationMb, 128);
if (yarnCluster == null)
{
yarnCluster = new MiniYARNCluster(typeof(TestUnmanagedAMLauncher).Name, 1, 1, 1);
yarnCluster.Init(conf);
yarnCluster.Start();
//get the address
Configuration yarnClusterConfig = yarnCluster.GetConfig();
Log.Info("MiniYARN ResourceManager published address: " + yarnClusterConfig.Get(YarnConfiguration
.RmAddress));
Log.Info("MiniYARN ResourceManager published web address: " + yarnClusterConfig.Get
(YarnConfiguration.RmWebappAddress));
string webapp = yarnClusterConfig.Get(YarnConfiguration.RmWebappAddress);
NUnit.Framework.Assert.IsTrue("Web app address still unbound to a host at " + webapp
, !webapp.StartsWith("0.0.0.0"));
Log.Info("Yarn webapp is at " + webapp);
Uri url = Sharpen.Thread.CurrentThread().GetContextClassLoader().GetResource("yarn-site.xml"
);
if (url == null)
{
throw new RuntimeException("Could not find 'yarn-site.xml' dummy file in classpath"
);
}
//write the document to a buffer (not directly to the file, as that
//can cause the file being written to get read -which will then fail.
ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
yarnClusterConfig.WriteXml(bytesOut);
bytesOut.Close();
//write the bytes to the file in the classpath
OutputStream os = new FileOutputStream(new FilePath(url.AbsolutePath));
os.Write(bytesOut.ToByteArray());
os.Close();
}
try
{
Sharpen.Thread.Sleep(2000);
}
catch (Exception e)
{
Log.Info("setup thread sleep interrupted. message=" + e.Message);
}
}
public virtual void TearDown()
{
if (yarnCluster != null)
{
try
{
yarnCluster.Stop();
}
finally
{
yarnCluster = null;
}
}
FileContext fsContext = FileContext.GetLocalFSFileContext();
fsContext.Delete(new Path(conf.Get("yarn.timeline-service.leveldb-timeline-store.path"
)), true);
}
public virtual void Setup()
{
// start minicluster
conf = new YarnConfiguration();
yarnCluster = new MiniYARNCluster(typeof(TestAMRMClient).FullName, nodeCount, 1,
1);
yarnCluster.Init(conf);
yarnCluster.Start();
NUnit.Framework.Assert.IsNotNull(yarnCluster);
NUnit.Framework.Assert.AreEqual(Service.STATE.Started, yarnCluster.GetServiceState
());
// start rm client
yarnClient = (YarnClientImpl)YarnClient.CreateYarnClient();
yarnClient.Init(conf);
yarnClient.Start();
NUnit.Framework.Assert.IsNotNull(yarnClient);
NUnit.Framework.Assert.AreEqual(Service.STATE.Started, yarnClient.GetServiceState
());
// get node info
nodeReports = yarnClient.GetNodeReports(NodeState.Running);
// submit new app
ApplicationSubmissionContext appContext = yarnClient.CreateApplication().GetApplicationSubmissionContext
();
ApplicationId appId = appContext.GetApplicationId();
// set the application name
appContext.SetApplicationName("Test");
// Set the priority for the application master
Priority pri = Priority.NewInstance(0);
appContext.SetPriority(pri);
// Set the queue to which this application is to be submitted in the RM
appContext.SetQueue("default");
// Set up the container launch context for the application master
ContainerLaunchContext amContainer = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<ContainerLaunchContext>();
appContext.SetAMContainerSpec(amContainer);
// unmanaged AM
appContext.SetUnmanagedAM(true);
// Create the request to send to the applications manager
SubmitApplicationRequest appRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<SubmitApplicationRequest>();
appRequest.SetApplicationSubmissionContext(appContext);
// Submit the application to the applications manager
yarnClient.SubmitApplication(appContext);
// wait for app to start
int iterationsLeft = 30;
RMAppAttempt appAttempt = null;
while (iterationsLeft > 0)
{
ApplicationReport appReport = yarnClient.GetApplicationReport(appId);
if (appReport.GetYarnApplicationState() == YarnApplicationState.Accepted)
{
attemptId = appReport.GetCurrentApplicationAttemptId();
appAttempt = yarnCluster.GetResourceManager().GetRMContext().GetRMApps()[attemptId
.GetApplicationId()].GetCurrentAppAttempt();
while (true)
{
if (appAttempt.GetAppAttemptState() == RMAppAttemptState.Launched)
{
break;
}
}
break;
}
Sleep(1000);
--iterationsLeft;
}
if (iterationsLeft == 0)
{
NUnit.Framework.Assert.Fail("Application hasn't bee started");
}
// Just dig into the ResourceManager and get the AMRMToken just for the sake
// of testing.
UserGroupInformation.SetLoginUser(UserGroupInformation.CreateRemoteUser(UserGroupInformation
.GetCurrentUser().GetUserName()));
UserGroupInformation.GetCurrentUser().AddToken(appAttempt.GetAMRMToken());
//creating an instance NMTokenCase
nmTokenCache = new NMTokenCache();
// start am rm client
rmClient = (AMRMClientImpl <AMRMClient.ContainerRequest>)AMRMClient.CreateAMRMClient
<AMRMClient.ContainerRequest>();
//setting an instance NMTokenCase
rmClient.SetNMTokenCache(nmTokenCache);
rmClient.Init(conf);
rmClient.Start();
NUnit.Framework.Assert.IsNotNull(rmClient);
NUnit.Framework.Assert.AreEqual(Service.STATE.Started, rmClient.GetServiceState()
);
// start am nm client
nmClient = (NMClientImpl)NMClient.CreateNMClient();
//propagating the AMRMClient NMTokenCache instance
nmClient.SetNMTokenCache(rmClient.GetNMTokenCache());
nmClient.Init(conf);
nmClient.Start();
NUnit.Framework.Assert.IsNotNull(nmClient);
NUnit.Framework.Assert.AreEqual(Service.STATE.Started, nmClient.GetServiceState()
);
}
