// nothing
/// <summary>Starts DFS and MR clusters, as specified in member-variable options.</summary>
/// <remarks>
/// Starts DFS and MR clusters, as specified in member-variable options. Also
/// writes out configuration and details, if requested.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.IO.FileNotFoundException"/>
/// <exception cref="Sharpen.URISyntaxException"/>
public virtual void Start()
{
if (!noDFS)
{
dfs = new MiniDFSCluster.Builder(conf).NameNodePort(nnPort).NumDataNodes(numDataNodes
).StartupOption(dfsOpts).Build();
Log.Info("Started MiniDFSCluster -- namenode on port " + dfs.GetNameNodePort());
}
if (!noMR)
{
if (fs == null && dfs != null)
{
fs = dfs.GetFileSystem().GetUri().ToString();
}
else
{
if (fs == null)
{
fs = "file:///tmp/minimr-" + Runtime.NanoTime();
}
}
FileSystem.SetDefaultUri(conf, new URI(fs));
// Instruct the minicluster to use fixed ports, so user will know which
// ports to use when communicating with the cluster.
conf.SetBoolean(YarnConfiguration.YarnMiniclusterFixedPorts, true);
conf.SetBoolean(JHAdminConfig.MrHistoryMiniclusterFixedPorts, true);
conf.Set(YarnConfiguration.RmAddress, MiniYARNCluster.GetHostname() + ":" + this.
rmPort);
conf.Set(JHAdminConfig.MrHistoryAddress, MiniYARNCluster.GetHostname() + ":" + this
.jhsPort);
mr = MiniMRClientClusterFactory.Create(this.GetType(), numNodeManagers, conf);
Log.Info("Started MiniMRCluster");
}
if (writeConfig != null)
{
FileOutputStream fos = new FileOutputStream(new FilePath(writeConfig));
conf.WriteXml(fos);
fos.Close();
}
if (writeDetails != null)
{
IDictionary <string, object> map = new SortedDictionary <string, object>();
if (dfs != null)
{
map["namenode_port"] = dfs.GetNameNodePort();
}
if (mr != null)
{
map["resourcemanager_port"] = mr.GetConfig().Get(YarnConfiguration.RmAddress).Split
(":")[1];
}
FileWriter fw = new FileWriter(new FilePath(writeDetails));
fw.Write(new JSON().ToJSON(map));
fw.Close();
}
}
public virtual void TestMRTimelineEventHandling()
{
Configuration conf = new YarnConfiguration();
conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, true);
conf.SetBoolean(MRJobConfig.MapreduceJobEmitTimelineData, true);
MiniMRYarnCluster cluster = null;
try
{
cluster = new MiniMRYarnCluster(typeof(TestJobHistoryEventHandler).Name, 1);
cluster.Init(conf);
cluster.Start();
conf.Set(YarnConfiguration.TimelineServiceWebappAddress, MiniYARNCluster.GetHostname
() + ":" + cluster.GetApplicationHistoryServer().GetPort());
TimelineStore ts = cluster.GetApplicationHistoryServer().GetTimelineStore();
Path inDir = new Path("input");
Path outDir = new Path("output");
RunningJob job = UtilsForTests.RunJobSucceed(new JobConf(conf), inDir, outDir);
NUnit.Framework.Assert.AreEqual(JobStatus.Succeeded, job.GetJobStatus().GetState(
).GetValue());
TimelineEntities entities = ts.GetEntities("MAPREDUCE_JOB", null, null, null, null
, null, null, null, null, null);
NUnit.Framework.Assert.AreEqual(1, entities.GetEntities().Count);
TimelineEntity tEntity = entities.GetEntities()[0];
NUnit.Framework.Assert.AreEqual(job.GetID().ToString(), tEntity.GetEntityId());
NUnit.Framework.Assert.AreEqual("MAPREDUCE_JOB", tEntity.GetEntityType());
NUnit.Framework.Assert.AreEqual(EventType.AmStarted.ToString(), tEntity.GetEvents
()[tEntity.GetEvents().Count - 1].GetEventType());
NUnit.Framework.Assert.AreEqual(EventType.JobFinished.ToString(), tEntity.GetEvents
()[0].GetEventType());
job = UtilsForTests.RunJobFail(new JobConf(conf), inDir, outDir);
NUnit.Framework.Assert.AreEqual(JobStatus.Failed, job.GetJobStatus().GetState().GetValue
());
entities = ts.GetEntities("MAPREDUCE_JOB", null, null, null, null, null, null, null
, null, null);
NUnit.Framework.Assert.AreEqual(2, entities.GetEntities().Count);
tEntity = entities.GetEntities()[0];
NUnit.Framework.Assert.AreEqual(job.GetID().ToString(), tEntity.GetEntityId());
NUnit.Framework.Assert.AreEqual("MAPREDUCE_JOB", tEntity.GetEntityType());
NUnit.Framework.Assert.AreEqual(EventType.AmStarted.ToString(), tEntity.GetEvents
()[tEntity.GetEvents().Count - 1].GetEventType());
NUnit.Framework.Assert.AreEqual(EventType.JobFailed.ToString(), tEntity.GetEvents
()[0].GetEventType());
}
finally
{
if (cluster != null)
{
cluster.Stop();
}
}
}
/// <exception cref="System.Exception"/>
protected internal virtual void SetupInternal(int numNodeManager)
{
Log.Info("Starting up YARN cluster");
conf = new YarnConfiguration();
conf.SetInt(YarnConfiguration.RmSchedulerMinimumAllocationMb, 128);
conf.Set("yarn.log.dir", "target");
conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, true);
conf.Set(YarnConfiguration.RmScheduler, typeof(CapacityScheduler).FullName);
conf.SetBoolean(YarnConfiguration.NodeLabelsEnabled, true);
if (yarnCluster == null)
{
yarnCluster = new MiniYARNCluster(typeof(TestDistributedShell).Name, 1, numNodeManager
, 1, 1);
yarnCluster.Init(conf);
yarnCluster.Start();
conf.Set(YarnConfiguration.TimelineServiceWebappAddress, MiniYARNCluster.GetHostname
() + ":" + yarnCluster.GetApplicationHistoryServer().GetPort());
WaitForNMsToRegister();
Uri url = Sharpen.Thread.CurrentThread().GetContextClassLoader().GetResource("yarn-site.xml"
);
if (url == null)
{
throw new RuntimeException("Could not find 'yarn-site.xml' dummy file in classpath"
);
}
Configuration yarnClusterConfig = yarnCluster.GetConfig();
yarnClusterConfig.Set("yarn.application.classpath", new FilePath(url.AbsolutePath
).GetParent());
//write the document to a buffer (not directly to the file, as that
//can cause the file being written to get read -which will then fail.
ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
yarnClusterConfig.WriteXml(bytesOut);
bytesOut.Close();
//write the bytes to the file in the classpath
OutputStream os = new FileOutputStream(new FilePath(url.AbsolutePath));
os.Write(bytesOut.ToByteArray());
os.Close();
}
FileContext fsContext = FileContext.GetLocalFSFileContext();
fsContext.Delete(new Path(conf.Get("yarn.timeline-service.leveldb-timeline-store.path"
)), true);
try
{
Sharpen.Thread.Sleep(2000);
}
catch (Exception e)
{
Log.Info("setup thread sleep interrupted. message=" + e.Message);
}
}
/// <exception cref="System.Exception"/>
protected override void ServiceStart()
{
lock (this)
{
try
{
if (!this.GetConfig().GetBoolean(JHAdminConfig.MrHistoryMiniclusterFixedPorts, JHAdminConfig
.DefaultMrHistoryMiniclusterFixedPorts))
{
string hostname = MiniYARNCluster.GetHostname();
// pick free random ports.
this.GetConfig().Set(JHAdminConfig.MrHistoryAddress, hostname + ":0");
MRWebAppUtil.SetJHSWebappURLWithoutScheme(this.GetConfig(), hostname + ":0");
this.GetConfig().Set(JHAdminConfig.JhsAdminAddress, hostname + ":0");
}
this._enclosing.historyServer = new JobHistoryServer();
this._enclosing.historyServer.Init(this.GetConfig());
new _Thread_212(this).Start();
while (!this.jhsStarted)
{
MiniMRYarnCluster.Log.Info("Waiting for HistoryServer to start...");
Sharpen.Thread.Sleep(1500);
}
//TODO Add a timeout. State.STOPPED check ?
if (this._enclosing.historyServer.GetServiceState() != Service.STATE.Started)
{
throw new IOException("HistoryServer failed to start");
}
base.ServiceStart();
}
catch (Exception t)
{
throw new YarnRuntimeException(t);
}
//need to do this because historyServer.init creates a new Configuration
this.GetConfig().Set(JHAdminConfig.MrHistoryAddress, this._enclosing.historyServer
.GetConfig().Get(JHAdminConfig.MrHistoryAddress));
MRWebAppUtil.SetJHSWebappURLWithoutScheme(this.GetConfig(), MRWebAppUtil.GetJHSWebappURLWithoutScheme
(this._enclosing.historyServer.GetConfig()));
MiniMRYarnCluster.Log.Info("MiniMRYARN ResourceManager address: " + this.GetConfig
().Get(YarnConfiguration.RmAddress));
MiniMRYarnCluster.Log.Info("MiniMRYARN ResourceManager web address: " + WebAppUtils
.GetRMWebAppURLWithoutScheme(this.GetConfig()));
MiniMRYarnCluster.Log.Info("MiniMRYARN HistoryServer address: " + this.GetConfig(
).Get(JHAdminConfig.MrHistoryAddress));
MiniMRYarnCluster.Log.Info("MiniMRYARN HistoryServer web address: " + MiniMRYarnCluster
.GetResolvedMRHistoryWebAppURLWithoutScheme(this.GetConfig(), MRWebAppUtil.GetJHSHttpPolicy
() == HttpConfig.Policy.HttpsOnly));
}
}
