// nothing /// <summary>Starts DFS and MR clusters, as specified in member-variable options.</summary> /// <remarks> /// Starts DFS and MR clusters, as specified in member-variable options. Also /// writes out configuration and details, if requested. /// </remarks> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.IO.FileNotFoundException"/> /// <exception cref="Sharpen.URISyntaxException"/> public virtual void Start() { if (!noDFS) { dfs = new MiniDFSCluster.Builder(conf).NameNodePort(nnPort).NumDataNodes(numDataNodes ).StartupOption(dfsOpts).Build(); Log.Info("Started MiniDFSCluster -- namenode on port " + dfs.GetNameNodePort()); } if (!noMR) { if (fs == null && dfs != null) { fs = dfs.GetFileSystem().GetUri().ToString(); } else { if (fs == null) { fs = "file:///tmp/minimr-" + Runtime.NanoTime(); } } FileSystem.SetDefaultUri(conf, new URI(fs)); // Instruct the minicluster to use fixed ports, so user will know which // ports to use when communicating with the cluster. conf.SetBoolean(YarnConfiguration.YarnMiniclusterFixedPorts, true); conf.SetBoolean(JHAdminConfig.MrHistoryMiniclusterFixedPorts, true); conf.Set(YarnConfiguration.RmAddress, MiniYARNCluster.GetHostname() + ":" + this. rmPort); conf.Set(JHAdminConfig.MrHistoryAddress, MiniYARNCluster.GetHostname() + ":" + this .jhsPort); mr = MiniMRClientClusterFactory.Create(this.GetType(), numNodeManagers, conf); Log.Info("Started MiniMRCluster"); } if (writeConfig != null) { FileOutputStream fos = new FileOutputStream(new FilePath(writeConfig)); conf.WriteXml(fos); fos.Close(); } if (writeDetails != null) { IDictionary <string, object> map = new SortedDictionary <string, object>(); if (dfs != null) { map["namenode_port"] = dfs.GetNameNodePort(); } if (mr != null) { map["resourcemanager_port"] = mr.GetConfig().Get(YarnConfiguration.RmAddress).Split (":")[1]; } FileWriter fw = new FileWriter(new FilePath(writeDetails)); fw.Write(new JSON().ToJSON(map)); fw.Close(); } }
public virtual void TestMRTimelineEventHandling() { Configuration conf = new YarnConfiguration(); conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, true); conf.SetBoolean(MRJobConfig.MapreduceJobEmitTimelineData, true); MiniMRYarnCluster cluster = null; try { cluster = new MiniMRYarnCluster(typeof(TestJobHistoryEventHandler).Name, 1); cluster.Init(conf); cluster.Start(); conf.Set(YarnConfiguration.TimelineServiceWebappAddress, MiniYARNCluster.GetHostname () + ":" + cluster.GetApplicationHistoryServer().GetPort()); TimelineStore ts = cluster.GetApplicationHistoryServer().GetTimelineStore(); Path inDir = new Path("input"); Path outDir = new Path("output"); RunningJob job = UtilsForTests.RunJobSucceed(new JobConf(conf), inDir, outDir); NUnit.Framework.Assert.AreEqual(JobStatus.Succeeded, job.GetJobStatus().GetState( ).GetValue()); TimelineEntities entities = ts.GetEntities("MAPREDUCE_JOB", null, null, null, null , null, null, null, null, null); NUnit.Framework.Assert.AreEqual(1, entities.GetEntities().Count); TimelineEntity tEntity = entities.GetEntities()[0]; NUnit.Framework.Assert.AreEqual(job.GetID().ToString(), tEntity.GetEntityId()); NUnit.Framework.Assert.AreEqual("MAPREDUCE_JOB", tEntity.GetEntityType()); NUnit.Framework.Assert.AreEqual(EventType.AmStarted.ToString(), tEntity.GetEvents ()[tEntity.GetEvents().Count - 1].GetEventType()); NUnit.Framework.Assert.AreEqual(EventType.JobFinished.ToString(), tEntity.GetEvents ()[0].GetEventType()); job = UtilsForTests.RunJobFail(new JobConf(conf), inDir, outDir); NUnit.Framework.Assert.AreEqual(JobStatus.Failed, job.GetJobStatus().GetState().GetValue ()); entities = ts.GetEntities("MAPREDUCE_JOB", null, null, null, null, null, null, null , null, null); NUnit.Framework.Assert.AreEqual(2, entities.GetEntities().Count); tEntity = entities.GetEntities()[0]; NUnit.Framework.Assert.AreEqual(job.GetID().ToString(), tEntity.GetEntityId()); NUnit.Framework.Assert.AreEqual("MAPREDUCE_JOB", tEntity.GetEntityType()); NUnit.Framework.Assert.AreEqual(EventType.AmStarted.ToString(), tEntity.GetEvents ()[tEntity.GetEvents().Count - 1].GetEventType()); NUnit.Framework.Assert.AreEqual(EventType.JobFailed.ToString(), tEntity.GetEvents ()[0].GetEventType()); } finally { if (cluster != null) { cluster.Stop(); } } }
/// <exception cref="System.Exception"/> protected internal virtual void SetupInternal(int numNodeManager) { Log.Info("Starting up YARN cluster"); conf = new YarnConfiguration(); conf.SetInt(YarnConfiguration.RmSchedulerMinimumAllocationMb, 128); conf.Set("yarn.log.dir", "target"); conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, true); conf.Set(YarnConfiguration.RmScheduler, typeof(CapacityScheduler).FullName); conf.SetBoolean(YarnConfiguration.NodeLabelsEnabled, true); if (yarnCluster == null) { yarnCluster = new MiniYARNCluster(typeof(TestDistributedShell).Name, 1, numNodeManager , 1, 1); yarnCluster.Init(conf); yarnCluster.Start(); conf.Set(YarnConfiguration.TimelineServiceWebappAddress, MiniYARNCluster.GetHostname () + ":" + yarnCluster.GetApplicationHistoryServer().GetPort()); WaitForNMsToRegister(); Uri url = Sharpen.Thread.CurrentThread().GetContextClassLoader().GetResource("yarn-site.xml" ); if (url == null) { throw new RuntimeException("Could not find 'yarn-site.xml' dummy file in classpath" ); } Configuration yarnClusterConfig = yarnCluster.GetConfig(); yarnClusterConfig.Set("yarn.application.classpath", new FilePath(url.AbsolutePath ).GetParent()); //write the document to a buffer (not directly to the file, as that //can cause the file being written to get read -which will then fail. ByteArrayOutputStream bytesOut = new ByteArrayOutputStream(); yarnClusterConfig.WriteXml(bytesOut); bytesOut.Close(); //write the bytes to the file in the classpath OutputStream os = new FileOutputStream(new FilePath(url.AbsolutePath)); os.Write(bytesOut.ToByteArray()); os.Close(); } FileContext fsContext = FileContext.GetLocalFSFileContext(); fsContext.Delete(new Path(conf.Get("yarn.timeline-service.leveldb-timeline-store.path" )), true); try { Sharpen.Thread.Sleep(2000); } catch (Exception e) { Log.Info("setup thread sleep interrupted. message=" + e.Message); } }
/// <exception cref="System.Exception"/> protected override void ServiceStart() { lock (this) { try { if (!this.GetConfig().GetBoolean(JHAdminConfig.MrHistoryMiniclusterFixedPorts, JHAdminConfig .DefaultMrHistoryMiniclusterFixedPorts)) { string hostname = MiniYARNCluster.GetHostname(); // pick free random ports. this.GetConfig().Set(JHAdminConfig.MrHistoryAddress, hostname + ":0"); MRWebAppUtil.SetJHSWebappURLWithoutScheme(this.GetConfig(), hostname + ":0"); this.GetConfig().Set(JHAdminConfig.JhsAdminAddress, hostname + ":0"); } this._enclosing.historyServer = new JobHistoryServer(); this._enclosing.historyServer.Init(this.GetConfig()); new _Thread_212(this).Start(); while (!this.jhsStarted) { MiniMRYarnCluster.Log.Info("Waiting for HistoryServer to start..."); Sharpen.Thread.Sleep(1500); } //TODO Add a timeout. State.STOPPED check ? if (this._enclosing.historyServer.GetServiceState() != Service.STATE.Started) { throw new IOException("HistoryServer failed to start"); } base.ServiceStart(); } catch (Exception t) { throw new YarnRuntimeException(t); } //need to do this because historyServer.init creates a new Configuration this.GetConfig().Set(JHAdminConfig.MrHistoryAddress, this._enclosing.historyServer .GetConfig().Get(JHAdminConfig.MrHistoryAddress)); MRWebAppUtil.SetJHSWebappURLWithoutScheme(this.GetConfig(), MRWebAppUtil.GetJHSWebappURLWithoutScheme (this._enclosing.historyServer.GetConfig())); MiniMRYarnCluster.Log.Info("MiniMRYARN ResourceManager address: " + this.GetConfig ().Get(YarnConfiguration.RmAddress)); MiniMRYarnCluster.Log.Info("MiniMRYARN ResourceManager web address: " + WebAppUtils .GetRMWebAppURLWithoutScheme(this.GetConfig())); MiniMRYarnCluster.Log.Info("MiniMRYARN HistoryServer address: " + this.GetConfig( ).Get(JHAdminConfig.MrHistoryAddress)); MiniMRYarnCluster.Log.Info("MiniMRYARN HistoryServer web address: " + MiniMRYarnCluster .GetResolvedMRHistoryWebAppURLWithoutScheme(this.GetConfig(), MRWebAppUtil.GetJHSHttpPolicy () == HttpConfig.Policy.HttpsOnly)); } }