// nothing /// <summary>Starts DFS and MR clusters, as specified in member-variable options.</summary> /// <remarks> /// Starts DFS and MR clusters, as specified in member-variable options. Also /// writes out configuration and details, if requested. /// </remarks> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.IO.FileNotFoundException"/> /// <exception cref="Sharpen.URISyntaxException"/> public virtual void Start() { if (!noDFS) { dfs = new MiniDFSCluster.Builder(conf).NameNodePort(nnPort).NumDataNodes(numDataNodes ).StartupOption(dfsOpts).Build(); Log.Info("Started MiniDFSCluster -- namenode on port " + dfs.GetNameNodePort()); } if (!noMR) { if (fs == null && dfs != null) { fs = dfs.GetFileSystem().GetUri().ToString(); } else { if (fs == null) { fs = "file:///tmp/minimr-" + Runtime.NanoTime(); } } FileSystem.SetDefaultUri(conf, new URI(fs)); // Instruct the minicluster to use fixed ports, so user will know which // ports to use when communicating with the cluster. conf.SetBoolean(YarnConfiguration.YarnMiniclusterFixedPorts, true); conf.SetBoolean(JHAdminConfig.MrHistoryMiniclusterFixedPorts, true); conf.Set(YarnConfiguration.RmAddress, MiniYARNCluster.GetHostname() + ":" + this. rmPort); conf.Set(JHAdminConfig.MrHistoryAddress, MiniYARNCluster.GetHostname() + ":" + this .jhsPort); mr = MiniMRClientClusterFactory.Create(this.GetType(), numNodeManagers, conf); Log.Info("Started MiniMRCluster"); } if (writeConfig != null) { FileOutputStream fos = new FileOutputStream(new FilePath(writeConfig)); conf.WriteXml(fos); fos.Close(); } if (writeDetails != null) { IDictionary <string, object> map = new SortedDictionary <string, object>(); if (dfs != null) { map["namenode_port"] = dfs.GetNameNodePort(); } if (mr != null) { map["resourcemanager_port"] = mr.GetConfig().Get(YarnConfiguration.RmAddress).Split (":")[1]; } FileWriter fw = new FileWriter(new FilePath(writeDetails)); fw.Write(new JSON().ToJSON(map)); fw.Close(); } }
public virtual void TestMRTimelineEventHandling() { Configuration conf = new YarnConfiguration(); conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, true); conf.SetBoolean(MRJobConfig.MapreduceJobEmitTimelineData, true); MiniMRYarnCluster cluster = null; try { cluster = new MiniMRYarnCluster(typeof(TestJobHistoryEventHandler).Name, 1); cluster.Init(conf); cluster.Start(); conf.Set(YarnConfiguration.TimelineServiceWebappAddress, MiniYARNCluster.GetHostname () + ":" + cluster.GetApplicationHistoryServer().GetPort()); TimelineStore ts = cluster.GetApplicationHistoryServer().GetTimelineStore(); Path inDir = new Path("input"); Path outDir = new Path("output"); RunningJob job = UtilsForTests.RunJobSucceed(new JobConf(conf), inDir, outDir); NUnit.Framework.Assert.AreEqual(JobStatus.Succeeded, job.GetJobStatus().GetState( ).GetValue()); TimelineEntities entities = ts.GetEntities("MAPREDUCE_JOB", null, null, null, null , null, null, null, null, null); NUnit.Framework.Assert.AreEqual(1, entities.GetEntities().Count); TimelineEntity tEntity = entities.GetEntities()[0]; NUnit.Framework.Assert.AreEqual(job.GetID().ToString(), tEntity.GetEntityId()); NUnit.Framework.Assert.AreEqual("MAPREDUCE_JOB", tEntity.GetEntityType()); NUnit.Framework.Assert.AreEqual(EventType.AmStarted.ToString(), tEntity.GetEvents ()[tEntity.GetEvents().Count - 1].GetEventType()); NUnit.Framework.Assert.AreEqual(EventType.JobFinished.ToString(), tEntity.GetEvents ()[0].GetEventType()); job = UtilsForTests.RunJobFail(new JobConf(conf), inDir, outDir); NUnit.Framework.Assert.AreEqual(JobStatus.Failed, job.GetJobStatus().GetState().GetValue ()); entities = ts.GetEntities("MAPREDUCE_JOB", null, null, null, null, null, null, null , null, null); NUnit.Framework.Assert.AreEqual(2, entities.GetEntities().Count); tEntity = entities.GetEntities()[0]; NUnit.Framework.Assert.AreEqual(job.GetID().ToString(), tEntity.GetEntityId()); NUnit.Framework.Assert.AreEqual("MAPREDUCE_JOB", tEntity.GetEntityType()); NUnit.Framework.Assert.AreEqual(EventType.AmStarted.ToString(), tEntity.GetEvents ()[tEntity.GetEvents().Count - 1].GetEventType()); NUnit.Framework.Assert.AreEqual(EventType.JobFailed.ToString(), tEntity.GetEvents ()[0].GetEventType()); } finally { if (cluster != null) { cluster.Stop(); } } }
public virtual void Setup() { fakeAppId = ApplicationId.NewInstance(Runtime.CurrentTimeMillis(), 0); conf = new YarnConfiguration(); conf.SetBoolean(YarnConfiguration.RmHaEnabled, true); conf.Set(YarnConfiguration.RmHaIds, Rm1NodeId + "," + Rm2NodeId); SetRpcAddressForRM(Rm1NodeId, Rm1PortBase); SetRpcAddressForRM(Rm2NodeId, Rm2PortBase); conf.SetLong(YarnConfiguration.ClientFailoverSleeptimeBaseMs, 100L); conf.SetBoolean(YarnConfiguration.YarnMiniclusterFixedPorts, true); conf.SetBoolean(YarnConfiguration.YarnMiniclusterUseRpc, true); cluster = new MiniYARNCluster(typeof(TestRMFailover).FullName, 2, 1, 1, 1); }
/// <exception cref="System.Exception"/> protected internal virtual void SetupInternal(int numNodeManager) { Log.Info("Starting up YARN cluster"); conf = new YarnConfiguration(); conf.SetInt(YarnConfiguration.RmSchedulerMinimumAllocationMb, 128); conf.Set("yarn.log.dir", "target"); conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, true); conf.Set(YarnConfiguration.RmScheduler, typeof(CapacityScheduler).FullName); conf.SetBoolean(YarnConfiguration.NodeLabelsEnabled, true); if (yarnCluster == null) { yarnCluster = new MiniYARNCluster(typeof(TestDistributedShell).Name, 1, numNodeManager , 1, 1); yarnCluster.Init(conf); yarnCluster.Start(); conf.Set(YarnConfiguration.TimelineServiceWebappAddress, MiniYARNCluster.GetHostname () + ":" + yarnCluster.GetApplicationHistoryServer().GetPort()); WaitForNMsToRegister(); Uri url = Sharpen.Thread.CurrentThread().GetContextClassLoader().GetResource("yarn-site.xml" ); if (url == null) { throw new RuntimeException("Could not find 'yarn-site.xml' dummy file in classpath" ); } Configuration yarnClusterConfig = yarnCluster.GetConfig(); yarnClusterConfig.Set("yarn.application.classpath", new FilePath(url.AbsolutePath ).GetParent()); //write the document to a buffer (not directly to the file, as that //can cause the file being written to get read -which will then fail. ByteArrayOutputStream bytesOut = new ByteArrayOutputStream(); yarnClusterConfig.WriteXml(bytesOut); bytesOut.Close(); //write the bytes to the file in the classpath OutputStream os = new FileOutputStream(new FilePath(url.AbsolutePath)); os.Write(bytesOut.ToByteArray()); os.Close(); } FileContext fsContext = FileContext.GetLocalFSFileContext(); fsContext.Delete(new Path(conf.Get("yarn.timeline-service.leveldb-timeline-store.path" )), true); try { Sharpen.Thread.Sleep(2000); } catch (Exception e) { Log.Info("setup thread sleep interrupted. message=" + e.Message); } }
public static void TearDown() { if (yarnCluster != null) { try { yarnCluster.Stop(); } finally { yarnCluster = null; } } }
/// <exception cref="System.Exception"/> protected override void ServiceStart() { lock (this) { try { if (!this.GetConfig().GetBoolean(JHAdminConfig.MrHistoryMiniclusterFixedPorts, JHAdminConfig .DefaultMrHistoryMiniclusterFixedPorts)) { string hostname = MiniYARNCluster.GetHostname(); // pick free random ports. this.GetConfig().Set(JHAdminConfig.MrHistoryAddress, hostname + ":0"); MRWebAppUtil.SetJHSWebappURLWithoutScheme(this.GetConfig(), hostname + ":0"); this.GetConfig().Set(JHAdminConfig.JhsAdminAddress, hostname + ":0"); } this._enclosing.historyServer = new JobHistoryServer(); this._enclosing.historyServer.Init(this.GetConfig()); new _Thread_212(this).Start(); while (!this.jhsStarted) { MiniMRYarnCluster.Log.Info("Waiting for HistoryServer to start..."); Sharpen.Thread.Sleep(1500); } //TODO Add a timeout. State.STOPPED check ? if (this._enclosing.historyServer.GetServiceState() != Service.STATE.Started) { throw new IOException("HistoryServer failed to start"); } base.ServiceStart(); } catch (Exception t) { throw new YarnRuntimeException(t); } //need to do this because historyServer.init creates a new Configuration this.GetConfig().Set(JHAdminConfig.MrHistoryAddress, this._enclosing.historyServer .GetConfig().Get(JHAdminConfig.MrHistoryAddress)); MRWebAppUtil.SetJHSWebappURLWithoutScheme(this.GetConfig(), MRWebAppUtil.GetJHSWebappURLWithoutScheme (this._enclosing.historyServer.GetConfig())); MiniMRYarnCluster.Log.Info("MiniMRYARN ResourceManager address: " + this.GetConfig ().Get(YarnConfiguration.RmAddress)); MiniMRYarnCluster.Log.Info("MiniMRYARN ResourceManager web address: " + WebAppUtils .GetRMWebAppURLWithoutScheme(this.GetConfig())); MiniMRYarnCluster.Log.Info("MiniMRYARN HistoryServer address: " + this.GetConfig( ).Get(JHAdminConfig.MrHistoryAddress)); MiniMRYarnCluster.Log.Info("MiniMRYARN HistoryServer web address: " + MiniMRYarnCluster .GetResolvedMRHistoryWebAppURLWithoutScheme(this.GetConfig(), MRWebAppUtil.GetJHSHttpPolicy () == HttpConfig.Policy.HttpsOnly)); } }
public virtual void TestRMWebAppRedirect() { cluster = new MiniYARNCluster(typeof(TestRMFailover).FullName, 2, 0, 1, 1); conf.SetBoolean(YarnConfiguration.AutoFailoverEnabled, false); cluster.Init(conf); cluster.Start(); GetAdminService(0).TransitionToActive(req); string rm1Url = "http://0.0.0.0:18088"; string rm2Url = "http://0.0.0.0:28088"; string redirectURL = GetRedirectURL(rm2Url); // if uri is null, RMWebAppFilter will append a slash at the trail of the redirection url NUnit.Framework.Assert.AreEqual(redirectURL, rm1Url + "/"); redirectURL = GetRedirectURL(rm2Url + "/metrics"); NUnit.Framework.Assert.AreEqual(redirectURL, rm1Url + "/metrics"); redirectURL = GetRedirectURL(rm2Url + "/jmx"); NUnit.Framework.Assert.AreEqual(redirectURL, rm1Url + "/jmx"); // standby RM links /conf, /stacks, /logLevel, /static, /logs, // /cluster/cluster as well as webService // /ws/v1/cluster/info should not be redirected to active RM redirectURL = GetRedirectURL(rm2Url + "/cluster/cluster"); NUnit.Framework.Assert.IsNull(redirectURL); redirectURL = GetRedirectURL(rm2Url + "/conf"); NUnit.Framework.Assert.IsNull(redirectURL); redirectURL = GetRedirectURL(rm2Url + "/stacks"); NUnit.Framework.Assert.IsNull(redirectURL); redirectURL = GetRedirectURL(rm2Url + "/logLevel"); NUnit.Framework.Assert.IsNull(redirectURL); redirectURL = GetRedirectURL(rm2Url + "/static"); NUnit.Framework.Assert.IsNull(redirectURL); redirectURL = GetRedirectURL(rm2Url + "/logs"); NUnit.Framework.Assert.IsNull(redirectURL); redirectURL = GetRedirectURL(rm2Url + "/ws/v1/cluster/info"); NUnit.Framework.Assert.IsNull(redirectURL); redirectURL = GetRedirectURL(rm2Url + "/ws/v1/cluster/apps"); NUnit.Framework.Assert.AreEqual(redirectURL, rm1Url + "/ws/v1/cluster/apps"); redirectURL = GetRedirectURL(rm2Url + "/proxy/" + fakeAppId); NUnit.Framework.Assert.IsNull(redirectURL); // transit the active RM to standby // Both of RMs are in standby mode GetAdminService(0).TransitionToStandby(req); // RM2 is expected to send the httpRequest to itself. // The Header Field: Refresh is expected to be set. redirectURL = GetRefreshURL(rm2Url); NUnit.Framework.Assert.IsTrue(redirectURL != null && redirectURL.Contains(YarnWebParams .NextRefreshInterval) && redirectURL.Contains(rm2Url)); }
public static void Setup() { Log.Info("Starting up YARN cluster"); conf.SetInt(YarnConfiguration.RmSchedulerMinimumAllocationMb, 128); if (yarnCluster == null) { yarnCluster = new MiniYARNCluster(typeof(TestUnmanagedAMLauncher).Name, 1, 1, 1); yarnCluster.Init(conf); yarnCluster.Start(); //get the address Configuration yarnClusterConfig = yarnCluster.GetConfig(); Log.Info("MiniYARN ResourceManager published address: " + yarnClusterConfig.Get(YarnConfiguration .RmAddress)); Log.Info("MiniYARN ResourceManager published web address: " + yarnClusterConfig.Get (YarnConfiguration.RmWebappAddress)); string webapp = yarnClusterConfig.Get(YarnConfiguration.RmWebappAddress); NUnit.Framework.Assert.IsTrue("Web app address still unbound to a host at " + webapp , !webapp.StartsWith("0.0.0.0")); Log.Info("Yarn webapp is at " + webapp); Uri url = Sharpen.Thread.CurrentThread().GetContextClassLoader().GetResource("yarn-site.xml" ); if (url == null) { throw new RuntimeException("Could not find 'yarn-site.xml' dummy file in classpath" ); } //write the document to a buffer (not directly to the file, as that //can cause the file being written to get read -which will then fail. ByteArrayOutputStream bytesOut = new ByteArrayOutputStream(); yarnClusterConfig.WriteXml(bytesOut); bytesOut.Close(); //write the bytes to the file in the classpath OutputStream os = new FileOutputStream(new FilePath(url.AbsolutePath)); os.Write(bytesOut.ToByteArray()); os.Close(); } try { Sharpen.Thread.Sleep(2000); } catch (Exception e) { Log.Info("setup thread sleep interrupted. message=" + e.Message); } }
public virtual void TearDown() { if (yarnCluster != null) { try { yarnCluster.Stop(); } finally { yarnCluster = null; } } FileContext fsContext = FileContext.GetLocalFSFileContext(); fsContext.Delete(new Path(conf.Get("yarn.timeline-service.leveldb-timeline-store.path" )), true); }
public virtual void Setup() { // start minicluster conf = new YarnConfiguration(); yarnCluster = new MiniYARNCluster(typeof(TestAMRMClient).FullName, nodeCount, 1, 1); yarnCluster.Init(conf); yarnCluster.Start(); NUnit.Framework.Assert.IsNotNull(yarnCluster); NUnit.Framework.Assert.AreEqual(Service.STATE.Started, yarnCluster.GetServiceState ()); // start rm client yarnClient = (YarnClientImpl)YarnClient.CreateYarnClient(); yarnClient.Init(conf); yarnClient.Start(); NUnit.Framework.Assert.IsNotNull(yarnClient); NUnit.Framework.Assert.AreEqual(Service.STATE.Started, yarnClient.GetServiceState ()); // get node info nodeReports = yarnClient.GetNodeReports(NodeState.Running); // submit new app ApplicationSubmissionContext appContext = yarnClient.CreateApplication().GetApplicationSubmissionContext (); ApplicationId appId = appContext.GetApplicationId(); // set the application name appContext.SetApplicationName("Test"); // Set the priority for the application master Priority pri = Priority.NewInstance(0); appContext.SetPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.SetQueue("default"); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <ContainerLaunchContext>(); appContext.SetAMContainerSpec(amContainer); // unmanaged AM appContext.SetUnmanagedAM(true); // Create the request to send to the applications manager SubmitApplicationRequest appRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <SubmitApplicationRequest>(); appRequest.SetApplicationSubmissionContext(appContext); // Submit the application to the applications manager yarnClient.SubmitApplication(appContext); // wait for app to start int iterationsLeft = 30; RMAppAttempt appAttempt = null; while (iterationsLeft > 0) { ApplicationReport appReport = yarnClient.GetApplicationReport(appId); if (appReport.GetYarnApplicationState() == YarnApplicationState.Accepted) { attemptId = appReport.GetCurrentApplicationAttemptId(); appAttempt = yarnCluster.GetResourceManager().GetRMContext().GetRMApps()[attemptId .GetApplicationId()].GetCurrentAppAttempt(); while (true) { if (appAttempt.GetAppAttemptState() == RMAppAttemptState.Launched) { break; } } break; } Sleep(1000); --iterationsLeft; } if (iterationsLeft == 0) { NUnit.Framework.Assert.Fail("Application hasn't bee started"); } // Just dig into the ResourceManager and get the AMRMToken just for the sake // of testing. UserGroupInformation.SetLoginUser(UserGroupInformation.CreateRemoteUser(UserGroupInformation .GetCurrentUser().GetUserName())); UserGroupInformation.GetCurrentUser().AddToken(appAttempt.GetAMRMToken()); //creating an instance NMTokenCase nmTokenCache = new NMTokenCache(); // start am rm client rmClient = (AMRMClientImpl <AMRMClient.ContainerRequest>)AMRMClient.CreateAMRMClient <AMRMClient.ContainerRequest>(); //setting an instance NMTokenCase rmClient.SetNMTokenCache(nmTokenCache); rmClient.Init(conf); rmClient.Start(); NUnit.Framework.Assert.IsNotNull(rmClient); NUnit.Framework.Assert.AreEqual(Service.STATE.Started, rmClient.GetServiceState() ); // start am nm client nmClient = (NMClientImpl)NMClient.CreateNMClient(); //propagating the AMRMClient NMTokenCache instance nmClient.SetNMTokenCache(rmClient.GetNMTokenCache()); nmClient.Init(conf); nmClient.Start(); NUnit.Framework.Assert.IsNotNull(nmClient); NUnit.Framework.Assert.AreEqual(Service.STATE.Started, nmClient.GetServiceState() ); }