public static IList <string> GetRMHAWebappAddresses(YarnConfiguration conf) { ICollection <string> rmIds = conf.GetStringCollection(YarnConfiguration.RmHaIds); IList <string> addrs = new AList <string>(); if (YarnConfiguration.UseHttps(conf)) { foreach (string id in rmIds) { string addr = conf.Get(YarnConfiguration.RmWebappHttpsAddress + "." + id); if (addr != null) { addrs.AddItem(addr); } } } else { foreach (string id in rmIds) { string addr = conf.Get(YarnConfiguration.RmWebappAddress + "." + id); if (addr != null) { addrs.AddItem(addr); } } } return(addrs); }
public static void SetUpResourceManager() { Configuration.AddDefaultResource("config-with-security.xml"); Configuration configuration = new YarnConfiguration(); resourceManager = new _ResourceManager_74(); resourceManager.Init(configuration); new _Thread_80().Start(); int waitCount = 0; while (resourceManager.GetServiceState() == Service.STATE.Inited && waitCount++ < 10) { Log.Info("Waiting for RM to start..."); Sharpen.Thread.Sleep(1000); } if (resourceManager.GetServiceState() != Service.STATE.Started) { throw new IOException("ResourceManager failed to start. Final state is " + resourceManager .GetServiceState()); } Log.Info("ResourceManager RMAdmin address: " + configuration.Get(YarnConfiguration .RmAdminAddress)); client = new ResourceManagerAdministrationProtocolPBClientImpl(1L, GetProtocolAddress (configuration), configuration); }
public virtual void TestHAIDLookup() { //test implicitly lookup HA-ID Configuration conf = new YarnConfiguration(configuration); rm = new MockRM(conf); rm.Init(conf); NUnit.Framework.Assert.AreEqual(conf.Get(YarnConfiguration.RmHaId), Rm2NodeId); //test explicitly lookup HA-ID configuration.Set(YarnConfiguration.RmHaId, Rm1NodeId); conf = new YarnConfiguration(configuration); rm = new MockRM(conf); rm.Init(conf); NUnit.Framework.Assert.AreEqual(conf.Get(YarnConfiguration.RmHaId), Rm1NodeId); //test if RM_HA_ID can not be found configuration.Set(YarnConfiguration.RmHaIds, Rm1NodeId + "," + Rm3NodeId); configuration.Unset(YarnConfiguration.RmHaId); conf = new YarnConfiguration(configuration); try { rm = new MockRM(conf); rm.Init(conf); NUnit.Framework.Assert.Fail("Should get an exception here."); } catch (Exception ex) { NUnit.Framework.Assert.IsTrue(ex.Message.Contains("Invalid configuration! Can not find valid RM_HA_ID." )); } }
/// <exception cref="Javax.Servlet.ServletException"/> protected internal virtual string FindRedirectUrl() { string addr; if (proxyUriBases.Count == 1) { // external proxy or not RM HA addr = proxyUriBases.Values.GetEnumerator().Next(); } else { // RM HA YarnConfiguration conf = new YarnConfiguration(); string activeRMId = RMHAUtils.FindActiveRMHAId(conf); string addressPropertyPrefix = YarnConfiguration.UseHttps(conf) ? YarnConfiguration .RmWebappHttpsAddress : YarnConfiguration.RmWebappAddress; string host = conf.Get(HAUtil.AddSuffix(addressPropertyPrefix, activeRMId)); addr = proxyUriBases[host]; } if (addr == null) { throw new ServletException("Could not determine the proxy server for redirection" ); } return(addr); }
/// <summary> /// Add the requisite security principal settings to the given Configuration, /// returning a copy. /// </summary> /// <param name="conf">the original config</param> /// <returns>a copy with the security settings added</returns> private static Configuration AddSecurityConfiguration(Configuration conf) { // Make a copy so we don't mutate it. Also use an YarnConfiguration to // force loading of yarn-site.xml. conf = new YarnConfiguration(conf); conf.Set(CommonConfigurationKeys.HadoopSecurityServiceUserNameKey, conf.Get(YarnConfiguration .RmPrincipal, string.Empty)); return(conf); }
/// <exception cref="System.Exception"/> protected internal virtual void SetupInternal(int numNodeManager) { Log.Info("Starting up YARN cluster"); conf = new YarnConfiguration(); conf.SetInt(YarnConfiguration.RmSchedulerMinimumAllocationMb, 128); conf.Set("yarn.log.dir", "target"); conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, true); conf.Set(YarnConfiguration.RmScheduler, typeof(CapacityScheduler).FullName); conf.SetBoolean(YarnConfiguration.NodeLabelsEnabled, true); if (yarnCluster == null) { yarnCluster = new MiniYARNCluster(typeof(TestDistributedShell).Name, 1, numNodeManager , 1, 1); yarnCluster.Init(conf); yarnCluster.Start(); conf.Set(YarnConfiguration.TimelineServiceWebappAddress, MiniYARNCluster.GetHostname () + ":" + yarnCluster.GetApplicationHistoryServer().GetPort()); WaitForNMsToRegister(); Uri url = Sharpen.Thread.CurrentThread().GetContextClassLoader().GetResource("yarn-site.xml" ); if (url == null) { throw new RuntimeException("Could not find 'yarn-site.xml' dummy file in classpath" ); } Configuration yarnClusterConfig = yarnCluster.GetConfig(); yarnClusterConfig.Set("yarn.application.classpath", new FilePath(url.AbsolutePath ).GetParent()); //write the document to a buffer (not directly to the file, as that //can cause the file being written to get read -which will then fail. ByteArrayOutputStream bytesOut = new ByteArrayOutputStream(); yarnClusterConfig.WriteXml(bytesOut); bytesOut.Close(); //write the bytes to the file in the classpath OutputStream os = new FileOutputStream(new FilePath(url.AbsolutePath)); os.Write(bytesOut.ToByteArray()); os.Close(); } FileContext fsContext = FileContext.GetLocalFSFileContext(); fsContext.Delete(new Path(conf.Get("yarn.timeline-service.leveldb-timeline-store.path" )), true); try { Sharpen.Thread.Sleep(2000); } catch (Exception e) { Log.Info("setup thread sleep interrupted. message=" + e.Message); } }
// Couldn't find an Active RM /// <exception cref="System.Exception"/> private static HAServiceProtocol.HAServiceState GetHAState(YarnConfiguration yarnConf ) { HAServiceTarget haServiceTarget; int rpcTimeoutForChecks = yarnConf.GetInt(CommonConfigurationKeys.HaFcCliCheckTimeoutKey , CommonConfigurationKeys.HaFcCliCheckTimeoutDefault); yarnConf.Set(CommonConfigurationKeys.HadoopSecurityServiceUserNameKey, yarnConf.Get (YarnConfiguration.RmPrincipal, string.Empty)); haServiceTarget = new RMHAServiceTarget(yarnConf); HAServiceProtocol proto = haServiceTarget.GetProxy(yarnConf, rpcTimeoutForChecks); HAServiceProtocol.HAServiceState haState = proto.GetServiceStatus().GetState(); return(haState); }
public virtual void TearDown() { if (yarnCluster != null) { try { yarnCluster.Stop(); } finally { yarnCluster = null; } } FileContext fsContext = FileContext.GetLocalFSFileContext(); fsContext.Delete(new Path(conf.Get("yarn.timeline-service.leveldb-timeline-store.path" )), true); }
public virtual void TestTimelineServiceStartInMiniCluster() { Configuration conf = new YarnConfiguration(); int numNodeManagers = 1; int numLocalDirs = 1; int numLogDirs = 1; bool enableAHS; /* * Timeline service should not start if TIMELINE_SERVICE_ENABLED == false * and enableAHS flag == false */ conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, false); enableAHS = false; MiniYARNCluster cluster = null; try { cluster = new MiniYARNCluster(typeof(TestMiniYarnCluster).Name, numNodeManagers, numLocalDirs, numLogDirs, numLogDirs, enableAHS); cluster.Init(conf); cluster.Start(); //verify that the timeline service is not started. NUnit.Framework.Assert.IsNull("Timeline Service should not have been started", cluster .GetApplicationHistoryServer()); } finally { if (cluster != null) { cluster.Stop(); } } /* * Timeline service should start if TIMELINE_SERVICE_ENABLED == true * and enableAHS == false */ conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, true); enableAHS = false; cluster = null; try { cluster = new MiniYARNCluster(typeof(TestMiniYarnCluster).Name, numNodeManagers, numLocalDirs, numLogDirs, numLogDirs, enableAHS); cluster.Init(conf); // Verify that the timeline-service starts on ephemeral ports by default string hostname = MiniYARNCluster.GetHostname(); NUnit.Framework.Assert.AreEqual(hostname + ":0", conf.Get(YarnConfiguration.TimelineServiceAddress )); NUnit.Framework.Assert.AreEqual(hostname + ":0", conf.Get(YarnConfiguration.TimelineServiceWebappAddress )); cluster.Start(); //Timeline service may sometime take a while to get started int wait = 0; while (cluster.GetApplicationHistoryServer() == null && wait < 20) { Sharpen.Thread.Sleep(500); wait++; } //verify that the timeline service is started. NUnit.Framework.Assert.IsNotNull("Timeline Service should have been started", cluster .GetApplicationHistoryServer()); } finally { if (cluster != null) { cluster.Stop(); } } /* * Timeline service should start if TIMELINE_SERVICE_ENABLED == false * and enableAHS == true */ conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, false); enableAHS = true; cluster = null; try { cluster = new MiniYARNCluster(typeof(TestMiniYarnCluster).Name, numNodeManagers, numLocalDirs, numLogDirs, numLogDirs, enableAHS); cluster.Init(conf); cluster.Start(); //Timeline service may sometime take a while to get started int wait = 0; while (cluster.GetApplicationHistoryServer() == null && wait < 20) { Sharpen.Thread.Sleep(500); wait++; } //verify that the timeline service is started. NUnit.Framework.Assert.IsNotNull("Timeline Service should have been started", cluster .GetApplicationHistoryServer()); } finally { if (cluster != null) { cluster.Stop(); } } }
public virtual void InnerTestHAWithRMHostName(bool includeBindHost) { //this is run two times, with and without a bind host configured if (includeBindHost) { configuration.Set(YarnConfiguration.RmBindHost, "9.9.9.9"); } //test if both RM_HOSTBANE_{rm_id} and RM_RPCADDRESS_{rm_id} are set //We should only read rpc addresses from RM_RPCADDRESS_{rm_id} configuration configuration.Set(HAUtil.AddSuffix(YarnConfiguration.RmHostname, Rm1NodeId), "1.1.1.1" ); configuration.Set(HAUtil.AddSuffix(YarnConfiguration.RmHostname, Rm2NodeId), "0.0.0.0" ); configuration.Set(HAUtil.AddSuffix(YarnConfiguration.RmHostname, Rm3NodeId), "2.2.2.2" ); try { Configuration conf = new YarnConfiguration(configuration); rm = new MockRM(conf); rm.Init(conf); foreach (string confKey in YarnConfiguration.GetServiceAddressConfKeys(conf)) { NUnit.Framework.Assert.AreEqual("RPC address not set for " + confKey, Rm1Address, conf.Get(HAUtil.AddSuffix(confKey, Rm1NodeId))); NUnit.Framework.Assert.AreEqual("RPC address not set for " + confKey, Rm2Address, conf.Get(HAUtil.AddSuffix(confKey, Rm2NodeId))); NUnit.Framework.Assert.AreEqual("RPC address not set for " + confKey, Rm3Address, conf.Get(HAUtil.AddSuffix(confKey, Rm3NodeId))); if (includeBindHost) { NUnit.Framework.Assert.AreEqual("Web address misconfigured WITH bind-host", Sharpen.Runtime.Substring (rm.webAppAddress, 0, 7), "9.9.9.9"); } else { //YarnConfiguration tries to figure out which rm host it's on by binding to it, //which doesn't happen for any of these fake addresses, so we end up with 0.0.0.0 NUnit.Framework.Assert.AreEqual("Web address misconfigured WITHOUT bind-host", Sharpen.Runtime.Substring (rm.webAppAddress, 0, 7), "0.0.0.0"); } } } catch (YarnRuntimeException) { NUnit.Framework.Assert.Fail("Should not throw any exceptions."); } //test if only RM_HOSTBANE_{rm_id} is set configuration.Clear(); configuration.SetBoolean(YarnConfiguration.RmHaEnabled, true); configuration.Set(YarnConfiguration.RmHaIds, Rm1NodeId + "," + Rm2NodeId); configuration.Set(HAUtil.AddSuffix(YarnConfiguration.RmHostname, Rm1NodeId), "1.1.1.1" ); configuration.Set(HAUtil.AddSuffix(YarnConfiguration.RmHostname, Rm2NodeId), "0.0.0.0" ); try { Configuration conf = new YarnConfiguration(configuration); rm = new MockRM(conf); rm.Init(conf); NUnit.Framework.Assert.AreEqual("RPC address not set for " + YarnConfiguration.RmAddress , "1.1.1.1:8032", conf.Get(HAUtil.AddSuffix(YarnConfiguration.RmAddress, Rm1NodeId ))); NUnit.Framework.Assert.AreEqual("RPC address not set for " + YarnConfiguration.RmAddress , "0.0.0.0:8032", conf.Get(HAUtil.AddSuffix(YarnConfiguration.RmAddress, Rm2NodeId ))); } catch (YarnRuntimeException) { NUnit.Framework.Assert.Fail("Should not throw any exceptions."); } }