private void VerifyClientConnection() { int numRetries = 3; while (numRetries-- > 0) { Configuration conf = new YarnConfiguration(this.conf); YarnClient client = YarnClient.CreateYarnClient(); client.Init(conf); client.Start(); try { client.GetApplications(); return; } catch (Exception e) { Log.Error(e); } finally { client.Stop(); } } NUnit.Framework.Assert.Fail("Client couldn't connect to the Active RM"); }
public YarnCLI() : base(new YarnConfiguration()) { client = YarnClient.CreateYarnClient(); client.Init(GetConf()); client.Start(); }
protected internal virtual YarnClient CreateYarnClient() { YarnClient yarnClient = YarnClient.CreateYarnClient(); yarnClient.Init(GetConf()); yarnClient.Start(); return(yarnClient); }
internal Client(string appMasterMainClass, Configuration conf) { this.conf = conf; this.appMasterMainClass = appMasterMainClass; yarnClient = YarnClient.CreateYarnClient(); yarnClient.Init(conf); opts = new Options(); opts.AddOption("appname", true, "Application Name. Default value - DistributedShell" ); opts.AddOption("priority", true, "Application Priority. Default 0"); opts.AddOption("queue", true, "RM Queue in which this application is to be submitted" ); opts.AddOption("timeout", true, "Application timeout in milliseconds"); opts.AddOption("master_memory", true, "Amount of memory in MB to be requested to run the application master" ); opts.AddOption("master_vcores", true, "Amount of virtual cores to be requested to run the application master" ); opts.AddOption("jar", true, "Jar file containing the application master"); opts.AddOption("shell_command", true, "Shell command to be executed by " + "the Application Master. Can only specify either --shell_command " + "or --shell_script"); opts.AddOption("shell_script", true, "Location of the shell script to be " + "executed. Can only specify either --shell_command or --shell_script" ); opts.AddOption("shell_args", true, "Command line args for the shell script." + "Multiple args can be separated by empty space." ); opts.GetOption("shell_args").SetArgs(Option.UnlimitedValues); opts.AddOption("shell_env", true, "Environment for shell script. Specified as env_key=env_val pairs" ); opts.AddOption("shell_cmd_priority", true, "Priority for the shell command containers" ); opts.AddOption("container_memory", true, "Amount of memory in MB to be requested to run the shell command" ); opts.AddOption("container_vcores", true, "Amount of virtual cores to be requested to run the shell command" ); opts.AddOption("num_containers", true, "No. of containers on which the shell command needs to be executed" ); opts.AddOption("log_properties", true, "log4j.properties file"); opts.AddOption("keep_containers_across_application_attempts", false, "Flag to indicate whether to keep containers across application attempts." + " If the flag is true, running containers will not be killed when" + " application attempt fails and these containers will be retrieved by" + " the new application attempt "); opts.AddOption("attempt_failures_validity_interval", true, "when attempt_failures_validity_interval in milliseconds is set to > 0," + "the failure number will not take failures which happen out of " + "the validityInterval into failure count. " + "If failure count reaches to maxAppAttempts, " + "the application will be failed." ); opts.AddOption("debug", false, "Dump out debug information"); opts.AddOption("domain", true, "ID of the timeline domain where the " + "timeline entities will be put" ); opts.AddOption("view_acls", true, "Users and groups that allowed to " + "view the timeline entities in the given domain" ); opts.AddOption("modify_acls", true, "Users and groups that allowed to " + "modify the timeline entities in the given domain" ); opts.AddOption("create", false, "Flag to indicate whether to create the " + "domain specified with -domain." ); opts.AddOption("help", false, "Print usage"); opts.AddOption("node_label_expression", true, "Node label expression to determine the nodes" + " where all the containers of this application" + " will be allocated, \"\" means containers" + " can be allocated anywhere, if you don't specify the option," + " default node_label_expression of queue will be used." ); }
protected internal virtual YarnClient CreateAndStartYarnClient(Configuration conf ) { Configuration configuration = new YarnConfiguration(conf); YarnClient client = YarnClient.CreateYarnClient(); client.Init(configuration); client.Start(); return(client); }
/// <exception cref="Org.Apache.Commons.Cli.ParseException"/> public virtual bool Init(string[] args) { Options opts = new Options(); opts.AddOption("appname", true, "Application Name. Default value - UnmanagedAM"); opts.AddOption("priority", true, "Application Priority. Default 0"); opts.AddOption("queue", true, "RM Queue in which this application is to be submitted" ); opts.AddOption("master_memory", true, "Amount of memory in MB to be requested to run the application master" ); opts.AddOption("cmd", true, "command to start unmanaged AM (required)"); opts.AddOption("classpath", true, "additional classpath"); opts.AddOption("help", false, "Print usage"); CommandLine cliParser = new GnuParser().Parse(opts, args); if (args.Length == 0) { PrintUsage(opts); throw new ArgumentException("No args specified for client to initialize"); } if (cliParser.HasOption("help")) { PrintUsage(opts); return(false); } appName = cliParser.GetOptionValue("appname", "UnmanagedAM"); amPriority = System.Convert.ToInt32(cliParser.GetOptionValue("priority", "0")); amQueue = cliParser.GetOptionValue("queue", "default"); classpath = cliParser.GetOptionValue("classpath", null); amCmd = cliParser.GetOptionValue("cmd"); if (amCmd == null) { PrintUsage(opts); throw new ArgumentException("No cmd specified for application master"); } YarnConfiguration yarnConf = new YarnConfiguration(conf); rmClient = YarnClient.CreateYarnClient(); rmClient.Init(yarnConf); return(true); }
/// <exception cref="System.Exception"/> public virtual void TestDSShell(bool haveDomain) { string[] args = new string[] { "--jar", AppmasterJar, "--num_containers", "2", "--shell_command" , Shell.Windows ? "dir" : "ls", "--master_memory", "512", "--master_vcores", "2" , "--container_memory", "128", "--container_vcores", "1" }; if (haveDomain) { string[] domainArgs = new string[] { "--domain", "TEST_DOMAIN", "--view_acls", "reader_user reader_group" , "--modify_acls", "writer_user writer_group", "--create" }; IList <string> argsList = new AList <string>(Arrays.AsList(args)); Sharpen.Collections.AddAll(argsList, Arrays.AsList(domainArgs)); args = Sharpen.Collections.ToArray(argsList, new string[argsList.Count]); } Log.Info("Initializing DS Client"); Client client = new Client(new Configuration(yarnCluster.GetConfig())); bool initSuccess = client.Init(args); NUnit.Framework.Assert.IsTrue(initSuccess); Log.Info("Running DS Client"); AtomicBoolean result = new AtomicBoolean(false); Sharpen.Thread t = new _Thread_194(result, client); t.Start(); YarnClient yarnClient = YarnClient.CreateYarnClient(); yarnClient.Init(new Configuration(yarnCluster.GetConfig())); yarnClient.Start(); string hostName = NetUtils.GetHostname(); bool verified = false; string errorMessage = string.Empty; while (!verified) { IList <ApplicationReport> apps = yarnClient.GetApplications(); if (apps.Count == 0) { Sharpen.Thread.Sleep(10); continue; } ApplicationReport appReport = apps[0]; if (appReport.GetHost().Equals("N/A")) { Sharpen.Thread.Sleep(10); continue; } errorMessage = "Expected host name to start with '" + hostName + "', was '" + appReport .GetHost() + "'. Expected rpc port to be '-1', was '" + appReport.GetRpcPort() + "'."; if (CheckHostname(appReport.GetHost()) && appReport.GetRpcPort() == -1) { verified = true; } if (appReport.GetYarnApplicationState() == YarnApplicationState.Finished) { break; } } NUnit.Framework.Assert.IsTrue(errorMessage, verified); t.Join(); Log.Info("Client run completed. Result=" + result); NUnit.Framework.Assert.IsTrue(result.Get()); TimelineDomain domain = null; if (haveDomain) { domain = yarnCluster.GetApplicationHistoryServer().GetTimelineStore().GetDomain("TEST_DOMAIN" ); NUnit.Framework.Assert.IsNotNull(domain); NUnit.Framework.Assert.AreEqual("reader_user reader_group", domain.GetReaders()); NUnit.Framework.Assert.AreEqual("writer_user writer_group", domain.GetWriters()); } TimelineEntities entitiesAttempts = yarnCluster.GetApplicationHistoryServer().GetTimelineStore ().GetEntities(ApplicationMaster.DSEntity.DsAppAttempt.ToString(), null, null, null , null, null, null, null, null, null); NUnit.Framework.Assert.IsNotNull(entitiesAttempts); NUnit.Framework.Assert.AreEqual(1, entitiesAttempts.GetEntities().Count); NUnit.Framework.Assert.AreEqual(2, entitiesAttempts.GetEntities()[0].GetEvents(). Count); NUnit.Framework.Assert.AreEqual(entitiesAttempts.GetEntities()[0].GetEntityType() .ToString(), ApplicationMaster.DSEntity.DsAppAttempt.ToString()); if (haveDomain) { NUnit.Framework.Assert.AreEqual(domain.GetId(), entitiesAttempts.GetEntities()[0] .GetDomainId()); } else { NUnit.Framework.Assert.AreEqual("DEFAULT", entitiesAttempts.GetEntities()[0].GetDomainId ()); } TimelineEntities entities = yarnCluster.GetApplicationHistoryServer().GetTimelineStore ().GetEntities(ApplicationMaster.DSEntity.DsContainer.ToString(), null, null, null , null, null, null, null, null, null); NUnit.Framework.Assert.IsNotNull(entities); NUnit.Framework.Assert.AreEqual(2, entities.GetEntities().Count); NUnit.Framework.Assert.AreEqual(entities.GetEntities()[0].GetEntityType().ToString (), ApplicationMaster.DSEntity.DsContainer.ToString()); if (haveDomain) { NUnit.Framework.Assert.AreEqual(domain.GetId(), entities.GetEntities()[0].GetDomainId ()); } else { NUnit.Framework.Assert.AreEqual("DEFAULT", entities.GetEntities()[0].GetDomainId( )); } }
/// <exception cref="System.Exception"/> protected override void ServiceInit(Configuration conf) { client.Init(conf); base.ServiceInit(conf); }