private void VerifyClientConnection()
{
int numRetries = 3;
while (numRetries-- > 0)
{
Configuration conf = new YarnConfiguration(this.conf);
YarnClient client = YarnClient.CreateYarnClient();
client.Init(conf);
client.Start();
try
{
client.GetApplications();
return;
}
catch (Exception e)
{
Log.Error(e);
}
finally
{
client.Stop();
}
}
NUnit.Framework.Assert.Fail("Client couldn't connect to the Active RM");
}
public YarnCLI()
: base(new YarnConfiguration())
{
client = YarnClient.CreateYarnClient();
client.Init(GetConf());
client.Start();
}
protected internal virtual YarnClient CreateYarnClient()
{
YarnClient yarnClient = YarnClient.CreateYarnClient();
yarnClient.Init(GetConf());
yarnClient.Start();
return(yarnClient);
}
/// <summary>
/// Delegate responsible for communicating with the Resource Manager's
/// <see cref="Org.Apache.Hadoop.Yarn.Api.ApplicationClientProtocol"/>
/// .
/// </summary>
/// <param name="conf">the configuration object.</param>
public ResourceMgrDelegate(YarnConfiguration conf)
: base(typeof(Org.Apache.Hadoop.Mapred.ResourceMgrDelegate).FullName)
{
this.conf = conf;
this.client = YarnClient.CreateYarnClient();
Init(conf);
Start();
}
internal Client(string appMasterMainClass, Configuration conf)
{
this.conf = conf;
this.appMasterMainClass = appMasterMainClass;
yarnClient = YarnClient.CreateYarnClient();
yarnClient.Init(conf);
opts = new Options();
opts.AddOption("appname", true, "Application Name. Default value - DistributedShell"
);
opts.AddOption("priority", true, "Application Priority. Default 0");
opts.AddOption("queue", true, "RM Queue in which this application is to be submitted"
);
opts.AddOption("timeout", true, "Application timeout in milliseconds");
opts.AddOption("master_memory", true, "Amount of memory in MB to be requested to run the application master"
);
opts.AddOption("master_vcores", true, "Amount of virtual cores to be requested to run the application master"
);
opts.AddOption("jar", true, "Jar file containing the application master");
opts.AddOption("shell_command", true, "Shell command to be executed by " + "the Application Master. Can only specify either --shell_command "
+ "or --shell_script");
opts.AddOption("shell_script", true, "Location of the shell script to be " + "executed. Can only specify either --shell_command or --shell_script"
);
opts.AddOption("shell_args", true, "Command line args for the shell script." + "Multiple args can be separated by empty space."
);
opts.GetOption("shell_args").SetArgs(Option.UnlimitedValues);
opts.AddOption("shell_env", true, "Environment for shell script. Specified as env_key=env_val pairs"
);
opts.AddOption("shell_cmd_priority", true, "Priority for the shell command containers"
);
opts.AddOption("container_memory", true, "Amount of memory in MB to be requested to run the shell command"
);
opts.AddOption("container_vcores", true, "Amount of virtual cores to be requested to run the shell command"
);
opts.AddOption("num_containers", true, "No. of containers on which the shell command needs to be executed"
);
opts.AddOption("log_properties", true, "log4j.properties file");
opts.AddOption("keep_containers_across_application_attempts", false, "Flag to indicate whether to keep containers across application attempts."
+ " If the flag is true, running containers will not be killed when" + " application attempt fails and these containers will be retrieved by"
+ " the new application attempt ");
opts.AddOption("attempt_failures_validity_interval", true, "when attempt_failures_validity_interval in milliseconds is set to > 0,"
+ "the failure number will not take failures which happen out of " + "the validityInterval into failure count. "
+ "If failure count reaches to maxAppAttempts, " + "the application will be failed."
);
opts.AddOption("debug", false, "Dump out debug information");
opts.AddOption("domain", true, "ID of the timeline domain where the " + "timeline entities will be put"
);
opts.AddOption("view_acls", true, "Users and groups that allowed to " + "view the timeline entities in the given domain"
);
opts.AddOption("modify_acls", true, "Users and groups that allowed to " + "modify the timeline entities in the given domain"
);
opts.AddOption("create", false, "Flag to indicate whether to create the " + "domain specified with -domain."
);
opts.AddOption("help", false, "Print usage");
opts.AddOption("node_label_expression", true, "Node label expression to determine the nodes"
+ " where all the containers of this application" + " will be allocated, \"\" means containers"
+ " can be allocated anywhere, if you don't specify the option," + " default node_label_expression of queue will be used."
);
}
protected internal virtual YarnClient CreateAndStartYarnClient(Configuration conf
)
{
Configuration configuration = new YarnConfiguration(conf);
YarnClient client = YarnClient.CreateYarnClient();
client.Init(configuration);
client.Start();
return(client);
}
/// <exception cref="Org.Apache.Commons.Cli.ParseException"/>
public virtual bool Init(string[] args)
{
Options opts = new Options();
opts.AddOption("appname", true, "Application Name. Default value - UnmanagedAM");
opts.AddOption("priority", true, "Application Priority. Default 0");
opts.AddOption("queue", true, "RM Queue in which this application is to be submitted"
);
opts.AddOption("master_memory", true, "Amount of memory in MB to be requested to run the application master"
);
opts.AddOption("cmd", true, "command to start unmanaged AM (required)");
opts.AddOption("classpath", true, "additional classpath");
opts.AddOption("help", false, "Print usage");
CommandLine cliParser = new GnuParser().Parse(opts, args);
if (args.Length == 0)
{
PrintUsage(opts);
throw new ArgumentException("No args specified for client to initialize");
}
if (cliParser.HasOption("help"))
{
PrintUsage(opts);
return(false);
}
appName = cliParser.GetOptionValue("appname", "UnmanagedAM");
amPriority = System.Convert.ToInt32(cliParser.GetOptionValue("priority", "0"));
amQueue = cliParser.GetOptionValue("queue", "default");
classpath = cliParser.GetOptionValue("classpath", null);
amCmd = cliParser.GetOptionValue("cmd");
if (amCmd == null)
{
PrintUsage(opts);
throw new ArgumentException("No cmd specified for application master");
}
YarnConfiguration yarnConf = new YarnConfiguration(conf);
rmClient = YarnClient.CreateYarnClient();
rmClient.Init(yarnConf);
return(true);
}
public virtual void Setup()
{
// start minicluster
conf = new YarnConfiguration();
yarnCluster = new MiniYARNCluster(typeof(TestAMRMClient).FullName, nodeCount, 1,
1);
yarnCluster.Init(conf);
yarnCluster.Start();
NUnit.Framework.Assert.IsNotNull(yarnCluster);
NUnit.Framework.Assert.AreEqual(Service.STATE.Started, yarnCluster.GetServiceState
());
// start rm client
yarnClient = (YarnClientImpl)YarnClient.CreateYarnClient();
yarnClient.Init(conf);
yarnClient.Start();
NUnit.Framework.Assert.IsNotNull(yarnClient);
NUnit.Framework.Assert.AreEqual(Service.STATE.Started, yarnClient.GetServiceState
());
// get node info
nodeReports = yarnClient.GetNodeReports(NodeState.Running);
// submit new app
ApplicationSubmissionContext appContext = yarnClient.CreateApplication().GetApplicationSubmissionContext
();
ApplicationId appId = appContext.GetApplicationId();
// set the application name
appContext.SetApplicationName("Test");
// Set the priority for the application master
Priority pri = Priority.NewInstance(0);
appContext.SetPriority(pri);
// Set the queue to which this application is to be submitted in the RM
appContext.SetQueue("default");
// Set up the container launch context for the application master
ContainerLaunchContext amContainer = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<ContainerLaunchContext>();
appContext.SetAMContainerSpec(amContainer);
// unmanaged AM
appContext.SetUnmanagedAM(true);
// Create the request to send to the applications manager
SubmitApplicationRequest appRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<SubmitApplicationRequest>();
appRequest.SetApplicationSubmissionContext(appContext);
// Submit the application to the applications manager
yarnClient.SubmitApplication(appContext);
// wait for app to start
int iterationsLeft = 30;
RMAppAttempt appAttempt = null;
while (iterationsLeft > 0)
{
ApplicationReport appReport = yarnClient.GetApplicationReport(appId);
if (appReport.GetYarnApplicationState() == YarnApplicationState.Accepted)
{
attemptId = appReport.GetCurrentApplicationAttemptId();
appAttempt = yarnCluster.GetResourceManager().GetRMContext().GetRMApps()[attemptId
.GetApplicationId()].GetCurrentAppAttempt();
while (true)
{
if (appAttempt.GetAppAttemptState() == RMAppAttemptState.Launched)
{
break;
}
}
break;
}
Sleep(1000);
--iterationsLeft;
}
if (iterationsLeft == 0)
{
NUnit.Framework.Assert.Fail("Application hasn't bee started");
}
// Just dig into the ResourceManager and get the AMRMToken just for the sake
// of testing.
UserGroupInformation.SetLoginUser(UserGroupInformation.CreateRemoteUser(UserGroupInformation
.GetCurrentUser().GetUserName()));
UserGroupInformation.GetCurrentUser().AddToken(appAttempt.GetAMRMToken());
//creating an instance NMTokenCase
nmTokenCache = new NMTokenCache();
// start am rm client
rmClient = (AMRMClientImpl <AMRMClient.ContainerRequest>)AMRMClient.CreateAMRMClient
<AMRMClient.ContainerRequest>();
//setting an instance NMTokenCase
rmClient.SetNMTokenCache(nmTokenCache);
rmClient.Init(conf);
rmClient.Start();
NUnit.Framework.Assert.IsNotNull(rmClient);
NUnit.Framework.Assert.AreEqual(Service.STATE.Started, rmClient.GetServiceState()
);
// start am nm client
nmClient = (NMClientImpl)NMClient.CreateNMClient();
//propagating the AMRMClient NMTokenCache instance
nmClient.SetNMTokenCache(rmClient.GetNMTokenCache());
nmClient.Init(conf);
nmClient.Start();
NUnit.Framework.Assert.IsNotNull(nmClient);
NUnit.Framework.Assert.AreEqual(Service.STATE.Started, nmClient.GetServiceState()
);
}
/// <exception cref="System.Exception"/>
public virtual void TestDSShell(bool haveDomain)
{
string[] args = new string[] { "--jar", AppmasterJar, "--num_containers", "2", "--shell_command"
, Shell.Windows ? "dir" : "ls", "--master_memory", "512", "--master_vcores", "2"
, "--container_memory", "128", "--container_vcores", "1" };
if (haveDomain)
{
string[] domainArgs = new string[] { "--domain", "TEST_DOMAIN", "--view_acls", "reader_user reader_group"
, "--modify_acls", "writer_user writer_group", "--create" };
IList <string> argsList = new AList <string>(Arrays.AsList(args));
Sharpen.Collections.AddAll(argsList, Arrays.AsList(domainArgs));
args = Sharpen.Collections.ToArray(argsList, new string[argsList.Count]);
}
Log.Info("Initializing DS Client");
Client client = new Client(new Configuration(yarnCluster.GetConfig()));
bool initSuccess = client.Init(args);
NUnit.Framework.Assert.IsTrue(initSuccess);
Log.Info("Running DS Client");
AtomicBoolean result = new AtomicBoolean(false);
Sharpen.Thread t = new _Thread_194(result, client);
t.Start();
YarnClient yarnClient = YarnClient.CreateYarnClient();
yarnClient.Init(new Configuration(yarnCluster.GetConfig()));
yarnClient.Start();
string hostName = NetUtils.GetHostname();
bool verified = false;
string errorMessage = string.Empty;
while (!verified)
{
IList <ApplicationReport> apps = yarnClient.GetApplications();
if (apps.Count == 0)
{
Sharpen.Thread.Sleep(10);
continue;
}
ApplicationReport appReport = apps[0];
if (appReport.GetHost().Equals("N/A"))
{
Sharpen.Thread.Sleep(10);
continue;
}
errorMessage = "Expected host name to start with '" + hostName + "', was '" + appReport
.GetHost() + "'. Expected rpc port to be '-1', was '" + appReport.GetRpcPort() +
"'.";
if (CheckHostname(appReport.GetHost()) && appReport.GetRpcPort() == -1)
{
verified = true;
}
if (appReport.GetYarnApplicationState() == YarnApplicationState.Finished)
{
break;
}
}
NUnit.Framework.Assert.IsTrue(errorMessage, verified);
t.Join();
Log.Info("Client run completed. Result=" + result);
NUnit.Framework.Assert.IsTrue(result.Get());
TimelineDomain domain = null;
if (haveDomain)
{
domain = yarnCluster.GetApplicationHistoryServer().GetTimelineStore().GetDomain("TEST_DOMAIN"
);
NUnit.Framework.Assert.IsNotNull(domain);
NUnit.Framework.Assert.AreEqual("reader_user reader_group", domain.GetReaders());
NUnit.Framework.Assert.AreEqual("writer_user writer_group", domain.GetWriters());
}
TimelineEntities entitiesAttempts = yarnCluster.GetApplicationHistoryServer().GetTimelineStore
().GetEntities(ApplicationMaster.DSEntity.DsAppAttempt.ToString(), null, null, null
, null, null, null, null, null, null);
NUnit.Framework.Assert.IsNotNull(entitiesAttempts);
NUnit.Framework.Assert.AreEqual(1, entitiesAttempts.GetEntities().Count);
NUnit.Framework.Assert.AreEqual(2, entitiesAttempts.GetEntities()[0].GetEvents().
Count);
NUnit.Framework.Assert.AreEqual(entitiesAttempts.GetEntities()[0].GetEntityType()
.ToString(), ApplicationMaster.DSEntity.DsAppAttempt.ToString());
if (haveDomain)
{
NUnit.Framework.Assert.AreEqual(domain.GetId(), entitiesAttempts.GetEntities()[0]
.GetDomainId());
}
else
{
NUnit.Framework.Assert.AreEqual("DEFAULT", entitiesAttempts.GetEntities()[0].GetDomainId
());
}
TimelineEntities entities = yarnCluster.GetApplicationHistoryServer().GetTimelineStore
().GetEntities(ApplicationMaster.DSEntity.DsContainer.ToString(), null, null, null
, null, null, null, null, null, null);
NUnit.Framework.Assert.IsNotNull(entities);
NUnit.Framework.Assert.AreEqual(2, entities.GetEntities().Count);
NUnit.Framework.Assert.AreEqual(entities.GetEntities()[0].GetEntityType().ToString
(), ApplicationMaster.DSEntity.DsContainer.ToString());
if (haveDomain)
{
NUnit.Framework.Assert.AreEqual(domain.GetId(), entities.GetEntities()[0].GetDomainId
());
}
else
{
NUnit.Framework.Assert.AreEqual("DEFAULT", entities.GetEntities()[0].GetDomainId(
));
}
}
public RemoteAppChecker()
: this(YarnClient.CreateYarnClient())
{
}