private void VerifyClientConnection()
{
int numRetries = 3;
while (numRetries-- > 0)
{
Configuration conf = new YarnConfiguration(this.conf);
YarnClient client = YarnClient.CreateYarnClient();
client.Init(conf);
client.Start();
try
{
client.GetApplications();
return;
}
catch (Exception e)
{
Log.Error(e);
}
finally
{
client.Stop();
}
}
NUnit.Framework.Assert.Fail("Client couldn't connect to the Active RM");
}
public virtual void TestGetClusterNodeLabelsWithLocalAccess()
{
YarnClient client = Org.Mockito.Mockito.Mock <YarnClient>();
Org.Mockito.Mockito.When(client.GetClusterNodeLabels()).ThenReturn(ImmutableSet.Of
("remote1", "remote2"));
ClusterCLI cli = new ClusterCLI();
cli.SetClient(client);
cli.SetSysOutPrintStream(sysOut);
cli.SetSysErrPrintStream(sysErr);
ClusterCLI.localNodeLabelsManager = Org.Mockito.Mockito.Mock <CommonNodeLabelsManager
>();
Org.Mockito.Mockito.When(ClusterCLI.localNodeLabelsManager.GetClusterNodeLabels()
).ThenReturn(ImmutableSet.Of("local1", "local2"));
int rc = cli.Run(new string[] { ClusterCLI.Cmd, "-" + ClusterCLI.ListLabelsCmd, "-"
+ ClusterCLI.DirectlyAccessNodeLabelStore });
NUnit.Framework.Assert.AreEqual(0, rc);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter pw = new PrintWriter(baos);
// it should return local* instead of remote*
pw.Write("Node Labels: local1,local2");
pw.Close();
Org.Mockito.Mockito.Verify(sysOut).WriteLine(baos.ToString("UTF-8"));
}
public virtual void Initiate()
{
StartHACluster(1, true, false, false);
Configuration conf = new YarnConfiguration(this.conf);
client = CreateAndStartYarnClient(conf);
}
public YarnCLI()
: base(new YarnConfiguration())
{
client = YarnClient.CreateYarnClient();
client.Init(GetConf());
client.Start();
}
/// <exception cref="System.Exception"/>
public virtual void TestHelpMessage()
{
Configuration conf = new YarnConfiguration();
YarnClient mockYarnClient = CreateMockYarnClient(YarnApplicationState.Finished);
LogsCLI dumper = new TestLogsCLI.LogsCLIForTest(mockYarnClient);
dumper.SetConf(conf);
int exitCode = dumper.Run(new string[] { });
NUnit.Framework.Assert.IsTrue(exitCode == -1);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter pw = new PrintWriter(baos);
pw.WriteLine("Retrieve logs for completed YARN applications.");
pw.WriteLine("usage: yarn logs -applicationId <application ID> [OPTIONS]");
pw.WriteLine();
pw.WriteLine("general options are:");
pw.WriteLine(" -appOwner <Application Owner> AppOwner (assumed to be current user if"
);
pw.WriteLine(" not specified)");
pw.WriteLine(" -containerId <Container ID> ContainerId (must be specified if node"
);
pw.WriteLine(" address is specified)");
pw.WriteLine(" -help Displays help for all commands.");
pw.WriteLine(" -nodeAddress <Node Address> NodeAddress in the format nodename:port"
);
pw.WriteLine(" (must be specified if container id is"
);
pw.WriteLine(" specified)");
pw.Close();
string appReportStr = baos.ToString("UTF-8");
NUnit.Framework.Assert.AreEqual(appReportStr, sysOutStream.ToString());
}
protected internal virtual void VerifyClientConnection()
{
int numRetries = 3;
while (numRetries-- > 0)
{
Configuration conf = new YarnConfiguration(this.conf);
YarnClient client = CreateAndStartYarnClient(conf);
try
{
Sharpen.Thread.Sleep(100);
client.GetApplications();
return;
}
catch (Exception e)
{
Log.Error(e.Message);
}
finally
{
client.Stop();
}
}
NUnit.Framework.Assert.Fail("Client couldn't connect to the Active RM");
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
private int VerifyApplicationState(ApplicationId appId)
{
YarnClient yarnClient = CreateYarnClient();
try
{
ApplicationReport appReport = yarnClient.GetApplicationReport(appId);
switch (appReport.GetYarnApplicationState())
{
case YarnApplicationState.New:
case YarnApplicationState.NewSaving:
case YarnApplicationState.Submitted:
{
return(-1);
}
case YarnApplicationState.Accepted:
case YarnApplicationState.Running:
case YarnApplicationState.Failed:
case YarnApplicationState.Finished:
case YarnApplicationState.Killed:
default:
{
break;
}
}
}
finally
{
yarnClient.Close();
}
return(0);
}
/// <summary>
/// Delegate responsible for communicating with the Resource Manager's
/// <see cref="Org.Apache.Hadoop.Yarn.Api.ApplicationClientProtocol"/>
/// .
/// </summary>
/// <param name="conf">the configuration object.</param>
public ResourceMgrDelegate(YarnConfiguration conf)
: base(typeof(Org.Apache.Hadoop.Mapred.ResourceMgrDelegate).FullName)
{
this.conf = conf;
this.client = YarnClient.CreateYarnClient();
Init(conf);
Start();
}
protected internal virtual YarnClient CreateYarnClient()
{
YarnClient yarnClient = YarnClient.CreateYarnClient();
yarnClient.Init(GetConf());
yarnClient.Start();
return(yarnClient);
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
private YarnClient CreateMockYarnClientUnknownApp()
{
YarnClient mockClient = Org.Mockito.Mockito.Mock <YarnClient>();
Org.Mockito.Mockito.DoThrow(new YarnException("Unknown AppId")).When(mockClient).
GetApplicationReport(Matchers.Any <ApplicationId>());
return(mockClient);
}
internal Client(string appMasterMainClass, Configuration conf)
{
this.conf = conf;
this.appMasterMainClass = appMasterMainClass;
yarnClient = YarnClient.CreateYarnClient();
yarnClient.Init(conf);
opts = new Options();
opts.AddOption("appname", true, "Application Name. Default value - DistributedShell"
);
opts.AddOption("priority", true, "Application Priority. Default 0");
opts.AddOption("queue", true, "RM Queue in which this application is to be submitted"
);
opts.AddOption("timeout", true, "Application timeout in milliseconds");
opts.AddOption("master_memory", true, "Amount of memory in MB to be requested to run the application master"
);
opts.AddOption("master_vcores", true, "Amount of virtual cores to be requested to run the application master"
);
opts.AddOption("jar", true, "Jar file containing the application master");
opts.AddOption("shell_command", true, "Shell command to be executed by " + "the Application Master. Can only specify either --shell_command "
+ "or --shell_script");
opts.AddOption("shell_script", true, "Location of the shell script to be " + "executed. Can only specify either --shell_command or --shell_script"
);
opts.AddOption("shell_args", true, "Command line args for the shell script." + "Multiple args can be separated by empty space."
);
opts.GetOption("shell_args").SetArgs(Option.UnlimitedValues);
opts.AddOption("shell_env", true, "Environment for shell script. Specified as env_key=env_val pairs"
);
opts.AddOption("shell_cmd_priority", true, "Priority for the shell command containers"
);
opts.AddOption("container_memory", true, "Amount of memory in MB to be requested to run the shell command"
);
opts.AddOption("container_vcores", true, "Amount of virtual cores to be requested to run the shell command"
);
opts.AddOption("num_containers", true, "No. of containers on which the shell command needs to be executed"
);
opts.AddOption("log_properties", true, "log4j.properties file");
opts.AddOption("keep_containers_across_application_attempts", false, "Flag to indicate whether to keep containers across application attempts."
+ " If the flag is true, running containers will not be killed when" + " application attempt fails and these containers will be retrieved by"
+ " the new application attempt ");
opts.AddOption("attempt_failures_validity_interval", true, "when attempt_failures_validity_interval in milliseconds is set to > 0,"
+ "the failure number will not take failures which happen out of " + "the validityInterval into failure count. "
+ "If failure count reaches to maxAppAttempts, " + "the application will be failed."
);
opts.AddOption("debug", false, "Dump out debug information");
opts.AddOption("domain", true, "ID of the timeline domain where the " + "timeline entities will be put"
);
opts.AddOption("view_acls", true, "Users and groups that allowed to " + "view the timeline entities in the given domain"
);
opts.AddOption("modify_acls", true, "Users and groups that allowed to " + "modify the timeline entities in the given domain"
);
opts.AddOption("create", false, "Flag to indicate whether to create the " + "domain specified with -domain."
);
opts.AddOption("help", false, "Print usage");
opts.AddOption("node_label_expression", true, "Node label expression to determine the nodes"
+ " where all the containers of this application" + " will be allocated, \"\" means containers"
+ " can be allocated anywhere, if you don't specify the option," + " default node_label_expression of queue will be used."
);
}
protected internal virtual YarnClient CreateAndStartYarnClient(Configuration conf
)
{
Configuration configuration = new YarnConfiguration(conf);
YarnClient client = YarnClient.CreateYarnClient();
client.Init(configuration);
client.Start();
return(client);
}
/// <summary>
/// Creates/initializes/starts a RemoteAppChecker with a spied
/// DummyYarnClientImpl.
/// </summary>
/// <returns>the spied DummyYarnClientImpl in the created AppChecker</returns>
private YarnClient CreateCheckerWithMockedClient()
{
YarnClient client = Org.Mockito.Mockito.Spy(new TestRemoteAppChecker.DummyYarnClientImpl
(this));
checker = new RemoteAppChecker(client);
checker.Init(new Configuration());
checker.Start();
return(client);
}
public virtual void TestRunningApp()
{
YarnClient client = CreateCheckerWithMockedClient();
ApplicationId id = ApplicationId.NewInstance(1, 1);
// create a report and set the state to an active one
ApplicationReport report = new ApplicationReportPBImpl();
report.SetYarnApplicationState(YarnApplicationState.Accepted);
Org.Mockito.Mockito.DoReturn(report).When(client).GetApplicationReport(id);
NUnit.Framework.Assert.IsTrue(checker.IsApplicationActive(id));
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
private YarnClient CreateMockYarnClient(YarnApplicationState appState)
{
YarnClient mockClient = Org.Mockito.Mockito.Mock <YarnClient>();
ApplicationReport mockAppReport = Org.Mockito.Mockito.Mock <ApplicationReport>();
Org.Mockito.Mockito.DoReturn(appState).When(mockAppReport).GetYarnApplicationState
();
Org.Mockito.Mockito.DoReturn(mockAppReport).When(mockClient).GetApplicationReport
(Matchers.Any <ApplicationId>());
return(mockClient);
}
/// <exception cref="System.Exception"/>
public virtual void TestInvalidApplicationId()
{
Configuration conf = new YarnConfiguration();
YarnClient mockYarnClient = CreateMockYarnClient(YarnApplicationState.Finished);
LogsCLI cli = new TestLogsCLI.LogsCLIForTest(mockYarnClient);
cli.SetConf(conf);
int exitCode = cli.Run(new string[] { "-applicationId", "not_an_app_id" });
NUnit.Framework.Assert.IsTrue(exitCode == -1);
NUnit.Framework.Assert.IsTrue(sysErrStream.ToString().StartsWith("Invalid ApplicationId specified"
));
}
public virtual void TestNonExistentApp()
{
YarnClient client = CreateCheckerWithMockedClient();
ApplicationId id = ApplicationId.NewInstance(1, 1);
// test for null
Org.Mockito.Mockito.DoReturn(null).When(client).GetApplicationReport(id);
NUnit.Framework.Assert.IsFalse(checker.IsApplicationActive(id));
// test for ApplicationNotFoundException
Org.Mockito.Mockito.DoThrow(new ApplicationNotFoundException("Throw!")).When(client
).GetApplicationReport(id);
NUnit.Framework.Assert.IsFalse(checker.IsApplicationActive(id));
}
/// <exception cref="System.Exception"/>
public virtual void TestUnknownApplicationId()
{
Configuration conf = new YarnConfiguration();
YarnClient mockYarnClient = CreateMockYarnClientUnknownApp();
LogsCLI cli = new TestLogsCLI.LogsCLIForTest(mockYarnClient);
cli.SetConf(conf);
int exitCode = cli.Run(new string[] { "-applicationId", ApplicationId.NewInstance
(1, 1).ToString() });
// Error since no logs present for the app.
NUnit.Framework.Assert.IsTrue(exitCode != 0);
NUnit.Framework.Assert.IsTrue(sysErrStream.ToString().StartsWith("Unable to get ApplicationState"
));
}
/// <exception cref="Org.Apache.Commons.Cli.ParseException"/>
public virtual bool Init(string[] args)
{
Options opts = new Options();
opts.AddOption("appname", true, "Application Name. Default value - UnmanagedAM");
opts.AddOption("priority", true, "Application Priority. Default 0");
opts.AddOption("queue", true, "RM Queue in which this application is to be submitted"
);
opts.AddOption("master_memory", true, "Amount of memory in MB to be requested to run the application master"
);
opts.AddOption("cmd", true, "command to start unmanaged AM (required)");
opts.AddOption("classpath", true, "additional classpath");
opts.AddOption("help", false, "Print usage");
CommandLine cliParser = new GnuParser().Parse(opts, args);
if (args.Length == 0)
{
PrintUsage(opts);
throw new ArgumentException("No args specified for client to initialize");
}
if (cliParser.HasOption("help"))
{
PrintUsage(opts);
return(false);
}
appName = cliParser.GetOptionValue("appname", "UnmanagedAM");
amPriority = System.Convert.ToInt32(cliParser.GetOptionValue("priority", "0"));
amQueue = cliParser.GetOptionValue("queue", "default");
classpath = cliParser.GetOptionValue("classpath", null);
amCmd = cliParser.GetOptionValue("cmd");
if (amCmd == null)
{
PrintUsage(opts);
throw new ArgumentException("No cmd specified for application master");
}
YarnConfiguration yarnConf = new YarnConfiguration(conf);
rmClient = YarnClient.CreateYarnClient();
rmClient.Init(yarnConf);
return(true);
}
/// <exception cref="System.Exception"/>
public virtual void TestFailResultCodes()
{
Configuration conf = new YarnConfiguration();
conf.SetClass("fs.file.impl", typeof(LocalFileSystem), typeof(FileSystem));
LogCLIHelpers cliHelper = new LogCLIHelpers();
cliHelper.SetConf(conf);
YarnClient mockYarnClient = CreateMockYarnClient(YarnApplicationState.Finished);
LogsCLI dumper = new TestLogsCLI.LogsCLIForTest(mockYarnClient);
dumper.SetConf(conf);
// verify dumping a non-existent application's logs returns a failure code
int exitCode = dumper.Run(new string[] { "-applicationId", "application_0_0" });
NUnit.Framework.Assert.IsTrue("Should return an error code", exitCode != 0);
// verify dumping a non-existent container log is a failure code
exitCode = cliHelper.DumpAContainersLogs("application_0_0", "container_0_0", "nonexistentnode:1234"
, "nobody");
NUnit.Framework.Assert.IsTrue("Should return an error code", exitCode != 0);
}
public virtual void TestGetEmptyClusterNodeLabels()
{
YarnClient client = Org.Mockito.Mockito.Mock <YarnClient>();
Org.Mockito.Mockito.When(client.GetClusterNodeLabels()).ThenReturn(new HashSet <string
>());
ClusterCLI cli = new ClusterCLI();
cli.SetClient(client);
cli.SetSysOutPrintStream(sysOut);
cli.SetSysErrPrintStream(sysErr);
int rc = cli.Run(new string[] { ClusterCLI.Cmd, "-" + ClusterCLI.ListLabelsCmd });
NUnit.Framework.Assert.AreEqual(0, rc);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter pw = new PrintWriter(baos);
pw.Write("Node Labels: ");
pw.Close();
Org.Mockito.Mockito.Verify(sysOut).WriteLine(baos.ToString("UTF-8"));
}
public virtual void SetClient(YarnClient client)
{
this.client = client;
}
public virtual void Setup()
{
// start minicluster
conf = new YarnConfiguration();
yarnCluster = new MiniYARNCluster(typeof(TestAMRMClient).FullName, nodeCount, 1,
1);
yarnCluster.Init(conf);
yarnCluster.Start();
NUnit.Framework.Assert.IsNotNull(yarnCluster);
NUnit.Framework.Assert.AreEqual(Service.STATE.Started, yarnCluster.GetServiceState
());
// start rm client
yarnClient = (YarnClientImpl)YarnClient.CreateYarnClient();
yarnClient.Init(conf);
yarnClient.Start();
NUnit.Framework.Assert.IsNotNull(yarnClient);
NUnit.Framework.Assert.AreEqual(Service.STATE.Started, yarnClient.GetServiceState
());
// get node info
nodeReports = yarnClient.GetNodeReports(NodeState.Running);
// submit new app
ApplicationSubmissionContext appContext = yarnClient.CreateApplication().GetApplicationSubmissionContext
();
ApplicationId appId = appContext.GetApplicationId();
// set the application name
appContext.SetApplicationName("Test");
// Set the priority for the application master
Priority pri = Priority.NewInstance(0);
appContext.SetPriority(pri);
// Set the queue to which this application is to be submitted in the RM
appContext.SetQueue("default");
// Set up the container launch context for the application master
ContainerLaunchContext amContainer = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<ContainerLaunchContext>();
appContext.SetAMContainerSpec(amContainer);
// unmanaged AM
appContext.SetUnmanagedAM(true);
// Create the request to send to the applications manager
SubmitApplicationRequest appRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<SubmitApplicationRequest>();
appRequest.SetApplicationSubmissionContext(appContext);
// Submit the application to the applications manager
yarnClient.SubmitApplication(appContext);
// wait for app to start
int iterationsLeft = 30;
RMAppAttempt appAttempt = null;
while (iterationsLeft > 0)
{
ApplicationReport appReport = yarnClient.GetApplicationReport(appId);
if (appReport.GetYarnApplicationState() == YarnApplicationState.Accepted)
{
attemptId = appReport.GetCurrentApplicationAttemptId();
appAttempt = yarnCluster.GetResourceManager().GetRMContext().GetRMApps()[attemptId
.GetApplicationId()].GetCurrentAppAttempt();
while (true)
{
if (appAttempt.GetAppAttemptState() == RMAppAttemptState.Launched)
{
break;
}
}
break;
}
Sleep(1000);
--iterationsLeft;
}
if (iterationsLeft == 0)
{
NUnit.Framework.Assert.Fail("Application hasn't bee started");
}
// Just dig into the ResourceManager and get the AMRMToken just for the sake
// of testing.
UserGroupInformation.SetLoginUser(UserGroupInformation.CreateRemoteUser(UserGroupInformation
.GetCurrentUser().GetUserName()));
UserGroupInformation.GetCurrentUser().AddToken(appAttempt.GetAMRMToken());
//creating an instance NMTokenCase
nmTokenCache = new NMTokenCache();
// start am rm client
rmClient = (AMRMClientImpl <AMRMClient.ContainerRequest>)AMRMClient.CreateAMRMClient
<AMRMClient.ContainerRequest>();
//setting an instance NMTokenCase
rmClient.SetNMTokenCache(nmTokenCache);
rmClient.Init(conf);
rmClient.Start();
NUnit.Framework.Assert.IsNotNull(rmClient);
NUnit.Framework.Assert.AreEqual(Service.STATE.Started, rmClient.GetServiceState()
);
// start am nm client
nmClient = (NMClientImpl)NMClient.CreateNMClient();
//propagating the AMRMClient NMTokenCache instance
nmClient.SetNMTokenCache(rmClient.GetNMTokenCache());
nmClient.Init(conf);
nmClient.Start();
NUnit.Framework.Assert.IsNotNull(nmClient);
NUnit.Framework.Assert.AreEqual(Service.STATE.Started, nmClient.GetServiceState()
);
}
public LogsCLIForTest(YarnClient yarnClient)
: base()
{
this.yarnClient = yarnClient;
}
public RemoteAppChecker()
: this(YarnClient.CreateYarnClient())
{
}
/// <exception cref="System.Exception"/>
public virtual void TestFetchApplictionLogs()
{
string remoteLogRootDir = "target/logs/";
Configuration configuration = new Configuration();
configuration.SetBoolean(YarnConfiguration.LogAggregationEnabled, true);
configuration.Set(YarnConfiguration.NmRemoteAppLogDir, remoteLogRootDir);
configuration.SetBoolean(YarnConfiguration.YarnAclEnable, true);
configuration.Set(YarnConfiguration.YarnAdminAcl, "admin");
FileSystem fs = FileSystem.Get(configuration);
UserGroupInformation ugi = UserGroupInformation.GetCurrentUser();
ApplicationId appId = ApplicationIdPBImpl.NewInstance(0, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptIdPBImpl.NewInstance(appId,
1);
ContainerId containerId0 = ContainerIdPBImpl.NewContainerId(appAttemptId, 0);
ContainerId containerId1 = ContainerIdPBImpl.NewContainerId(appAttemptId, 1);
ContainerId containerId2 = ContainerIdPBImpl.NewContainerId(appAttemptId, 2);
NodeId nodeId = NodeId.NewInstance("localhost", 1234);
// create local logs
string rootLogDir = "target/LocalLogs";
Path rootLogDirPath = new Path(rootLogDir);
if (fs.Exists(rootLogDirPath))
{
fs.Delete(rootLogDirPath, true);
}
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(rootLogDirPath));
Path appLogsDir = new Path(rootLogDirPath, appId.ToString());
if (fs.Exists(appLogsDir))
{
fs.Delete(appLogsDir, true);
}
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(appLogsDir));
IList <string> rootLogDirs = Arrays.AsList(rootLogDir);
// create container logs in localLogDir
CreateContainerLogInLocalDir(appLogsDir, containerId1, fs);
CreateContainerLogInLocalDir(appLogsDir, containerId2, fs);
Path path = new Path(remoteLogRootDir + ugi.GetShortUserName() + "/logs/application_0_0001"
);
if (fs.Exists(path))
{
fs.Delete(path, true);
}
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(path));
// upload container logs into remote directory
// the first two logs is empty. When we try to read first two logs,
// we will meet EOF exception, but it will not impact other logs.
// Other logs should be read successfully.
UploadEmptyContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId0
, path, fs);
UploadEmptyContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId1
, path, fs);
UploadContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId1
, path, fs);
UploadContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId2
, path, fs);
YarnClient mockYarnClient = CreateMockYarnClient(YarnApplicationState.Finished);
LogsCLI cli = new TestLogsCLI.LogsCLIForTest(mockYarnClient);
cli.SetConf(configuration);
int exitCode = cli.Run(new string[] { "-applicationId", appId.ToString() });
NUnit.Framework.Assert.IsTrue(exitCode == 0);
NUnit.Framework.Assert.IsTrue(sysOutStream.ToString().Contains("Hello container_0_0001_01_000001!"
));
NUnit.Framework.Assert.IsTrue(sysOutStream.ToString().Contains("Hello container_0_0001_01_000002!"
));
sysOutStream.Reset();
// uploaded two logs for container1. The first log is empty.
// The second one is not empty.
// We can still successfully read logs for container1.
exitCode = cli.Run(new string[] { "-applicationId", appId.ToString(), "-nodeAddress"
, nodeId.ToString(), "-containerId", containerId1.ToString() });
NUnit.Framework.Assert.IsTrue(exitCode == 0);
NUnit.Framework.Assert.IsTrue(sysOutStream.ToString().Contains("Hello container_0_0001_01_000001!"
));
NUnit.Framework.Assert.IsTrue(sysOutStream.ToString().Contains("Log Upload Time")
);
NUnit.Framework.Assert.IsTrue(!sysOutStream.ToString().Contains("Logs for container "
+ containerId1.ToString() + " are not present in this log-file."));
sysOutStream.Reset();
// Uploaded the empty log for container0.
// We should see the message showing the log for container0
// are not present.
exitCode = cli.Run(new string[] { "-applicationId", appId.ToString(), "-nodeAddress"
, nodeId.ToString(), "-containerId", containerId0.ToString() });
NUnit.Framework.Assert.IsTrue(exitCode == -1);
NUnit.Framework.Assert.IsTrue(sysOutStream.ToString().Contains("Logs for container "
+ containerId0.ToString() + " are not present in this log-file."));
fs.Delete(new Path(remoteLogRootDir), true);
fs.Delete(new Path(rootLogDir), true);
}
internal RemoteAppChecker(YarnClient client)
: base("RemoteAppChecker")
{
this.client = client;
}
/// <exception cref="System.Exception"/>
public virtual void TestDSShell(bool haveDomain)
{
string[] args = new string[] { "--jar", AppmasterJar, "--num_containers", "2", "--shell_command"
, Shell.Windows ? "dir" : "ls", "--master_memory", "512", "--master_vcores", "2"
, "--container_memory", "128", "--container_vcores", "1" };
if (haveDomain)
{
string[] domainArgs = new string[] { "--domain", "TEST_DOMAIN", "--view_acls", "reader_user reader_group"
, "--modify_acls", "writer_user writer_group", "--create" };
IList <string> argsList = new AList <string>(Arrays.AsList(args));
Sharpen.Collections.AddAll(argsList, Arrays.AsList(domainArgs));
args = Sharpen.Collections.ToArray(argsList, new string[argsList.Count]);
}
Log.Info("Initializing DS Client");
Client client = new Client(new Configuration(yarnCluster.GetConfig()));
bool initSuccess = client.Init(args);
NUnit.Framework.Assert.IsTrue(initSuccess);
Log.Info("Running DS Client");
AtomicBoolean result = new AtomicBoolean(false);
Sharpen.Thread t = new _Thread_194(result, client);
t.Start();
YarnClient yarnClient = YarnClient.CreateYarnClient();
yarnClient.Init(new Configuration(yarnCluster.GetConfig()));
yarnClient.Start();
string hostName = NetUtils.GetHostname();
bool verified = false;
string errorMessage = string.Empty;
while (!verified)
{
IList <ApplicationReport> apps = yarnClient.GetApplications();
if (apps.Count == 0)
{
Sharpen.Thread.Sleep(10);
continue;
}
ApplicationReport appReport = apps[0];
if (appReport.GetHost().Equals("N/A"))
{
Sharpen.Thread.Sleep(10);
continue;
}
errorMessage = "Expected host name to start with '" + hostName + "', was '" + appReport
.GetHost() + "'. Expected rpc port to be '-1', was '" + appReport.GetRpcPort() +
"'.";
if (CheckHostname(appReport.GetHost()) && appReport.GetRpcPort() == -1)
{
verified = true;
}
if (appReport.GetYarnApplicationState() == YarnApplicationState.Finished)
{
break;
}
}
NUnit.Framework.Assert.IsTrue(errorMessage, verified);
t.Join();
Log.Info("Client run completed. Result=" + result);
NUnit.Framework.Assert.IsTrue(result.Get());
TimelineDomain domain = null;
if (haveDomain)
{
domain = yarnCluster.GetApplicationHistoryServer().GetTimelineStore().GetDomain("TEST_DOMAIN"
);
NUnit.Framework.Assert.IsNotNull(domain);
NUnit.Framework.Assert.AreEqual("reader_user reader_group", domain.GetReaders());
NUnit.Framework.Assert.AreEqual("writer_user writer_group", domain.GetWriters());
}
TimelineEntities entitiesAttempts = yarnCluster.GetApplicationHistoryServer().GetTimelineStore
().GetEntities(ApplicationMaster.DSEntity.DsAppAttempt.ToString(), null, null, null
, null, null, null, null, null, null);
NUnit.Framework.Assert.IsNotNull(entitiesAttempts);
NUnit.Framework.Assert.AreEqual(1, entitiesAttempts.GetEntities().Count);
NUnit.Framework.Assert.AreEqual(2, entitiesAttempts.GetEntities()[0].GetEvents().
Count);
NUnit.Framework.Assert.AreEqual(entitiesAttempts.GetEntities()[0].GetEntityType()
.ToString(), ApplicationMaster.DSEntity.DsAppAttempt.ToString());
if (haveDomain)
{
NUnit.Framework.Assert.AreEqual(domain.GetId(), entitiesAttempts.GetEntities()[0]
.GetDomainId());
}
else
{
NUnit.Framework.Assert.AreEqual("DEFAULT", entitiesAttempts.GetEntities()[0].GetDomainId
());
}
TimelineEntities entities = yarnCluster.GetApplicationHistoryServer().GetTimelineStore
().GetEntities(ApplicationMaster.DSEntity.DsContainer.ToString(), null, null, null
, null, null, null, null, null, null);
NUnit.Framework.Assert.IsNotNull(entities);
NUnit.Framework.Assert.AreEqual(2, entities.GetEntities().Count);
NUnit.Framework.Assert.AreEqual(entities.GetEntities()[0].GetEntityType().ToString
(), ApplicationMaster.DSEntity.DsContainer.ToString());
if (haveDomain)
{
NUnit.Framework.Assert.AreEqual(domain.GetId(), entities.GetEntities()[0].GetDomainId
());
}
else
{
NUnit.Framework.Assert.AreEqual("DEFAULT", entities.GetEntities()[0].GetDomainId(
));
}
}