/// <exception cref="System.Exception"/>
protected internal virtual void SetupInternal(int numNodeManager)
{
Log.Info("Starting up YARN cluster");
conf = new YarnConfiguration();
conf.SetInt(YarnConfiguration.RmSchedulerMinimumAllocationMb, 128);
conf.Set("yarn.log.dir", "target");
conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, true);
conf.Set(YarnConfiguration.RmScheduler, typeof(CapacityScheduler).FullName);
conf.SetBoolean(YarnConfiguration.NodeLabelsEnabled, true);
if (yarnCluster == null)
{
yarnCluster = new MiniYARNCluster(typeof(TestDistributedShell).Name, 1, numNodeManager
, 1, 1);
yarnCluster.Init(conf);
yarnCluster.Start();
conf.Set(YarnConfiguration.TimelineServiceWebappAddress, MiniYARNCluster.GetHostname
() + ":" + yarnCluster.GetApplicationHistoryServer().GetPort());
WaitForNMsToRegister();
Uri url = Sharpen.Thread.CurrentThread().GetContextClassLoader().GetResource("yarn-site.xml"
);
if (url == null)
{
throw new RuntimeException("Could not find 'yarn-site.xml' dummy file in classpath"
);
}
Configuration yarnClusterConfig = yarnCluster.GetConfig();
yarnClusterConfig.Set("yarn.application.classpath", new FilePath(url.AbsolutePath
).GetParent());
//write the document to a buffer (not directly to the file, as that
//can cause the file being written to get read -which will then fail.
ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
yarnClusterConfig.WriteXml(bytesOut);
bytesOut.Close();
//write the bytes to the file in the classpath
OutputStream os = new FileOutputStream(new FilePath(url.AbsolutePath));
os.Write(bytesOut.ToByteArray());
os.Close();
}
FileContext fsContext = FileContext.GetLocalFSFileContext();
fsContext.Delete(new Path(conf.Get("yarn.timeline-service.leveldb-timeline-store.path"
)), true);
try
{
Sharpen.Thread.Sleep(2000);
}
catch (Exception e)
{
Log.Info("setup thread sleep interrupted. message=" + e.Message);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestDSShell(bool haveDomain)
{
string[] args = new string[] { "--jar", AppmasterJar, "--num_containers", "2", "--shell_command"
, Shell.Windows ? "dir" : "ls", "--master_memory", "512", "--master_vcores", "2"
, "--container_memory", "128", "--container_vcores", "1" };
if (haveDomain)
{
string[] domainArgs = new string[] { "--domain", "TEST_DOMAIN", "--view_acls", "reader_user reader_group"
, "--modify_acls", "writer_user writer_group", "--create" };
IList <string> argsList = new AList <string>(Arrays.AsList(args));
Sharpen.Collections.AddAll(argsList, Arrays.AsList(domainArgs));
args = Sharpen.Collections.ToArray(argsList, new string[argsList.Count]);
}
Log.Info("Initializing DS Client");
Client client = new Client(new Configuration(yarnCluster.GetConfig()));
bool initSuccess = client.Init(args);
NUnit.Framework.Assert.IsTrue(initSuccess);
Log.Info("Running DS Client");
AtomicBoolean result = new AtomicBoolean(false);
Sharpen.Thread t = new _Thread_194(result, client);
t.Start();
YarnClient yarnClient = YarnClient.CreateYarnClient();
yarnClient.Init(new Configuration(yarnCluster.GetConfig()));
yarnClient.Start();
string hostName = NetUtils.GetHostname();
bool verified = false;
string errorMessage = string.Empty;
while (!verified)
{
IList <ApplicationReport> apps = yarnClient.GetApplications();
if (apps.Count == 0)
{
Sharpen.Thread.Sleep(10);
continue;
}
ApplicationReport appReport = apps[0];
if (appReport.GetHost().Equals("N/A"))
{
Sharpen.Thread.Sleep(10);
continue;
}
errorMessage = "Expected host name to start with '" + hostName + "', was '" + appReport
.GetHost() + "'. Expected rpc port to be '-1', was '" + appReport.GetRpcPort() +
"'.";
if (CheckHostname(appReport.GetHost()) && appReport.GetRpcPort() == -1)
{
verified = true;
}
if (appReport.GetYarnApplicationState() == YarnApplicationState.Finished)
{
break;
}
}
NUnit.Framework.Assert.IsTrue(errorMessage, verified);
t.Join();
Log.Info("Client run completed. Result=" + result);
NUnit.Framework.Assert.IsTrue(result.Get());
TimelineDomain domain = null;
if (haveDomain)
{
domain = yarnCluster.GetApplicationHistoryServer().GetTimelineStore().GetDomain("TEST_DOMAIN"
);
NUnit.Framework.Assert.IsNotNull(domain);
NUnit.Framework.Assert.AreEqual("reader_user reader_group", domain.GetReaders());
NUnit.Framework.Assert.AreEqual("writer_user writer_group", domain.GetWriters());
}
TimelineEntities entitiesAttempts = yarnCluster.GetApplicationHistoryServer().GetTimelineStore
().GetEntities(ApplicationMaster.DSEntity.DsAppAttempt.ToString(), null, null, null
, null, null, null, null, null, null);
NUnit.Framework.Assert.IsNotNull(entitiesAttempts);
NUnit.Framework.Assert.AreEqual(1, entitiesAttempts.GetEntities().Count);
NUnit.Framework.Assert.AreEqual(2, entitiesAttempts.GetEntities()[0].GetEvents().
Count);
NUnit.Framework.Assert.AreEqual(entitiesAttempts.GetEntities()[0].GetEntityType()
.ToString(), ApplicationMaster.DSEntity.DsAppAttempt.ToString());
if (haveDomain)
{
NUnit.Framework.Assert.AreEqual(domain.GetId(), entitiesAttempts.GetEntities()[0]
.GetDomainId());
}
else
{
NUnit.Framework.Assert.AreEqual("DEFAULT", entitiesAttempts.GetEntities()[0].GetDomainId
());
}
TimelineEntities entities = yarnCluster.GetApplicationHistoryServer().GetTimelineStore
().GetEntities(ApplicationMaster.DSEntity.DsContainer.ToString(), null, null, null
, null, null, null, null, null, null);
NUnit.Framework.Assert.IsNotNull(entities);
NUnit.Framework.Assert.AreEqual(2, entities.GetEntities().Count);
NUnit.Framework.Assert.AreEqual(entities.GetEntities()[0].GetEntityType().ToString
(), ApplicationMaster.DSEntity.DsContainer.ToString());
if (haveDomain)
{
NUnit.Framework.Assert.AreEqual(domain.GetId(), entities.GetEntities()[0].GetDomainId
());
}
else
{
NUnit.Framework.Assert.AreEqual("DEFAULT", entities.GetEntities()[0].GetDomainId(
));
}
}
