public virtual void Setup()
{
localFS.Delete(new Path(localDir.GetAbsolutePath()), true);
localFS.Delete(new Path(tmpDir.GetAbsolutePath()), true);
localFS.Delete(new Path(localLogDir.GetAbsolutePath()), true);
localFS.Delete(new Path(remoteLogDir.GetAbsolutePath()), true);
localDir.Mkdir();
tmpDir.Mkdir();
localLogDir.Mkdir();
remoteLogDir.Mkdir();
Log.Info("Created localDir in " + localDir.GetAbsolutePath());
Log.Info("Created tmpDir in " + tmpDir.GetAbsolutePath());
string bindAddress = "0.0.0.0:12345";
conf.Set(YarnConfiguration.NmAddress, bindAddress);
conf.Set(YarnConfiguration.NmLocalDirs, localDir.GetAbsolutePath());
conf.Set(YarnConfiguration.NmLogDirs, localLogDir.GetAbsolutePath());
conf.Set(YarnConfiguration.NmRemoteAppLogDir, remoteLogDir.GetAbsolutePath());
conf.SetLong(YarnConfiguration.NmLogRetainSeconds, 1);
// Default delSrvc
delSrvc = CreateDeletionService();
delSrvc.Init(conf);
exec = CreateContainerExecutor();
nodeHealthChecker = new NodeHealthCheckerService();
nodeHealthChecker.Init(conf);
dirsHandler = nodeHealthChecker.GetDiskHandler();
containerManager = CreateContainerManager(delSrvc);
((NodeManager.NMContext)context).SetContainerManager(containerManager);
nodeStatusUpdater.Init(conf);
containerManager.Init(conf);
nodeStatusUpdater.Start();
}
public _NodeStatusUpdaterImpl_121(BaseContainerManagerTest _enclosing, Context baseArg1
, Dispatcher baseArg2, NodeHealthCheckerService baseArg3, NodeManagerMetrics baseArg4
)
: base(baseArg1, baseArg2, baseArg3, baseArg4)
{
this._enclosing = _enclosing;
}
public _NodeStatusUpdaterImpl_601(ShortCircuitedNodeManager _enclosing, Context baseArg1
, Dispatcher baseArg2, NodeHealthCheckerService baseArg3, NodeManagerMetrics baseArg4
)
: base(baseArg1, baseArg2, baseArg3, baseArg4)
{
this._enclosing = _enclosing;
}
private int StartNMWebAppServer(string webAddr)
{
Context nmContext = new NodeManager.NMContext(null, null, null, null, null);
ResourceView resourceView = new _ResourceView_84();
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.NmLocalDirs, testRootDir.GetAbsolutePath());
conf.Set(YarnConfiguration.NmLogDirs, testLogDir.GetAbsolutePath());
NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
healthChecker.Init(conf);
LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler();
conf.Set(YarnConfiguration.NmWebappAddress, webAddr);
WebServer server = new WebServer(nmContext, resourceView, new ApplicationACLsManager
(conf), dirsHandler);
try
{
server.Init(conf);
server.Start();
return(server.GetPort());
}
finally
{
server.Stop();
healthChecker.Stop();
}
}
public virtual void TestNMWebApp()
{
Context nmContext = new NodeManager.NMContext(null, null, null, null, null);
ResourceView resourceView = new _ResourceView_147();
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.NmLocalDirs, testRootDir.GetAbsolutePath());
conf.Set(YarnConfiguration.NmLogDirs, testLogDir.GetAbsolutePath());
NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
healthChecker.Init(conf);
LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler();
WebServer server = new WebServer(nmContext, resourceView, new ApplicationACLsManager
(conf), dirsHandler);
server.Init(conf);
server.Start();
// Add an application and the corresponding containers
RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(conf);
Dispatcher dispatcher = new AsyncDispatcher();
string user = "******";
long clusterTimeStamp = 1234;
ApplicationId appId = BuilderUtils.NewApplicationId(recordFactory, clusterTimeStamp
, 1);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
>();
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user);
Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
nmContext.GetApplications()[appId] = app;
ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
);
ContainerId container1 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId
, 0);
ContainerId container2 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId
, 1);
NodeManagerMetrics metrics = Org.Mockito.Mockito.Mock <NodeManagerMetrics>();
NMStateStoreService stateStore = new NMNullStateStoreService();
foreach (ContainerId containerId in new ContainerId[] { container1, container2 })
{
// TODO: Use builder utils
ContainerLaunchContext launchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
long currentTime = Runtime.CurrentTimeMillis();
Token containerToken = BuilderUtils.NewContainerToken(containerId, "127.0.0.1", 1234
, user, BuilderUtils.NewResource(1024, 1), currentTime + 10000L, 123, Sharpen.Runtime.GetBytesForString
("password"), currentTime);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= new _ContainerImpl_214(conf, dispatcher, stateStore, launchContext, null, metrics
, BuilderUtils.NewContainerTokenIdentifier(containerToken));
nmContext.GetContainers()[containerId] = container;
//TODO: Gross hack. Fix in code.
ApplicationId applicationId = containerId.GetApplicationAttemptId().GetApplicationId
();
nmContext.GetApplications()[applicationId].GetContainers()[containerId] = container;
WriteContainerLogs(nmContext, containerId, dirsHandler);
}
}
protected override void ConfigureServlets()
{
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps.conf.Set(YarnConfiguration
.NmLocalDirs, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
.testRootDir.GetAbsolutePath());
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps.conf.Set(YarnConfiguration
.NmLogDirs, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
.testLogDir.GetAbsolutePath());
NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
healthChecker.Init(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
.conf);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps.dirsHandler
= healthChecker.GetDiskHandler();
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps.aclsManager
= new ApplicationACLsManager(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
.conf);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps.nmContext
= new NodeManager.NMContext(null, null, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
.dirsHandler, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
.aclsManager, null);
NodeId nodeId = NodeId.NewInstance("testhost.foo.com", 9999);
((NodeManager.NMContext)Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
.nmContext).SetNodeId(nodeId);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps.resourceView
= new _ResourceView_106();
// 15.5G in bytes
// 16G in bytes
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps.nmWebApp =
new WebServer.NMWebApp(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
.resourceView, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
.aclsManager, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
.dirsHandler);
this.Bind <JAXBContextResolver>();
this.Bind <NMWebServices>();
this.Bind <GenericExceptionHandler>();
this.Bind <Context>().ToInstance(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
.nmContext);
this.Bind <WebApp>().ToInstance(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
.nmWebApp);
this.Bind <ResourceView>().ToInstance(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
.resourceView);
this.Bind <ApplicationACLsManager>().ToInstance(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
.aclsManager);
this.Bind <LocalDirsHandlerService>().ToInstance(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
.dirsHandler);
this.Serve("/*").With(typeof(GuiceContainer));
}
protected override NodeStatusUpdater CreateNodeStatusUpdater(Context context, Dispatcher
dispatcher, NodeHealthCheckerService healthChecker)
{
return(new _NodeStatusUpdaterImpl_601(this, context, dispatcher, healthChecker, this
.metrics));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public virtual void TestContainerLogDirs()
{
FilePath absLogDir = new FilePath("target", typeof(TestNMWebServer).Name + "LogDir"
).GetAbsoluteFile();
string logdirwithFile = absLogDir.ToURI().ToString();
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.NmLogDirs, logdirwithFile);
NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
healthChecker.Init(conf);
LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler();
NodeManager.NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler
, new ApplicationACLsManager(conf), new NMNullStateStoreService());
// Add an application and the corresponding containers
RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(conf);
string user = "******";
long clusterTimeStamp = 1234;
ApplicationId appId = BuilderUtils.NewApplicationId(recordFactory, clusterTimeStamp
, 1);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
>();
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user);
Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
);
ContainerId container1 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId
, 0);
nmContext.GetApplications()[appId] = app;
MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(),
conf, user, appId, 1);
container.SetState(ContainerState.Running);
nmContext.GetContainers()[container1] = container;
IList <FilePath> files = null;
files = ContainerLogsUtils.GetContainerLogDirs(container1, user, nmContext);
NUnit.Framework.Assert.IsTrue(!(files[0].ToString().Contains("file:")));
// After container is completed, it is removed from nmContext
Sharpen.Collections.Remove(nmContext.GetContainers(), container1);
NUnit.Framework.Assert.IsNull(nmContext.GetContainers()[container1]);
files = ContainerLogsUtils.GetContainerLogDirs(container1, user, nmContext);
NUnit.Framework.Assert.IsTrue(!(files[0].ToString().Contains("file:")));
// Create a new context to check if correct container log dirs are fetched
// on full disk.
LocalDirsHandlerService dirsHandlerForFullDisk = Org.Mockito.Mockito.Spy(dirsHandler
);
// good log dirs are empty and nm log dir is in the full log dir list.
Org.Mockito.Mockito.When(dirsHandlerForFullDisk.GetLogDirs()).ThenReturn(new AList
<string>());
Org.Mockito.Mockito.When(dirsHandlerForFullDisk.GetLogDirsForRead()).ThenReturn(Arrays
.AsList(new string[] { absLogDir.GetAbsolutePath() }));
nmContext = new NodeManager.NMContext(null, null, dirsHandlerForFullDisk, new ApplicationACLsManager
(conf), new NMNullStateStoreService());
nmContext.GetApplications()[appId] = app;
container.SetState(ContainerState.Running);
nmContext.GetContainers()[container1] = container;
IList <FilePath> dirs = ContainerLogsUtils.GetContainerLogDirs(container1, user, nmContext
);
FilePath containerLogDir = new FilePath(absLogDir, appId + "/" + container1);
NUnit.Framework.Assert.IsTrue(dirs.Contains(containerLogDir));
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestContainerLogPageAccess()
{
// SecureIOUtils require Native IO to be enabled. This test will run
// only if it is enabled.
Assume.AssumeTrue(NativeIO.IsAvailable());
string user = "******" + Runtime.CurrentTimeMillis();
FilePath absLogDir = null;
FilePath appDir = null;
FilePath containerDir = null;
FilePath syslog = null;
try
{
// target log directory
absLogDir = new FilePath("target", typeof(TestContainerLogsPage).Name + "LogDir")
.GetAbsoluteFile();
absLogDir.Mkdir();
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.NmLogDirs, absLogDir.ToURI().ToString());
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
healthChecker.Init(conf);
LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler();
// Add an application and the corresponding containers
RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(conf);
long clusterTimeStamp = 1234;
ApplicationId appId = BuilderUtils.NewApplicationId(recordFactory, clusterTimeStamp
, 1);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
>();
Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
// Making sure that application returns a random user. This is required
// for SecureIOUtils' file owner check.
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user);
ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
);
ContainerId container1 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId
, 0);
// Testing secure read access for log files
// Creating application and container directory and syslog file.
appDir = new FilePath(absLogDir, appId.ToString());
appDir.Mkdir();
containerDir = new FilePath(appDir, container1.ToString());
containerDir.Mkdir();
syslog = new FilePath(containerDir, "syslog");
syslog.CreateNewFile();
BufferedOutputStream @out = new BufferedOutputStream(new FileOutputStream(syslog)
);
@out.Write(Sharpen.Runtime.GetBytesForString("Log file Content"));
@out.Close();
Context context = Org.Mockito.Mockito.Mock <Context>();
ConcurrentMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
> appMap = new ConcurrentHashMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
>();
appMap[appId] = app;
Org.Mockito.Mockito.When(context.GetApplications()).ThenReturn(appMap);
ConcurrentHashMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
> containers = new ConcurrentHashMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
>();
Org.Mockito.Mockito.When(context.GetContainers()).ThenReturn(containers);
Org.Mockito.Mockito.When(context.GetLocalDirsHandler()).ThenReturn(dirsHandler);
MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(),
conf, user, appId, 1);
container.SetState(ContainerState.Running);
context.GetContainers()[container1] = container;
ContainerLogsPage.ContainersLogsBlock cLogsBlock = new ContainerLogsPage.ContainersLogsBlock
(context);
IDictionary <string, string> @params = new Dictionary <string, string>();
@params[YarnWebParams.ContainerId] = container1.ToString();
@params[YarnWebParams.ContainerLogType] = "syslog";
Injector injector = WebAppTests.TestPage <ContainerLogsPage.ContainersLogsBlock>(typeof(
ContainerLogsPage), cLogsBlock, @params, (Module[])null);
PrintWriter spyPw = WebAppTests.GetPrintWriter(injector);
Org.Mockito.Mockito.Verify(spyPw).Write("Exception reading log file. Application submitted by '"
+ user + "' doesn't own requested log file : syslog");
}
finally
{
if (syslog != null)
{
syslog.Delete();
}
if (containerDir != null)
{
containerDir.Delete();
}
if (appDir != null)
{
appDir.Delete();
}
if (absLogDir != null)
{
absLogDir.Delete();
}
}
}
