public virtual void TestContainerLogs()
{
WebResource r = Resource();
ContainerId containerId = BuilderUtils.NewContainerId(0, 0, 0, 0);
string containerIdStr = BuilderUtils.NewContainerId(0, 0, 0, 0).ToString();
ApplicationAttemptId appAttemptId = containerId.GetApplicationAttemptId();
ApplicationId appId = appAttemptId.GetApplicationId();
string appIdStr = appId.ToString();
string filename = "logfile1";
string logMessage = "log message\n";
nmContext.GetApplications()[appId] = new ApplicationImpl(null, "user", appId, null
, nmContext);
MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(),
new Configuration(), "user", appId, 1);
container.SetState(ContainerState.Running);
nmContext.GetContainers()[containerId] = container;
// write out log file
Path path = dirsHandler.GetLogPathForWrite(ContainerLaunch.GetRelativeContainerLogDir
(appIdStr, containerIdStr) + "/" + filename, false);
FilePath logFile = new FilePath(path.ToUri().GetPath());
logFile.DeleteOnExit();
NUnit.Framework.Assert.IsTrue("Failed to create log dir", logFile.GetParentFile()
.Mkdirs());
PrintWriter pw = new PrintWriter(logFile);
pw.Write(logMessage);
pw.Close();
// ask for it
ClientResponse response = r.Path("ws").Path("v1").Path("node").Path("containerlogs"
).Path(containerIdStr).Path(filename).Accept(MediaType.TextPlain).Get <ClientResponse
>();
string responseText = response.GetEntity <string>();
NUnit.Framework.Assert.AreEqual(logMessage, responseText);
// ask for file that doesn't exist
response = r.Path("ws").Path("v1").Path("node").Path("containerlogs").Path(containerIdStr
).Path("uhhh").Accept(MediaType.TextPlain).Get <ClientResponse>();
NUnit.Framework.Assert.AreEqual(ClientResponse.Status.NotFound.GetStatusCode(), response
.GetStatus());
responseText = response.GetEntity <string>();
NUnit.Framework.Assert.IsTrue(responseText.Contains("Cannot find this log on the local disk."
));
// After container is completed, it is removed from nmContext
Sharpen.Collections.Remove(nmContext.GetContainers(), containerId);
NUnit.Framework.Assert.IsNull(nmContext.GetContainers()[containerId]);
response = r.Path("ws").Path("v1").Path("node").Path("containerlogs").Path(containerIdStr
).Path(filename).Accept(MediaType.TextPlain).Get <ClientResponse>();
responseText = response.GetEntity <string>();
NUnit.Framework.Assert.AreEqual(logMessage, responseText);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public virtual void TestContainerLogFile()
{
FilePath absLogDir = new FilePath("target", typeof(TestNMWebServer).Name + "LogDir"
).GetAbsoluteFile();
string logdirwithFile = absLogDir.ToURI().ToString();
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.NmLogDirs, logdirwithFile);
conf.SetFloat(YarnConfiguration.NmMaxPerDiskUtilizationPercentage, 0.0f);
LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
dirsHandler.Init(conf);
NodeManager.NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler
, new ApplicationACLsManager(conf), new NMNullStateStoreService());
// Add an application and the corresponding containers
string user = "******";
long clusterTimeStamp = 1234;
ApplicationId appId = BuilderUtils.NewApplicationId(clusterTimeStamp, 1);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
>();
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user);
Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
);
ContainerId containerId = BuilderUtils.NewContainerId(appAttemptId, 1);
nmContext.GetApplications()[appId] = app;
MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(),
conf, user, appId, 1);
container.SetState(ContainerState.Running);
nmContext.GetContainers()[containerId] = container;
FilePath containerLogDir = new FilePath(absLogDir, ContainerLaunch.GetRelativeContainerLogDir
(appId.ToString(), containerId.ToString()));
containerLogDir.Mkdirs();
string fileName = "fileName";
FilePath containerLogFile = new FilePath(containerLogDir, fileName);
containerLogFile.CreateNewFile();
FilePath file = ContainerLogsUtils.GetContainerLogFile(containerId, fileName, user
, nmContext);
NUnit.Framework.Assert.AreEqual(containerLogFile.ToURI().ToString(), file.ToURI()
.ToString());
FileUtil.FullyDelete(absLogDir);
}
/// <exception cref="System.IO.IOException"/>
private Dictionary <string, string> AddAppContainers(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app)
{
Dispatcher dispatcher = new AsyncDispatcher();
ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(app.GetAppId
(), 1);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container1
= new MockContainer(appAttemptId, dispatcher, conf, app.GetUser(), app.GetAppId
(), 1);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container2
= new MockContainer(appAttemptId, dispatcher, conf, app.GetUser(), app.GetAppId
(), 2);
nmContext.GetContainers()[container1.GetContainerId()] = container1;
nmContext.GetContainers()[container2.GetContainerId()] = container2;
app.GetContainers()[container1.GetContainerId()] = container1;
app.GetContainers()[container2.GetContainerId()] = container2;
Dictionary <string, string> hash = new Dictionary <string, string>();
hash[container1.GetContainerId().ToString()] = container1.GetContainerId().ToString
();
hash[container2.GetContainerId().ToString()] = container2.GetContainerId().ToString
();
return(hash);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public virtual void TestContainerLogDirs()
{
FilePath absLogDir = new FilePath("target", typeof(TestNMWebServer).Name + "LogDir"
).GetAbsoluteFile();
string logdirwithFile = absLogDir.ToURI().ToString();
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.NmLogDirs, logdirwithFile);
NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
healthChecker.Init(conf);
LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler();
NodeManager.NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler
, new ApplicationACLsManager(conf), new NMNullStateStoreService());
// Add an application and the corresponding containers
RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(conf);
string user = "******";
long clusterTimeStamp = 1234;
ApplicationId appId = BuilderUtils.NewApplicationId(recordFactory, clusterTimeStamp
, 1);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
>();
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user);
Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
);
ContainerId container1 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId
, 0);
nmContext.GetApplications()[appId] = app;
MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(),
conf, user, appId, 1);
container.SetState(ContainerState.Running);
nmContext.GetContainers()[container1] = container;
IList <FilePath> files = null;
files = ContainerLogsUtils.GetContainerLogDirs(container1, user, nmContext);
NUnit.Framework.Assert.IsTrue(!(files[0].ToString().Contains("file:")));
// After container is completed, it is removed from nmContext
Sharpen.Collections.Remove(nmContext.GetContainers(), container1);
NUnit.Framework.Assert.IsNull(nmContext.GetContainers()[container1]);
files = ContainerLogsUtils.GetContainerLogDirs(container1, user, nmContext);
NUnit.Framework.Assert.IsTrue(!(files[0].ToString().Contains("file:")));
// Create a new context to check if correct container log dirs are fetched
// on full disk.
LocalDirsHandlerService dirsHandlerForFullDisk = Org.Mockito.Mockito.Spy(dirsHandler
);
// good log dirs are empty and nm log dir is in the full log dir list.
Org.Mockito.Mockito.When(dirsHandlerForFullDisk.GetLogDirs()).ThenReturn(new AList
<string>());
Org.Mockito.Mockito.When(dirsHandlerForFullDisk.GetLogDirsForRead()).ThenReturn(Arrays
.AsList(new string[] { absLogDir.GetAbsolutePath() }));
nmContext = new NodeManager.NMContext(null, null, dirsHandlerForFullDisk, new ApplicationACLsManager
(conf), new NMNullStateStoreService());
nmContext.GetApplications()[appId] = app;
container.SetState(ContainerState.Running);
nmContext.GetContainers()[container1] = container;
IList <FilePath> dirs = ContainerLogsUtils.GetContainerLogDirs(container1, user, nmContext
);
FilePath containerLogDir = new FilePath(absLogDir, appId + "/" + container1);
NUnit.Framework.Assert.IsTrue(dirs.Contains(containerLogDir));
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestContainerLogPageAccess()
{
// SecureIOUtils require Native IO to be enabled. This test will run
// only if it is enabled.
Assume.AssumeTrue(NativeIO.IsAvailable());
string user = "******" + Runtime.CurrentTimeMillis();
FilePath absLogDir = null;
FilePath appDir = null;
FilePath containerDir = null;
FilePath syslog = null;
try
{
// target log directory
absLogDir = new FilePath("target", typeof(TestContainerLogsPage).Name + "LogDir")
.GetAbsoluteFile();
absLogDir.Mkdir();
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.NmLogDirs, absLogDir.ToURI().ToString());
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
healthChecker.Init(conf);
LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler();
// Add an application and the corresponding containers
RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(conf);
long clusterTimeStamp = 1234;
ApplicationId appId = BuilderUtils.NewApplicationId(recordFactory, clusterTimeStamp
, 1);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
>();
Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
// Making sure that application returns a random user. This is required
// for SecureIOUtils' file owner check.
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user);
ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
);
ContainerId container1 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId
, 0);
// Testing secure read access for log files
// Creating application and container directory and syslog file.
appDir = new FilePath(absLogDir, appId.ToString());
appDir.Mkdir();
containerDir = new FilePath(appDir, container1.ToString());
containerDir.Mkdir();
syslog = new FilePath(containerDir, "syslog");
syslog.CreateNewFile();
BufferedOutputStream @out = new BufferedOutputStream(new FileOutputStream(syslog)
);
@out.Write(Sharpen.Runtime.GetBytesForString("Log file Content"));
@out.Close();
Context context = Org.Mockito.Mockito.Mock <Context>();
ConcurrentMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
> appMap = new ConcurrentHashMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
>();
appMap[appId] = app;
Org.Mockito.Mockito.When(context.GetApplications()).ThenReturn(appMap);
ConcurrentHashMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
> containers = new ConcurrentHashMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
>();
Org.Mockito.Mockito.When(context.GetContainers()).ThenReturn(containers);
Org.Mockito.Mockito.When(context.GetLocalDirsHandler()).ThenReturn(dirsHandler);
MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(),
conf, user, appId, 1);
container.SetState(ContainerState.Running);
context.GetContainers()[container1] = container;
ContainerLogsPage.ContainersLogsBlock cLogsBlock = new ContainerLogsPage.ContainersLogsBlock
(context);
IDictionary <string, string> @params = new Dictionary <string, string>();
@params[YarnWebParams.ContainerId] = container1.ToString();
@params[YarnWebParams.ContainerLogType] = "syslog";
Injector injector = WebAppTests.TestPage <ContainerLogsPage.ContainersLogsBlock>(typeof(
ContainerLogsPage), cLogsBlock, @params, (Module[])null);
PrintWriter spyPw = WebAppTests.GetPrintWriter(injector);
Org.Mockito.Mockito.Verify(spyPw).Write("Exception reading log file. Application submitted by '"
+ user + "' doesn't own requested log file : syslog");
}
finally
{
if (syslog != null)
{
syslog.Delete();
}
if (containerDir != null)
{
containerDir.Delete();
}
if (appDir != null)
{
appDir.Delete();
}
if (absLogDir != null)
{
absLogDir.Delete();
}
}
}
