public virtual void TestNMWebApp()
{
Context nmContext = new NodeManager.NMContext(null, null, null, null, null);
ResourceView resourceView = new _ResourceView_147();
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.NmLocalDirs, testRootDir.GetAbsolutePath());
conf.Set(YarnConfiguration.NmLogDirs, testLogDir.GetAbsolutePath());
NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
healthChecker.Init(conf);
LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler();
WebServer server = new WebServer(nmContext, resourceView, new ApplicationACLsManager
(conf), dirsHandler);
server.Init(conf);
server.Start();
// Add an application and the corresponding containers
RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(conf);
Dispatcher dispatcher = new AsyncDispatcher();
string user = "******";
long clusterTimeStamp = 1234;
ApplicationId appId = BuilderUtils.NewApplicationId(recordFactory, clusterTimeStamp
, 1);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
>();
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user);
Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
nmContext.GetApplications()[appId] = app;
ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
);
ContainerId container1 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId
, 0);
ContainerId container2 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId
, 1);
NodeManagerMetrics metrics = Org.Mockito.Mockito.Mock <NodeManagerMetrics>();
NMStateStoreService stateStore = new NMNullStateStoreService();
foreach (ContainerId containerId in new ContainerId[] { container1, container2 })
{
// TODO: Use builder utils
ContainerLaunchContext launchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
long currentTime = Runtime.CurrentTimeMillis();
Token containerToken = BuilderUtils.NewContainerToken(containerId, "127.0.0.1", 1234
, user, BuilderUtils.NewResource(1024, 1), currentTime + 10000L, 123, Sharpen.Runtime.GetBytesForString
("password"), currentTime);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= new _ContainerImpl_214(conf, dispatcher, stateStore, launchContext, null, metrics
, BuilderUtils.NewContainerTokenIdentifier(containerToken));
nmContext.GetContainers()[containerId] = container;
//TODO: Gross hack. Fix in code.
ApplicationId applicationId = containerId.GetApplicationAttemptId().GetApplicationId
();
nmContext.GetApplications()[applicationId].GetContainers()[containerId] = container;
WriteContainerLogs(nmContext, containerId, dirsHandler);
}
}
/// <summary>Finds the log file with the given filename for the given container.</summary>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public static FilePath GetContainerLogFile(ContainerId containerId, string fileName
, string remoteUser, Context context)
{
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= context.GetContainers()[containerId];
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
application = GetApplicationForContainer(containerId, context);
CheckAccess(remoteUser, application, context);
if (container != null)
{
CheckState(container.GetContainerState());
}
try
{
LocalDirsHandlerService dirsHandler = context.GetLocalDirsHandler();
string relativeContainerLogDir = ContainerLaunch.GetRelativeContainerLogDir(application
.GetAppId().ToString(), containerId.ToString());
Path logPath = dirsHandler.GetLogPathToRead(relativeContainerLogDir + Path.Separator
+ fileName);
URI logPathURI = new FilePath(logPath.ToString()).ToURI();
FilePath logFile = new FilePath(logPathURI.GetPath());
return(logFile);
}
catch (IOException e)
{
Log.Warn("Failed to find log file", e);
throw new NotFoundException("Cannot find this log on the local disk.");
}
}
/// <exception cref="System.IO.IOException"/>
private Dictionary <string, string> AddAppContainers(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app)
{
Dispatcher dispatcher = new AsyncDispatcher();
ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(app.GetAppId
(), 1);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container1
= new MockContainer(appAttemptId, dispatcher, conf, app.GetUser(), app.GetAppId
(), 1);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container2
= new MockContainer(appAttemptId, dispatcher, conf, app.GetUser(), app.GetAppId
(), 2);
nmContext.GetContainers()[container1.GetContainerId()] = container1;
nmContext.GetContainers()[container2.GetContainerId()] = container2;
app.GetContainers()[container1.GetContainerId()] = container1;
app.GetContainers()[container2.GetContainerId()] = container2;
Dictionary <string, string> hash = new Dictionary <string, string>();
hash[container1.GetContainerId().ToString()] = container1.GetContainerId().ToString
();
hash[container2.GetContainerId().ToString()] = container2.GetContainerId().ToString
();
return(hash);
}
internal WrappedApplication(TestApplication _enclosing, int id, long timestamp, string
user, int numContainers)
{
this._enclosing = _enclosing;
Configuration conf = new Configuration();
this.dispatcher = new DrainDispatcher();
this.containerTokenIdentifierMap = new Dictionary <ContainerId, ContainerTokenIdentifier
>();
this.dispatcher.Init(conf);
this.localizerBus = Org.Mockito.Mockito.Mock <EventHandler>();
this.launcherBus = Org.Mockito.Mockito.Mock <EventHandler>();
this.monitorBus = Org.Mockito.Mockito.Mock <EventHandler>();
this.auxBus = Org.Mockito.Mockito.Mock <EventHandler>();
this.containerBus = Org.Mockito.Mockito.Mock <EventHandler>();
this.logAggregationBus = Org.Mockito.Mockito.Mock <EventHandler>();
this.dispatcher.Register(typeof(LocalizationEventType), this.localizerBus);
this.dispatcher.Register(typeof(ContainersLauncherEventType), this.launcherBus);
this.dispatcher.Register(typeof(ContainersMonitorEventType), this.monitorBus);
this.dispatcher.Register(typeof(AuxServicesEventType), this.auxBus);
this.dispatcher.Register(typeof(ContainerEventType), this.containerBus);
this.dispatcher.Register(typeof(LogHandlerEventType), this.logAggregationBus);
this.nmTokenSecretMgr = Org.Mockito.Mockito.Mock <NMTokenSecretManagerInNM>();
this.context = Org.Mockito.Mockito.Mock <Context>();
Org.Mockito.Mockito.When(this.context.GetContainerTokenSecretManager()).ThenReturn
(new NMContainerTokenSecretManager(conf));
Org.Mockito.Mockito.When(this.context.GetApplicationACLsManager()).ThenReturn(new
ApplicationACLsManager(conf));
Org.Mockito.Mockito.When(this.context.GetNMTokenSecretManager()).ThenReturn(this.
nmTokenSecretMgr);
// Setting master key
MasterKey masterKey = new MasterKeyPBImpl();
masterKey.SetKeyId(123);
masterKey.SetBytes(ByteBuffer.Wrap(new byte[] { (123) }));
this.context.GetContainerTokenSecretManager().SetMasterKey(masterKey);
this.user = user;
this.appId = BuilderUtils.NewApplicationId(timestamp, id);
this.app = new ApplicationImpl(this.dispatcher, this.user, this.appId, null, this
.context);
this.containers = new AList <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
>();
for (int i = 0; i < numContainers; i++)
{
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= this._enclosing.CreateMockedContainer(this.appId, i);
this.containers.AddItem(container);
long currentTime = Runtime.CurrentTimeMillis();
ContainerTokenIdentifier identifier = new ContainerTokenIdentifier(container.GetContainerId
(), string.Empty, string.Empty, null, currentTime + 2000, masterKey.GetKeyId(),
currentTime, Priority.NewInstance(0), 0);
this.containerTokenIdentifierMap[identifier.GetContainerID()] = identifier;
this.context.GetContainerTokenSecretManager().StartContainerSuccessful(identifier
);
NUnit.Framework.Assert.IsFalse(this.context.GetContainerTokenSecretManager().IsValidStartContainerRequest
(identifier));
}
this.dispatcher.Start();
}
public RecoveredContainerLaunch(Context context, Configuration configuration, Dispatcher
dispatcher, ContainerExecutor exec, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
container, LocalDirsHandlerService dirsHandler, ContainerManagerImpl containerManager
)
: base(context, configuration, dispatcher, exec, app, container, dirsHandler, containerManager
)
{
this.shouldLaunchContainer.Set(true);
}
public virtual void Handle(ContainersLauncherEvent @event)
{
// TODO: ContainersLauncher launches containers one by one!!
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= @event.GetContainer();
ContainerId containerId = container.GetContainerId();
switch (@event.GetType())
{
case ContainersLauncherEventType.LaunchContainer:
{
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = context.GetApplications()[containerId.GetApplicationAttemptId().GetApplicationId
()];
ContainerLaunch launch = new ContainerLaunch(context, GetConfig(), dispatcher, exec
, app, @event.GetContainer(), dirsHandler, containerManager);
containerLauncher.Submit(launch);
running[containerId] = launch;
break;
}
case ContainersLauncherEventType.RecoverContainer:
{
app = context.GetApplications()[containerId.GetApplicationAttemptId().GetApplicationId
()];
launch = new RecoveredContainerLaunch(context, GetConfig(), dispatcher, exec, app
, @event.GetContainer(), dirsHandler, containerManager);
containerLauncher.Submit(launch);
running[containerId] = launch;
break;
}
case ContainersLauncherEventType.CleanupContainer:
{
ContainerLaunch launcher = Sharpen.Collections.Remove(running, containerId);
if (launcher == null)
{
// Container not launched. So nothing needs to be done.
return;
}
// Cleanup a container whether it is running/killed/completed, so that
// no sub-processes are alive.
try
{
launcher.CleanupContainer();
}
catch (IOException)
{
Log.Warn("Got exception while cleaning container " + containerId + ". Ignoring.");
}
break;
}
}
}
GetApplicationForContainer(ContainerId containerId, Context context)
{
ApplicationId applicationId = containerId.GetApplicationAttemptId().GetApplicationId
();
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
application = context.GetApplications()[applicationId];
if (application == null)
{
throw new NotFoundException("Unknown container. Container either has not started or "
+ "has already completed or " + "doesn't belong to this node at all.");
}
return(application);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public virtual void TestContainerLogFile()
{
FilePath absLogDir = new FilePath("target", typeof(TestNMWebServer).Name + "LogDir"
).GetAbsoluteFile();
string logdirwithFile = absLogDir.ToURI().ToString();
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.NmLogDirs, logdirwithFile);
conf.SetFloat(YarnConfiguration.NmMaxPerDiskUtilizationPercentage, 0.0f);
LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
dirsHandler.Init(conf);
NodeManager.NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler
, new ApplicationACLsManager(conf), new NMNullStateStoreService());
// Add an application and the corresponding containers
string user = "******";
long clusterTimeStamp = 1234;
ApplicationId appId = BuilderUtils.NewApplicationId(clusterTimeStamp, 1);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
>();
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user);
Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
);
ContainerId containerId = BuilderUtils.NewContainerId(appAttemptId, 1);
nmContext.GetApplications()[appId] = app;
MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(),
conf, user, appId, 1);
container.SetState(ContainerState.Running);
nmContext.GetContainers()[containerId] = container;
FilePath containerLogDir = new FilePath(absLogDir, ContainerLaunch.GetRelativeContainerLogDir
(appId.ToString(), containerId.ToString()));
containerLogDir.Mkdirs();
string fileName = "fileName";
FilePath containerLogFile = new FilePath(containerLogDir, fileName);
containerLogFile.CreateNewFile();
FilePath file = ContainerLogsUtils.GetContainerLogFile(containerId, fileName, user
, nmContext);
NUnit.Framework.Assert.AreEqual(containerLogFile.ToURI().ToString(), file.ToURI()
.ToString());
FileUtil.FullyDelete(absLogDir);
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
private static void CheckAccess(string remoteUser, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
application, Context context)
{
UserGroupInformation callerUGI = null;
if (remoteUser != null)
{
callerUGI = UserGroupInformation.CreateRemoteUser(remoteUser);
}
if (callerUGI != null && !context.GetApplicationACLsManager().CheckAccess(callerUGI
, ApplicationAccessType.ViewApp, application.GetUser(), application.GetAppId()))
{
throw new YarnException("User [" + remoteUser + "] is not authorized to view the logs for application "
+ application.GetAppId());
}
}
public AppInfo(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app)
{
// JAXB needs this
this.id = ConverterUtils.ToString(app.GetAppId());
this.state = app.GetApplicationState().ToString();
this.user = app.GetUser();
this.containerids = new AList <string>();
IDictionary <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
> appContainers = app.GetContainers();
foreach (ContainerId containerId in appContainers.Keys)
{
string containerIdStr = ConverterUtils.ToString(containerId);
containerids.AddItem(containerIdStr);
}
}
public virtual void TestLogFileWithDriveLetter()
{
ContainerImpl container = Org.Mockito.Mockito.Mock <ContainerImpl>();
ApplicationIdPBImpl appId = Org.Mockito.Mockito.Mock <ApplicationIdPBImpl>();
Org.Mockito.Mockito.When(appId.ToString()).ThenReturn("appId");
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
>();
Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
ApplicationAttemptIdPBImpl appAttemptId = Org.Mockito.Mockito.Mock <ApplicationAttemptIdPBImpl
>();
Org.Mockito.Mockito.When(appAttemptId.GetApplicationId()).ThenReturn(appId);
ConcurrentMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
> applications = new ConcurrentHashMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
>();
applications[appId] = app;
ContainerId containerId = Org.Mockito.Mockito.Mock <ContainerIdPBImpl>();
Org.Mockito.Mockito.When(containerId.ToString()).ThenReturn("containerId");
Org.Mockito.Mockito.When(containerId.GetApplicationAttemptId()).ThenReturn(appAttemptId
);
ConcurrentMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
> containers = new ConcurrentHashMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
>();
containers[containerId] = container;
LocalDirsHandlerService localDirs = Org.Mockito.Mockito.Mock <LocalDirsHandlerService
>();
Org.Mockito.Mockito.When(localDirs.GetLogPathToRead("appId" + Path.Separator + "containerId"
+ Path.Separator + "fileName")).ThenReturn(new Path("F:/nmlogs/appId/containerId/fileName"
));
NodeManager.NMContext context = Org.Mockito.Mockito.Mock <NodeManager.NMContext>();
Org.Mockito.Mockito.When(context.GetLocalDirsHandler()).ThenReturn(localDirs);
Org.Mockito.Mockito.When(context.GetApplications()).ThenReturn(applications);
Org.Mockito.Mockito.When(context.GetContainers()).ThenReturn(containers);
FilePath logFile = ContainerLogsUtils.GetContainerLogFile(containerId, "fileName"
, null, context);
NUnit.Framework.Assert.IsTrue("logFile lost drive letter " + logFile, logFile.ToString
().IndexOf("F:" + FilePath.separator + "nmlogs") > -1);
}
public ContainerLaunch(Context context, Configuration configuration, Dispatcher dispatcher
, ContainerExecutor exec, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
container, LocalDirsHandlerService dirsHandler, ContainerManagerImpl containerManager
)
{
this.context = context;
this.conf = configuration;
this.app = app;
this.exec = exec;
this.container = container;
this.dispatcher = dispatcher;
this.dirsHandler = dirsHandler;
this.containerManager = containerManager;
this.sleepDelayBeforeSigKill = conf.GetLong(YarnConfiguration.NmSleepDelayBeforeSigkillMs
, YarnConfiguration.DefaultNmSleepDelayBeforeSigkillMs);
this.maxKillWaitTime = conf.GetLong(YarnConfiguration.NmProcessKillWaitMs, YarnConfiguration
.DefaultNmProcessKillWaitMs);
}
protected override void Render(HtmlBlock.Block html)
{
ApplicationId applicationID = ConverterUtils.ToApplicationId(this.recordFactory,
$(ApplicationId));
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = this.nmContext.GetApplications()[applicationID];
AppInfo info = new AppInfo(app);
Info("Application's information").("ApplicationId", info.GetId()).("ApplicationState"
, info.GetState()).("User", info.GetUser());
Hamlet.TABLE <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> containersListBody = html
.(typeof(InfoBlock)).Table("#containers");
foreach (string containerIdStr in info.GetContainers())
{
containersListBody.Tr().Td().A(Url("container", containerIdStr), containerIdStr).
().();
}
containersListBody.();
}
/// <summary>
/// Finds the local directories that logs for the given container are stored
/// on.
/// </summary>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public static IList <FilePath> GetContainerLogDirs(ContainerId containerId, string
remoteUser, Context context)
{
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= context.GetContainers()[containerId];
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
application = GetApplicationForContainer(containerId, context);
CheckAccess(remoteUser, application, context);
// It is not required to have null check for container ( container == null )
// and throw back exception.Because when container is completed, NodeManager
// remove container information from its NMContext.Configuring log
// aggregation to false, container log view request is forwarded to NM. NM
// does not have completed container information,but still NM serve request for
// reading container logs.
if (container != null)
{
CheckState(container.GetContainerState());
}
return(GetContainerLogDirs(containerId, context.GetLocalDirsHandler()));
}
public virtual void Logs()
{
string containerIdStr = $(ContainerId);
ContainerId containerId = null;
try
{
containerId = ConverterUtils.ToContainerId(containerIdStr);
}
catch (ArgumentException)
{
Render(typeof(ContainerLogsPage));
return;
}
ApplicationId appId = containerId.GetApplicationAttemptId().GetApplicationId();
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = nmContext.GetApplications()[appId];
if (app == null && nmConf.GetBoolean(YarnConfiguration.LogAggregationEnabled, YarnConfiguration
.DefaultLogAggregationEnabled))
{
string logServerUrl = nmConf.Get(YarnConfiguration.YarnLogServerUrl);
string redirectUrl = null;
if (logServerUrl == null || logServerUrl.IsEmpty())
{
redirectUrl = "false";
}
else
{
redirectUrl = Url(logServerUrl, nmContext.GetNodeId().ToString(), containerIdStr,
containerIdStr, $(AppOwner));
}
Set(ContainerLogsPage.RedirectUrl, redirectUrl);
}
Render(typeof(ContainerLogsPage));
}
public ApplicationLocalizationEvent(LocalizationEventType type, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app)
: base(type)
{
this.app = app;
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestContainerLogPageAccess()
{
// SecureIOUtils require Native IO to be enabled. This test will run
// only if it is enabled.
Assume.AssumeTrue(NativeIO.IsAvailable());
string user = "******" + Runtime.CurrentTimeMillis();
FilePath absLogDir = null;
FilePath appDir = null;
FilePath containerDir = null;
FilePath syslog = null;
try
{
// target log directory
absLogDir = new FilePath("target", typeof(TestContainerLogsPage).Name + "LogDir")
.GetAbsoluteFile();
absLogDir.Mkdir();
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.NmLogDirs, absLogDir.ToURI().ToString());
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
healthChecker.Init(conf);
LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler();
// Add an application and the corresponding containers
RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(conf);
long clusterTimeStamp = 1234;
ApplicationId appId = BuilderUtils.NewApplicationId(recordFactory, clusterTimeStamp
, 1);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
>();
Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
// Making sure that application returns a random user. This is required
// for SecureIOUtils' file owner check.
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user);
ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
);
ContainerId container1 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId
, 0);
// Testing secure read access for log files
// Creating application and container directory and syslog file.
appDir = new FilePath(absLogDir, appId.ToString());
appDir.Mkdir();
containerDir = new FilePath(appDir, container1.ToString());
containerDir.Mkdir();
syslog = new FilePath(containerDir, "syslog");
syslog.CreateNewFile();
BufferedOutputStream @out = new BufferedOutputStream(new FileOutputStream(syslog)
);
@out.Write(Sharpen.Runtime.GetBytesForString("Log file Content"));
@out.Close();
Context context = Org.Mockito.Mockito.Mock <Context>();
ConcurrentMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
> appMap = new ConcurrentHashMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
>();
appMap[appId] = app;
Org.Mockito.Mockito.When(context.GetApplications()).ThenReturn(appMap);
ConcurrentHashMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
> containers = new ConcurrentHashMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
>();
Org.Mockito.Mockito.When(context.GetContainers()).ThenReturn(containers);
Org.Mockito.Mockito.When(context.GetLocalDirsHandler()).ThenReturn(dirsHandler);
MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(),
conf, user, appId, 1);
container.SetState(ContainerState.Running);
context.GetContainers()[container1] = container;
ContainerLogsPage.ContainersLogsBlock cLogsBlock = new ContainerLogsPage.ContainersLogsBlock
(context);
IDictionary <string, string> @params = new Dictionary <string, string>();
@params[YarnWebParams.ContainerId] = container1.ToString();
@params[YarnWebParams.ContainerLogType] = "syslog";
Injector injector = WebAppTests.TestPage <ContainerLogsPage.ContainersLogsBlock>(typeof(
ContainerLogsPage), cLogsBlock, @params, (Module[])null);
PrintWriter spyPw = WebAppTests.GetPrintWriter(injector);
Org.Mockito.Mockito.Verify(spyPw).Write("Exception reading log file. Application submitted by '"
+ user + "' doesn't own requested log file : syslog");
}
finally
{
if (syslog != null)
{
syslog.Delete();
}
if (containerDir != null)
{
containerDir.Delete();
}
if (appDir != null)
{
appDir.Delete();
}
if (absLogDir != null)
{
absLogDir.Delete();
}
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public virtual void TestContainerLogDirs()
{
FilePath absLogDir = new FilePath("target", typeof(TestNMWebServer).Name + "LogDir"
).GetAbsoluteFile();
string logdirwithFile = absLogDir.ToURI().ToString();
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.NmLogDirs, logdirwithFile);
NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
healthChecker.Init(conf);
LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler();
NodeManager.NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler
, new ApplicationACLsManager(conf), new NMNullStateStoreService());
// Add an application and the corresponding containers
RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(conf);
string user = "******";
long clusterTimeStamp = 1234;
ApplicationId appId = BuilderUtils.NewApplicationId(recordFactory, clusterTimeStamp
, 1);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
>();
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user);
Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
);
ContainerId container1 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId
, 0);
nmContext.GetApplications()[appId] = app;
MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(),
conf, user, appId, 1);
container.SetState(ContainerState.Running);
nmContext.GetContainers()[container1] = container;
IList <FilePath> files = null;
files = ContainerLogsUtils.GetContainerLogDirs(container1, user, nmContext);
NUnit.Framework.Assert.IsTrue(!(files[0].ToString().Contains("file:")));
// After container is completed, it is removed from nmContext
Sharpen.Collections.Remove(nmContext.GetContainers(), container1);
NUnit.Framework.Assert.IsNull(nmContext.GetContainers()[container1]);
files = ContainerLogsUtils.GetContainerLogDirs(container1, user, nmContext);
NUnit.Framework.Assert.IsTrue(!(files[0].ToString().Contains("file:")));
// Create a new context to check if correct container log dirs are fetched
// on full disk.
LocalDirsHandlerService dirsHandlerForFullDisk = Org.Mockito.Mockito.Spy(dirsHandler
);
// good log dirs are empty and nm log dir is in the full log dir list.
Org.Mockito.Mockito.When(dirsHandlerForFullDisk.GetLogDirs()).ThenReturn(new AList
<string>());
Org.Mockito.Mockito.When(dirsHandlerForFullDisk.GetLogDirsForRead()).ThenReturn(Arrays
.AsList(new string[] { absLogDir.GetAbsolutePath() }));
nmContext = new NodeManager.NMContext(null, null, dirsHandlerForFullDisk, new ApplicationACLsManager
(conf), new NMNullStateStoreService());
nmContext.GetApplications()[appId] = app;
container.SetState(ContainerState.Running);
nmContext.GetContainers()[container1] = container;
IList <FilePath> dirs = ContainerLogsUtils.GetContainerLogDirs(container1, user, nmContext
);
FilePath containerLogDir = new FilePath(absLogDir, appId + "/" + container1);
NUnit.Framework.Assert.IsTrue(dirs.Contains(containerLogDir));
}
public ContainerLocalizationImpl(Dispatcher dispatcher, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app, LocalizationProtocol localization)
{
}
