private int StartNMWebAppServer(string webAddr)
{
Context nmContext = new NodeManager.NMContext(null, null, null, null, null);
ResourceView resourceView = new _ResourceView_84();
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.NmLocalDirs, testRootDir.GetAbsolutePath());
conf.Set(YarnConfiguration.NmLogDirs, testLogDir.GetAbsolutePath());
NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
healthChecker.Init(conf);
LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler();
conf.Set(YarnConfiguration.NmWebappAddress, webAddr);
WebServer server = new WebServer(nmContext, resourceView, new ApplicationACLsManager
(conf), dirsHandler);
try
{
server.Init(conf);
server.Start();
return(server.GetPort());
}
finally
{
server.Stop();
healthChecker.Stop();
}
}
public virtual void TestLogDirWithDriveLetter()
{
//To verify that logs paths which include drive letters (Windows)
//do not lose their drive letter specification
LocalDirsHandlerService localDirs = Org.Mockito.Mockito.Mock <LocalDirsHandlerService
>();
IList <string> logDirs = new AList <string>();
logDirs.AddItem("F:/nmlogs");
Org.Mockito.Mockito.When(localDirs.GetLogDirsForRead()).ThenReturn(logDirs);
ApplicationIdPBImpl appId = Org.Mockito.Mockito.Mock <ApplicationIdPBImpl>();
Org.Mockito.Mockito.When(appId.ToString()).ThenReturn("app_id_1");
ApplicationAttemptIdPBImpl appAttemptId = Org.Mockito.Mockito.Mock <ApplicationAttemptIdPBImpl
>();
Org.Mockito.Mockito.When(appAttemptId.GetApplicationId()).ThenReturn(appId);
ContainerId containerId = Org.Mockito.Mockito.Mock <ContainerIdPBImpl>();
Org.Mockito.Mockito.When(containerId.GetApplicationAttemptId()).ThenReturn(appAttemptId
);
IList <FilePath> logDirFiles = ContainerLogsUtils.GetContainerLogDirs(containerId,
localDirs);
NUnit.Framework.Assert.IsTrue("logDir lost drive letter " + logDirFiles[0], logDirFiles
[0].ToString().IndexOf("F:" + FilePath.separator + "nmlogs") > -1);
}
public virtual void TestNMWebApp()
{
Context nmContext = new NodeManager.NMContext(null, null, null, null, null);
ResourceView resourceView = new _ResourceView_147();
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.NmLocalDirs, testRootDir.GetAbsolutePath());
conf.Set(YarnConfiguration.NmLogDirs, testLogDir.GetAbsolutePath());
NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
healthChecker.Init(conf);
LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler();
WebServer server = new WebServer(nmContext, resourceView, new ApplicationACLsManager
(conf), dirsHandler);
server.Init(conf);
server.Start();
// Add an application and the corresponding containers
RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(conf);
Dispatcher dispatcher = new AsyncDispatcher();
string user = "******";
long clusterTimeStamp = 1234;
ApplicationId appId = BuilderUtils.NewApplicationId(recordFactory, clusterTimeStamp
, 1);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
>();
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user);
Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
nmContext.GetApplications()[appId] = app;
ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
);
ContainerId container1 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId
, 0);
ContainerId container2 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId
, 1);
NodeManagerMetrics metrics = Org.Mockito.Mockito.Mock <NodeManagerMetrics>();
NMStateStoreService stateStore = new NMNullStateStoreService();
foreach (ContainerId containerId in new ContainerId[] { container1, container2 })
{
// TODO: Use builder utils
ContainerLaunchContext launchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
long currentTime = Runtime.CurrentTimeMillis();
Token containerToken = BuilderUtils.NewContainerToken(containerId, "127.0.0.1", 1234
, user, BuilderUtils.NewResource(1024, 1), currentTime + 10000L, 123, Sharpen.Runtime.GetBytesForString
("password"), currentTime);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= new _ContainerImpl_214(conf, dispatcher, stateStore, launchContext, null, metrics
, BuilderUtils.NewContainerTokenIdentifier(containerToken));
nmContext.GetContainers()[containerId] = container;
//TODO: Gross hack. Fix in code.
ApplicationId applicationId = containerId.GetApplicationAttemptId().GetApplicationId
();
nmContext.GetApplications()[applicationId].GetContainers()[containerId] = container;
WriteContainerLogs(nmContext, containerId, dirsHandler);
}
}
/// <summary>Finds the log file with the given filename for the given container.</summary>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public static FilePath GetContainerLogFile(ContainerId containerId, string fileName
, string remoteUser, Context context)
{
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= context.GetContainers()[containerId];
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
application = GetApplicationForContainer(containerId, context);
CheckAccess(remoteUser, application, context);
if (container != null)
{
CheckState(container.GetContainerState());
}
try
{
LocalDirsHandlerService dirsHandler = context.GetLocalDirsHandler();
string relativeContainerLogDir = ContainerLaunch.GetRelativeContainerLogDir(application
.GetAppId().ToString(), containerId.ToString());
Path logPath = dirsHandler.GetLogPathToRead(relativeContainerLogDir + Path.Separator
+ fileName);
URI logPathURI = new FilePath(logPath.ToString()).ToURI();
FilePath logFile = new FilePath(logPathURI.GetPath());
return(logFile);
}
catch (IOException e)
{
Log.Warn("Failed to find log file", e);
throw new NotFoundException("Cannot find this log on the local disk.");
}
}
public virtual void TestFailedDirLogDeletion()
{
FilePath[] localLogDirs = GetLocalLogDirFiles(this.GetType().FullName, 7);
IList <string> localLogDirPaths = new AList <string>(localLogDirs.Length);
for (int i = 0; i < localLogDirs.Length; i++)
{
localLogDirPaths.AddItem(localLogDirs[i].GetAbsolutePath());
}
string localLogDirsString = StringUtils.Join(localLogDirPaths, ",");
conf.Set(YarnConfiguration.NmLogDirs, localLogDirsString);
conf.SetBoolean(YarnConfiguration.LogAggregationEnabled, false);
conf.SetLong(YarnConfiguration.NmLogRetainSeconds, 0l);
LocalDirsHandlerService mockDirsHandler = Org.Mockito.Mockito.Mock <LocalDirsHandlerService
>();
NonAggregatingLogHandler rawLogHandler = new NonAggregatingLogHandler(dispatcher,
mockDelService, mockDirsHandler, new NMNullStateStoreService());
NonAggregatingLogHandler logHandler = Org.Mockito.Mockito.Spy(rawLogHandler);
AbstractFileSystem spylfs = Org.Mockito.Mockito.Spy(FileContext.GetLocalFSFileContext
().GetDefaultFileSystem());
FileContext lfs = FileContext.GetFileContext(spylfs, conf);
Org.Mockito.Mockito.DoReturn(lfs).When(logHandler).GetLocalFileContext(Matchers.IsA
<Configuration>());
logHandler.Init(conf);
logHandler.Start();
RunMockedFailedDirs(logHandler, appId, user, mockDelService, mockDirsHandler, conf
, spylfs, lfs, localLogDirs);
logHandler.Close();
}
public virtual void Setup()
{
localFS.Delete(new Path(localDir.GetAbsolutePath()), true);
localFS.Delete(new Path(tmpDir.GetAbsolutePath()), true);
localFS.Delete(new Path(localLogDir.GetAbsolutePath()), true);
localFS.Delete(new Path(remoteLogDir.GetAbsolutePath()), true);
localDir.Mkdir();
tmpDir.Mkdir();
localLogDir.Mkdir();
remoteLogDir.Mkdir();
Log.Info("Created localDir in " + localDir.GetAbsolutePath());
Log.Info("Created tmpDir in " + tmpDir.GetAbsolutePath());
string bindAddress = "0.0.0.0:12345";
conf.Set(YarnConfiguration.NmAddress, bindAddress);
conf.Set(YarnConfiguration.NmLocalDirs, localDir.GetAbsolutePath());
conf.Set(YarnConfiguration.NmLogDirs, localLogDir.GetAbsolutePath());
conf.Set(YarnConfiguration.NmRemoteAppLogDir, remoteLogDir.GetAbsolutePath());
conf.SetLong(YarnConfiguration.NmLogRetainSeconds, 1);
// Default delSrvc
delSrvc = CreateDeletionService();
delSrvc.Init(conf);
exec = CreateContainerExecutor();
nodeHealthChecker = new NodeHealthCheckerService();
nodeHealthChecker.Init(conf);
dirsHandler = nodeHealthChecker.GetDiskHandler();
containerManager = CreateContainerManager(delSrvc);
((NodeManager.NMContext)context).SetContainerManager(containerManager);
nodeStatusUpdater.Init(conf);
containerManager.Init(conf);
nodeStatusUpdater.Start();
}
public NMWebApp(ResourceView resourceView, ApplicationACLsManager aclsManager, LocalDirsHandlerService
dirsHandler)
{
this.resourceView = resourceView;
this.aclsManager = aclsManager;
this.dirsHandler = dirsHandler;
}
/// <exception cref="System.IO.IOException"/>
private void TestDirsFailures(bool localORLogDirs)
{
string dirType = localORLogDirs ? "local" : "log";
string dirsProperty = localORLogDirs ? YarnConfiguration.NmLocalDirs : YarnConfiguration
.NmLogDirs;
Configuration conf = new Configuration();
// set disk health check interval to a small value (say 1 sec).
conf.SetLong(YarnConfiguration.NmDiskHealthCheckIntervalMs, DiskHealthCheckInterval
);
// If 2 out of the total 4 local-dirs fail OR if 2 Out of the total 4
// log-dirs fail, then the node's health status should become unhealthy.
conf.SetFloat(YarnConfiguration.NmMinHealthyDisksFraction, 0.60F);
if (yarnCluster != null)
{
yarnCluster.Stop();
FileUtil.FullyDelete(localFSDirBase);
localFSDirBase.Mkdirs();
}
Log.Info("Starting up YARN cluster");
yarnCluster = new MiniYARNCluster(typeof(TestDiskFailures).FullName, 1, numLocalDirs
, numLogDirs);
yarnCluster.Init(conf);
yarnCluster.Start();
NodeManager nm = yarnCluster.GetNodeManager(0);
Log.Info("Configured nm-" + dirType + "-dirs=" + nm.GetConfig().Get(dirsProperty)
);
dirsHandler = nm.GetNodeHealthChecker().GetDiskHandler();
IList <string> list = localORLogDirs ? dirsHandler.GetLocalDirs() : dirsHandler.GetLogDirs
();
string[] dirs = Sharpen.Collections.ToArray(list, new string[list.Count]);
NUnit.Framework.Assert.AreEqual("Number of nm-" + dirType + "-dirs is wrong.", numLocalDirs
, dirs.Length);
string expectedDirs = StringUtils.Join(",", list);
// validate the health of disks initially
VerifyDisksHealth(localORLogDirs, expectedDirs, true);
// Make 1 nm-local-dir fail and verify if "the nodemanager can identify
// the disk failure(s) and can update the list of good nm-local-dirs.
PrepareDirToFail(dirs[2]);
expectedDirs = dirs[0] + "," + dirs[1] + "," + dirs[3];
VerifyDisksHealth(localORLogDirs, expectedDirs, true);
// Now, make 1 more nm-local-dir/nm-log-dir fail and verify if "the
// nodemanager can identify the disk failures and can update the list of
// good nm-local-dirs/nm-log-dirs and can update the overall health status
// of the node to unhealthy".
PrepareDirToFail(dirs[0]);
expectedDirs = dirs[1] + "," + dirs[3];
VerifyDisksHealth(localORLogDirs, expectedDirs, false);
// Fail the remaining 2 local-dirs/log-dirs and verify if NM remains with
// empty list of local-dirs/log-dirs and the overall health status is
// unhealthy.
PrepareDirToFail(dirs[1]);
PrepareDirToFail(dirs[3]);
expectedDirs = string.Empty;
VerifyDisksHealth(localORLogDirs, expectedDirs, false);
}
public WebServer(Context nmContext, ResourceView resourceView, ApplicationACLsManager
aclsManager, LocalDirsHandlerService dirsHandler)
: base(typeof(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.WebServer).FullName
)
{
this.nmContext = nmContext;
this.nmWebApp = new WebServer.NMWebApp(resourceView, aclsManager, dirsHandler);
}
public RecoveredContainerLaunch(Context context, Configuration configuration, Dispatcher
dispatcher, ContainerExecutor exec, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
container, LocalDirsHandlerService dirsHandler, ContainerManagerImpl containerManager
)
: base(context, configuration, dispatcher, exec, app, container, dirsHandler, containerManager
)
{
this.shouldLaunchContainer.Set(true);
}
public _ContainerManagerImpl_319(LogHandler logHandler, ResourceLocalizationService
rsrcSrv, ContainersLauncher launcher, Context baseArg1, ContainerExecutor baseArg2
, DeletionService baseArg3, NodeStatusUpdater baseArg4, NodeManagerMetrics baseArg5
, ApplicationACLsManager baseArg6, LocalDirsHandlerService baseArg7)
: base(baseArg1, baseArg2, baseArg3, baseArg4, baseArg5, baseArg6, baseArg7)
{
this.logHandler = logHandler;
this.rsrcSrv = rsrcSrv;
this.launcher = launcher;
}
public NonAggregatingLogHandler(Dispatcher dispatcher, DeletionService delService
, LocalDirsHandlerService dirsHandler, NMStateStoreService stateStore)
: base(typeof(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Loghandler.NonAggregatingLogHandler
).FullName)
{
this.dispatcher = dispatcher;
this.delService = delService;
this.dirsHandler = dirsHandler;
this.stateStore = stateStore;
this.appOwners = new ConcurrentHashMap <ApplicationId, string>();
}
public ContainersLauncher(Context context, Dispatcher dispatcher, ContainerExecutor
exec, LocalDirsHandlerService dirsHandler, ContainerManagerImpl containerManager
)
: base("containers-launcher")
{
this.exec = exec;
this.context = context;
this.dispatcher = dispatcher;
this.dirsHandler = dirsHandler;
this.containerManager = containerManager;
}
public virtual void Setup()
{
mockDelService = Org.Mockito.Mockito.Mock <DeletionService>();
conf = new YarnConfiguration();
dispatcher = CreateDispatcher(conf);
appEventHandler = Org.Mockito.Mockito.Mock <EventHandler>();
dispatcher.Register(typeof(ApplicationEventType), appEventHandler);
appId = BuilderUtils.NewApplicationId(1234, 1);
appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1);
container11 = BuilderUtils.NewContainerId(appAttemptId, 1);
dirsHandler = new LocalDirsHandlerService();
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public virtual void TestContainerLogFile()
{
FilePath absLogDir = new FilePath("target", typeof(TestNMWebServer).Name + "LogDir"
).GetAbsoluteFile();
string logdirwithFile = absLogDir.ToURI().ToString();
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.NmLogDirs, logdirwithFile);
conf.SetFloat(YarnConfiguration.NmMaxPerDiskUtilizationPercentage, 0.0f);
LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
dirsHandler.Init(conf);
NodeManager.NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler
, new ApplicationACLsManager(conf), new NMNullStateStoreService());
// Add an application and the corresponding containers
string user = "******";
long clusterTimeStamp = 1234;
ApplicationId appId = BuilderUtils.NewApplicationId(clusterTimeStamp, 1);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
>();
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user);
Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
);
ContainerId containerId = BuilderUtils.NewContainerId(appAttemptId, 1);
nmContext.GetApplications()[appId] = app;
MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(),
conf, user, appId, 1);
container.SetState(ContainerState.Running);
nmContext.GetContainers()[containerId] = container;
FilePath containerLogDir = new FilePath(absLogDir, ContainerLaunch.GetRelativeContainerLogDir
(appId.ToString(), containerId.ToString()));
containerLogDir.Mkdirs();
string fileName = "fileName";
FilePath containerLogFile = new FilePath(containerLogDir, fileName);
containerLogFile.CreateNewFile();
FilePath file = ContainerLogsUtils.GetContainerLogFile(containerId, fileName, user
, nmContext);
NUnit.Framework.Assert.AreEqual(containerLogFile.ToURI().ToString(), file.ToURI()
.ToString());
FileUtil.FullyDelete(absLogDir);
}
// TODO: Pull logs and test contents.
// Thread.sleep(1000000);
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
private void WriteContainerLogs(Context nmContext, ContainerId containerId, LocalDirsHandlerService
dirsHandler)
{
// ContainerLogDir should be created
FilePath containerLogDir = ContainerLogsUtils.GetContainerLogDirs(containerId, dirsHandler
)[0];
containerLogDir.Mkdirs();
foreach (string fileType in new string[] { "stdout", "stderr", "syslog" })
{
TextWriter writer = new FileWriter(new FilePath(containerLogDir, fileType));
writer.Write(ConverterUtils.ToString(containerId) + "\n Hello " + fileType + "!");
writer.Close();
}
}
public virtual void TestLogFileWithDriveLetter()
{
ContainerImpl container = Org.Mockito.Mockito.Mock <ContainerImpl>();
ApplicationIdPBImpl appId = Org.Mockito.Mockito.Mock <ApplicationIdPBImpl>();
Org.Mockito.Mockito.When(appId.ToString()).ThenReturn("appId");
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
>();
Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
ApplicationAttemptIdPBImpl appAttemptId = Org.Mockito.Mockito.Mock <ApplicationAttemptIdPBImpl
>();
Org.Mockito.Mockito.When(appAttemptId.GetApplicationId()).ThenReturn(appId);
ConcurrentMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
> applications = new ConcurrentHashMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
>();
applications[appId] = app;
ContainerId containerId = Org.Mockito.Mockito.Mock <ContainerIdPBImpl>();
Org.Mockito.Mockito.When(containerId.ToString()).ThenReturn("containerId");
Org.Mockito.Mockito.When(containerId.GetApplicationAttemptId()).ThenReturn(appAttemptId
);
ConcurrentMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
> containers = new ConcurrentHashMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
>();
containers[containerId] = container;
LocalDirsHandlerService localDirs = Org.Mockito.Mockito.Mock <LocalDirsHandlerService
>();
Org.Mockito.Mockito.When(localDirs.GetLogPathToRead("appId" + Path.Separator + "containerId"
+ Path.Separator + "fileName")).ThenReturn(new Path("F:/nmlogs/appId/containerId/fileName"
));
NodeManager.NMContext context = Org.Mockito.Mockito.Mock <NodeManager.NMContext>();
Org.Mockito.Mockito.When(context.GetLocalDirsHandler()).ThenReturn(localDirs);
Org.Mockito.Mockito.When(context.GetApplications()).ThenReturn(applications);
Org.Mockito.Mockito.When(context.GetContainers()).ThenReturn(containers);
FilePath logFile = ContainerLogsUtils.GetContainerLogFile(containerId, "fileName"
, null, context);
NUnit.Framework.Assert.IsTrue("logFile lost drive letter " + logFile, logFile.ToString
().IndexOf("F:" + FilePath.separator + "nmlogs") > -1);
}
public ContainerLaunch(Context context, Configuration configuration, Dispatcher dispatcher
, ContainerExecutor exec, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
container, LocalDirsHandlerService dirsHandler, ContainerManagerImpl containerManager
)
{
this.context = context;
this.conf = configuration;
this.app = app;
this.exec = exec;
this.container = container;
this.dispatcher = dispatcher;
this.dirsHandler = dirsHandler;
this.containerManager = containerManager;
this.sleepDelayBeforeSigKill = conf.GetLong(YarnConfiguration.NmSleepDelayBeforeSigkillMs
, YarnConfiguration.DefaultNmSleepDelayBeforeSigkillMs);
this.maxKillWaitTime = conf.GetLong(YarnConfiguration.NmProcessKillWaitMs, YarnConfiguration
.DefaultNmProcessKillWaitMs);
}
public LogAggregationService(Dispatcher dispatcher, Context context, DeletionService
deletionService, LocalDirsHandlerService dirsHandler)
: base(typeof(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Logaggregation.LogAggregationService
).FullName)
{
/*
* Expected deployment TLD will be 1777, owner=<NMOwner>, group=<NMGroup -
* Group to which NMOwner belongs> App dirs will be created as 770,
* owner=<AppOwner>, group=<NMGroup>: so that the owner and <NMOwner> can
* access / modify the files.
* <NMGroup> should obviously be a limited access group.
*/
this.dispatcher = dispatcher;
this.context = context;
this.deletionService = deletionService;
this.dirsHandler = dirsHandler;
this.appLogAggregators = new ConcurrentHashMap <ApplicationId, AppLogAggregator>();
this.threadPool = Executors.NewCachedThreadPool(new ThreadFactoryBuilder().SetNameFormat
("LogAggregationService #%d").Build());
}
public virtual void TestDirFailuresOnStartup()
{
Configuration conf = new YarnConfiguration();
string localDir1 = new FilePath(testDir, "localDir1").GetPath();
string localDir2 = new FilePath(testDir, "localDir2").GetPath();
string logDir1 = new FilePath(testDir, "logDir1").GetPath();
string logDir2 = new FilePath(testDir, "logDir2").GetPath();
conf.Set(YarnConfiguration.NmLocalDirs, localDir1 + "," + localDir2);
conf.Set(YarnConfiguration.NmLogDirs, logDir1 + "," + logDir2);
PrepareDirToFail(localDir1);
PrepareDirToFail(logDir2);
LocalDirsHandlerService dirSvc = new LocalDirsHandlerService();
dirSvc.Init(conf);
IList <string> localDirs = dirSvc.GetLocalDirs();
NUnit.Framework.Assert.AreEqual(1, localDirs.Count);
NUnit.Framework.Assert.AreEqual(new Path(localDir2).ToString(), localDirs[0]);
IList <string> logDirs = dirSvc.GetLogDirs();
NUnit.Framework.Assert.AreEqual(1, logDirs.Count);
NUnit.Framework.Assert.AreEqual(new Path(logDir1).ToString(), logDirs[0]);
}
public AppLogAggregatorImpl(Dispatcher dispatcher, DeletionService deletionService
, Configuration conf, ApplicationId appId, UserGroupInformation userUgi, NodeId
nodeId, LocalDirsHandlerService dirsHandler, Path remoteNodeLogFileForApp, ContainerLogsRetentionPolicy
retentionPolicy, IDictionary <ApplicationAccessType, string> appAcls, LogAggregationContext
logAggregationContext, Context context, FileContext lfs)
{
// This is temporary solution. The configuration will be deleted once
// we find a more scalable method to only write a single log file per LRS.
// This configuration is for debug and test purpose. By setting
// this configuration as true. We can break the lower bound of
// NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS.
// This variable is only for testing
this.dispatcher = dispatcher;
this.conf = conf;
this.delService = deletionService;
this.appId = appId;
this.applicationId = ConverterUtils.ToString(appId);
this.userUgi = userUgi;
this.dirsHandler = dirsHandler;
this.remoteNodeLogFileForApp = remoteNodeLogFileForApp;
this.remoteNodeTmpLogFileForApp = GetRemoteNodeTmpLogFileForApp();
this.retentionPolicy = retentionPolicy;
this.pendingContainers = new LinkedBlockingQueue <ContainerId>();
this.appAcls = appAcls;
this.lfs = lfs;
this.logAggregationContext = logAggregationContext;
this.context = context;
this.nodeId = nodeId;
int configuredRentionSize = conf.GetInt(NmLogAggregationNumLogFilesSizePerApp, DefaultNmLogAggregationNumLogFilesSizePerApp
);
if (configuredRentionSize <= 0)
{
this.retentionSize = DefaultNmLogAggregationNumLogFilesSizePerApp;
}
else
{
this.retentionSize = configuredRentionSize;
}
long configuredRollingMonitorInterval = conf.GetLong(YarnConfiguration.NmLogAggregationRollMonitoringIntervalSeconds
, YarnConfiguration.DefaultNmLogAggregationRollMonitoringIntervalSeconds);
bool debug_mode = conf.GetBoolean(NmLogAggregationDebugEnabled, DefaultNmLogAggregationDebugEnabled
);
if (configuredRollingMonitorInterval > 0 && configuredRollingMonitorInterval < NmLogAggregationMinRollMonitoringIntervalSeconds)
{
if (debug_mode)
{
this.rollingMonitorInterval = configuredRollingMonitorInterval;
}
else
{
Log.Warn("rollingMonitorIntervall should be more than or equal to " + NmLogAggregationMinRollMonitoringIntervalSeconds
+ " seconds. Using " + NmLogAggregationMinRollMonitoringIntervalSeconds + " seconds instead."
);
this.rollingMonitorInterval = NmLogAggregationMinRollMonitoringIntervalSeconds;
}
}
else
{
if (configuredRollingMonitorInterval <= 0)
{
Log.Warn("rollingMonitorInterval is set as " + configuredRollingMonitorInterval +
". " + "The log rolling mornitoring interval is disabled. " + "The logs will be aggregated after this application is finished."
);
}
else
{
Log.Warn("rollingMonitorInterval is set as " + configuredRollingMonitorInterval +
". " + "The logs will be aggregated every " + configuredRollingMonitorInterval
+ " seconds");
}
this.rollingMonitorInterval = configuredRollingMonitorInterval;
}
this.logAggregationInRolling = this.rollingMonitorInterval <= 0 || this.logAggregationContext
== null || this.logAggregationContext.GetRolledLogsIncludePattern() == null ||
this.logAggregationContext.GetRolledLogsIncludePattern().IsEmpty() ? false : true;
}
public _ContainersLauncher_309(Context baseArg1, Dispatcher baseArg2, ContainerExecutor
baseArg3, LocalDirsHandlerService baseArg4, ContainerManagerImpl baseArg5)
: base(baseArg1, baseArg2, baseArg3, baseArg4, baseArg5)
{
}
public _ResourceLocalizationService_287(Dispatcher baseArg1, ContainerExecutor baseArg2
, DeletionService baseArg3, LocalDirsHandlerService baseArg4, Context baseArg5)
: base(baseArg1, baseArg2, baseArg3, baseArg4, baseArg5)
{
}
public _NMContext_108(NMContainerTokenSecretManager baseArg1, NMTokenSecretManagerInNM
baseArg2, LocalDirsHandlerService baseArg3, ApplicationACLsManager baseArg4, NMStateStoreService
baseArg5)
: base(baseArg1, baseArg2, baseArg3, baseArg4, baseArg5)
{
}
public _ContainerManagerImpl_191(Context baseArg1, ContainerExecutor baseArg2, DeletionService
baseArg3, NodeStatusUpdater baseArg4, NodeManagerMetrics baseArg5, ApplicationACLsManager
baseArg6, LocalDirsHandlerService baseArg7)
: base(baseArg1, baseArg2, baseArg3, baseArg4, baseArg5, baseArg6, baseArg7)
{
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
internal static IList <FilePath> GetContainerLogDirs(ContainerId containerId, LocalDirsHandlerService
dirsHandler)
{
IList <string> logDirs = dirsHandler.GetLogDirsForRead();
IList <FilePath> containerLogDirs = new AList <FilePath>(logDirs.Count);
foreach (string logDir in logDirs)
{
logDir = new FilePath(logDir).ToURI().GetPath();
string appIdStr = ConverterUtils.ToString(containerId.GetApplicationAttemptId().GetApplicationId
());
FilePath appLogDir = new FilePath(logDir, appIdStr);
containerLogDirs.AddItem(new FilePath(appLogDir, containerId.ToString()));
}
return(containerLogDirs);
}
public _ContainerManagerImpl_115(TestContainerManager _enclosing, Context baseArg1
, ContainerExecutor baseArg2, DeletionService baseArg3, NodeStatusUpdater baseArg4
, NodeManagerMetrics baseArg5, ApplicationACLsManager baseArg6, LocalDirsHandlerService
baseArg7)
: base(baseArg1, baseArg2, baseArg3, baseArg4, baseArg5, baseArg6, baseArg7)
{
this._enclosing = _enclosing;
}
public NonAggregatingLogHandlerWithMockExecutor(TestNonAggregatingLogHandler _enclosing
, Dispatcher dispatcher, DeletionService delService, LocalDirsHandlerService dirsHandler
)
: this(dispatcher, delService, dirsHandler, new NMNullStateStoreService())
{
this._enclosing = _enclosing;
}
/// <summary>
/// Function to run a log handler with directories failing the getFileStatus
/// call.
/// </summary>
/// <remarks>
/// Function to run a log handler with directories failing the getFileStatus
/// call. The function accepts the log handler, setup the mocks to fail with
/// specific exceptions and ensures the deletion service has the correct calls.
/// </remarks>
/// <param name="logHandler">the logHandler implementation to test</param>
/// <param name="appId">
/// the application id that we wish when sending events to the log
/// handler
/// </param>
/// <param name="user">the user name to use</param>
/// <param name="mockDelService">
/// a mock of the DeletionService which we will verify
/// the delete calls against
/// </param>
/// <param name="dirsHandler">
/// a spy or mock on the LocalDirsHandler service used to
/// when creating the logHandler. It needs to be a spy so that we can intercept
/// the getAllLogDirs() call.
/// </param>
/// <param name="conf">the configuration used</param>
/// <param name="spylfs">a spy on the AbstractFileSystem object used when creating lfs
/// </param>
/// <param name="lfs">
/// the FileContext object to be used to mock the getFileStatus()
/// calls
/// </param>
/// <param name="localLogDirs">
/// list of the log dirs to run the test against, must have
/// at least 7 entries
/// </param>
/// <exception cref="System.Exception"/>
public static void RunMockedFailedDirs(LogHandler logHandler, ApplicationId appId
, string user, DeletionService mockDelService, LocalDirsHandlerService dirsHandler
, Configuration conf, AbstractFileSystem spylfs, FileContext lfs, FilePath[] localLogDirs
)
{
IDictionary <ApplicationAccessType, string> appAcls = new Dictionary <ApplicationAccessType
, string>();
if (localLogDirs.Length < 7)
{
throw new ArgumentException("Argument localLogDirs must be at least of length 7");
}
Path[] localAppLogDirPaths = new Path[localLogDirs.Length];
for (int i = 0; i < localAppLogDirPaths.Length; i++)
{
localAppLogDirPaths[i] = new Path(localLogDirs[i].GetAbsolutePath(), appId.ToString
());
}
IList <string> localLogDirPaths = new AList <string>(localLogDirs.Length);
for (int i_1 = 0; i_1 < localLogDirs.Length; i_1++)
{
localLogDirPaths.AddItem(localLogDirs[i_1].GetAbsolutePath());
}
// setup mocks
FsPermission defaultPermission = FsPermission.GetDirDefault().ApplyUMask(lfs.GetUMask
());
FileStatus fs = new FileStatus(0, true, 1, 0, Runtime.CurrentTimeMillis(), 0, defaultPermission
, string.Empty, string.Empty, new Path(localLogDirs[0].GetAbsolutePath()));
Org.Mockito.Mockito.DoReturn(fs).When(spylfs).GetFileStatus(Matchers.IsA <Path>());
Org.Mockito.Mockito.DoReturn(localLogDirPaths).When(dirsHandler).GetLogDirsForCleanup
();
logHandler.Handle(new LogHandlerAppStartedEvent(appId, user, null, ContainerLogsRetentionPolicy
.AllContainers, appAcls));
// test case where some dirs have the log dir to delete
// mock some dirs throwing various exceptions
// verify deletion happens only on the others
Org.Mockito.Mockito.DoThrow(new FileNotFoundException()).When(spylfs).GetFileStatus
(Matchers.Eq(localAppLogDirPaths[0]));
Org.Mockito.Mockito.DoReturn(fs).When(spylfs).GetFileStatus(Matchers.Eq(localAppLogDirPaths
[1]));
Org.Mockito.Mockito.DoThrow(new AccessControlException()).When(spylfs).GetFileStatus
(Matchers.Eq(localAppLogDirPaths[2]));
Org.Mockito.Mockito.DoReturn(fs).When(spylfs).GetFileStatus(Matchers.Eq(localAppLogDirPaths
[3]));
Org.Mockito.Mockito.DoThrow(new IOException()).When(spylfs).GetFileStatus(Matchers.Eq
(localAppLogDirPaths[4]));
Org.Mockito.Mockito.DoThrow(new UnsupportedFileSystemException("test")).When(spylfs
).GetFileStatus(Matchers.Eq(localAppLogDirPaths[5]));
Org.Mockito.Mockito.DoReturn(fs).When(spylfs).GetFileStatus(Matchers.Eq(localAppLogDirPaths
[6]));
logHandler.Handle(new LogHandlerAppFinishedEvent(appId));
TestDeletionServiceCall(mockDelService, user, 5000, localAppLogDirPaths[1], localAppLogDirPaths
[3], localAppLogDirPaths[6]);
return;
}
public NonAggregatingLogHandlerWithMockExecutor(TestNonAggregatingLogHandler _enclosing
, Dispatcher dispatcher, DeletionService delService, LocalDirsHandlerService dirsHandler
, NMStateStoreService stateStore)
: base(dispatcher, delService, dirsHandler, stateStore)
{
this._enclosing = _enclosing;
}
