public virtual void TestMinimumPerDirectoryFileLimit()
{
YarnConfiguration conf = new YarnConfiguration();
conf.Set(YarnConfiguration.NmLocalCacheMaxFilesPerDirectory, "1");
Exception e = null;
NodeManager.NMContext nmContext = new NodeManager.NMContext(new NMContainerTokenSecretManager
(conf), new NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf),
new NMNullStateStoreService());
ResourceLocalizationService service = new ResourceLocalizationService(null, null,
null, null, nmContext);
try
{
service.Init(conf);
}
catch (Exception e1)
{
e = e1;
}
NUnit.Framework.Assert.IsNotNull(e);
NUnit.Framework.Assert.AreEqual(typeof(YarnRuntimeException), e.GetType());
NUnit.Framework.Assert.AreEqual(e.Message, YarnConfiguration.NmLocalCacheMaxFilesPerDirectory
+ " parameter is configured with a value less than 37.");
}
private int StartNMWebAppServer(string webAddr)
{
Context nmContext = new NodeManager.NMContext(null, null, null, null, null);
ResourceView resourceView = new _ResourceView_84();
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.NmLocalDirs, testRootDir.GetAbsolutePath());
conf.Set(YarnConfiguration.NmLogDirs, testLogDir.GetAbsolutePath());
NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
healthChecker.Init(conf);
LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler();
conf.Set(YarnConfiguration.NmWebappAddress, webAddr);
WebServer server = new WebServer(nmContext, resourceView, new ApplicationACLsManager
(conf), dirsHandler);
try
{
server.Init(conf);
server.Start();
return(server.GetPort());
}
finally
{
server.Stop();
healthChecker.Stop();
}
}
public virtual void TestNMWebApp()
{
Context nmContext = new NodeManager.NMContext(null, null, null, null, null);
ResourceView resourceView = new _ResourceView_147();
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.NmLocalDirs, testRootDir.GetAbsolutePath());
conf.Set(YarnConfiguration.NmLogDirs, testLogDir.GetAbsolutePath());
NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
healthChecker.Init(conf);
LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler();
WebServer server = new WebServer(nmContext, resourceView, new ApplicationACLsManager
(conf), dirsHandler);
server.Init(conf);
server.Start();
// Add an application and the corresponding containers
RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(conf);
Dispatcher dispatcher = new AsyncDispatcher();
string user = "******";
long clusterTimeStamp = 1234;
ApplicationId appId = BuilderUtils.NewApplicationId(recordFactory, clusterTimeStamp
, 1);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
>();
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user);
Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
nmContext.GetApplications()[appId] = app;
ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
);
ContainerId container1 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId
, 0);
ContainerId container2 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId
, 1);
NodeManagerMetrics metrics = Org.Mockito.Mockito.Mock <NodeManagerMetrics>();
NMStateStoreService stateStore = new NMNullStateStoreService();
foreach (ContainerId containerId in new ContainerId[] { container1, container2 })
{
// TODO: Use builder utils
ContainerLaunchContext launchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
long currentTime = Runtime.CurrentTimeMillis();
Token containerToken = BuilderUtils.NewContainerToken(containerId, "127.0.0.1", 1234
, user, BuilderUtils.NewResource(1024, 1), currentTime + 10000L, 123, Sharpen.Runtime.GetBytesForString
("password"), currentTime);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= new _ContainerImpl_214(conf, dispatcher, stateStore, launchContext, null, metrics
, BuilderUtils.NewContainerTokenIdentifier(containerToken));
nmContext.GetContainers()[containerId] = container;
//TODO: Gross hack. Fix in code.
ApplicationId applicationId = containerId.GetApplicationAttemptId().GetApplicationId
();
nmContext.GetApplications()[applicationId].GetContainers()[containerId] = container;
WriteContainerLogs(nmContext, containerId, dirsHandler);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public virtual void TestContainerLogFile()
{
FilePath absLogDir = new FilePath("target", typeof(TestNMWebServer).Name + "LogDir"
).GetAbsoluteFile();
string logdirwithFile = absLogDir.ToURI().ToString();
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.NmLogDirs, logdirwithFile);
conf.SetFloat(YarnConfiguration.NmMaxPerDiskUtilizationPercentage, 0.0f);
LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
dirsHandler.Init(conf);
NodeManager.NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler
, new ApplicationACLsManager(conf), new NMNullStateStoreService());
// Add an application and the corresponding containers
string user = "******";
long clusterTimeStamp = 1234;
ApplicationId appId = BuilderUtils.NewApplicationId(clusterTimeStamp, 1);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
>();
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user);
Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
);
ContainerId containerId = BuilderUtils.NewContainerId(appAttemptId, 1);
nmContext.GetApplications()[appId] = app;
MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(),
conf, user, appId, 1);
container.SetState(ContainerState.Running);
nmContext.GetContainers()[containerId] = container;
FilePath containerLogDir = new FilePath(absLogDir, ContainerLaunch.GetRelativeContainerLogDir
(appId.ToString(), containerId.ToString()));
containerLogDir.Mkdirs();
string fileName = "fileName";
FilePath containerLogFile = new FilePath(containerLogDir, fileName);
containerLogFile.CreateNewFile();
FilePath file = ContainerLogsUtils.GetContainerLogFile(containerId, fileName, user
, nmContext);
NUnit.Framework.Assert.AreEqual(containerLogFile.ToURI().ToString(), file.ToURI()
.ToString());
FileUtil.FullyDelete(absLogDir);
}
public virtual void TestLogFileWithDriveLetter()
{
ContainerImpl container = Org.Mockito.Mockito.Mock <ContainerImpl>();
ApplicationIdPBImpl appId = Org.Mockito.Mockito.Mock <ApplicationIdPBImpl>();
Org.Mockito.Mockito.When(appId.ToString()).ThenReturn("appId");
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
>();
Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
ApplicationAttemptIdPBImpl appAttemptId = Org.Mockito.Mockito.Mock <ApplicationAttemptIdPBImpl
>();
Org.Mockito.Mockito.When(appAttemptId.GetApplicationId()).ThenReturn(appId);
ConcurrentMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
> applications = new ConcurrentHashMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
>();
applications[appId] = app;
ContainerId containerId = Org.Mockito.Mockito.Mock <ContainerIdPBImpl>();
Org.Mockito.Mockito.When(containerId.ToString()).ThenReturn("containerId");
Org.Mockito.Mockito.When(containerId.GetApplicationAttemptId()).ThenReturn(appAttemptId
);
ConcurrentMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
> containers = new ConcurrentHashMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
>();
containers[containerId] = container;
LocalDirsHandlerService localDirs = Org.Mockito.Mockito.Mock <LocalDirsHandlerService
>();
Org.Mockito.Mockito.When(localDirs.GetLogPathToRead("appId" + Path.Separator + "containerId"
+ Path.Separator + "fileName")).ThenReturn(new Path("F:/nmlogs/appId/containerId/fileName"
));
NodeManager.NMContext context = Org.Mockito.Mockito.Mock <NodeManager.NMContext>();
Org.Mockito.Mockito.When(context.GetLocalDirsHandler()).ThenReturn(localDirs);
Org.Mockito.Mockito.When(context.GetApplications()).ThenReturn(applications);
Org.Mockito.Mockito.When(context.GetContainers()).ThenReturn(containers);
FilePath logFile = ContainerLogsUtils.GetContainerLogFile(containerId, "fileName"
, null, context);
NUnit.Framework.Assert.IsTrue("logFile lost drive letter " + logFile, logFile.ToString
().IndexOf("F:" + FilePath.separator + "nmlogs") > -1);
}
public virtual void TestApplicationRecovery()
{
YarnConfiguration conf = new YarnConfiguration();
conf.SetBoolean(YarnConfiguration.NmRecoveryEnabled, true);
conf.Set(YarnConfiguration.NmAddress, "localhost:1234");
conf.SetBoolean(YarnConfiguration.YarnAclEnable, true);
conf.Set(YarnConfiguration.YarnAdminAcl, "yarn_admin_user");
NMStateStoreService stateStore = new NMMemoryStateStoreService();
stateStore.Init(conf);
stateStore.Start();
Context context = new NodeManager.NMContext(new NMContainerTokenSecretManager(conf
), new NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), stateStore
);
ContainerManagerImpl cm = CreateContainerManager(context);
cm.Init(conf);
cm.Start();
// simulate registration with RM
MasterKey masterKey = new MasterKeyPBImpl();
masterKey.SetKeyId(123);
masterKey.SetBytes(ByteBuffer.Wrap(new byte[] { 123 }));
context.GetContainerTokenSecretManager().SetMasterKey(masterKey);
context.GetNMTokenSecretManager().SetMasterKey(masterKey);
// add an application by starting a container
string appUser = "******";
string modUser = "******";
string viewUser = "******";
string enemyUser = "******";
ApplicationId appId = ApplicationId.NewInstance(0, 1);
ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1);
ContainerId cid = ContainerId.NewContainerId(attemptId, 1);
IDictionary <string, LocalResource> localResources = Collections.EmptyMap();
IDictionary <string, string> containerEnv = Sharpen.Collections.EmptyMap();
IList <string> containerCmds = Sharpen.Collections.EmptyList();
IDictionary <string, ByteBuffer> serviceData = Sharpen.Collections.EmptyMap();
Credentials containerCreds = new Credentials();
DataOutputBuffer dob = new DataOutputBuffer();
containerCreds.WriteTokenStorageToStream(dob);
ByteBuffer containerTokens = ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength());
IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
, string>();
acls[ApplicationAccessType.ModifyApp] = modUser;
acls[ApplicationAccessType.ViewApp] = viewUser;
ContainerLaunchContext clc = ContainerLaunchContext.NewInstance(localResources, containerEnv
, containerCmds, serviceData, containerTokens, acls);
// create the logAggregationContext
LogAggregationContext logAggregationContext = LogAggregationContext.NewInstance("includePattern"
, "excludePattern", "includePatternInRollingAggregation", "excludePatternInRollingAggregation"
);
StartContainersResponse startResponse = StartContainer(context, cm, cid, clc, logAggregationContext
);
NUnit.Framework.Assert.IsTrue(startResponse.GetFailedRequests().IsEmpty());
NUnit.Framework.Assert.AreEqual(1, context.GetApplications().Count);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = context.GetApplications()[appId];
NUnit.Framework.Assert.IsNotNull(app);
WaitForAppState(app, ApplicationState.Initing);
NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(modUser), ApplicationAccessType.ModifyApp, appUser, appId));
NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(viewUser), ApplicationAccessType.ModifyApp, appUser, appId));
NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(viewUser), ApplicationAccessType.ViewApp, appUser, appId));
NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(enemyUser), ApplicationAccessType.ViewApp, appUser, appId));
// reset container manager and verify app recovered with proper acls
cm.Stop();
context = new NodeManager.NMContext(new NMContainerTokenSecretManager(conf), new
NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), stateStore);
cm = CreateContainerManager(context);
cm.Init(conf);
cm.Start();
NUnit.Framework.Assert.AreEqual(1, context.GetApplications().Count);
app = context.GetApplications()[appId];
NUnit.Framework.Assert.IsNotNull(app);
// check whether LogAggregationContext is recovered correctly
LogAggregationContext recovered = ((ApplicationImpl)app).GetLogAggregationContext
();
NUnit.Framework.Assert.IsNotNull(recovered);
NUnit.Framework.Assert.AreEqual(logAggregationContext.GetIncludePattern(), recovered
.GetIncludePattern());
NUnit.Framework.Assert.AreEqual(logAggregationContext.GetExcludePattern(), recovered
.GetExcludePattern());
NUnit.Framework.Assert.AreEqual(logAggregationContext.GetRolledLogsIncludePattern
(), recovered.GetRolledLogsIncludePattern());
NUnit.Framework.Assert.AreEqual(logAggregationContext.GetRolledLogsExcludePattern
(), recovered.GetRolledLogsExcludePattern());
WaitForAppState(app, ApplicationState.Initing);
NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(modUser), ApplicationAccessType.ModifyApp, appUser, appId));
NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(viewUser), ApplicationAccessType.ModifyApp, appUser, appId));
NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(viewUser), ApplicationAccessType.ViewApp, appUser, appId));
NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(enemyUser), ApplicationAccessType.ViewApp, appUser, appId));
// simulate application completion
IList <ApplicationId> finishedApps = new AList <ApplicationId>();
finishedApps.AddItem(appId);
cm.Handle(new CMgrCompletedAppsEvent(finishedApps, CMgrCompletedAppsEvent.Reason.
ByResourcemanager));
WaitForAppState(app, ApplicationState.ApplicationResourcesCleaningup);
// restart and verify app is marked for finishing
cm.Stop();
context = new NodeManager.NMContext(new NMContainerTokenSecretManager(conf), new
NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), stateStore);
cm = CreateContainerManager(context);
cm.Init(conf);
cm.Start();
NUnit.Framework.Assert.AreEqual(1, context.GetApplications().Count);
app = context.GetApplications()[appId];
NUnit.Framework.Assert.IsNotNull(app);
WaitForAppState(app, ApplicationState.ApplicationResourcesCleaningup);
NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(modUser), ApplicationAccessType.ModifyApp, appUser, appId));
NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(viewUser), ApplicationAccessType.ModifyApp, appUser, appId));
NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(viewUser), ApplicationAccessType.ViewApp, appUser, appId));
NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(enemyUser), ApplicationAccessType.ViewApp, appUser, appId));
// simulate log aggregation completion
app.Handle(new ApplicationEvent(app.GetAppId(), ApplicationEventType.ApplicationResourcesCleanedup
));
NUnit.Framework.Assert.AreEqual(app.GetApplicationState(), ApplicationState.Finished
);
app.Handle(new ApplicationEvent(app.GetAppId(), ApplicationEventType.ApplicationLogHandlingFinished
));
// restart and verify app is no longer present after recovery
cm.Stop();
context = new NodeManager.NMContext(new NMContainerTokenSecretManager(conf), new
NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), stateStore);
cm = CreateContainerManager(context);
cm.Init(conf);
cm.Start();
NUnit.Framework.Assert.IsTrue(context.GetApplications().IsEmpty());
cm.Stop();
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public virtual void TestContainerLogDirs()
{
FilePath absLogDir = new FilePath("target", typeof(TestNMWebServer).Name + "LogDir"
).GetAbsoluteFile();
string logdirwithFile = absLogDir.ToURI().ToString();
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.NmLogDirs, logdirwithFile);
NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
healthChecker.Init(conf);
LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler();
NodeManager.NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler
, new ApplicationACLsManager(conf), new NMNullStateStoreService());
// Add an application and the corresponding containers
RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(conf);
string user = "******";
long clusterTimeStamp = 1234;
ApplicationId appId = BuilderUtils.NewApplicationId(recordFactory, clusterTimeStamp
, 1);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
>();
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user);
Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
);
ContainerId container1 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId
, 0);
nmContext.GetApplications()[appId] = app;
MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(),
conf, user, appId, 1);
container.SetState(ContainerState.Running);
nmContext.GetContainers()[container1] = container;
IList <FilePath> files = null;
files = ContainerLogsUtils.GetContainerLogDirs(container1, user, nmContext);
NUnit.Framework.Assert.IsTrue(!(files[0].ToString().Contains("file:")));
// After container is completed, it is removed from nmContext
Sharpen.Collections.Remove(nmContext.GetContainers(), container1);
NUnit.Framework.Assert.IsNull(nmContext.GetContainers()[container1]);
files = ContainerLogsUtils.GetContainerLogDirs(container1, user, nmContext);
NUnit.Framework.Assert.IsTrue(!(files[0].ToString().Contains("file:")));
// Create a new context to check if correct container log dirs are fetched
// on full disk.
LocalDirsHandlerService dirsHandlerForFullDisk = Org.Mockito.Mockito.Spy(dirsHandler
);
// good log dirs are empty and nm log dir is in the full log dir list.
Org.Mockito.Mockito.When(dirsHandlerForFullDisk.GetLogDirs()).ThenReturn(new AList
<string>());
Org.Mockito.Mockito.When(dirsHandlerForFullDisk.GetLogDirsForRead()).ThenReturn(Arrays
.AsList(new string[] { absLogDir.GetAbsolutePath() }));
nmContext = new NodeManager.NMContext(null, null, dirsHandlerForFullDisk, new ApplicationACLsManager
(conf), new NMNullStateStoreService());
nmContext.GetApplications()[appId] = app;
container.SetState(ContainerState.Running);
nmContext.GetContainers()[container1] = container;
IList <FilePath> dirs = ContainerLogsUtils.GetContainerLogDirs(container1, user, nmContext
);
FilePath containerLogDir = new FilePath(absLogDir, appId + "/" + container1);
NUnit.Framework.Assert.IsTrue(dirs.Contains(containerLogDir));
}