/// <exception cref="System.Exception"/>
public virtual void TestPostExecuteAfterReacquisition()
{
// make up some bogus container ID
ApplicationId appId = ApplicationId.NewInstance(12345, 67890);
ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 54321);
ContainerId cid = ContainerId.NewContainerId(attemptId, 9876);
Configuration conf = new YarnConfiguration();
conf.SetClass(YarnConfiguration.NmLinuxContainerResourcesHandler, typeof(TestLinuxContainerExecutor.TestResourceHandler
), typeof(LCEResourcesHandler));
LinuxContainerExecutor lce = new LinuxContainerExecutor();
lce.SetConf(conf);
try
{
lce.Init();
}
catch (IOException)
{
}
// expected if LCE isn't setup right, but not necessary for this test
lce.ReacquireContainer("foouser", cid);
NUnit.Framework.Assert.IsTrue("postExec not called after reacquisition", TestLinuxContainerExecutor.TestResourceHandler
.postExecContainers.Contains(cid));
}
public virtual void TestNonsecureUsernamePattern()
{
try
{
//nonsecure default
Configuration conf = new YarnConfiguration();
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "simple");
UserGroupInformation.SetConfiguration(conf);
LinuxContainerExecutor lce = new LinuxContainerExecutor();
lce.SetConf(conf);
lce.VerifyUsernamePattern("foo");
try
{
lce.VerifyUsernamePattern("foo/x");
NUnit.Framework.Assert.Fail();
}
catch (ArgumentException)
{
}
catch (Exception ex)
{
//NOP
NUnit.Framework.Assert.Fail(ex.ToString());
}
//nonsecure custom setting
conf.Set(YarnConfiguration.NmNonsecureModeUserPatternKey, "foo");
lce = new LinuxContainerExecutor();
lce.SetConf(conf);
lce.VerifyUsernamePattern("foo");
try
{
lce.VerifyUsernamePattern("bar");
NUnit.Framework.Assert.Fail();
}
catch (ArgumentException)
{
}
catch (Exception ex)
{
//NOP
NUnit.Framework.Assert.Fail(ex.ToString());
}
//secure, pattern matching does not kick in.
conf = new YarnConfiguration();
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
lce = new LinuxContainerExecutor();
lce.SetConf(conf);
lce.VerifyUsernamePattern("foo");
lce.VerifyUsernamePattern("foo/w");
}
finally
{
Configuration conf = new YarnConfiguration();
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "simple");
UserGroupInformation.SetConfiguration(conf);
}
}
protected internal override ContainerExecutor CreateContainerExecutor()
{
base.conf.Set(YarnConfiguration.NmLinuxContainerExecutorPath, Runtime.GetProperty
(YarnConfiguration.NmLinuxContainerExecutorPath));
LinuxContainerExecutor linuxContainerExecutor = new LinuxContainerExecutor();
linuxContainerExecutor.SetConf(base.conf);
return(linuxContainerExecutor);
}
public virtual void TestContainerLaunchError()
{
// reinitialize executer
FilePath f = new FilePath("./src/test/resources/mock-container-executer-with-error"
);
if (!FileUtil.CanExecute(f))
{
FileUtil.SetExecutable(f, true);
}
string executorPath = f.GetAbsolutePath();
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.NmLinuxContainerExecutorPath, executorPath);
conf.Set(YarnConfiguration.NmLocalDirs, "file:///bin/echo");
conf.Set(YarnConfiguration.NmLogDirs, "file:///dev/null");
mockExec = Org.Mockito.Mockito.Spy(new LinuxContainerExecutor());
Org.Mockito.Mockito.DoAnswer(new _Answer_226()).When(mockExec).LogOutput(Matchers.Any
<string>());
dirsHandler = new LocalDirsHandlerService();
dirsHandler.Init(conf);
mockExec.SetConf(conf);
string appSubmitter = "nobody";
string cmd = LinuxContainerExecutor.Commands.LaunchContainer.GetValue().ToString(
);
string appId = "APP_ID";
string containerId = "CONTAINER_ID";
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
>();
ContainerId cId = Org.Mockito.Mockito.Mock <ContainerId>();
ContainerLaunchContext context = Org.Mockito.Mockito.Mock <ContainerLaunchContext>
();
Dictionary <string, string> env = new Dictionary <string, string>();
Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(cId);
Org.Mockito.Mockito.When(container.GetLaunchContext()).ThenReturn(context);
Org.Mockito.Mockito.DoAnswer(new _Answer_254()).When(container).Handle(Matchers.Any
<ContainerDiagnosticsUpdateEvent>());
Org.Mockito.Mockito.When(cId.ToString()).ThenReturn(containerId);
Org.Mockito.Mockito.When(context.GetEnvironment()).ThenReturn(env);
Path scriptPath = new Path("file:///bin/echo");
Path tokensPath = new Path("file:///dev/null");
Path workDir = new Path("/tmp");
Path pidFile = new Path(workDir, "pid.txt");
mockExec.ActivateContainer(cId, pidFile);
int ret = mockExec.LaunchContainer(container, scriptPath, tokensPath, appSubmitter
, appId, workDir, dirsHandler.GetLocalDirs(), dirsHandler.GetLogDirs());
NUnit.Framework.Assert.AreNotSame(0, ret);
NUnit.Framework.Assert.AreEqual(Arrays.AsList(YarnConfiguration.DefaultNmNonsecureModeLocalUser
, appSubmitter, cmd, appId, containerId, workDir.ToString(), "/bin/echo", "/dev/null"
, pidFile.ToString(), StringUtils.Join(",", dirsHandler.GetLocalDirs()), StringUtils
.Join(",", dirsHandler.GetLogDirs()), "cgroups=none"), ReadMockParams());
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestContainerLaunchWithPriority()
{
// set the scheduler priority to make sure still works with nice -n prio
FilePath f = new FilePath("./src/test/resources/mock-container-executor");
if (!FileUtil.CanExecute(f))
{
FileUtil.SetExecutable(f, true);
}
string executorPath = f.GetAbsolutePath();
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.NmLinuxContainerExecutorPath, executorPath);
conf.SetInt(YarnConfiguration.NmContainerExecutorSchedPriority, 2);
mockExec.SetConf(conf);
IList <string> command = new AList <string>();
mockExec.AddSchedPriorityCommand(command);
NUnit.Framework.Assert.AreEqual("first should be nice", "nice", command[0]);
NUnit.Framework.Assert.AreEqual("second should be -n", "-n", command[1]);
NUnit.Framework.Assert.AreEqual("third should be the priority", Sharpen.Extensions.ToString
(2), command[2]);
TestContainerLaunch();
}
public virtual void Setup()
{
Assume.AssumeTrue(!Path.Windows);
FilePath f = new FilePath("./src/test/resources/mock-container-executor");
if (!FileUtil.CanExecute(f))
{
FileUtil.SetExecutable(f, true);
}
string executorPath = f.GetAbsolutePath();
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.NmLinuxContainerExecutorPath, executorPath);
mockExec = new LinuxContainerExecutor();
dirsHandler = new LocalDirsHandlerService();
dirsHandler.Init(conf);
mockExec.SetConf(conf);
}
public virtual void TestContainerLocalizer()
{
if (!ShouldRun())
{
return;
}
IList <string> localDirs = dirsHandler.GetLocalDirs();
IList <string> logDirs = dirsHandler.GetLogDirs();
foreach (string localDir in localDirs)
{
Path userDir = new Path(localDir, ContainerLocalizer.Usercache);
files.Mkdir(userDir, new FsPermission("777"), false);
// $local/filecache
Path fileDir = new Path(localDir, ContainerLocalizer.Filecache);
files.Mkdir(fileDir, new FsPermission("777"), false);
}
string locId = "container_01_01";
Path nmPrivateContainerTokensPath = dirsHandler.GetLocalPathForWrite(ResourceLocalizationService
.NmPrivateDir + Path.Separator + string.Format(ContainerLocalizer.TokenFileNameFmt
, locId));
files.Create(nmPrivateContainerTokensPath, EnumSet.Of(CreateFlag.Create, CreateFlag
.Overwrite));
Configuration config = new YarnConfiguration(conf);
IPEndPoint nmAddr = config.GetSocketAddr(YarnConfiguration.NmBindHost, YarnConfiguration
.NmLocalizerAddress, YarnConfiguration.DefaultNmLocalizerAddress, YarnConfiguration
.DefaultNmLocalizerPort);
string appId = "application_01_01";
exec = new _LinuxContainerExecutor_258();
exec.SetConf(conf);
exec.StartLocalizer(nmPrivateContainerTokensPath, nmAddr, appSubmitter, appId, locId
, dirsHandler);
string locId2 = "container_01_02";
Path nmPrivateContainerTokensPath2 = dirsHandler.GetLocalPathForWrite(ResourceLocalizationService
.NmPrivateDir + Path.Separator + string.Format(ContainerLocalizer.TokenFileNameFmt
, locId2));
files.Create(nmPrivateContainerTokensPath2, EnumSet.Of(CreateFlag.Create, CreateFlag
.Overwrite));
exec.StartLocalizer(nmPrivateContainerTokensPath2, nmAddr, appSubmitter, appId, locId2
, dirsHandler);
}
public virtual void TestLocalUser()
{
try
{
//nonsecure default
Configuration conf = new YarnConfiguration();
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "simple");
UserGroupInformation.SetConfiguration(conf);
LinuxContainerExecutor lce = new LinuxContainerExecutor();
lce.SetConf(conf);
NUnit.Framework.Assert.AreEqual(YarnConfiguration.DefaultNmNonsecureModeLocalUser
, lce.GetRunAsUser("foo"));
//nonsecure custom setting
conf.Set(YarnConfiguration.NmNonsecureModeLocalUserKey, "bar");
lce = new LinuxContainerExecutor();
lce.SetConf(conf);
NUnit.Framework.Assert.AreEqual("bar", lce.GetRunAsUser("foo"));
//nonsecure without limits
conf.Set(YarnConfiguration.NmNonsecureModeLocalUserKey, "bar");
conf.Set(YarnConfiguration.NmNonsecureModeLimitUsers, "false");
lce = new LinuxContainerExecutor();
lce.SetConf(conf);
NUnit.Framework.Assert.AreEqual("foo", lce.GetRunAsUser("foo"));
//secure
conf = new YarnConfiguration();
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
lce = new LinuxContainerExecutor();
lce.SetConf(conf);
NUnit.Framework.Assert.AreEqual("foo", lce.GetRunAsUser("foo"));
}
finally
{
Configuration conf = new YarnConfiguration();
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "simple");
UserGroupInformation.SetConfiguration(conf);
}
}
public virtual void Setup()
{
files = FileContext.GetLocalFSFileContext();
Path workSpacePath = new Path(workSpace.GetAbsolutePath());
files.Mkdir(workSpacePath, null, true);
FileUtil.Chmod(workSpace.GetAbsolutePath(), "777");
FilePath localDir = new FilePath(workSpace.GetAbsoluteFile(), "localDir");
files.Mkdir(new Path(localDir.GetAbsolutePath()), new FsPermission("777"), false);
FilePath logDir = new FilePath(workSpace.GetAbsoluteFile(), "logDir");
files.Mkdir(new Path(logDir.GetAbsolutePath()), new FsPermission("777"), false);
string exec_path = Runtime.GetProperty("container-executor.path");
if (exec_path != null && !exec_path.IsEmpty())
{
conf = new Configuration(false);
conf.SetClass("fs.AbstractFileSystem.file.impl", typeof(LocalFs), typeof(AbstractFileSystem
));
conf.Set(YarnConfiguration.NmNonsecureModeLocalUserKey, "xuan");
Log.Info("Setting " + YarnConfiguration.NmLinuxContainerExecutorPath + "=" + exec_path
);
conf.Set(YarnConfiguration.NmLinuxContainerExecutorPath, exec_path);
exec = new LinuxContainerExecutor();
exec.SetConf(conf);
conf.Set(YarnConfiguration.NmLocalDirs, localDir.GetAbsolutePath());
conf.Set(YarnConfiguration.NmLogDirs, logDir.GetAbsolutePath());
dirsHandler = new LocalDirsHandlerService();
dirsHandler.Init(conf);
}
appSubmitter = Runtime.GetProperty("application.submitter");
if (appSubmitter == null || appSubmitter.IsEmpty())
{
appSubmitter = "nobody";
}
}