public virtual void TestGetFullDirs()
{
Configuration conf = new YarnConfiguration();
conf.Set(CommonConfigurationKeys.FsPermissionsUmaskKey, "077");
FileContext localFs = FileContext.GetLocalFSFileContext(conf);
string localDir1 = new FilePath(testDir, "localDir1").GetPath();
string localDir2 = new FilePath(testDir, "localDir2").GetPath();
string logDir1 = new FilePath(testDir, "logDir1").GetPath();
string logDir2 = new FilePath(testDir, "logDir2").GetPath();
Path localDir1Path = new Path(localDir1);
Path logDir1Path = new Path(logDir1);
FsPermission dirPermissions = new FsPermission((short)0x108);
localFs.Mkdir(localDir1Path, dirPermissions, true);
localFs.Mkdir(logDir1Path, dirPermissions, true);
conf.Set(YarnConfiguration.NmLocalDirs, localDir1 + "," + localDir2);
conf.Set(YarnConfiguration.NmLogDirs, logDir1 + "," + logDir2);
conf.SetFloat(YarnConfiguration.NmMaxPerDiskUtilizationPercentage, 0.0f);
LocalDirsHandlerService dirSvc = new LocalDirsHandlerService();
dirSvc.Init(conf);
NUnit.Framework.Assert.AreEqual(0, dirSvc.GetLocalDirs().Count);
NUnit.Framework.Assert.AreEqual(0, dirSvc.GetLogDirs().Count);
NUnit.Framework.Assert.AreEqual(1, dirSvc.GetDiskFullLocalDirs().Count);
NUnit.Framework.Assert.AreEqual(1, dirSvc.GetDiskFullLogDirs().Count);
FileUtils.DeleteDirectory(new FilePath(localDir1));
FileUtils.DeleteDirectory(new FilePath(localDir2));
FileUtils.DeleteDirectory(new FilePath(logDir1));
FileUtils.DeleteDirectory(new FilePath(logDir1));
dirSvc.Close();
}
public virtual void TestValidPathsDirHandlerService()
{
Configuration conf = new YarnConfiguration();
string localDir1 = new FilePath("file:///" + testDir, "localDir1").GetPath();
string localDir2 = new FilePath("hdfs:///" + testDir, "localDir2").GetPath();
conf.Set(YarnConfiguration.NmLocalDirs, localDir1 + "," + localDir2);
string logDir1 = new FilePath("file:///" + testDir, "logDir1").GetPath();
conf.Set(YarnConfiguration.NmLogDirs, logDir1);
LocalDirsHandlerService dirSvc = new LocalDirsHandlerService();
try
{
dirSvc.Init(conf);
NUnit.Framework.Assert.Fail("Service should have thrown an exception due to wrong URI"
);
}
catch (YarnRuntimeException)
{
}
NUnit.Framework.Assert.AreEqual("Service should not be inited", Service.STATE.Stopped
, dirSvc.GetServiceState());
dirSvc.Close();
}
public virtual void TestContainerLaunchError()
{
// reinitialize executer
FilePath f = new FilePath("./src/test/resources/mock-container-executer-with-error"
);
if (!FileUtil.CanExecute(f))
{
FileUtil.SetExecutable(f, true);
}
string executorPath = f.GetAbsolutePath();
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.NmLinuxContainerExecutorPath, executorPath);
conf.Set(YarnConfiguration.NmLocalDirs, "file:///bin/echo");
conf.Set(YarnConfiguration.NmLogDirs, "file:///dev/null");
mockExec = Org.Mockito.Mockito.Spy(new LinuxContainerExecutor());
Org.Mockito.Mockito.DoAnswer(new _Answer_226()).When(mockExec).LogOutput(Matchers.Any
<string>());
dirsHandler = new LocalDirsHandlerService();
dirsHandler.Init(conf);
mockExec.SetConf(conf);
string appSubmitter = "nobody";
string cmd = LinuxContainerExecutor.Commands.LaunchContainer.GetValue().ToString(
);
string appId = "APP_ID";
string containerId = "CONTAINER_ID";
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
>();
ContainerId cId = Org.Mockito.Mockito.Mock <ContainerId>();
ContainerLaunchContext context = Org.Mockito.Mockito.Mock <ContainerLaunchContext>
();
Dictionary <string, string> env = new Dictionary <string, string>();
Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(cId);
Org.Mockito.Mockito.When(container.GetLaunchContext()).ThenReturn(context);
Org.Mockito.Mockito.DoAnswer(new _Answer_254()).When(container).Handle(Matchers.Any
<ContainerDiagnosticsUpdateEvent>());
Org.Mockito.Mockito.When(cId.ToString()).ThenReturn(containerId);
Org.Mockito.Mockito.When(context.GetEnvironment()).ThenReturn(env);
Path scriptPath = new Path("file:///bin/echo");
Path tokensPath = new Path("file:///dev/null");
Path workDir = new Path("/tmp");
Path pidFile = new Path(workDir, "pid.txt");
mockExec.ActivateContainer(cId, pidFile);
int ret = mockExec.LaunchContainer(container, scriptPath, tokensPath, appSubmitter
, appId, workDir, dirsHandler.GetLocalDirs(), dirsHandler.GetLogDirs());
NUnit.Framework.Assert.AreNotSame(0, ret);
NUnit.Framework.Assert.AreEqual(Arrays.AsList(YarnConfiguration.DefaultNmNonsecureModeLocalUser
, appSubmitter, cmd, appId, containerId, workDir.ToString(), "/bin/echo", "/dev/null"
, pidFile.ToString(), StringUtils.Join(",", dirsHandler.GetLocalDirs()), StringUtils
.Join(",", dirsHandler.GetLogDirs()), "cgroups=none"), ReadMockParams());
}
public virtual void TestDirStructure()
{
Configuration conf = new YarnConfiguration();
string localDir1 = new FilePath("file:///" + testDir, "localDir1").GetPath();
conf.Set(YarnConfiguration.NmLocalDirs, localDir1);
string logDir1 = new FilePath("file:///" + testDir, "logDir1").GetPath();
conf.Set(YarnConfiguration.NmLogDirs, logDir1);
LocalDirsHandlerService dirSvc = new LocalDirsHandlerService();
dirSvc.Init(conf);
NUnit.Framework.Assert.AreEqual(1, dirSvc.GetLocalDirs().Count);
dirSvc.Close();
}
public virtual void Setup()
{
Assume.AssumeTrue(!Path.Windows);
FilePath f = new FilePath("./src/test/resources/mock-container-executor");
if (!FileUtil.CanExecute(f))
{
FileUtil.SetExecutable(f, true);
}
string executorPath = f.GetAbsolutePath();
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.NmLinuxContainerExecutorPath, executorPath);
mockExec = new LinuxContainerExecutor();
dirsHandler = new LocalDirsHandlerService();
dirsHandler.Init(conf);
mockExec.SetConf(conf);
}
public virtual void Setup()
{
try
{
lfs = FileContext.GetLocalFSFileContext();
workDir = new Path("/tmp/temp-" + Runtime.CurrentTimeMillis());
workSpace = new FilePath(workDir.ToUri().GetPath());
lfs.Mkdir(workDir, FsPermission.GetDirDefault(), true);
}
catch (IOException e)
{
throw new RuntimeException(e);
}
Configuration conf = new Configuration();
yarnImage = "yarnImage";
long time = Runtime.CurrentTimeMillis();
conf.Set(YarnConfiguration.NmLocalDirs, "/tmp/nm-local-dir" + time);
conf.Set(YarnConfiguration.NmLogDirs, "/tmp/userlogs" + time);
dockerUrl = Runtime.GetProperty("docker-service-url");
Log.Info("dockerUrl: " + dockerUrl);
if (Strings.IsNullOrEmpty(dockerUrl))
{
return;
}
dockerUrl = " -H " + dockerUrl;
dockerExec = "docker " + dockerUrl;
conf.Set(YarnConfiguration.NmDockerContainerExecutorImageName, yarnImage);
conf.Set(YarnConfiguration.NmDockerContainerExecutorExecName, dockerExec);
exec = new DockerContainerExecutor();
dirsHandler = new LocalDirsHandlerService();
dirsHandler.Init(conf);
exec.SetConf(conf);
appSubmitter = Runtime.GetProperty("application.submitter");
if (appSubmitter == null || appSubmitter.IsEmpty())
{
appSubmitter = "nobody";
}
ShellExec(dockerExec + " pull " + testImage);
}
public virtual void Setup()
{
files = FileContext.GetLocalFSFileContext();
Path workSpacePath = new Path(workSpace.GetAbsolutePath());
files.Mkdir(workSpacePath, null, true);
FileUtil.Chmod(workSpace.GetAbsolutePath(), "777");
FilePath localDir = new FilePath(workSpace.GetAbsoluteFile(), "localDir");
files.Mkdir(new Path(localDir.GetAbsolutePath()), new FsPermission("777"), false);
FilePath logDir = new FilePath(workSpace.GetAbsoluteFile(), "logDir");
files.Mkdir(new Path(logDir.GetAbsolutePath()), new FsPermission("777"), false);
string exec_path = Runtime.GetProperty("container-executor.path");
if (exec_path != null && !exec_path.IsEmpty())
{
conf = new Configuration(false);
conf.SetClass("fs.AbstractFileSystem.file.impl", typeof(LocalFs), typeof(AbstractFileSystem
));
conf.Set(YarnConfiguration.NmNonsecureModeLocalUserKey, "xuan");
Log.Info("Setting " + YarnConfiguration.NmLinuxContainerExecutorPath + "=" + exec_path
);
conf.Set(YarnConfiguration.NmLinuxContainerExecutorPath, exec_path);
exec = new LinuxContainerExecutor();
exec.SetConf(conf);
conf.Set(YarnConfiguration.NmLocalDirs, localDir.GetAbsolutePath());
conf.Set(YarnConfiguration.NmLogDirs, logDir.GetAbsolutePath());
dirsHandler = new LocalDirsHandlerService();
dirsHandler.Init(conf);
}
appSubmitter = Runtime.GetProperty("application.submitter");
if (appSubmitter == null || appSubmitter.IsEmpty())
{
appSubmitter = "nobody";
}
}
public virtual void Setup()
{
Assume.AssumeTrue(Shell.Linux);
FilePath f = new FilePath("./src/test/resources/mock-container-executor");
if (!FileUtil.CanExecute(f))
{
FileUtil.SetExecutable(f, true);
}
string executorPath = f.GetAbsolutePath();
Configuration conf = new Configuration();
yarnImage = "yarnImage";
long time = Runtime.CurrentTimeMillis();
conf.Set(YarnConfiguration.NmLinuxContainerExecutorPath, executorPath);
conf.Set(YarnConfiguration.NmLocalDirs, "/tmp/nm-local-dir" + time);
conf.Set(YarnConfiguration.NmLogDirs, "/tmp/userlogs" + time);
conf.Set(YarnConfiguration.NmDockerContainerExecutorImageName, yarnImage);
conf.Set(YarnConfiguration.NmDockerContainerExecutorExecName, DockerLaunchCommand
);
dockerContainerExecutor = new DockerContainerExecutor();
dirsHandler = new LocalDirsHandlerService();
dirsHandler.Init(conf);
dockerContainerExecutor.SetConf(conf);
lfs = null;
try
{
lfs = FileContext.GetLocalFSFileContext();
workDir = new Path("/tmp/temp-" + Runtime.CurrentTimeMillis());
lfs.Mkdir(workDir, FsPermission.GetDirDefault(), true);
}
catch (IOException e)
{
throw new RuntimeException(e);
}
}
