public virtual void TestGetFullDirs() { Configuration conf = new YarnConfiguration(); conf.Set(CommonConfigurationKeys.FsPermissionsUmaskKey, "077"); FileContext localFs = FileContext.GetLocalFSFileContext(conf); string localDir1 = new FilePath(testDir, "localDir1").GetPath(); string localDir2 = new FilePath(testDir, "localDir2").GetPath(); string logDir1 = new FilePath(testDir, "logDir1").GetPath(); string logDir2 = new FilePath(testDir, "logDir2").GetPath(); Path localDir1Path = new Path(localDir1); Path logDir1Path = new Path(logDir1); FsPermission dirPermissions = new FsPermission((short)0x108); localFs.Mkdir(localDir1Path, dirPermissions, true); localFs.Mkdir(logDir1Path, dirPermissions, true); conf.Set(YarnConfiguration.NmLocalDirs, localDir1 + "," + localDir2); conf.Set(YarnConfiguration.NmLogDirs, logDir1 + "," + logDir2); conf.SetFloat(YarnConfiguration.NmMaxPerDiskUtilizationPercentage, 0.0f); LocalDirsHandlerService dirSvc = new LocalDirsHandlerService(); dirSvc.Init(conf); NUnit.Framework.Assert.AreEqual(0, dirSvc.GetLocalDirs().Count); NUnit.Framework.Assert.AreEqual(0, dirSvc.GetLogDirs().Count); NUnit.Framework.Assert.AreEqual(1, dirSvc.GetDiskFullLocalDirs().Count); NUnit.Framework.Assert.AreEqual(1, dirSvc.GetDiskFullLogDirs().Count); FileUtils.DeleteDirectory(new FilePath(localDir1)); FileUtils.DeleteDirectory(new FilePath(localDir2)); FileUtils.DeleteDirectory(new FilePath(logDir1)); FileUtils.DeleteDirectory(new FilePath(logDir1)); dirSvc.Close(); }
public virtual void TestValidPathsDirHandlerService() { Configuration conf = new YarnConfiguration(); string localDir1 = new FilePath("file:///" + testDir, "localDir1").GetPath(); string localDir2 = new FilePath("hdfs:///" + testDir, "localDir2").GetPath(); conf.Set(YarnConfiguration.NmLocalDirs, localDir1 + "," + localDir2); string logDir1 = new FilePath("file:///" + testDir, "logDir1").GetPath(); conf.Set(YarnConfiguration.NmLogDirs, logDir1); LocalDirsHandlerService dirSvc = new LocalDirsHandlerService(); try { dirSvc.Init(conf); NUnit.Framework.Assert.Fail("Service should have thrown an exception due to wrong URI" ); } catch (YarnRuntimeException) { } NUnit.Framework.Assert.AreEqual("Service should not be inited", Service.STATE.Stopped , dirSvc.GetServiceState()); dirSvc.Close(); }
public virtual void TestContainerLaunchError() { // reinitialize executer FilePath f = new FilePath("./src/test/resources/mock-container-executer-with-error" ); if (!FileUtil.CanExecute(f)) { FileUtil.SetExecutable(f, true); } string executorPath = f.GetAbsolutePath(); Configuration conf = new Configuration(); conf.Set(YarnConfiguration.NmLinuxContainerExecutorPath, executorPath); conf.Set(YarnConfiguration.NmLocalDirs, "file:///bin/echo"); conf.Set(YarnConfiguration.NmLogDirs, "file:///dev/null"); mockExec = Org.Mockito.Mockito.Spy(new LinuxContainerExecutor()); Org.Mockito.Mockito.DoAnswer(new _Answer_226()).When(mockExec).LogOutput(Matchers.Any <string>()); dirsHandler = new LocalDirsHandlerService(); dirsHandler.Init(conf); mockExec.SetConf(conf); string appSubmitter = "nobody"; string cmd = LinuxContainerExecutor.Commands.LaunchContainer.GetValue().ToString( ); string appId = "APP_ID"; string containerId = "CONTAINER_ID"; Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container >(); ContainerId cId = Org.Mockito.Mockito.Mock <ContainerId>(); ContainerLaunchContext context = Org.Mockito.Mockito.Mock <ContainerLaunchContext> (); Dictionary <string, string> env = new Dictionary <string, string>(); Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(cId); Org.Mockito.Mockito.When(container.GetLaunchContext()).ThenReturn(context); Org.Mockito.Mockito.DoAnswer(new _Answer_254()).When(container).Handle(Matchers.Any <ContainerDiagnosticsUpdateEvent>()); Org.Mockito.Mockito.When(cId.ToString()).ThenReturn(containerId); Org.Mockito.Mockito.When(context.GetEnvironment()).ThenReturn(env); Path scriptPath = new Path("file:///bin/echo"); Path tokensPath = new Path("file:///dev/null"); Path workDir = new Path("/tmp"); Path pidFile = new Path(workDir, "pid.txt"); mockExec.ActivateContainer(cId, pidFile); int ret = mockExec.LaunchContainer(container, scriptPath, tokensPath, appSubmitter , appId, workDir, dirsHandler.GetLocalDirs(), dirsHandler.GetLogDirs()); NUnit.Framework.Assert.AreNotSame(0, ret); NUnit.Framework.Assert.AreEqual(Arrays.AsList(YarnConfiguration.DefaultNmNonsecureModeLocalUser , appSubmitter, cmd, appId, containerId, workDir.ToString(), "/bin/echo", "/dev/null" , pidFile.ToString(), StringUtils.Join(",", dirsHandler.GetLocalDirs()), StringUtils .Join(",", dirsHandler.GetLogDirs()), "cgroups=none"), ReadMockParams()); }
public virtual void TestDirStructure() { Configuration conf = new YarnConfiguration(); string localDir1 = new FilePath("file:///" + testDir, "localDir1").GetPath(); conf.Set(YarnConfiguration.NmLocalDirs, localDir1); string logDir1 = new FilePath("file:///" + testDir, "logDir1").GetPath(); conf.Set(YarnConfiguration.NmLogDirs, logDir1); LocalDirsHandlerService dirSvc = new LocalDirsHandlerService(); dirSvc.Init(conf); NUnit.Framework.Assert.AreEqual(1, dirSvc.GetLocalDirs().Count); dirSvc.Close(); }
public virtual void Setup() { Assume.AssumeTrue(!Path.Windows); FilePath f = new FilePath("./src/test/resources/mock-container-executor"); if (!FileUtil.CanExecute(f)) { FileUtil.SetExecutable(f, true); } string executorPath = f.GetAbsolutePath(); Configuration conf = new Configuration(); conf.Set(YarnConfiguration.NmLinuxContainerExecutorPath, executorPath); mockExec = new LinuxContainerExecutor(); dirsHandler = new LocalDirsHandlerService(); dirsHandler.Init(conf); mockExec.SetConf(conf); }
public virtual void Setup() { try { lfs = FileContext.GetLocalFSFileContext(); workDir = new Path("/tmp/temp-" + Runtime.CurrentTimeMillis()); workSpace = new FilePath(workDir.ToUri().GetPath()); lfs.Mkdir(workDir, FsPermission.GetDirDefault(), true); } catch (IOException e) { throw new RuntimeException(e); } Configuration conf = new Configuration(); yarnImage = "yarnImage"; long time = Runtime.CurrentTimeMillis(); conf.Set(YarnConfiguration.NmLocalDirs, "/tmp/nm-local-dir" + time); conf.Set(YarnConfiguration.NmLogDirs, "/tmp/userlogs" + time); dockerUrl = Runtime.GetProperty("docker-service-url"); Log.Info("dockerUrl: " + dockerUrl); if (Strings.IsNullOrEmpty(dockerUrl)) { return; } dockerUrl = " -H " + dockerUrl; dockerExec = "docker " + dockerUrl; conf.Set(YarnConfiguration.NmDockerContainerExecutorImageName, yarnImage); conf.Set(YarnConfiguration.NmDockerContainerExecutorExecName, dockerExec); exec = new DockerContainerExecutor(); dirsHandler = new LocalDirsHandlerService(); dirsHandler.Init(conf); exec.SetConf(conf); appSubmitter = Runtime.GetProperty("application.submitter"); if (appSubmitter == null || appSubmitter.IsEmpty()) { appSubmitter = "nobody"; } ShellExec(dockerExec + " pull " + testImage); }
public virtual void Setup() { files = FileContext.GetLocalFSFileContext(); Path workSpacePath = new Path(workSpace.GetAbsolutePath()); files.Mkdir(workSpacePath, null, true); FileUtil.Chmod(workSpace.GetAbsolutePath(), "777"); FilePath localDir = new FilePath(workSpace.GetAbsoluteFile(), "localDir"); files.Mkdir(new Path(localDir.GetAbsolutePath()), new FsPermission("777"), false); FilePath logDir = new FilePath(workSpace.GetAbsoluteFile(), "logDir"); files.Mkdir(new Path(logDir.GetAbsolutePath()), new FsPermission("777"), false); string exec_path = Runtime.GetProperty("container-executor.path"); if (exec_path != null && !exec_path.IsEmpty()) { conf = new Configuration(false); conf.SetClass("fs.AbstractFileSystem.file.impl", typeof(LocalFs), typeof(AbstractFileSystem )); conf.Set(YarnConfiguration.NmNonsecureModeLocalUserKey, "xuan"); Log.Info("Setting " + YarnConfiguration.NmLinuxContainerExecutorPath + "=" + exec_path ); conf.Set(YarnConfiguration.NmLinuxContainerExecutorPath, exec_path); exec = new LinuxContainerExecutor(); exec.SetConf(conf); conf.Set(YarnConfiguration.NmLocalDirs, localDir.GetAbsolutePath()); conf.Set(YarnConfiguration.NmLogDirs, logDir.GetAbsolutePath()); dirsHandler = new LocalDirsHandlerService(); dirsHandler.Init(conf); } appSubmitter = Runtime.GetProperty("application.submitter"); if (appSubmitter == null || appSubmitter.IsEmpty()) { appSubmitter = "nobody"; } }
public virtual void Setup() { Assume.AssumeTrue(Shell.Linux); FilePath f = new FilePath("./src/test/resources/mock-container-executor"); if (!FileUtil.CanExecute(f)) { FileUtil.SetExecutable(f, true); } string executorPath = f.GetAbsolutePath(); Configuration conf = new Configuration(); yarnImage = "yarnImage"; long time = Runtime.CurrentTimeMillis(); conf.Set(YarnConfiguration.NmLinuxContainerExecutorPath, executorPath); conf.Set(YarnConfiguration.NmLocalDirs, "/tmp/nm-local-dir" + time); conf.Set(YarnConfiguration.NmLogDirs, "/tmp/userlogs" + time); conf.Set(YarnConfiguration.NmDockerContainerExecutorImageName, yarnImage); conf.Set(YarnConfiguration.NmDockerContainerExecutorExecName, DockerLaunchCommand ); dockerContainerExecutor = new DockerContainerExecutor(); dirsHandler = new LocalDirsHandlerService(); dirsHandler.Init(conf); dockerContainerExecutor.SetConf(conf); lfs = null; try { lfs = FileContext.GetLocalFSFileContext(); workDir = new Path("/tmp/temp-" + Runtime.CurrentTimeMillis()); lfs.Mkdir(workDir, FsPermission.GetDirDefault(), true); } catch (IOException e) { throw new RuntimeException(e); } }