protected internal LocalWrapperScriptBuilder(DefaultContainerExecutor _enclosing, Path containerWorkDir) { this._enclosing = _enclosing; this.wrapperScriptPath = new Path(containerWorkDir, Shell.AppendScriptExtension("default_container_executor" )); }
public WindowsLocalWrapperScriptBuilder(DefaultContainerExecutor _enclosing, string containerIdStr, Path containerWorkDir) : base(_enclosing) { this._enclosing = _enclosing; this.containerIdStr = containerIdStr; }
public UnixLocalWrapperScriptBuilder(DefaultContainerExecutor _enclosing, Path containerWorkDir ) : base(_enclosing) { this._enclosing = _enclosing; this.sessionScriptPath = new Path(containerWorkDir, Shell.AppendScriptExtension("default_container_executor_session" )); }
public virtual void TestKillContainersOnShutdown() { nm = new TestNodeManagerShutdown.TestNodeManager(this); nm.Init(CreateNMConfig()); nm.Start(); StartContainer(nm, cId, localFS, tmpDir, processStartFile); int MaxTries = 20; int numTries = 0; while (!processStartFile.Exists() && numTries < MaxTries) { try { Sharpen.Thread.Sleep(500); } catch (Exception ex) { Sharpen.Runtime.PrintStackTrace(ex); } numTries++; } nm.Stop(); // Now verify the contents of the file. Script generates a message when it // receives a sigterm so we look for that. We cannot perform this check on // Windows, because the process is not notified when killed by winutils. // There is no way for the process to trap and respond. Instead, we can // verify that the job object with ID matching container ID no longer exists. if (Shell.Windows) { NUnit.Framework.Assert.IsFalse("Process is still alive!", DefaultContainerExecutor .ContainerIsAlive(cId.ToString())); } else { BufferedReader reader = new BufferedReader(new FileReader(processStartFile)); bool foundSigTermMessage = false; while (true) { string line = reader.ReadLine(); if (line == null) { break; } if (line.Contains("SIGTERM")) { foundSigTermMessage = true; break; } } NUnit.Framework.Assert.IsTrue("Did not find sigterm message", foundSigTermMessage ); reader.Close(); } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public virtual void TestStartLocalizer() { IPEndPoint localizationServerAddress; Path firstDir = new Path(BaseTmpPath, "localDir1"); IList <string> localDirs = new AList <string>(); Path secondDir = new Path(BaseTmpPath, "localDir2"); IList <string> logDirs = new AList <string>(); Path logDir = new Path(BaseTmpPath, "logDir"); Path tokenDir = new Path(BaseTmpPath, "tokenDir"); FsPermission perms = new FsPermission((short)0x1f8); Configuration conf = new Configuration(); localizationServerAddress = conf.GetSocketAddr(YarnConfiguration.NmBindHost, YarnConfiguration .NmLocalizerAddress, YarnConfiguration.DefaultNmLocalizerAddress, YarnConfiguration .DefaultNmLocalizerPort); FileContext mockLfs = Org.Mockito.Mockito.Spy(FileContext.GetLocalFSFileContext(conf )); FileContext.Util mockUtil = Org.Mockito.Mockito.Spy(mockLfs.Util()); Org.Mockito.Mockito.DoAnswer(new _Answer_344(mockUtil)).When(mockLfs).Util(); Org.Mockito.Mockito.DoAnswer(new _Answer_351(firstDir, mockLfs)).When(mockUtil).Copy (Matchers.Any <Path>(), Matchers.Any <Path>()); // throw an Exception when copy token to the first local dir // to simulate no space on the first drive // copy token to the second local dir Org.Mockito.Mockito.DoAnswer(new _Answer_378(firstDir)).When(mockLfs).GetFsStatus (Matchers.Any <Path>()); // let second local directory return more free space than // first local directory DefaultContainerExecutor mockExec = Org.Mockito.Mockito.Spy(new DefaultContainerExecutor (mockLfs)); mockExec.SetConf(conf); localDirs.AddItem(mockLfs.MakeQualified(firstDir).ToString()); localDirs.AddItem(mockLfs.MakeQualified(secondDir).ToString()); logDirs.AddItem(mockLfs.MakeQualified(logDir).ToString()); conf.SetStrings(YarnConfiguration.NmLocalDirs, Sharpen.Collections.ToArray(localDirs , new string[localDirs.Count])); conf.Set(YarnConfiguration.NmLogDirs, logDir.ToString()); mockLfs.Mkdir(tokenDir, perms, true); Path nmPrivateCTokensPath = new Path(tokenDir, "test.tokens"); string appSubmitter = "nobody"; string appId = "APP_ID"; string locId = "LOC_ID"; LocalDirsHandlerService dirsHandler = Org.Mockito.Mockito.Mock <LocalDirsHandlerService >(); Org.Mockito.Mockito.When(dirsHandler.GetLocalDirs()).ThenReturn(localDirs); Org.Mockito.Mockito.When(dirsHandler.GetLogDirs()).ThenReturn(logDirs); try { mockExec.StartLocalizer(nmPrivateCTokensPath, localizationServerAddress, appSubmitter , appId, locId, dirsHandler); } catch (IOException e) { NUnit.Framework.Assert.Fail("StartLocalizer failed to copy token file " + e); } finally { mockExec.DeleteAsUser(appSubmitter, firstDir); mockExec.DeleteAsUser(appSubmitter, secondDir); mockExec.DeleteAsUser(appSubmitter, logDir); DeleteTmpFiles(); } }
public virtual void TestContainerLaunchError() { if (Shell.Windows) { BaseTmpPath = new Path(new FilePath("target").GetAbsolutePath(), typeof(TestDefaultContainerExecutor ).Name); } Path localDir = new Path(BaseTmpPath, "localDir"); IList <string> localDirs = new AList <string>(); localDirs.AddItem(localDir.ToString()); IList <string> logDirs = new AList <string>(); Path logDir = new Path(BaseTmpPath, "logDir"); logDirs.AddItem(logDir.ToString()); Configuration conf = new Configuration(); conf.Set(CommonConfigurationKeys.FsPermissionsUmaskKey, "077"); conf.Set(YarnConfiguration.NmLocalDirs, localDir.ToString()); conf.Set(YarnConfiguration.NmLogDirs, logDir.ToString()); FileContext lfs = FileContext.GetLocalFSFileContext(conf); DefaultContainerExecutor mockExec = Org.Mockito.Mockito.Spy(new DefaultContainerExecutor (lfs)); mockExec.SetConf(conf); Org.Mockito.Mockito.DoAnswer(new _Answer_245()).When(mockExec).LogOutput(Matchers.Any <string>()); string appSubmitter = "nobody"; string appId = "APP_ID"; string containerId = "CONTAINER_ID"; Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container >(); ContainerId cId = Org.Mockito.Mockito.Mock <ContainerId>(); ContainerLaunchContext context = Org.Mockito.Mockito.Mock <ContainerLaunchContext> (); Dictionary <string, string> env = new Dictionary <string, string>(); Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(cId); Org.Mockito.Mockito.When(container.GetLaunchContext()).ThenReturn(context); try { Org.Mockito.Mockito.DoAnswer(new _Answer_268()).When(container).Handle(Matchers.Any <ContainerDiagnosticsUpdateEvent>()); Org.Mockito.Mockito.When(cId.ToString()).ThenReturn(containerId); Org.Mockito.Mockito.When(cId.GetApplicationAttemptId()).ThenReturn(ApplicationAttemptId .NewInstance(ApplicationId.NewInstance(0, 1), 0)); Org.Mockito.Mockito.When(context.GetEnvironment()).ThenReturn(env); mockExec.CreateUserLocalDirs(localDirs, appSubmitter); mockExec.CreateUserCacheDirs(localDirs, appSubmitter); mockExec.CreateAppDirs(localDirs, appSubmitter, appId); mockExec.CreateAppLogDirs(appId, logDirs, appSubmitter); Path scriptPath = new Path("file:///bin/echo"); Path tokensPath = new Path("file:///dev/null"); if (Shell.Windows) { FilePath tmp = new FilePath(BaseTmpPath.ToString(), "test_echo.cmd"); BufferedWriter output = new BufferedWriter(new FileWriter(tmp)); output.Write("Exit 1"); output.Write("Echo No such file or directory 1>&2"); output.Close(); scriptPath = new Path(tmp.GetAbsolutePath()); tmp = new FilePath(BaseTmpPath.ToString(), "tokens"); tmp.CreateNewFile(); tokensPath = new Path(tmp.GetAbsolutePath()); } Path workDir = localDir; Path pidFile = new Path(workDir, "pid.txt"); mockExec.Init(); mockExec.ActivateContainer(cId, pidFile); int ret = mockExec.LaunchContainer(container, scriptPath, tokensPath, appSubmitter , appId, workDir, localDirs, localDirs); NUnit.Framework.Assert.AreNotSame(0, ret); } finally { mockExec.DeleteAsUser(appSubmitter, localDir); mockExec.DeleteAsUser(appSubmitter, logDir); } }
public virtual void TestDirPermissions() { DeleteTmpFiles(); string user = "******"; string appId = "app_12345_123"; FsPermission userCachePerm = new FsPermission(DefaultContainerExecutor.UserPerm); FsPermission appCachePerm = new FsPermission(DefaultContainerExecutor.AppcachePerm ); FsPermission fileCachePerm = new FsPermission(DefaultContainerExecutor.FilecachePerm ); FsPermission appDirPerm = new FsPermission(DefaultContainerExecutor.AppdirPerm); FsPermission logDirPerm = new FsPermission(DefaultContainerExecutor.LogdirPerm); IList <string> localDirs = new AList <string>(); localDirs.AddItem(new Path(BaseTmpPath, "localDirA").ToString()); localDirs.AddItem(new Path(BaseTmpPath, "localDirB").ToString()); IList <string> logDirs = new AList <string>(); logDirs.AddItem(new Path(BaseTmpPath, "logDirA").ToString()); logDirs.AddItem(new Path(BaseTmpPath, "logDirB").ToString()); Configuration conf = new Configuration(); conf.Set(CommonConfigurationKeys.FsPermissionsUmaskKey, "077"); FileContext lfs = FileContext.GetLocalFSFileContext(conf); DefaultContainerExecutor executor = new DefaultContainerExecutor(lfs); executor.Init(); try { executor.CreateUserLocalDirs(localDirs, user); executor.CreateUserCacheDirs(localDirs, user); executor.CreateAppDirs(localDirs, user, appId); foreach (string dir in localDirs) { FileStatus stats = lfs.GetFileStatus(new Path(new Path(dir, ContainerLocalizer.Usercache ), user)); NUnit.Framework.Assert.AreEqual(userCachePerm, stats.GetPermission()); } foreach (string dir_1 in localDirs) { Path userCachePath = new Path(new Path(dir_1, ContainerLocalizer.Usercache), user ); Path appCachePath = new Path(userCachePath, ContainerLocalizer.Appcache); FileStatus stats = lfs.GetFileStatus(appCachePath); NUnit.Framework.Assert.AreEqual(appCachePerm, stats.GetPermission()); stats = lfs.GetFileStatus(new Path(userCachePath, ContainerLocalizer.Filecache)); NUnit.Framework.Assert.AreEqual(fileCachePerm, stats.GetPermission()); stats = lfs.GetFileStatus(new Path(appCachePath, appId)); NUnit.Framework.Assert.AreEqual(appDirPerm, stats.GetPermission()); } executor.CreateAppLogDirs(appId, logDirs, user); foreach (string dir_2 in logDirs) { FileStatus stats = lfs.GetFileStatus(new Path(dir_2, appId)); NUnit.Framework.Assert.AreEqual(logDirPerm, stats.GetPermission()); } } finally { DeleteTmpFiles(); } }
public virtual void TestSuccessfulContainerLaunch() { FileContext localFS = FileContext.GetLocalFSFileContext(); localFS.Delete(new Path(localDir.GetAbsolutePath()), true); localFS.Delete(new Path(localLogDir.GetAbsolutePath()), true); localFS.Delete(new Path(remoteLogDir.GetAbsolutePath()), true); localDir.Mkdir(); localLogDir.Mkdir(); remoteLogDir.Mkdir(); YarnConfiguration conf = new YarnConfiguration(); Context context = new _NMContext_84(new NMContainerTokenSecretManager(conf), new NMTokenSecretManagerInNM(), null, null, new NMNullStateStoreService()); conf.Set(YarnConfiguration.NmLocalDirs, localDir.GetAbsolutePath()); conf.Set(YarnConfiguration.NmLogDirs, localLogDir.GetAbsolutePath()); conf.Set(YarnConfiguration.NmRemoteAppLogDir, remoteLogDir.GetAbsolutePath()); ContainerExecutor exec = new DefaultContainerExecutor(); exec.SetConf(conf); DeletionService del = new DeletionService(exec); Dispatcher dispatcher = new AsyncDispatcher(); NodeHealthCheckerService healthChecker = new NodeHealthCheckerService(); healthChecker.Init(conf); LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler(); NodeManagerMetrics metrics = NodeManagerMetrics.Create(); NodeStatusUpdater nodeStatusUpdater = new _NodeStatusUpdaterImpl_106(context, dispatcher , healthChecker, metrics); // Don't start any updating thread. DummyContainerManager containerManager = new DummyContainerManager(context, exec, del, nodeStatusUpdater, metrics, new ApplicationACLsManager(conf), dirsHandler); nodeStatusUpdater.Init(conf); ((NodeManager.NMContext)context).SetContainerManager(containerManager); nodeStatusUpdater.Start(); containerManager.Init(conf); containerManager.Start(); ContainerLaunchContext launchContext = recordFactory.NewRecordInstance <ContainerLaunchContext >(); ApplicationId applicationId = ApplicationId.NewInstance(0, 0); ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.NewInstance(applicationId , 0); ContainerId cID = ContainerId.NewContainerId(applicationAttemptId, 0); string user = "******"; StartContainerRequest scRequest = StartContainerRequest.NewInstance(launchContext , TestContainerManager.CreateContainerToken(cID, SimulatedRmIdentifier, context. GetNodeId(), user, context.GetContainerTokenSecretManager())); IList <StartContainerRequest> list = new AList <StartContainerRequest>(); list.AddItem(scRequest); StartContainersRequest allRequests = StartContainersRequest.NewInstance(list); containerManager.StartContainers(allRequests); BaseContainerManagerTest.WaitForContainerState(containerManager, cID, ContainerState .Running); IList <ContainerId> containerIds = new AList <ContainerId>(); containerIds.AddItem(cID); StopContainersRequest stopRequest = StopContainersRequest.NewInstance(containerIds ); containerManager.StopContainers(stopRequest); BaseContainerManagerTest.WaitForContainerState(containerManager, cID, ContainerState .Complete); containerManager.Stop(); }