public virtual void TestGetFullDirs() { Configuration conf = new YarnConfiguration(); conf.Set(CommonConfigurationKeys.FsPermissionsUmaskKey, "077"); FileContext localFs = FileContext.GetLocalFSFileContext(conf); string localDir1 = new FilePath(testDir, "localDir1").GetPath(); string localDir2 = new FilePath(testDir, "localDir2").GetPath(); string logDir1 = new FilePath(testDir, "logDir1").GetPath(); string logDir2 = new FilePath(testDir, "logDir2").GetPath(); Path localDir1Path = new Path(localDir1); Path logDir1Path = new Path(logDir1); FsPermission dirPermissions = new FsPermission((short)0x108); localFs.Mkdir(localDir1Path, dirPermissions, true); localFs.Mkdir(logDir1Path, dirPermissions, true); conf.Set(YarnConfiguration.NmLocalDirs, localDir1 + "," + localDir2); conf.Set(YarnConfiguration.NmLogDirs, logDir1 + "," + logDir2); conf.SetFloat(YarnConfiguration.NmMaxPerDiskUtilizationPercentage, 0.0f); LocalDirsHandlerService dirSvc = new LocalDirsHandlerService(); dirSvc.Init(conf); NUnit.Framework.Assert.AreEqual(0, dirSvc.GetLocalDirs().Count); NUnit.Framework.Assert.AreEqual(0, dirSvc.GetLogDirs().Count); NUnit.Framework.Assert.AreEqual(1, dirSvc.GetDiskFullLocalDirs().Count); NUnit.Framework.Assert.AreEqual(1, dirSvc.GetDiskFullLogDirs().Count); FileUtils.DeleteDirectory(new FilePath(localDir1)); FileUtils.DeleteDirectory(new FilePath(localDir2)); FileUtils.DeleteDirectory(new FilePath(logDir1)); FileUtils.DeleteDirectory(new FilePath(logDir1)); dirSvc.Close(); }
public virtual void TestValidPathsDirHandlerService() { Configuration conf = new YarnConfiguration(); string localDir1 = new FilePath("file:///" + testDir, "localDir1").GetPath(); string localDir2 = new FilePath("hdfs:///" + testDir, "localDir2").GetPath(); conf.Set(YarnConfiguration.NmLocalDirs, localDir1 + "," + localDir2); string logDir1 = new FilePath("file:///" + testDir, "logDir1").GetPath(); conf.Set(YarnConfiguration.NmLogDirs, logDir1); LocalDirsHandlerService dirSvc = new LocalDirsHandlerService(); try { dirSvc.Init(conf); NUnit.Framework.Assert.Fail("Service should have thrown an exception due to wrong URI" ); } catch (YarnRuntimeException) { } NUnit.Framework.Assert.AreEqual("Service should not be inited", Service.STATE.Stopped , dirSvc.GetServiceState()); dirSvc.Close(); }
/// <exception cref="System.Exception"/> protected override void ServiceInit(Configuration conf) { conf.SetBoolean(Dispatcher.DispatcherExitOnErrorKey, true); rmWorkPreservingRestartEnabled = conf.GetBoolean(YarnConfiguration.RmWorkPreservingRecoveryEnabled , YarnConfiguration.DefaultRmWorkPreservingRecoveryEnabled); InitAndStartRecoveryStore(conf); NMContainerTokenSecretManager containerTokenSecretManager = new NMContainerTokenSecretManager (conf, nmStore); NMTokenSecretManagerInNM nmTokenSecretManager = new NMTokenSecretManagerInNM(nmStore ); RecoverTokens(nmTokenSecretManager, containerTokenSecretManager); this.aclsManager = new ApplicationACLsManager(conf); ContainerExecutor exec = ReflectionUtils.NewInstance(conf.GetClass <ContainerExecutor >(YarnConfiguration.NmContainerExecutor, typeof(DefaultContainerExecutor)), conf ); try { exec.Init(); } catch (IOException e) { throw new YarnRuntimeException("Failed to initialize container executor", e); } DeletionService del = CreateDeletionService(exec); AddService(del); // NodeManager level dispatcher this.dispatcher = new AsyncDispatcher(); nodeHealthChecker = new NodeHealthCheckerService(); AddService(nodeHealthChecker); dirsHandler = nodeHealthChecker.GetDiskHandler(); this.context = CreateNMContext(containerTokenSecretManager, nmTokenSecretManager, nmStore); nodeStatusUpdater = CreateNodeStatusUpdater(context, dispatcher, nodeHealthChecker ); NodeResourceMonitor nodeResourceMonitor = CreateNodeResourceMonitor(); AddService(nodeResourceMonitor); containerManager = CreateContainerManager(context, exec, del, nodeStatusUpdater, this.aclsManager, dirsHandler); AddService(containerManager); ((NodeManager.NMContext)context).SetContainerManager(containerManager); WebServer webServer = CreateWebServer(context, containerManager.GetContainersMonitor (), this.aclsManager, dirsHandler); AddService(webServer); ((NodeManager.NMContext)context).SetWebServer(webServer); dispatcher.Register(typeof(ContainerManagerEventType), containerManager); dispatcher.Register(typeof(NodeManagerEventType), this); AddService(dispatcher); DefaultMetricsSystem.Initialize("NodeManager"); // StatusUpdater should be added last so that it get started last // so that we make sure everything is up before registering with RM. AddService(nodeStatusUpdater); base.ServiceInit(conf); }
public virtual void TestContainerLaunchError() { // reinitialize executer FilePath f = new FilePath("./src/test/resources/mock-container-executer-with-error" ); if (!FileUtil.CanExecute(f)) { FileUtil.SetExecutable(f, true); } string executorPath = f.GetAbsolutePath(); Configuration conf = new Configuration(); conf.Set(YarnConfiguration.NmLinuxContainerExecutorPath, executorPath); conf.Set(YarnConfiguration.NmLocalDirs, "file:///bin/echo"); conf.Set(YarnConfiguration.NmLogDirs, "file:///dev/null"); mockExec = Org.Mockito.Mockito.Spy(new LinuxContainerExecutor()); Org.Mockito.Mockito.DoAnswer(new _Answer_226()).When(mockExec).LogOutput(Matchers.Any <string>()); dirsHandler = new LocalDirsHandlerService(); dirsHandler.Init(conf); mockExec.SetConf(conf); string appSubmitter = "nobody"; string cmd = LinuxContainerExecutor.Commands.LaunchContainer.GetValue().ToString( ); string appId = "APP_ID"; string containerId = "CONTAINER_ID"; Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container >(); ContainerId cId = Org.Mockito.Mockito.Mock <ContainerId>(); ContainerLaunchContext context = Org.Mockito.Mockito.Mock <ContainerLaunchContext> (); Dictionary <string, string> env = new Dictionary <string, string>(); Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(cId); Org.Mockito.Mockito.When(container.GetLaunchContext()).ThenReturn(context); Org.Mockito.Mockito.DoAnswer(new _Answer_254()).When(container).Handle(Matchers.Any <ContainerDiagnosticsUpdateEvent>()); Org.Mockito.Mockito.When(cId.ToString()).ThenReturn(containerId); Org.Mockito.Mockito.When(context.GetEnvironment()).ThenReturn(env); Path scriptPath = new Path("file:///bin/echo"); Path tokensPath = new Path("file:///dev/null"); Path workDir = new Path("/tmp"); Path pidFile = new Path(workDir, "pid.txt"); mockExec.ActivateContainer(cId, pidFile); int ret = mockExec.LaunchContainer(container, scriptPath, tokensPath, appSubmitter , appId, workDir, dirsHandler.GetLocalDirs(), dirsHandler.GetLogDirs()); NUnit.Framework.Assert.AreNotSame(0, ret); NUnit.Framework.Assert.AreEqual(Arrays.AsList(YarnConfiguration.DefaultNmNonsecureModeLocalUser , appSubmitter, cmd, appId, containerId, workDir.ToString(), "/bin/echo", "/dev/null" , pidFile.ToString(), StringUtils.Join(",", dirsHandler.GetLocalDirs()), StringUtils .Join(",", dirsHandler.GetLogDirs()), "cgroups=none"), ReadMockParams()); }
public NMContext(NMContainerTokenSecretManager containerTokenSecretManager, NMTokenSecretManagerInNM nmTokenSecretManager, LocalDirsHandlerService dirsHandler, ApplicationACLsManager aclsManager, NMStateStoreService stateStore) { this.containerTokenSecretManager = containerTokenSecretManager; this.nmTokenSecretManager = nmTokenSecretManager; this.dirsHandler = dirsHandler; this.aclsManager = aclsManager; this.nodeHealthStatus.SetIsNodeHealthy(true); this.nodeHealthStatus.SetHealthReport("Healthy"); this.nodeHealthStatus.SetLastHealthReportTime(Runtime.CurrentTimeMillis()); this.stateStore = stateStore; }
public virtual void TestDirStructure() { Configuration conf = new YarnConfiguration(); string localDir1 = new FilePath("file:///" + testDir, "localDir1").GetPath(); conf.Set(YarnConfiguration.NmLocalDirs, localDir1); string logDir1 = new FilePath("file:///" + testDir, "logDir1").GetPath(); conf.Set(YarnConfiguration.NmLogDirs, logDir1); LocalDirsHandlerService dirSvc = new LocalDirsHandlerService(); dirSvc.Init(conf); NUnit.Framework.Assert.AreEqual(1, dirSvc.GetLocalDirs().Count); dirSvc.Close(); }
public virtual void Setup() { Assume.AssumeTrue(!Path.Windows); FilePath f = new FilePath("./src/test/resources/mock-container-executor"); if (!FileUtil.CanExecute(f)) { FileUtil.SetExecutable(f, true); } string executorPath = f.GetAbsolutePath(); Configuration conf = new Configuration(); conf.Set(YarnConfiguration.NmLinuxContainerExecutorPath, executorPath); mockExec = new LinuxContainerExecutor(); dirsHandler = new LocalDirsHandlerService(); dirsHandler.Init(conf); mockExec.SetConf(conf); }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnRuntimeException"/> public MonitoringTimerTask(LocalDirsHandlerService _enclosing, Configuration conf ) { this._enclosing = _enclosing; float maxUsableSpacePercentagePerDisk = conf.GetFloat(YarnConfiguration.NmMaxPerDiskUtilizationPercentage , YarnConfiguration.DefaultNmMaxPerDiskUtilizationPercentage); long minFreeSpacePerDiskMB = conf.GetLong(YarnConfiguration.NmMinPerDiskFreeSpaceMb , YarnConfiguration.DefaultNmMinPerDiskFreeSpaceMb); this._enclosing.localDirs = new DirectoryCollection(LocalDirsHandlerService.ValidatePaths (conf.GetTrimmedStrings(YarnConfiguration.NmLocalDirs)), maxUsableSpacePercentagePerDisk , minFreeSpacePerDiskMB); this._enclosing.logDirs = new DirectoryCollection(LocalDirsHandlerService.ValidatePaths (conf.GetTrimmedStrings(YarnConfiguration.NmLogDirs)), maxUsableSpacePercentagePerDisk , minFreeSpacePerDiskMB); this._enclosing.localDirsAllocator = new LocalDirAllocator(YarnConfiguration.NmLocalDirs ); this._enclosing.logDirsAllocator = new LocalDirAllocator(YarnConfiguration.NmLogDirs ); }
public virtual void Setup() { try { lfs = FileContext.GetLocalFSFileContext(); workDir = new Path("/tmp/temp-" + Runtime.CurrentTimeMillis()); workSpace = new FilePath(workDir.ToUri().GetPath()); lfs.Mkdir(workDir, FsPermission.GetDirDefault(), true); } catch (IOException e) { throw new RuntimeException(e); } Configuration conf = new Configuration(); yarnImage = "yarnImage"; long time = Runtime.CurrentTimeMillis(); conf.Set(YarnConfiguration.NmLocalDirs, "/tmp/nm-local-dir" + time); conf.Set(YarnConfiguration.NmLogDirs, "/tmp/userlogs" + time); dockerUrl = Runtime.GetProperty("docker-service-url"); Log.Info("dockerUrl: " + dockerUrl); if (Strings.IsNullOrEmpty(dockerUrl)) { return; } dockerUrl = " -H " + dockerUrl; dockerExec = "docker " + dockerUrl; conf.Set(YarnConfiguration.NmDockerContainerExecutorImageName, yarnImage); conf.Set(YarnConfiguration.NmDockerContainerExecutorExecName, dockerExec); exec = new DockerContainerExecutor(); dirsHandler = new LocalDirsHandlerService(); dirsHandler.Init(conf); exec.SetConf(conf); appSubmitter = Runtime.GetProperty("application.submitter"); if (appSubmitter == null || appSubmitter.IsEmpty()) { appSubmitter = "nobody"; } ShellExec(dockerExec + " pull " + testImage); }
public virtual void Setup() { Assume.AssumeTrue(Shell.Linux); FilePath f = new FilePath("./src/test/resources/mock-container-executor"); if (!FileUtil.CanExecute(f)) { FileUtil.SetExecutable(f, true); } string executorPath = f.GetAbsolutePath(); Configuration conf = new Configuration(); yarnImage = "yarnImage"; long time = Runtime.CurrentTimeMillis(); conf.Set(YarnConfiguration.NmLinuxContainerExecutorPath, executorPath); conf.Set(YarnConfiguration.NmLocalDirs, "/tmp/nm-local-dir" + time); conf.Set(YarnConfiguration.NmLogDirs, "/tmp/userlogs" + time); conf.Set(YarnConfiguration.NmDockerContainerExecutorImageName, yarnImage); conf.Set(YarnConfiguration.NmDockerContainerExecutorExecName, DockerLaunchCommand ); dockerContainerExecutor = new DockerContainerExecutor(); dirsHandler = new LocalDirsHandlerService(); dirsHandler.Init(conf); dockerContainerExecutor.SetConf(conf); lfs = null; try { lfs = FileContext.GetLocalFSFileContext(); workDir = new Path("/tmp/temp-" + Runtime.CurrentTimeMillis()); lfs.Mkdir(workDir, FsPermission.GetDirDefault(), true); } catch (IOException e) { throw new RuntimeException(e); } }
public virtual void Setup() { files = FileContext.GetLocalFSFileContext(); Path workSpacePath = new Path(workSpace.GetAbsolutePath()); files.Mkdir(workSpacePath, null, true); FileUtil.Chmod(workSpace.GetAbsolutePath(), "777"); FilePath localDir = new FilePath(workSpace.GetAbsoluteFile(), "localDir"); files.Mkdir(new Path(localDir.GetAbsolutePath()), new FsPermission("777"), false); FilePath logDir = new FilePath(workSpace.GetAbsoluteFile(), "logDir"); files.Mkdir(new Path(logDir.GetAbsolutePath()), new FsPermission("777"), false); string exec_path = Runtime.GetProperty("container-executor.path"); if (exec_path != null && !exec_path.IsEmpty()) { conf = new Configuration(false); conf.SetClass("fs.AbstractFileSystem.file.impl", typeof(LocalFs), typeof(AbstractFileSystem )); conf.Set(YarnConfiguration.NmNonsecureModeLocalUserKey, "xuan"); Log.Info("Setting " + YarnConfiguration.NmLinuxContainerExecutorPath + "=" + exec_path ); conf.Set(YarnConfiguration.NmLinuxContainerExecutorPath, exec_path); exec = new LinuxContainerExecutor(); exec.SetConf(conf); conf.Set(YarnConfiguration.NmLocalDirs, localDir.GetAbsolutePath()); conf.Set(YarnConfiguration.NmLogDirs, logDir.GetAbsolutePath()); dirsHandler = new LocalDirsHandlerService(); dirsHandler.Init(conf); } appSubmitter = Runtime.GetProperty("application.submitter"); if (appSubmitter == null || appSubmitter.IsEmpty()) { appSubmitter = "nobody"; } }
protected internal virtual WebServer CreateWebServer(Context nmContext, ResourceView resourceView, ApplicationACLsManager aclsManager, LocalDirsHandlerService dirsHandler ) { return(new WebServer(nmContext, resourceView, aclsManager, dirsHandler)); }
protected internal virtual ContainerManagerImpl CreateContainerManager(Context context , ContainerExecutor exec, DeletionService del, NodeStatusUpdater nodeStatusUpdater , ApplicationACLsManager aclsManager, LocalDirsHandlerService dirsHandler) { return(new ContainerManagerImpl(context, exec, del, nodeStatusUpdater, metrics, aclsManager , dirsHandler)); }
/// <summary>Prepare the environment for containers in this application to execute.</summary> /// <remarks> /// Prepare the environment for containers in this application to execute. /// <pre> /// For $x in local.dirs /// create $x/$user/$appId /// Copy $nmLocal/appTokens /// <literal>-></literal> /// $N/$user/$appId /// For $rsrc in private resources /// Copy $rsrc /// <literal>-></literal> /// $N/$user/filecache/[idef] /// For $rsrc in job resources /// Copy $rsrc /// <literal>-></literal> /// $N/$user/$appId/filecache/idef /// </pre> /// </remarks> /// <param name="user">user name of application owner</param> /// <param name="appId">id of the application</param> /// <param name="nmPrivateContainerTokens">path to localized credentials, rsrc by NM</param> /// <param name="nmAddr">RPC address to contact NM</param> /// <param name="dirsHandler">NM local dirs service, for nm-local-dirs and nm-log-dirs /// </param> /// <exception cref="System.IO.IOException">For most application init failures</exception> /// <exception cref="System.Exception">If application init thread is halted by NM</exception> public abstract void StartLocalizer(Path nmPrivateContainerTokens, IPEndPoint nmAddr , string user, string appId, string locId, LocalDirsHandlerService dirsHandler);
public _NMContext_84(NMContainerTokenSecretManager baseArg1, NMTokenSecretManagerInNM baseArg2, LocalDirsHandlerService baseArg3, ApplicationACLsManager baseArg4, NMStateStoreService baseArg5) : base(baseArg1, baseArg2, baseArg3, baseArg4, baseArg5) { }
public _ContainerManagerImpl_438(TestNodeManager2 _enclosing, Context baseArg1, ContainerExecutor baseArg2, DeletionService baseArg3, NodeStatusUpdater baseArg4, NodeManagerMetrics baseArg5, ApplicationACLsManager baseArg6, LocalDirsHandlerService baseArg7) : base(baseArg1, baseArg2, baseArg3, baseArg4, baseArg5, baseArg6, baseArg7) { this._enclosing = _enclosing; }
protected internal override ContainerManagerImpl CreateContainerManager(Context context , ContainerExecutor exec, DeletionService del, NodeStatusUpdater nodeStatusUpdater , ApplicationACLsManager aclsManager, LocalDirsHandlerService dirsHandler) { return(new _ContainerManagerImpl_438(this, context, exec, del, nodeStatusUpdater, this.metrics, aclsManager, dirsHandler)); }
public virtual void TestSuccessfulContainerLaunch() { FileContext localFS = FileContext.GetLocalFSFileContext(); localFS.Delete(new Path(localDir.GetAbsolutePath()), true); localFS.Delete(new Path(localLogDir.GetAbsolutePath()), true); localFS.Delete(new Path(remoteLogDir.GetAbsolutePath()), true); localDir.Mkdir(); localLogDir.Mkdir(); remoteLogDir.Mkdir(); YarnConfiguration conf = new YarnConfiguration(); Context context = new _NMContext_84(new NMContainerTokenSecretManager(conf), new NMTokenSecretManagerInNM(), null, null, new NMNullStateStoreService()); conf.Set(YarnConfiguration.NmLocalDirs, localDir.GetAbsolutePath()); conf.Set(YarnConfiguration.NmLogDirs, localLogDir.GetAbsolutePath()); conf.Set(YarnConfiguration.NmRemoteAppLogDir, remoteLogDir.GetAbsolutePath()); ContainerExecutor exec = new DefaultContainerExecutor(); exec.SetConf(conf); DeletionService del = new DeletionService(exec); Dispatcher dispatcher = new AsyncDispatcher(); NodeHealthCheckerService healthChecker = new NodeHealthCheckerService(); healthChecker.Init(conf); LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler(); NodeManagerMetrics metrics = NodeManagerMetrics.Create(); NodeStatusUpdater nodeStatusUpdater = new _NodeStatusUpdaterImpl_106(context, dispatcher , healthChecker, metrics); // Don't start any updating thread. DummyContainerManager containerManager = new DummyContainerManager(context, exec, del, nodeStatusUpdater, metrics, new ApplicationACLsManager(conf), dirsHandler); nodeStatusUpdater.Init(conf); ((NodeManager.NMContext)context).SetContainerManager(containerManager); nodeStatusUpdater.Start(); containerManager.Init(conf); containerManager.Start(); ContainerLaunchContext launchContext = recordFactory.NewRecordInstance <ContainerLaunchContext >(); ApplicationId applicationId = ApplicationId.NewInstance(0, 0); ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.NewInstance(applicationId , 0); ContainerId cID = ContainerId.NewContainerId(applicationAttemptId, 0); string user = "******"; StartContainerRequest scRequest = StartContainerRequest.NewInstance(launchContext , TestContainerManager.CreateContainerToken(cID, SimulatedRmIdentifier, context. GetNodeId(), user, context.GetContainerTokenSecretManager())); IList <StartContainerRequest> list = new AList <StartContainerRequest>(); list.AddItem(scRequest); StartContainersRequest allRequests = StartContainersRequest.NewInstance(list); containerManager.StartContainers(allRequests); BaseContainerManagerTest.WaitForContainerState(containerManager, cID, ContainerState .Running); IList <ContainerId> containerIds = new AList <ContainerId>(); containerIds.AddItem(cID); StopContainersRequest stopRequest = StopContainersRequest.NewInstance(containerIds ); containerManager.StopContainers(stopRequest); BaseContainerManagerTest.WaitForContainerState(containerManager, cID, ContainerState .Complete); containerManager.Stop(); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public override void StartLocalizer(Path nmPrivateContainerTokensPath, IPEndPoint nmAddr, string user, string appId, string locId, LocalDirsHandlerService dirsHandler ) { IList <string> localDirs = dirsHandler.GetLocalDirs(); IList <string> logDirs = dirsHandler.GetLogDirs(); VerifyUsernamePattern(user); string runAsUser = GetRunAsUser(user); IList <string> command = new AList <string>(); AddSchedPriorityCommand(command); Sharpen.Collections.AddAll(command, Arrays.AsList(containerExecutorExe, runAsUser , user, Sharpen.Extensions.ToString(LinuxContainerExecutor.Commands.InitializeContainer .GetValue()), appId, nmPrivateContainerTokensPath.ToUri().GetPath().ToString(), StringUtils.Join(",", localDirs), StringUtils.Join(",", logDirs))); FilePath jvm = new FilePath(new FilePath(Runtime.GetProperty("java.home"), "bin") , "java"); // use same jvm as parent command.AddItem(jvm.ToString()); command.AddItem("-classpath"); command.AddItem(Runtime.GetProperty("java.class.path")); string javaLibPath = Runtime.GetProperty("java.library.path"); if (javaLibPath != null) { command.AddItem("-Djava.library.path=" + javaLibPath); } BuildMainArgs(command, user, appId, locId, nmAddr, localDirs); string[] commandArray = Sharpen.Collections.ToArray(command, new string[command.Count ]); Shell.ShellCommandExecutor shExec = new Shell.ShellCommandExecutor(commandArray); if (Log.IsDebugEnabled()) { Log.Debug("initApplication: " + Arrays.ToString(commandArray)); } try { shExec.Execute(); if (Log.IsDebugEnabled()) { LogOutput(shExec.GetOutput()); } } catch (Shell.ExitCodeException e) { int exitCode = shExec.GetExitCode(); Log.Warn("Exit code from container " + locId + " startLocalizer is : " + exitCode , e); LogOutput(shExec.GetOutput()); throw new IOException("Application " + appId + " initialization failed" + " (exitCode=" + exitCode + ") with output: " + shExec.GetOutput(), e); } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public override void StartLocalizer(Path nmPrivateContainerTokens, IPEndPoint nmAddr , string user, string appId, string locId, LocalDirsHandlerService dirsHandler) { IList <string> localDirs = dirsHandler.GetLocalDirs(); IList <string> logDirs = dirsHandler.GetLogDirs(); Path classpathJarPrivateDir = dirsHandler.GetLocalPathForWrite(ResourceLocalizationService .NmPrivateDir); CreateUserLocalDirs(localDirs, user); CreateUserCacheDirs(localDirs, user); CreateAppDirs(localDirs, user, appId); CreateAppLogDirs(appId, logDirs, user); Path appStorageDir = GetWorkingDir(localDirs, user, appId); string tokenFn = string.Format(ContainerLocalizer.TokenFileNameFmt, locId); Path tokenDst = new Path(appStorageDir, tokenFn); CopyFile(nmPrivateContainerTokens, tokenDst, user); FilePath cwdApp = new FilePath(appStorageDir.ToString()); if (Log.IsDebugEnabled()) { Log.Debug(string.Format("cwdApp: %s", cwdApp)); } IList <string> command; command = new AList <string>(); //use same jvm as parent FilePath jvm = new FilePath(new FilePath(Runtime.GetProperty("java.home"), "bin") , "java.exe"); command.AddItem(jvm.ToString()); Path cwdPath = new Path(cwdApp.GetPath()); // Build a temp classpath jar. See ContainerLaunch.sanitizeEnv(). // Passing CLASSPATH explicitly is *way* too long for command line. string classPath = Runtime.GetProperty("java.class.path"); IDictionary <string, string> env = new Dictionary <string, string>(Sharpen.Runtime.GetEnv ()); string[] jarCp = FileUtil.CreateJarWithClassPath(classPath, classpathJarPrivateDir , cwdPath, env); string classPathJar = LocalizeClasspathJar(new Path(jarCp[0]), cwdPath, user).ToString (); command.AddItem("-classpath"); command.AddItem(classPathJar + jarCp[1]); string javaLibPath = Runtime.GetProperty("java.library.path"); if (javaLibPath != null) { command.AddItem("-Djava.library.path=" + javaLibPath); } ContainerLocalizer.BuildMainArgs(command, user, appId, locId, nmAddr, localDirs); string cmdLine = StringUtils.Join(command, " "); string localizerPid = string.Format(LocalizerPidFormat, locId); WindowsSecureContainerExecutor.WintuilsProcessStubExecutor stubExecutor = new WindowsSecureContainerExecutor.WintuilsProcessStubExecutor (cwdApp.GetAbsolutePath(), localizerPid, user, "nul:", cmdLine); try { stubExecutor.Execute(); stubExecutor.ValidateResult(); } finally { stubExecutor.Close(); try { KillContainer(localizerPid, ContainerExecutor.Signal.Kill); } catch (Exception e) { Log.Warn(string.Format("An exception occured during the cleanup of localizer job %s:%n%s" , localizerPid, StringUtils.StringifyException(e))); } } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public override void StartLocalizer(Path nmPrivateContainerTokensPath, IPEndPoint nmAddr, string user, string appId, string locId, LocalDirsHandlerService dirsHandler ) { lock (this) { IList <string> localDirs = dirsHandler.GetLocalDirs(); IList <string> logDirs = dirsHandler.GetLogDirs(); ContainerLocalizer localizer = new ContainerLocalizer(lfs, user, appId, locId, GetPaths (localDirs), RecordFactoryProvider.GetRecordFactory(GetConf())); CreateUserLocalDirs(localDirs, user); CreateUserCacheDirs(localDirs, user); CreateAppDirs(localDirs, user, appId); CreateAppLogDirs(appId, logDirs, user); // randomly choose the local directory Path appStorageDir = GetWorkingDir(localDirs, user, appId); string tokenFn = string.Format(ContainerLocalizer.TokenFileNameFmt, locId); Path tokenDst = new Path(appStorageDir, tokenFn); CopyFile(nmPrivateContainerTokensPath, tokenDst, user); Log.Info("Copying from " + nmPrivateContainerTokensPath + " to " + tokenDst); lfs.SetWorkingDirectory(appStorageDir); Log.Info("CWD set to " + appStorageDir + " = " + lfs.GetWorkingDirectory()); // TODO: DO it over RPC for maintaining similarity? localizer.RunLocalization(nmAddr); } }
public NodeHealthCheckerService() : base(typeof(Org.Apache.Hadoop.Yarn.Server.Nodemanager.NodeHealthCheckerService) .FullName) { dirsHandler = new LocalDirsHandlerService(); }
public _ContainersLauncher_148(DummyContainerManager _enclosing, Context baseArg1 , Dispatcher baseArg2, ContainerExecutor baseArg3, LocalDirsHandlerService baseArg4 , ContainerManagerImpl baseArg5) : base(baseArg1, baseArg2, baseArg3, baseArg4, baseArg5) { this._enclosing = _enclosing; }
public DummyContainerManager(Context context, ContainerExecutor exec, DeletionService deletionContext, NodeStatusUpdater nodeStatusUpdater, NodeManagerMetrics metrics , ApplicationACLsManager applicationACLsManager, LocalDirsHandlerService dirsHandler ) : base(context, exec, deletionContext, nodeStatusUpdater, metrics, applicationACLsManager , dirsHandler) { }
public _ResourceLocalizationService_78(Dispatcher baseArg1, ContainerExecutor baseArg2 , DeletionService baseArg3, LocalDirsHandlerService baseArg4, Context baseArg5) : base(baseArg1, baseArg2, baseArg3, baseArg4, baseArg5) { }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public virtual void TestStartLocalizer() { IPEndPoint localizationServerAddress; Path firstDir = new Path(BaseTmpPath, "localDir1"); IList <string> localDirs = new AList <string>(); Path secondDir = new Path(BaseTmpPath, "localDir2"); IList <string> logDirs = new AList <string>(); Path logDir = new Path(BaseTmpPath, "logDir"); Path tokenDir = new Path(BaseTmpPath, "tokenDir"); FsPermission perms = new FsPermission((short)0x1f8); Configuration conf = new Configuration(); localizationServerAddress = conf.GetSocketAddr(YarnConfiguration.NmBindHost, YarnConfiguration .NmLocalizerAddress, YarnConfiguration.DefaultNmLocalizerAddress, YarnConfiguration .DefaultNmLocalizerPort); FileContext mockLfs = Org.Mockito.Mockito.Spy(FileContext.GetLocalFSFileContext(conf )); FileContext.Util mockUtil = Org.Mockito.Mockito.Spy(mockLfs.Util()); Org.Mockito.Mockito.DoAnswer(new _Answer_344(mockUtil)).When(mockLfs).Util(); Org.Mockito.Mockito.DoAnswer(new _Answer_351(firstDir, mockLfs)).When(mockUtil).Copy (Matchers.Any <Path>(), Matchers.Any <Path>()); // throw an Exception when copy token to the first local dir // to simulate no space on the first drive // copy token to the second local dir Org.Mockito.Mockito.DoAnswer(new _Answer_378(firstDir)).When(mockLfs).GetFsStatus (Matchers.Any <Path>()); // let second local directory return more free space than // first local directory DefaultContainerExecutor mockExec = Org.Mockito.Mockito.Spy(new DefaultContainerExecutor (mockLfs)); mockExec.SetConf(conf); localDirs.AddItem(mockLfs.MakeQualified(firstDir).ToString()); localDirs.AddItem(mockLfs.MakeQualified(secondDir).ToString()); logDirs.AddItem(mockLfs.MakeQualified(logDir).ToString()); conf.SetStrings(YarnConfiguration.NmLocalDirs, Sharpen.Collections.ToArray(localDirs , new string[localDirs.Count])); conf.Set(YarnConfiguration.NmLogDirs, logDir.ToString()); mockLfs.Mkdir(tokenDir, perms, true); Path nmPrivateCTokensPath = new Path(tokenDir, "test.tokens"); string appSubmitter = "nobody"; string appId = "APP_ID"; string locId = "LOC_ID"; LocalDirsHandlerService dirsHandler = Org.Mockito.Mockito.Mock <LocalDirsHandlerService >(); Org.Mockito.Mockito.When(dirsHandler.GetLocalDirs()).ThenReturn(localDirs); Org.Mockito.Mockito.When(dirsHandler.GetLogDirs()).ThenReturn(logDirs); try { mockExec.StartLocalizer(nmPrivateCTokensPath, localizationServerAddress, appSubmitter , appId, locId, dirsHandler); } catch (IOException e) { NUnit.Framework.Assert.Fail("StartLocalizer failed to copy token file " + e); } finally { mockExec.DeleteAsUser(appSubmitter, firstDir); mockExec.DeleteAsUser(appSubmitter, secondDir); mockExec.DeleteAsUser(appSubmitter, logDir); DeleteTmpFiles(); } }