/// <summary> /// Create the ResourceCalculatorProcessTree rooted to specified process /// from the class name and configure it. /// </summary> /// <remarks> /// Create the ResourceCalculatorProcessTree rooted to specified process /// from the class name and configure it. If class name is null, this method /// will try and return a process tree plugin available for this system. /// </remarks> /// <param name="pid">process pid of the root of the process tree</param> /// <param name="clazz">class-name</param> /// <param name="conf">configure the plugin with this.</param> /// <returns> /// ResourceCalculatorProcessTree or null if ResourceCalculatorPluginTree /// is not available for this system. /// </returns> public static Org.Apache.Hadoop.Yarn.Util.ResourceCalculatorProcessTree GetResourceCalculatorProcessTree (string pid, Type clazz, Configuration conf) { if (clazz != null) { try { Constructor <Org.Apache.Hadoop.Yarn.Util.ResourceCalculatorProcessTree> c = clazz. GetConstructor(typeof(string)); Org.Apache.Hadoop.Yarn.Util.ResourceCalculatorProcessTree rctree = c.NewInstance( pid); rctree.SetConf(conf); return(rctree); } catch (Exception e) { throw new RuntimeException(e); } } // No class given, try a os specific class if (ProcfsBasedProcessTree.IsAvailable()) { return(new ProcfsBasedProcessTree(pid)); } if (WindowsBasedProcessTree.IsAvailable()) { return(new WindowsBasedProcessTree(pid)); } // Not supported on this system. return(null); }
public virtual void TestProcessTreeLimits() { // set up a dummy proc file system FilePath procfsRootDir = new FilePath(localDir, "proc"); string[] pids = new string[] { "100", "200", "300", "400", "500", "600", "700" }; try { TestProcfsBasedProcessTree.SetupProcfsRootDir(procfsRootDir); // create pid dirs. TestProcfsBasedProcessTree.SetupPidDirs(procfsRootDir, pids); // create process infos. TestProcfsBasedProcessTree.ProcessStatInfo[] procs = new TestProcfsBasedProcessTree.ProcessStatInfo [7]; // assume pids 100, 500 are in 1 tree // 200,300,400 are in another // 600,700 are in a third procs[0] = new TestProcfsBasedProcessTree.ProcessStatInfo(new string[] { "100", "proc1" , "1", "100", "100", "100000" }); procs[1] = new TestProcfsBasedProcessTree.ProcessStatInfo(new string[] { "200", "proc2" , "1", "200", "200", "200000" }); procs[2] = new TestProcfsBasedProcessTree.ProcessStatInfo(new string[] { "300", "proc3" , "200", "200", "200", "300000" }); procs[3] = new TestProcfsBasedProcessTree.ProcessStatInfo(new string[] { "400", "proc4" , "200", "200", "200", "400000" }); procs[4] = new TestProcfsBasedProcessTree.ProcessStatInfo(new string[] { "500", "proc5" , "100", "100", "100", "1500000" }); procs[5] = new TestProcfsBasedProcessTree.ProcessStatInfo(new string[] { "600", "proc6" , "1", "600", "600", "100000" }); procs[6] = new TestProcfsBasedProcessTree.ProcessStatInfo(new string[] { "700", "proc7" , "600", "600", "600", "100000" }); // write stat files. TestProcfsBasedProcessTree.WriteStatFiles(procfsRootDir, pids, procs, null); // vmem limit long limit = 700000; ContainersMonitorImpl test = new ContainersMonitorImpl(null, null, null); // create process trees // tree rooted at 100 is over limit immediately, as it is // twice over the mem limit. ProcfsBasedProcessTree pTree = new ProcfsBasedProcessTree("100", procfsRootDir.GetAbsolutePath ()); pTree.UpdateProcessTree(); NUnit.Framework.Assert.IsTrue("tree rooted at 100 should be over limit " + "after first iteration." , test.IsProcessTreeOverLimit(pTree, "dummyId", limit)); // the tree rooted at 200 is initially below limit. pTree = new ProcfsBasedProcessTree("200", procfsRootDir.GetAbsolutePath()); pTree.UpdateProcessTree(); NUnit.Framework.Assert.IsFalse("tree rooted at 200 shouldn't be over limit " + "after one iteration." , test.IsProcessTreeOverLimit(pTree, "dummyId", limit)); // second iteration - now the tree has been over limit twice, // hence it should be declared over limit. pTree.UpdateProcessTree(); NUnit.Framework.Assert.IsTrue("tree rooted at 200 should be over limit after 2 iterations" , test.IsProcessTreeOverLimit(pTree, "dummyId", limit)); // the tree rooted at 600 is never over limit. pTree = new ProcfsBasedProcessTree("600", procfsRootDir.GetAbsolutePath()); pTree.UpdateProcessTree(); NUnit.Framework.Assert.IsFalse("tree rooted at 600 should never be over limit.", test.IsProcessTreeOverLimit(pTree, "dummyId", limit)); // another iteration does not make any difference. pTree.UpdateProcessTree(); NUnit.Framework.Assert.IsFalse("tree rooted at 600 should never be over limit.", test.IsProcessTreeOverLimit(pTree, "dummyId", limit)); } finally { FileUtil.FullyDelete(procfsRootDir); } }
public virtual void TestContainerKillOnMemoryOverflow() { if (!ProcfsBasedProcessTree.IsAvailable()) { return; } containerManager.Start(); FilePath scriptFile = new FilePath(tmpDir, "scriptFile.sh"); PrintWriter fileWriter = new PrintWriter(scriptFile); FilePath processStartFile = new FilePath(tmpDir, "start_file.txt").GetAbsoluteFile (); fileWriter.Write("\numask 0"); // So that start file is readable by the // test. fileWriter.Write("\necho Hello World! > " + processStartFile); fileWriter.Write("\necho $$ >> " + processStartFile); fileWriter.Write("\nsleep 15"); fileWriter.Close(); ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext >(); // ////// Construct the Container-id ApplicationId appId = ApplicationId.NewInstance(0, 0); ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, 1); ContainerId cId = ContainerId.NewContainerId(appAttemptId, 0); int port = 12345; URL resource_alpha = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new Path(scriptFile.GetAbsolutePath()))); LocalResource rsrc_alpha = recordFactory.NewRecordInstance <LocalResource>(); rsrc_alpha.SetResource(resource_alpha); rsrc_alpha.SetSize(-1); rsrc_alpha.SetVisibility(LocalResourceVisibility.Application); rsrc_alpha.SetType(LocalResourceType.File); rsrc_alpha.SetTimestamp(scriptFile.LastModified()); string destinationFile = "dest_file"; IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource >(); localResources[destinationFile] = rsrc_alpha; containerLaunchContext.SetLocalResources(localResources); IList <string> commands = new AList <string>(); commands.AddItem("/bin/bash"); commands.AddItem(scriptFile.GetAbsolutePath()); containerLaunchContext.SetCommands(commands); Resource r = BuilderUtils.NewResource(8 * 1024 * 1024, 1); ContainerTokenIdentifier containerIdentifier = new ContainerTokenIdentifier(cId, context.GetNodeId().ToString(), user, r, Runtime.CurrentTimeMillis() + 120000, 123 , DummyRmIdentifier, Priority.NewInstance(0), 0); Token containerToken = BuilderUtils.NewContainerToken(context.GetNodeId(), containerManager .GetContext().GetContainerTokenSecretManager().CreatePassword(containerIdentifier ), containerIdentifier); StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext , containerToken); IList <StartContainerRequest> list = new AList <StartContainerRequest>(); list.AddItem(scRequest); StartContainersRequest allRequests = StartContainersRequest.NewInstance(list); containerManager.StartContainers(allRequests); int timeoutSecs = 0; while (!processStartFile.Exists() && timeoutSecs++ < 20) { Sharpen.Thread.Sleep(1000); Log.Info("Waiting for process start-file to be created"); } NUnit.Framework.Assert.IsTrue("ProcessStartFile doesn't exist!", processStartFile .Exists()); // Now verify the contents of the file BufferedReader reader = new BufferedReader(new FileReader(processStartFile)); NUnit.Framework.Assert.AreEqual("Hello World!", reader.ReadLine()); // Get the pid of the process string pid = reader.ReadLine().Trim(); // No more lines NUnit.Framework.Assert.AreEqual(null, reader.ReadLine()); BaseContainerManagerTest.WaitForContainerState(containerManager, cId, ContainerState .Complete, 60); IList <ContainerId> containerIds = new AList <ContainerId>(); containerIds.AddItem(cId); GetContainerStatusesRequest gcsRequest = GetContainerStatusesRequest.NewInstance( containerIds); ContainerStatus containerStatus = containerManager.GetContainerStatuses(gcsRequest ).GetContainerStatuses()[0]; NUnit.Framework.Assert.AreEqual(ContainerExitStatus.KilledExceededVmem, containerStatus .GetExitStatus()); string expectedMsgPattern = "Container \\[pid=" + pid + ",containerID=" + cId + "\\] is running beyond virtual memory limits. Current usage: " + "[0-9.]+ ?[KMGTPE]?B of [0-9.]+ ?[KMGTPE]?B physical memory used; " + "[0-9.]+ ?[KMGTPE]?B of [0-9.]+ ?[KMGTPE]?B virtual memory used. " + "Killing container.\nDump of the process-tree for " + cId + " :\n"; Sharpen.Pattern pat = Sharpen.Pattern.Compile(expectedMsgPattern); NUnit.Framework.Assert.AreEqual("Expected message pattern is: " + expectedMsgPattern + "\n\nObserved message is: " + containerStatus.GetDiagnostics(), true, pat.Matcher (containerStatus.GetDiagnostics()).Find()); // Assert that the process is not alive anymore NUnit.Framework.Assert.IsFalse("Process is still alive!", exec.SignalContainer(user , pid, ContainerExecutor.Signal.Null)); }