/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
private void TestGetContainerStatus(Container container, int index, ContainerState
state, string diagnostics, IList <int> exitStatuses)
{
while (true)
{
try
{
ContainerStatus status = nmClient.GetContainerStatus(container.GetId(), container
.GetNodeId());
// NodeManager may still need some time to get the stable
// container status
if (status.GetState() == state)
{
NUnit.Framework.Assert.AreEqual(container.GetId(), status.GetContainerId());
NUnit.Framework.Assert.IsTrue(string.Empty + index + ": " + status.GetDiagnostics
(), status.GetDiagnostics().Contains(diagnostics));
NUnit.Framework.Assert.IsTrue("Exit Statuses are supposed to be in: " + exitStatuses
+ ", but the actual exit status code is: " + status.GetExitStatus(), exitStatuses
.Contains(status.GetExitStatus()));
break;
}
Sharpen.Thread.Sleep(100);
}
catch (Exception e)
{
Sharpen.Runtime.PrintStackTrace(e);
}
}
}
public virtual void TestCreateAbnormalContainerStatus()
{
ContainerStatus cd = SchedulerUtils.CreateAbnormalContainerStatus(ContainerId.NewContainerId
(ApplicationAttemptId.NewInstance(ApplicationId.NewInstance(Runtime.CurrentTimeMillis
(), 1), 1), 1), "x");
NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Aborted, cd.GetExitStatus());
}
public ContainerInfo(Context nmContext, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
container, string requestUri, string pathPrefix)
{
// JAXB needs this
this.id = container.GetContainerId().ToString();
this.nodeId = nmContext.GetNodeId().ToString();
ContainerStatus containerData = container.CloneAndGetContainerStatus();
this.exitCode = containerData.GetExitStatus();
this.exitStatus = (this.exitCode == ContainerExitStatus.Invalid) ? "N/A" : exitCode
.ToString();
this.state = container.GetContainerState().ToString();
this.diagnostics = containerData.GetDiagnostics();
if (this.diagnostics == null || this.diagnostics.IsEmpty())
{
this.diagnostics = string.Empty;
}
this.user = container.GetUser();
Resource res = container.GetResource();
if (res != null)
{
this.totalMemoryNeededMB = res.GetMemory();
this.totalVCoresNeeded = res.GetVirtualCores();
}
this.containerLogsShortLink = StringHelper.Ujoin("containerlogs", this.id, container
.GetUser());
if (requestUri == null)
{
requestUri = string.Empty;
}
if (pathPrefix == null)
{
pathPrefix = string.Empty;
}
this.containerLogsLink = StringHelper.Join(requestUri, pathPrefix, this.containerLogsShortLink
);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
private void TestContainerLaunchAndExit(int exitCode)
{
FilePath scriptFile = Shell.AppendScriptExtension(tmpDir, "scriptFile");
PrintWriter fileWriter = new PrintWriter(scriptFile);
FilePath processStartFile = new FilePath(tmpDir, "start_file.txt").GetAbsoluteFile
();
// ////// Construct the Container-id
ContainerId cId = CreateContainerId(0);
if (Shell.Windows)
{
fileWriter.WriteLine("@echo Hello World!> " + processStartFile);
fileWriter.WriteLine("@echo " + cId + ">> " + processStartFile);
if (exitCode != 0)
{
fileWriter.WriteLine("@exit " + exitCode);
}
}
else
{
fileWriter.Write("\numask 0");
// So that start file is readable by the test
fileWriter.Write("\necho Hello World! > " + processStartFile);
fileWriter.Write("\necho $$ >> " + processStartFile);
// Have script throw an exit code at the end
if (exitCode != 0)
{
fileWriter.Write("\nexit " + exitCode);
}
}
fileWriter.Close();
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
URL resource_alpha = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new
Path(scriptFile.GetAbsolutePath())));
LocalResource rsrc_alpha = recordFactory.NewRecordInstance <LocalResource>();
rsrc_alpha.SetResource(resource_alpha);
rsrc_alpha.SetSize(-1);
rsrc_alpha.SetVisibility(LocalResourceVisibility.Application);
rsrc_alpha.SetType(LocalResourceType.File);
rsrc_alpha.SetTimestamp(scriptFile.LastModified());
string destinationFile = "dest_file";
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
localResources[destinationFile] = rsrc_alpha;
containerLaunchContext.SetLocalResources(localResources);
IList <string> commands = Arrays.AsList(Shell.GetRunScriptCommand(scriptFile));
containerLaunchContext.SetCommands(commands);
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, CreateContainerToken(cId, DummyRmIdentifier, context.GetNodeId(), user, context
.GetContainerTokenSecretManager()));
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
containerManager.StartContainers(allRequests);
BaseContainerManagerTest.WaitForContainerState(containerManager, cId, ContainerState
.Complete);
IList <ContainerId> containerIds = new AList <ContainerId>();
containerIds.AddItem(cId);
GetContainerStatusesRequest gcsRequest = GetContainerStatusesRequest.NewInstance(
containerIds);
ContainerStatus containerStatus = containerManager.GetContainerStatuses(gcsRequest
).GetContainerStatuses()[0];
// Verify exit status matches exit state of script
NUnit.Framework.Assert.AreEqual(exitCode, containerStatus.GetExitStatus());
}
public virtual void TestContainerLaunchAndStop()
{
containerManager.Start();
FilePath scriptFile = Shell.AppendScriptExtension(tmpDir, "scriptFile");
PrintWriter fileWriter = new PrintWriter(scriptFile);
FilePath processStartFile = new FilePath(tmpDir, "start_file.txt").GetAbsoluteFile
();
// ////// Construct the Container-id
ContainerId cId = CreateContainerId(0);
if (Shell.Windows)
{
fileWriter.WriteLine("@echo Hello World!> " + processStartFile);
fileWriter.WriteLine("@echo " + cId + ">> " + processStartFile);
fileWriter.WriteLine("@ping -n 100 127.0.0.1 >nul");
}
else
{
fileWriter.Write("\numask 0");
// So that start file is readable by the test
fileWriter.Write("\necho Hello World! > " + processStartFile);
fileWriter.Write("\necho $$ >> " + processStartFile);
fileWriter.Write("\nexec sleep 100");
}
fileWriter.Close();
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
URL resource_alpha = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new
Path(scriptFile.GetAbsolutePath())));
LocalResource rsrc_alpha = recordFactory.NewRecordInstance <LocalResource>();
rsrc_alpha.SetResource(resource_alpha);
rsrc_alpha.SetSize(-1);
rsrc_alpha.SetVisibility(LocalResourceVisibility.Application);
rsrc_alpha.SetType(LocalResourceType.File);
rsrc_alpha.SetTimestamp(scriptFile.LastModified());
string destinationFile = "dest_file";
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
localResources[destinationFile] = rsrc_alpha;
containerLaunchContext.SetLocalResources(localResources);
IList <string> commands = Arrays.AsList(Shell.GetRunScriptCommand(scriptFile));
containerLaunchContext.SetCommands(commands);
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, CreateContainerToken(cId, DummyRmIdentifier, context.GetNodeId(), user, context
.GetContainerTokenSecretManager()));
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
containerManager.StartContainers(allRequests);
int timeoutSecs = 0;
while (!processStartFile.Exists() && timeoutSecs++ < 20)
{
Sharpen.Thread.Sleep(1000);
Log.Info("Waiting for process start-file to be created");
}
NUnit.Framework.Assert.IsTrue("ProcessStartFile doesn't exist!", processStartFile
.Exists());
// Now verify the contents of the file
BufferedReader reader = new BufferedReader(new FileReader(processStartFile));
NUnit.Framework.Assert.AreEqual("Hello World!", reader.ReadLine());
// Get the pid of the process
string pid = reader.ReadLine().Trim();
// No more lines
NUnit.Framework.Assert.AreEqual(null, reader.ReadLine());
// Now test the stop functionality.
// Assert that the process is alive
NUnit.Framework.Assert.IsTrue("Process is not alive!", DefaultContainerExecutor.ContainerIsAlive
(pid));
// Once more
NUnit.Framework.Assert.IsTrue("Process is not alive!", DefaultContainerExecutor.ContainerIsAlive
(pid));
IList <ContainerId> containerIds = new AList <ContainerId>();
containerIds.AddItem(cId);
StopContainersRequest stopRequest = StopContainersRequest.NewInstance(containerIds
);
containerManager.StopContainers(stopRequest);
BaseContainerManagerTest.WaitForContainerState(containerManager, cId, ContainerState
.Complete);
GetContainerStatusesRequest gcsRequest = GetContainerStatusesRequest.NewInstance(
containerIds);
ContainerStatus containerStatus = containerManager.GetContainerStatuses(gcsRequest
).GetContainerStatuses()[0];
int expectedExitCode = ContainerExitStatus.KilledByAppmaster;
NUnit.Framework.Assert.AreEqual(expectedExitCode, containerStatus.GetExitStatus()
);
// Assert that the process is not alive anymore
NUnit.Framework.Assert.IsFalse("Process is still alive!", DefaultContainerExecutor
.ContainerIsAlive(pid));
}
public virtual void TestContainerKillOnMemoryOverflow()
{
if (!ProcfsBasedProcessTree.IsAvailable())
{
return;
}
containerManager.Start();
FilePath scriptFile = new FilePath(tmpDir, "scriptFile.sh");
PrintWriter fileWriter = new PrintWriter(scriptFile);
FilePath processStartFile = new FilePath(tmpDir, "start_file.txt").GetAbsoluteFile
();
fileWriter.Write("\numask 0");
// So that start file is readable by the
// test.
fileWriter.Write("\necho Hello World! > " + processStartFile);
fileWriter.Write("\necho $$ >> " + processStartFile);
fileWriter.Write("\nsleep 15");
fileWriter.Close();
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
// ////// Construct the Container-id
ApplicationId appId = ApplicationId.NewInstance(0, 0);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, 1);
ContainerId cId = ContainerId.NewContainerId(appAttemptId, 0);
int port = 12345;
URL resource_alpha = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new
Path(scriptFile.GetAbsolutePath())));
LocalResource rsrc_alpha = recordFactory.NewRecordInstance <LocalResource>();
rsrc_alpha.SetResource(resource_alpha);
rsrc_alpha.SetSize(-1);
rsrc_alpha.SetVisibility(LocalResourceVisibility.Application);
rsrc_alpha.SetType(LocalResourceType.File);
rsrc_alpha.SetTimestamp(scriptFile.LastModified());
string destinationFile = "dest_file";
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
localResources[destinationFile] = rsrc_alpha;
containerLaunchContext.SetLocalResources(localResources);
IList <string> commands = new AList <string>();
commands.AddItem("/bin/bash");
commands.AddItem(scriptFile.GetAbsolutePath());
containerLaunchContext.SetCommands(commands);
Resource r = BuilderUtils.NewResource(8 * 1024 * 1024, 1);
ContainerTokenIdentifier containerIdentifier = new ContainerTokenIdentifier(cId,
context.GetNodeId().ToString(), user, r, Runtime.CurrentTimeMillis() + 120000, 123
, DummyRmIdentifier, Priority.NewInstance(0), 0);
Token containerToken = BuilderUtils.NewContainerToken(context.GetNodeId(), containerManager
.GetContext().GetContainerTokenSecretManager().CreatePassword(containerIdentifier
), containerIdentifier);
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, containerToken);
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
containerManager.StartContainers(allRequests);
int timeoutSecs = 0;
while (!processStartFile.Exists() && timeoutSecs++ < 20)
{
Sharpen.Thread.Sleep(1000);
Log.Info("Waiting for process start-file to be created");
}
NUnit.Framework.Assert.IsTrue("ProcessStartFile doesn't exist!", processStartFile
.Exists());
// Now verify the contents of the file
BufferedReader reader = new BufferedReader(new FileReader(processStartFile));
NUnit.Framework.Assert.AreEqual("Hello World!", reader.ReadLine());
// Get the pid of the process
string pid = reader.ReadLine().Trim();
// No more lines
NUnit.Framework.Assert.AreEqual(null, reader.ReadLine());
BaseContainerManagerTest.WaitForContainerState(containerManager, cId, ContainerState
.Complete, 60);
IList <ContainerId> containerIds = new AList <ContainerId>();
containerIds.AddItem(cId);
GetContainerStatusesRequest gcsRequest = GetContainerStatusesRequest.NewInstance(
containerIds);
ContainerStatus containerStatus = containerManager.GetContainerStatuses(gcsRequest
).GetContainerStatuses()[0];
NUnit.Framework.Assert.AreEqual(ContainerExitStatus.KilledExceededVmem, containerStatus
.GetExitStatus());
string expectedMsgPattern = "Container \\[pid=" + pid + ",containerID=" + cId + "\\] is running beyond virtual memory limits. Current usage: "
+ "[0-9.]+ ?[KMGTPE]?B of [0-9.]+ ?[KMGTPE]?B physical memory used; " + "[0-9.]+ ?[KMGTPE]?B of [0-9.]+ ?[KMGTPE]?B virtual memory used. "
+ "Killing container.\nDump of the process-tree for " + cId + " :\n";
Sharpen.Pattern pat = Sharpen.Pattern.Compile(expectedMsgPattern);
NUnit.Framework.Assert.AreEqual("Expected message pattern is: " + expectedMsgPattern
+ "\n\nObserved message is: " + containerStatus.GetDiagnostics(), true, pat.Matcher
(containerStatus.GetDiagnostics()).Find());
// Assert that the process is not alive anymore
NUnit.Framework.Assert.IsFalse("Process is still alive!", exec.SignalContainer(user
, pid, ContainerExecutor.Signal.Null));
}
