public ApplicationContainerInitEvent(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
container)
: base(container.GetContainerId().GetApplicationAttemptId().GetApplicationId(), ApplicationEventType
.InitContainer)
{
this.container = container;
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestContainerLaunchNullImage()
{
string appSubmitter = "nobody";
string appId = "APP_ID";
string containerId = "CONTAINER_ID";
string testImage = string.Empty;
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
>(Org.Mockito.Mockito.ReturnsDeepStubs);
ContainerId cId = Org.Mockito.Mockito.Mock <ContainerId>(Org.Mockito.Mockito.ReturnsDeepStubs
);
ContainerLaunchContext context = Org.Mockito.Mockito.Mock <ContainerLaunchContext>
();
Dictionary <string, string> env = new Dictionary <string, string>();
Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(cId);
Org.Mockito.Mockito.When(container.GetLaunchContext()).ThenReturn(context);
Org.Mockito.Mockito.When(cId.GetApplicationAttemptId().GetApplicationId().ToString
()).ThenReturn(appId);
Org.Mockito.Mockito.When(cId.ToString()).ThenReturn(containerId);
Org.Mockito.Mockito.When(context.GetEnvironment()).ThenReturn(env);
env[YarnConfiguration.NmDockerContainerExecutorImageName] = testImage;
dockerContainerExecutor.GetConf().Set(YarnConfiguration.NmDockerContainerExecutorImageName
, testImage);
Path scriptPath = new Path("file:///bin/echo");
Path tokensPath = new Path("file:///dev/null");
Path pidFile = new Path(workDir, "pid.txt");
dockerContainerExecutor.ActivateContainer(cId, pidFile);
dockerContainerExecutor.LaunchContainer(container, scriptPath, tokensPath, appSubmitter
, appId, workDir, dirsHandler.GetLocalDirs(), dirsHandler.GetLogDirs());
}
/// <exception cref="System.IO.IOException"/>
private int RunAndBlock(ContainerId cId, IDictionary <string, string> launchCtxEnv
, params string[] cmd)
{
string appId = "APP_" + Runtime.CurrentTimeMillis();
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
>();
ContainerLaunchContext context = Org.Mockito.Mockito.Mock <ContainerLaunchContext>
();
Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(cId);
Org.Mockito.Mockito.When(container.GetLaunchContext()).ThenReturn(context);
Org.Mockito.Mockito.When(cId.GetApplicationAttemptId().GetApplicationId().ToString
()).ThenReturn(appId);
Org.Mockito.Mockito.When(context.GetEnvironment()).ThenReturn(launchCtxEnv);
string script = WriteScriptFile(launchCtxEnv, cmd);
Path scriptPath = new Path(script);
Path tokensPath = new Path("/dev/null");
Path workDir = new Path(workSpace.GetAbsolutePath());
Path pidFile = new Path(workDir, "pid.txt");
exec.ActivateContainer(cId, pidFile);
return(exec.LaunchContainer(container, scriptPath, tokensPath, appSubmitter, appId
, workDir, dirsHandler.GetLocalDirs(), dirsHandler.GetLogDirs()));
}
public virtual void Handle(ContainersLauncherEvent @event)
{
// TODO: ContainersLauncher launches containers one by one!!
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= @event.GetContainer();
ContainerId containerId = container.GetContainerId();
switch (@event.GetType())
{
case ContainersLauncherEventType.LaunchContainer:
{
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = context.GetApplications()[containerId.GetApplicationAttemptId().GetApplicationId
()];
ContainerLaunch launch = new ContainerLaunch(context, GetConfig(), dispatcher, exec
, app, @event.GetContainer(), dirsHandler, containerManager);
containerLauncher.Submit(launch);
running[containerId] = launch;
break;
}
case ContainersLauncherEventType.RecoverContainer:
{
app = context.GetApplications()[containerId.GetApplicationAttemptId().GetApplicationId
()];
launch = new RecoveredContainerLaunch(context, GetConfig(), dispatcher, exec, app
, @event.GetContainer(), dirsHandler, containerManager);
containerLauncher.Submit(launch);
running[containerId] = launch;
break;
}
case ContainersLauncherEventType.CleanupContainer:
{
ContainerLaunch launcher = Sharpen.Collections.Remove(running, containerId);
if (launcher == null)
{
// Container not launched. So nothing needs to be done.
return;
}
// Cleanup a container whether it is running/killed/completed, so that
// no sub-processes are alive.
try
{
launcher.CleanupContainer();
}
catch (IOException)
{
Log.Warn("Got exception while cleaning container " + containerId + ". Ignoring.");
}
break;
}
}
}
/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/>
/// <exception cref="System.Exception"/>
public virtual void VerifyNodeContainerInfoGeneric(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
cont, string id, string state, string user, int exitCode, string diagnostics, string
nodeId, int totalMemoryNeededMB, int totalVCoresNeeded, string logsLink)
{
WebServicesTestUtils.CheckStringMatch("id", cont.GetContainerId().ToString(), id);
WebServicesTestUtils.CheckStringMatch("state", cont.GetContainerState().ToString(
), state);
WebServicesTestUtils.CheckStringMatch("user", cont.GetUser().ToString(), user);
NUnit.Framework.Assert.AreEqual("exitCode wrong", 0, exitCode);
WebServicesTestUtils.CheckStringMatch("diagnostics", "testing", diagnostics);
WebServicesTestUtils.CheckStringMatch("nodeId", nmContext.GetNodeId().ToString(),
nodeId);
NUnit.Framework.Assert.AreEqual("totalMemoryNeededMB wrong", YarnConfiguration.DefaultRmSchedulerMinimumAllocationMb
, totalMemoryNeededMB);
NUnit.Framework.Assert.AreEqual("totalVCoresNeeded wrong", YarnConfiguration.DefaultRmSchedulerMinimumAllocationVcores
, totalVCoresNeeded);
string shortLink = StringHelper.Ujoin("containerlogs", cont.GetContainerId().ToString
(), cont.GetUser());
NUnit.Framework.Assert.IsTrue("containerLogsLink wrong", logsLink.Contains(shortLink
));
}
public ContainerInfo(Context nmContext, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
container, string requestUri, string pathPrefix)
{
// JAXB needs this
this.id = container.GetContainerId().ToString();
this.nodeId = nmContext.GetNodeId().ToString();
ContainerStatus containerData = container.CloneAndGetContainerStatus();
this.exitCode = containerData.GetExitStatus();
this.exitStatus = (this.exitCode == ContainerExitStatus.Invalid) ? "N/A" : exitCode
.ToString();
this.state = container.GetContainerState().ToString();
this.diagnostics = containerData.GetDiagnostics();
if (this.diagnostics == null || this.diagnostics.IsEmpty())
{
this.diagnostics = string.Empty;
}
this.user = container.GetUser();
Resource res = container.GetResource();
if (res != null)
{
this.totalMemoryNeededMB = res.GetMemory();
this.totalVCoresNeeded = res.GetVirtualCores();
}
this.containerLogsShortLink = StringHelper.Ujoin("containerlogs", this.id, container
.GetUser());
if (requestUri == null)
{
requestUri = string.Empty;
}
if (pathPrefix == null)
{
pathPrefix = string.Empty;
}
this.containerLogsLink = StringHelper.Join(requestUri, pathPrefix, this.containerLogsShortLink
);
}
public virtual void TestContainerLaunch()
{
string appSubmitter = "nobody";
string appId = "APP_ID";
string containerId = "CONTAINER_ID";
string testImage = "\"sequenceiq/hadoop-docker:2.4.1\"";
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
>(Org.Mockito.Mockito.ReturnsDeepStubs);
ContainerId cId = Org.Mockito.Mockito.Mock <ContainerId>(Org.Mockito.Mockito.ReturnsDeepStubs
);
ContainerLaunchContext context = Org.Mockito.Mockito.Mock <ContainerLaunchContext>
();
Dictionary <string, string> env = new Dictionary <string, string>();
Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(cId);
Org.Mockito.Mockito.When(container.GetLaunchContext()).ThenReturn(context);
Org.Mockito.Mockito.When(cId.GetApplicationAttemptId().GetApplicationId().ToString
()).ThenReturn(appId);
Org.Mockito.Mockito.When(cId.ToString()).ThenReturn(containerId);
Org.Mockito.Mockito.When(context.GetEnvironment()).ThenReturn(env);
env[YarnConfiguration.NmDockerContainerExecutorImageName] = testImage;
Path scriptPath = new Path("file:///bin/echo");
Path tokensPath = new Path("file:///dev/null");
Path pidFile = new Path(workDir, "pid");
dockerContainerExecutor.ActivateContainer(cId, pidFile);
int ret = dockerContainerExecutor.LaunchContainer(container, scriptPath, tokensPath
, appSubmitter, appId, workDir, dirsHandler.GetLocalDirs(), dirsHandler.GetLogDirs
());
NUnit.Framework.Assert.AreEqual(0, ret);
//get the script
Path sessionScriptPath = new Path(workDir, Shell.AppendScriptExtension(DockerContainerExecutor
.DockerContainerExecutorSessionScript));
LineNumberReader lnr = new LineNumberReader(new FileReader(sessionScriptPath.ToString
()));
bool cmdFound = false;
IList <string> localDirs = DirsToMount(dirsHandler.GetLocalDirs());
IList <string> logDirs = DirsToMount(dirsHandler.GetLogDirs());
IList <string> workDirMount = DirsToMount(Sharpen.Collections.SingletonList(workDir
.ToUri().GetPath()));
IList <string> expectedCommands = new AList <string>(Arrays.AsList(DockerLaunchCommand
, "run", "--rm", "--net=host", "--name", containerId));
Sharpen.Collections.AddAll(expectedCommands, localDirs);
Sharpen.Collections.AddAll(expectedCommands, logDirs);
Sharpen.Collections.AddAll(expectedCommands, workDirMount);
string shellScript = workDir + "/launch_container.sh";
Sharpen.Collections.AddAll(expectedCommands, Arrays.AsList(testImage.ReplaceAll("['\"]"
, string.Empty), "bash", "\"" + shellScript + "\""));
string expectedPidString = "echo `/bin/true inspect --format {{.State.Pid}} " + containerId
+ "` > " + pidFile.ToString() + ".tmp";
bool pidSetterFound = false;
while (lnr.Ready())
{
string line = lnr.ReadLine();
Log.Debug("line: " + line);
if (line.StartsWith(DockerLaunchCommand))
{
IList <string> command = new AList <string>();
foreach (string s in line.Split("\\s+"))
{
command.AddItem(s.Trim());
}
NUnit.Framework.Assert.AreEqual(expectedCommands, command);
cmdFound = true;
}
else
{
if (line.StartsWith("echo"))
{
NUnit.Framework.Assert.AreEqual(expectedPidString, line);
pidSetterFound = true;
}
}
}
NUnit.Framework.Assert.IsTrue(cmdFound);
NUnit.Framework.Assert.IsTrue(pidSetterFound);
}
public AuxServicesEvent(AuxServicesEventType eventType, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
container)
: this(eventType, null, container.GetContainerId().GetApplicationAttemptId().GetApplicationId
(), null, null, container)
{
}
