public ApplicationContainerInitEvent(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
container)
: base(container.GetContainerId().GetApplicationAttemptId().GetApplicationId(), ApplicationEventType
.InitContainer)
{
this.container = container;
}
/// <summary>Event requesting the cleanup of the rsrc.</summary>
/// <param name="c"/>
/// <param name="rsrc"/>
public ContainerLocalizationCleanupEvent(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
c, IDictionary <LocalResourceVisibility, ICollection <LocalResourceRequest> > rsrc
)
: base(LocalizationEventType.CleanupContainerResources, c)
{
this.rsrc = rsrc;
}
public _MockResourceTracker_235(_TestNodeStatusUpdaterResync_232 _enclosing, ContainerStatus
testCompleteContainer, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
container)
{
this._enclosing = _enclosing;
this.testCompleteContainer = testCompleteContainer;
this.container = container;
}
/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/>
/// <exception cref="System.Exception"/>
public virtual void VerifyNodeContainerInfo(JSONObject info, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
cont)
{
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 9, info.Length());
VerifyNodeContainerInfoGeneric(cont, info.GetString("id"), info.GetString("state"
), info.GetString("user"), info.GetInt("exitCode"), info.GetString("diagnostics"
), info.GetString("nodeId"), info.GetInt("totalMemoryNeededMB"), info.GetInt("totalVCoresNeeded"
), info.GetString("containerLogsLink"));
}
public _NodeManager_225(TestNodeManagerResync _enclosing, ContainerStatus testCompleteContainer
, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
container)
{
this._enclosing = _enclosing;
this.testCompleteContainer = testCompleteContainer;
this.container = container;
this.registerCount = 0;
}
public _TestNodeStatusUpdaterResync_232(_NodeManager_225 _enclosing, ContainerStatus
testCompleteContainer, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
container, Context baseArg1, Dispatcher baseArg2, NodeHealthCheckerService baseArg3
, NodeManagerMetrics baseArg4)
: base(_enclosing, baseArg1, baseArg2, baseArg3, baseArg4)
{
this._enclosing = _enclosing;
this.testCompleteContainer = testCompleteContainer;
this.container = container;
}
public AuxServicesEvent(AuxServicesEventType eventType, string user, ApplicationId
appId, string serviceId, ByteBuffer serviceData, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
container)
: base(eventType)
{
this.user = user;
this.appId = appId;
this.serviceId = serviceId;
this.serviceData = serviceData;
this.container = container;
}
/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/>
/// <exception cref="System.Exception"/>
public virtual void VerifyContainersInfoXML(NodeList nodes, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
cont)
{
for (int i = 0; i < nodes.GetLength(); i++)
{
Element element = (Element)nodes.Item(i);
VerifyNodeContainerInfoGeneric(cont, WebServicesTestUtils.GetXmlString(element, "id"
), WebServicesTestUtils.GetXmlString(element, "state"), WebServicesTestUtils.GetXmlString
(element, "user"), WebServicesTestUtils.GetXmlInt(element, "exitCode"), WebServicesTestUtils
.GetXmlString(element, "diagnostics"), WebServicesTestUtils.GetXmlString(element
, "nodeId"), WebServicesTestUtils.GetXmlInt(element, "totalMemoryNeededMB"), WebServicesTestUtils
.GetXmlInt(element, "totalVCoresNeeded"), WebServicesTestUtils.GetXmlString(element
, "containerLogsLink"));
}
}
/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/>
/// <exception cref="System.Exception"/>
public virtual void VerifyNodeContainerInfoGeneric(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
cont, string id, string state, string user, int exitCode, string diagnostics, string
nodeId, int totalMemoryNeededMB, int totalVCoresNeeded, string logsLink)
{
WebServicesTestUtils.CheckStringMatch("id", cont.GetContainerId().ToString(), id);
WebServicesTestUtils.CheckStringMatch("state", cont.GetContainerState().ToString(
), state);
WebServicesTestUtils.CheckStringMatch("user", cont.GetUser().ToString(), user);
NUnit.Framework.Assert.AreEqual("exitCode wrong", 0, exitCode);
WebServicesTestUtils.CheckStringMatch("diagnostics", "testing", diagnostics);
WebServicesTestUtils.CheckStringMatch("nodeId", nmContext.GetNodeId().ToString(),
nodeId);
NUnit.Framework.Assert.AreEqual("totalMemoryNeededMB wrong", YarnConfiguration.DefaultRmSchedulerMinimumAllocationMb
, totalMemoryNeededMB);
NUnit.Framework.Assert.AreEqual("totalVCoresNeeded wrong", YarnConfiguration.DefaultRmSchedulerMinimumAllocationVcores
, totalVCoresNeeded);
string shortLink = StringHelper.Ujoin("containerlogs", cont.GetContainerId().ToString
(), cont.GetUser());
NUnit.Framework.Assert.IsTrue("containerLogsLink wrong", logsLink.Contains(shortLink
));
}
public ContainerInfo(Context nmContext, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
container, string requestUri, string pathPrefix)
{
// JAXB needs this
this.id = container.GetContainerId().ToString();
this.nodeId = nmContext.GetNodeId().ToString();
ContainerStatus containerData = container.CloneAndGetContainerStatus();
this.exitCode = containerData.GetExitStatus();
this.exitStatus = (this.exitCode == ContainerExitStatus.Invalid) ? "N/A" : exitCode
.ToString();
this.state = container.GetContainerState().ToString();
this.diagnostics = containerData.GetDiagnostics();
if (this.diagnostics == null || this.diagnostics.IsEmpty())
{
this.diagnostics = string.Empty;
}
this.user = container.GetUser();
Resource res = container.GetResource();
if (res != null)
{
this.totalMemoryNeededMB = res.GetMemory();
this.totalVCoresNeeded = res.GetVirtualCores();
}
this.containerLogsShortLink = StringHelper.Ujoin("containerlogs", this.id, container
.GetUser());
if (requestUri == null)
{
requestUri = string.Empty;
}
if (pathPrefix == null)
{
pathPrefix = string.Empty;
}
this.containerLogsLink = StringHelper.Join(requestUri, pathPrefix, this.containerLogsShortLink
);
}
public ContainersLauncherEvent(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
container, ContainersLauncherEventType eventType)
: base(eventType)
{
this.container = container;
}
protected override void Render(HtmlBlock.Block html)
{
ContainerId containerID;
try
{
containerID = ConverterUtils.ToContainerId($(ContainerId));
}
catch (ArgumentException)
{
html.P().("Invalid containerId " + $(ContainerId)).();
return;
}
Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> div = html.Div("#content"
);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= this.nmContext.GetContainers()[containerID];
if (container == null)
{
div.H1("Unknown Container. Container might have completed, " + "please go back to the previous page and retry."
).();
return;
}
ContainerInfo info = new ContainerInfo(this.nmContext, container);
Info("Container information").("ContainerID", info.GetId()).("ContainerState", info
.GetState()).("ExitStatus", info.GetExitStatus()).("Diagnostics", info.GetDiagnostics
()).("User", info.GetUser()).("TotalMemoryNeeded", info.GetMemoryNeeded()).("TotalVCoresNeeded"
, info.GetVCoresNeeded()).("logs", info.GetShortLogLink(), "Link to logs");
html.(typeof(InfoBlock));
}
public virtual ContainerInfo GetNodeContainer(string id)
{
ContainerId containerId = null;
Init();
try
{
containerId = ConverterUtils.ToContainerId(id);
}
catch (Exception)
{
throw new BadRequestException("invalid container id, " + id);
}
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= nmContext.GetContainers()[containerId];
if (container == null)
{
throw new NotFoundException("container with id, " + id + ", not found");
}
return(new ContainerInfo(this.nmContext, container, uriInfo.GetBaseUri().ToString
(), webapp.Name()));
}
public virtual void TestNMSentContainerStatusOnResync()
{
ContainerStatus testCompleteContainer = TestNodeStatusUpdater.CreateContainerStatus
(2, ContainerState.Complete);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= TestNodeStatusUpdater.GetMockContainer(testCompleteContainer);
NMContainerStatus report = CreateNMContainerStatus(2, ContainerState.Complete);
Org.Mockito.Mockito.When(container.GetNMContainerStatus()).ThenReturn(report);
NodeManager nm = new _NodeManager_225(this, testCompleteContainer, container);
// first register, no containers info.
// put the completed container into the context
// second register contains the completed container info.
// first heartBeat contains the completed container info
// notify RESYNC on first heartbeat.
YarnConfiguration conf = CreateNMConfig();
nm.Init(conf);
nm.Start();
try
{
syncBarrier.Await();
}
catch (BrokenBarrierException)
{
}
NUnit.Framework.Assert.IsFalse(assertionFailedInThread.Get());
nm.Stop();
}
// do nothing
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
protected internal override void AuthorizeGetAndStopContainerRequest(ContainerId
containerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
container, bool stopRequest, NMTokenIdentifier identifier)
{
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
protected internal override void AuthorizeGetAndStopContainerRequest(ContainerId
containerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
container, bool stopRequest, NMTokenIdentifier identifier)
{
if (container == null || container.GetUser().Equals("Fail"))
{
throw new YarnException("Reject this container");
}
}
/// <exception cref="System.IO.IOException"/>
private int RunAndBlock(ContainerId cId, params string[] cmd)
{
string appId = "APP_" + GetNextId();
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
>();
ContainerLaunchContext context = Org.Mockito.Mockito.Mock <ContainerLaunchContext>
();
Dictionary <string, string> env = new Dictionary <string, string>();
Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(cId);
Org.Mockito.Mockito.When(container.GetLaunchContext()).ThenReturn(context);
Org.Mockito.Mockito.When(context.GetEnvironment()).ThenReturn(env);
string script = WriteScriptFile(cmd);
Path scriptPath = new Path(script);
Path tokensPath = new Path("/dev/null");
Path workDir = new Path(workSpace.GetAbsolutePath());
Path pidFile = new Path(workDir, "pid.txt");
exec.ActivateContainer(cId, pidFile);
return(exec.LaunchContainer(container, scriptPath, tokensPath, appSubmitter, appId
, workDir, dirsHandler.GetLocalDirs(), dirsHandler.GetLogDirs()));
}
public ContainerInfo(Context nmContext, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
container)
: this(nmContext, container, string.Empty, string.Empty)
{
}
public AuxServicesEvent(AuxServicesEventType eventType, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
container)
: this(eventType, null, container.GetContainerId().GetApplicationAttemptId().GetApplicationId
(), null, null, container)
{
}
public virtual void Handle(ContainersLauncherEvent @event)
{
// TODO: ContainersLauncher launches containers one by one!!
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= @event.GetContainer();
ContainerId containerId = container.GetContainerId();
switch (@event.GetType())
{
case ContainersLauncherEventType.LaunchContainer:
{
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = context.GetApplications()[containerId.GetApplicationAttemptId().GetApplicationId
()];
ContainerLaunch launch = new ContainerLaunch(context, GetConfig(), dispatcher, exec
, app, @event.GetContainer(), dirsHandler, containerManager);
containerLauncher.Submit(launch);
running[containerId] = launch;
break;
}
case ContainersLauncherEventType.RecoverContainer:
{
app = context.GetApplications()[containerId.GetApplicationAttemptId().GetApplicationId
()];
launch = new RecoveredContainerLaunch(context, GetConfig(), dispatcher, exec, app
, @event.GetContainer(), dirsHandler, containerManager);
containerLauncher.Submit(launch);
running[containerId] = launch;
break;
}
case ContainersLauncherEventType.CleanupContainer:
{
ContainerLaunch launcher = Sharpen.Collections.Remove(running, containerId);
if (launcher == null)
{
// Container not launched. So nothing needs to be done.
return;
}
// Cleanup a container whether it is running/killed/completed, so that
// no sub-processes are alive.
try
{
launcher.CleanupContainer();
}
catch (IOException)
{
Log.Warn("Got exception while cleaning container " + containerId + ". Ignoring.");
}
break;
}
}
}
public ContainerLocalizationEvent(LocalizationEventType @event, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
c)
: base(@event)
{
this.container = c;
}
public virtual void TestContainerLaunch()
{
string appSubmitter = "nobody";
string appId = "APP_ID";
string containerId = "CONTAINER_ID";
string testImage = "\"sequenceiq/hadoop-docker:2.4.1\"";
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
>(Org.Mockito.Mockito.ReturnsDeepStubs);
ContainerId cId = Org.Mockito.Mockito.Mock <ContainerId>(Org.Mockito.Mockito.ReturnsDeepStubs
);
ContainerLaunchContext context = Org.Mockito.Mockito.Mock <ContainerLaunchContext>
();
Dictionary <string, string> env = new Dictionary <string, string>();
Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(cId);
Org.Mockito.Mockito.When(container.GetLaunchContext()).ThenReturn(context);
Org.Mockito.Mockito.When(cId.GetApplicationAttemptId().GetApplicationId().ToString
()).ThenReturn(appId);
Org.Mockito.Mockito.When(cId.ToString()).ThenReturn(containerId);
Org.Mockito.Mockito.When(context.GetEnvironment()).ThenReturn(env);
env[YarnConfiguration.NmDockerContainerExecutorImageName] = testImage;
Path scriptPath = new Path("file:///bin/echo");
Path tokensPath = new Path("file:///dev/null");
Path pidFile = new Path(workDir, "pid");
dockerContainerExecutor.ActivateContainer(cId, pidFile);
int ret = dockerContainerExecutor.LaunchContainer(container, scriptPath, tokensPath
, appSubmitter, appId, workDir, dirsHandler.GetLocalDirs(), dirsHandler.GetLogDirs
());
NUnit.Framework.Assert.AreEqual(0, ret);
//get the script
Path sessionScriptPath = new Path(workDir, Shell.AppendScriptExtension(DockerContainerExecutor
.DockerContainerExecutorSessionScript));
LineNumberReader lnr = new LineNumberReader(new FileReader(sessionScriptPath.ToString
()));
bool cmdFound = false;
IList <string> localDirs = DirsToMount(dirsHandler.GetLocalDirs());
IList <string> logDirs = DirsToMount(dirsHandler.GetLogDirs());
IList <string> workDirMount = DirsToMount(Sharpen.Collections.SingletonList(workDir
.ToUri().GetPath()));
IList <string> expectedCommands = new AList <string>(Arrays.AsList(DockerLaunchCommand
, "run", "--rm", "--net=host", "--name", containerId));
Sharpen.Collections.AddAll(expectedCommands, localDirs);
Sharpen.Collections.AddAll(expectedCommands, logDirs);
Sharpen.Collections.AddAll(expectedCommands, workDirMount);
string shellScript = workDir + "/launch_container.sh";
Sharpen.Collections.AddAll(expectedCommands, Arrays.AsList(testImage.ReplaceAll("['\"]"
, string.Empty), "bash", "\"" + shellScript + "\""));
string expectedPidString = "echo `/bin/true inspect --format {{.State.Pid}} " + containerId
+ "` > " + pidFile.ToString() + ".tmp";
bool pidSetterFound = false;
while (lnr.Ready())
{
string line = lnr.ReadLine();
Log.Debug("line: " + line);
if (line.StartsWith(DockerLaunchCommand))
{
IList <string> command = new AList <string>();
foreach (string s in line.Split("\\s+"))
{
command.AddItem(s.Trim());
}
NUnit.Framework.Assert.AreEqual(expectedCommands, command);
cmdFound = true;
}
else
{
if (line.StartsWith("echo"))
{
NUnit.Framework.Assert.AreEqual(expectedPidString, line);
pidSetterFound = true;
}
}
}
NUnit.Framework.Assert.IsTrue(cmdFound);
NUnit.Framework.Assert.IsTrue(pidSetterFound);
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestContainerLaunchNullImage()
{
string appSubmitter = "nobody";
string appId = "APP_ID";
string containerId = "CONTAINER_ID";
string testImage = string.Empty;
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
>(Org.Mockito.Mockito.ReturnsDeepStubs);
ContainerId cId = Org.Mockito.Mockito.Mock <ContainerId>(Org.Mockito.Mockito.ReturnsDeepStubs
);
ContainerLaunchContext context = Org.Mockito.Mockito.Mock <ContainerLaunchContext>
();
Dictionary <string, string> env = new Dictionary <string, string>();
Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(cId);
Org.Mockito.Mockito.When(container.GetLaunchContext()).ThenReturn(context);
Org.Mockito.Mockito.When(cId.GetApplicationAttemptId().GetApplicationId().ToString
()).ThenReturn(appId);
Org.Mockito.Mockito.When(cId.ToString()).ThenReturn(containerId);
Org.Mockito.Mockito.When(context.GetEnvironment()).ThenReturn(env);
env[YarnConfiguration.NmDockerContainerExecutorImageName] = testImage;
dockerContainerExecutor.GetConf().Set(YarnConfiguration.NmDockerContainerExecutorImageName
, testImage);
Path scriptPath = new Path("file:///bin/echo");
Path tokensPath = new Path("file:///dev/null");
Path pidFile = new Path(workDir, "pid.txt");
dockerContainerExecutor.ActivateContainer(cId, pidFile);
dockerContainerExecutor.LaunchContainer(container, scriptPath, tokensPath, appSubmitter
, appId, workDir, dirsHandler.GetLocalDirs(), dirsHandler.GetLogDirs());
}
