public ApplicationContainerInitEvent(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container) : base(container.GetContainerId().GetApplicationAttemptId().GetApplicationId(), ApplicationEventType .InitContainer) { this.container = container; }
/// <summary>Event requesting the cleanup of the rsrc.</summary> /// <param name="c"/> /// <param name="rsrc"/> public ContainerLocalizationCleanupEvent(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container c, IDictionary <LocalResourceVisibility, ICollection <LocalResourceRequest> > rsrc ) : base(LocalizationEventType.CleanupContainerResources, c) { this.rsrc = rsrc; }
public _MockResourceTracker_235(_TestNodeStatusUpdaterResync_232 _enclosing, ContainerStatus testCompleteContainer, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container) { this._enclosing = _enclosing; this.testCompleteContainer = testCompleteContainer; this.container = container; }
/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/> /// <exception cref="System.Exception"/> public virtual void VerifyNodeContainerInfo(JSONObject info, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container cont) { NUnit.Framework.Assert.AreEqual("incorrect number of elements", 9, info.Length()); VerifyNodeContainerInfoGeneric(cont, info.GetString("id"), info.GetString("state" ), info.GetString("user"), info.GetInt("exitCode"), info.GetString("diagnostics" ), info.GetString("nodeId"), info.GetInt("totalMemoryNeededMB"), info.GetInt("totalVCoresNeeded" ), info.GetString("containerLogsLink")); }
public _NodeManager_225(TestNodeManagerResync _enclosing, ContainerStatus testCompleteContainer , Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container) { this._enclosing = _enclosing; this.testCompleteContainer = testCompleteContainer; this.container = container; this.registerCount = 0; }
public _TestNodeStatusUpdaterResync_232(_NodeManager_225 _enclosing, ContainerStatus testCompleteContainer, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container, Context baseArg1, Dispatcher baseArg2, NodeHealthCheckerService baseArg3 , NodeManagerMetrics baseArg4) : base(_enclosing, baseArg1, baseArg2, baseArg3, baseArg4) { this._enclosing = _enclosing; this.testCompleteContainer = testCompleteContainer; this.container = container; }
public AuxServicesEvent(AuxServicesEventType eventType, string user, ApplicationId appId, string serviceId, ByteBuffer serviceData, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container) : base(eventType) { this.user = user; this.appId = appId; this.serviceId = serviceId; this.serviceData = serviceData; this.container = container; }
/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/> /// <exception cref="System.Exception"/> public virtual void VerifyContainersInfoXML(NodeList nodes, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container cont) { for (int i = 0; i < nodes.GetLength(); i++) { Element element = (Element)nodes.Item(i); VerifyNodeContainerInfoGeneric(cont, WebServicesTestUtils.GetXmlString(element, "id" ), WebServicesTestUtils.GetXmlString(element, "state"), WebServicesTestUtils.GetXmlString (element, "user"), WebServicesTestUtils.GetXmlInt(element, "exitCode"), WebServicesTestUtils .GetXmlString(element, "diagnostics"), WebServicesTestUtils.GetXmlString(element , "nodeId"), WebServicesTestUtils.GetXmlInt(element, "totalMemoryNeededMB"), WebServicesTestUtils .GetXmlInt(element, "totalVCoresNeeded"), WebServicesTestUtils.GetXmlString(element , "containerLogsLink")); } }
/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/> /// <exception cref="System.Exception"/> public virtual void VerifyNodeContainerInfoGeneric(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container cont, string id, string state, string user, int exitCode, string diagnostics, string nodeId, int totalMemoryNeededMB, int totalVCoresNeeded, string logsLink) { WebServicesTestUtils.CheckStringMatch("id", cont.GetContainerId().ToString(), id); WebServicesTestUtils.CheckStringMatch("state", cont.GetContainerState().ToString( ), state); WebServicesTestUtils.CheckStringMatch("user", cont.GetUser().ToString(), user); NUnit.Framework.Assert.AreEqual("exitCode wrong", 0, exitCode); WebServicesTestUtils.CheckStringMatch("diagnostics", "testing", diagnostics); WebServicesTestUtils.CheckStringMatch("nodeId", nmContext.GetNodeId().ToString(), nodeId); NUnit.Framework.Assert.AreEqual("totalMemoryNeededMB wrong", YarnConfiguration.DefaultRmSchedulerMinimumAllocationMb , totalMemoryNeededMB); NUnit.Framework.Assert.AreEqual("totalVCoresNeeded wrong", YarnConfiguration.DefaultRmSchedulerMinimumAllocationVcores , totalVCoresNeeded); string shortLink = StringHelper.Ujoin("containerlogs", cont.GetContainerId().ToString (), cont.GetUser()); NUnit.Framework.Assert.IsTrue("containerLogsLink wrong", logsLink.Contains(shortLink )); }
public ContainerInfo(Context nmContext, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container, string requestUri, string pathPrefix) { // JAXB needs this this.id = container.GetContainerId().ToString(); this.nodeId = nmContext.GetNodeId().ToString(); ContainerStatus containerData = container.CloneAndGetContainerStatus(); this.exitCode = containerData.GetExitStatus(); this.exitStatus = (this.exitCode == ContainerExitStatus.Invalid) ? "N/A" : exitCode .ToString(); this.state = container.GetContainerState().ToString(); this.diagnostics = containerData.GetDiagnostics(); if (this.diagnostics == null || this.diagnostics.IsEmpty()) { this.diagnostics = string.Empty; } this.user = container.GetUser(); Resource res = container.GetResource(); if (res != null) { this.totalMemoryNeededMB = res.GetMemory(); this.totalVCoresNeeded = res.GetVirtualCores(); } this.containerLogsShortLink = StringHelper.Ujoin("containerlogs", this.id, container .GetUser()); if (requestUri == null) { requestUri = string.Empty; } if (pathPrefix == null) { pathPrefix = string.Empty; } this.containerLogsLink = StringHelper.Join(requestUri, pathPrefix, this.containerLogsShortLink ); }
public ContainersLauncherEvent(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container, ContainersLauncherEventType eventType) : base(eventType) { this.container = container; }
protected override void Render(HtmlBlock.Block html) { ContainerId containerID; try { containerID = ConverterUtils.ToContainerId($(ContainerId)); } catch (ArgumentException) { html.P().("Invalid containerId " + $(ContainerId)).(); return; } Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> div = html.Div("#content" ); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container = this.nmContext.GetContainers()[containerID]; if (container == null) { div.H1("Unknown Container. Container might have completed, " + "please go back to the previous page and retry." ).(); return; } ContainerInfo info = new ContainerInfo(this.nmContext, container); Info("Container information").("ContainerID", info.GetId()).("ContainerState", info .GetState()).("ExitStatus", info.GetExitStatus()).("Diagnostics", info.GetDiagnostics ()).("User", info.GetUser()).("TotalMemoryNeeded", info.GetMemoryNeeded()).("TotalVCoresNeeded" , info.GetVCoresNeeded()).("logs", info.GetShortLogLink(), "Link to logs"); html.(typeof(InfoBlock)); }
public virtual ContainerInfo GetNodeContainer(string id) { ContainerId containerId = null; Init(); try { containerId = ConverterUtils.ToContainerId(id); } catch (Exception) { throw new BadRequestException("invalid container id, " + id); } Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container = nmContext.GetContainers()[containerId]; if (container == null) { throw new NotFoundException("container with id, " + id + ", not found"); } return(new ContainerInfo(this.nmContext, container, uriInfo.GetBaseUri().ToString (), webapp.Name())); }
public virtual void TestNMSentContainerStatusOnResync() { ContainerStatus testCompleteContainer = TestNodeStatusUpdater.CreateContainerStatus (2, ContainerState.Complete); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container = TestNodeStatusUpdater.GetMockContainer(testCompleteContainer); NMContainerStatus report = CreateNMContainerStatus(2, ContainerState.Complete); Org.Mockito.Mockito.When(container.GetNMContainerStatus()).ThenReturn(report); NodeManager nm = new _NodeManager_225(this, testCompleteContainer, container); // first register, no containers info. // put the completed container into the context // second register contains the completed container info. // first heartBeat contains the completed container info // notify RESYNC on first heartbeat. YarnConfiguration conf = CreateNMConfig(); nm.Init(conf); nm.Start(); try { syncBarrier.Await(); } catch (BrokenBarrierException) { } NUnit.Framework.Assert.IsFalse(assertionFailedInThread.Get()); nm.Stop(); }
// do nothing /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> protected internal override void AuthorizeGetAndStopContainerRequest(ContainerId containerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container, bool stopRequest, NMTokenIdentifier identifier) { }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> protected internal override void AuthorizeGetAndStopContainerRequest(ContainerId containerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container, bool stopRequest, NMTokenIdentifier identifier) { if (container == null || container.GetUser().Equals("Fail")) { throw new YarnException("Reject this container"); } }
/// <exception cref="System.IO.IOException"/> private int RunAndBlock(ContainerId cId, params string[] cmd) { string appId = "APP_" + GetNextId(); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container >(); ContainerLaunchContext context = Org.Mockito.Mockito.Mock <ContainerLaunchContext> (); Dictionary <string, string> env = new Dictionary <string, string>(); Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(cId); Org.Mockito.Mockito.When(container.GetLaunchContext()).ThenReturn(context); Org.Mockito.Mockito.When(context.GetEnvironment()).ThenReturn(env); string script = WriteScriptFile(cmd); Path scriptPath = new Path(script); Path tokensPath = new Path("/dev/null"); Path workDir = new Path(workSpace.GetAbsolutePath()); Path pidFile = new Path(workDir, "pid.txt"); exec.ActivateContainer(cId, pidFile); return(exec.LaunchContainer(container, scriptPath, tokensPath, appSubmitter, appId , workDir, dirsHandler.GetLocalDirs(), dirsHandler.GetLogDirs())); }
public ContainerInfo(Context nmContext, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container) : this(nmContext, container, string.Empty, string.Empty) { }
public AuxServicesEvent(AuxServicesEventType eventType, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container) : this(eventType, null, container.GetContainerId().GetApplicationAttemptId().GetApplicationId (), null, null, container) { }
public virtual void Handle(ContainersLauncherEvent @event) { // TODO: ContainersLauncher launches containers one by one!! Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container = @event.GetContainer(); ContainerId containerId = container.GetContainerId(); switch (@event.GetType()) { case ContainersLauncherEventType.LaunchContainer: { Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app = context.GetApplications()[containerId.GetApplicationAttemptId().GetApplicationId ()]; ContainerLaunch launch = new ContainerLaunch(context, GetConfig(), dispatcher, exec , app, @event.GetContainer(), dirsHandler, containerManager); containerLauncher.Submit(launch); running[containerId] = launch; break; } case ContainersLauncherEventType.RecoverContainer: { app = context.GetApplications()[containerId.GetApplicationAttemptId().GetApplicationId ()]; launch = new RecoveredContainerLaunch(context, GetConfig(), dispatcher, exec, app , @event.GetContainer(), dirsHandler, containerManager); containerLauncher.Submit(launch); running[containerId] = launch; break; } case ContainersLauncherEventType.CleanupContainer: { ContainerLaunch launcher = Sharpen.Collections.Remove(running, containerId); if (launcher == null) { // Container not launched. So nothing needs to be done. return; } // Cleanup a container whether it is running/killed/completed, so that // no sub-processes are alive. try { launcher.CleanupContainer(); } catch (IOException) { Log.Warn("Got exception while cleaning container " + containerId + ". Ignoring."); } break; } } }
public ContainerLocalizationEvent(LocalizationEventType @event, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container c) : base(@event) { this.container = c; }
public virtual void TestContainerLaunch() { string appSubmitter = "nobody"; string appId = "APP_ID"; string containerId = "CONTAINER_ID"; string testImage = "\"sequenceiq/hadoop-docker:2.4.1\""; Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container >(Org.Mockito.Mockito.ReturnsDeepStubs); ContainerId cId = Org.Mockito.Mockito.Mock <ContainerId>(Org.Mockito.Mockito.ReturnsDeepStubs ); ContainerLaunchContext context = Org.Mockito.Mockito.Mock <ContainerLaunchContext> (); Dictionary <string, string> env = new Dictionary <string, string>(); Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(cId); Org.Mockito.Mockito.When(container.GetLaunchContext()).ThenReturn(context); Org.Mockito.Mockito.When(cId.GetApplicationAttemptId().GetApplicationId().ToString ()).ThenReturn(appId); Org.Mockito.Mockito.When(cId.ToString()).ThenReturn(containerId); Org.Mockito.Mockito.When(context.GetEnvironment()).ThenReturn(env); env[YarnConfiguration.NmDockerContainerExecutorImageName] = testImage; Path scriptPath = new Path("file:///bin/echo"); Path tokensPath = new Path("file:///dev/null"); Path pidFile = new Path(workDir, "pid"); dockerContainerExecutor.ActivateContainer(cId, pidFile); int ret = dockerContainerExecutor.LaunchContainer(container, scriptPath, tokensPath , appSubmitter, appId, workDir, dirsHandler.GetLocalDirs(), dirsHandler.GetLogDirs ()); NUnit.Framework.Assert.AreEqual(0, ret); //get the script Path sessionScriptPath = new Path(workDir, Shell.AppendScriptExtension(DockerContainerExecutor .DockerContainerExecutorSessionScript)); LineNumberReader lnr = new LineNumberReader(new FileReader(sessionScriptPath.ToString ())); bool cmdFound = false; IList <string> localDirs = DirsToMount(dirsHandler.GetLocalDirs()); IList <string> logDirs = DirsToMount(dirsHandler.GetLogDirs()); IList <string> workDirMount = DirsToMount(Sharpen.Collections.SingletonList(workDir .ToUri().GetPath())); IList <string> expectedCommands = new AList <string>(Arrays.AsList(DockerLaunchCommand , "run", "--rm", "--net=host", "--name", containerId)); Sharpen.Collections.AddAll(expectedCommands, localDirs); Sharpen.Collections.AddAll(expectedCommands, logDirs); Sharpen.Collections.AddAll(expectedCommands, workDirMount); string shellScript = workDir + "/launch_container.sh"; Sharpen.Collections.AddAll(expectedCommands, Arrays.AsList(testImage.ReplaceAll("['\"]" , string.Empty), "bash", "\"" + shellScript + "\"")); string expectedPidString = "echo `/bin/true inspect --format {{.State.Pid}} " + containerId + "` > " + pidFile.ToString() + ".tmp"; bool pidSetterFound = false; while (lnr.Ready()) { string line = lnr.ReadLine(); Log.Debug("line: " + line); if (line.StartsWith(DockerLaunchCommand)) { IList <string> command = new AList <string>(); foreach (string s in line.Split("\\s+")) { command.AddItem(s.Trim()); } NUnit.Framework.Assert.AreEqual(expectedCommands, command); cmdFound = true; } else { if (line.StartsWith("echo")) { NUnit.Framework.Assert.AreEqual(expectedPidString, line); pidSetterFound = true; } } } NUnit.Framework.Assert.IsTrue(cmdFound); NUnit.Framework.Assert.IsTrue(pidSetterFound); }
/// <exception cref="System.IO.IOException"/> public virtual void TestContainerLaunchNullImage() { string appSubmitter = "nobody"; string appId = "APP_ID"; string containerId = "CONTAINER_ID"; string testImage = string.Empty; Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container >(Org.Mockito.Mockito.ReturnsDeepStubs); ContainerId cId = Org.Mockito.Mockito.Mock <ContainerId>(Org.Mockito.Mockito.ReturnsDeepStubs ); ContainerLaunchContext context = Org.Mockito.Mockito.Mock <ContainerLaunchContext> (); Dictionary <string, string> env = new Dictionary <string, string>(); Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(cId); Org.Mockito.Mockito.When(container.GetLaunchContext()).ThenReturn(context); Org.Mockito.Mockito.When(cId.GetApplicationAttemptId().GetApplicationId().ToString ()).ThenReturn(appId); Org.Mockito.Mockito.When(cId.ToString()).ThenReturn(containerId); Org.Mockito.Mockito.When(context.GetEnvironment()).ThenReturn(env); env[YarnConfiguration.NmDockerContainerExecutorImageName] = testImage; dockerContainerExecutor.GetConf().Set(YarnConfiguration.NmDockerContainerExecutorImageName , testImage); Path scriptPath = new Path("file:///bin/echo"); Path tokensPath = new Path("file:///dev/null"); Path pidFile = new Path(workDir, "pid.txt"); dockerContainerExecutor.ActivateContainer(cId, pidFile); dockerContainerExecutor.LaunchContainer(container, scriptPath, tokensPath, appSubmitter , appId, workDir, dirsHandler.GetLocalDirs(), dirsHandler.GetLogDirs()); }