public virtual void TestContainerManagerInitialization() { containerManager.Start(); IPAddress localAddr = Sharpen.Runtime.GetLocalHost(); string fqdn = localAddr.ToString(); if (!localAddr.GetHostAddress().Equals(fqdn)) { // only check if fqdn is not same as ip // api returns ip in case of resolution failure NUnit.Framework.Assert.AreEqual(fqdn, context.GetNodeId().GetHost()); } // Just do a query for a non-existing container. bool throwsException = false; try { IList <ContainerId> containerIds = new AList <ContainerId>(); ContainerId id = CreateContainerId(0); containerIds.AddItem(id); GetContainerStatusesRequest request = GetContainerStatusesRequest.NewInstance(containerIds ); GetContainerStatusesResponse response = containerManager.GetContainerStatuses(request ); if (response.GetFailedRequests().Contains(id)) { throw response.GetFailedRequests()[id].DeSerialize(); } } catch { throwsException = true; } NUnit.Framework.Assert.IsTrue(throwsException); }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> public virtual GetContainerStatusesResponse GetContainerStatuses(GetContainerStatusesRequest request) { lock (this) { IList <ContainerStatus> statuses = new AList <ContainerStatus>(); foreach (ContainerId containerId in request.GetContainerIds()) { IList <Container> appContainers = containers[containerId.GetApplicationAttemptId() .GetApplicationId()]; Container container = null; foreach (Container c in appContainers) { if (c.GetId().Equals(containerId)) { container = c; } } if (container != null && containerStatusMap[container].GetState() != null) { statuses.AddItem(containerStatusMap[container]); } } return(GetContainerStatusesResponse.NewInstance(statuses, null)); } }
/// <exception cref="System.Exception"/> private void GetContainerStatus(YarnRPC rpc, Token nmToken, ContainerId containerId , ApplicationAttemptId appAttemptId, NodeId nodeId, bool isExceptionExpected) { IList <ContainerId> containerIds = new AList <ContainerId>(); containerIds.AddItem(containerId); GetContainerStatusesRequest request = GetContainerStatusesRequest.NewInstance(containerIds ); ContainerManagementProtocol proxy = null; try { proxy = GetContainerManagementProtocolProxy(rpc, nmToken, nodeId, appAttemptId.ToString ()); GetContainerStatusesResponse statuses = proxy.GetContainerStatuses(request); if (statuses.GetFailedRequests() != null && statuses.GetFailedRequests().Contains (containerId)) { ParseAndThrowException(statuses.GetFailedRequests()[containerId].DeSerialize()); } } finally { if (proxy != null) { rpc.StopProxy(proxy, conf); } } }
/// <exception cref="System.Exception"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> public static void WaitForContainerState(ContainerManagementProtocol containerManager , ContainerId containerID, ContainerState finalState, int timeOutMax) { IList <ContainerId> list = new AList <ContainerId>(); list.AddItem(containerID); GetContainerStatusesRequest request = GetContainerStatusesRequest.NewInstance(list ); ContainerStatus containerStatus = containerManager.GetContainerStatuses(request). GetContainerStatuses()[0]; int timeoutSecs = 0; while (!containerStatus.GetState().Equals(finalState) && timeoutSecs++ < timeOutMax ) { Sharpen.Thread.Sleep(1000); Log.Info("Waiting for container to get into state " + finalState + ". Current state is " + containerStatus.GetState()); containerStatus = containerManager.GetContainerStatuses(request).GetContainerStatuses ()[0]; } Log.Info("Container state is " + containerStatus.GetState()); NUnit.Framework.Assert.AreEqual("ContainerState is not correct (timedout)", finalState , containerStatus.GetState()); }
/// <exception cref="System.IO.IOException"/> public virtual GetContainerStatusesResponse GetContainerStatuses(GetContainerStatusesRequest request) { IList<ContainerStatus> statuses = new AList<ContainerStatus>(); statuses.AddItem(this.status); return GetContainerStatusesResponse.NewInstance(statuses, null); }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> public override ContainerStatus GetContainerStatus(ContainerId containerId, NodeId nodeId) { ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData proxy = null; IList <ContainerId> containerIds = new AList <ContainerId>(); containerIds.AddItem(containerId); try { proxy = cmProxy.GetProxy(nodeId.ToString(), containerId); GetContainerStatusesResponse response = proxy.GetContainerManagementProtocol().GetContainerStatuses (GetContainerStatusesRequest.NewInstance(containerIds)); if (response.GetFailedRequests() != null && response.GetFailedRequests().Contains (containerId)) { Exception t = response.GetFailedRequests()[containerId].DeSerialize(); ParseAndThrowException(t); } ContainerStatus containerStatus = response.GetContainerStatuses()[0]; return(containerStatus); } finally { if (proxy != null) { cmProxy.MayBeCloseProxy(proxy); } } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> public static void StartContainer(NodeManager nm, ContainerId cId, FileContext localFS , FilePath scriptFileDir, FilePath processStartFile) { FilePath scriptFile = CreateUnhaltingScriptFile(cId, scriptFileDir, processStartFile ); ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext >(); NodeId nodeId = BuilderUtils.NewNodeId(Sharpen.Extensions.GetAddressByName("localhost" ).ToString(), 12345); URL localResourceUri = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new Path(scriptFile.GetAbsolutePath()))); LocalResource localResource = recordFactory.NewRecordInstance <LocalResource>(); localResource.SetResource(localResourceUri); localResource.SetSize(-1); localResource.SetVisibility(LocalResourceVisibility.Application); localResource.SetType(LocalResourceType.File); localResource.SetTimestamp(scriptFile.LastModified()); string destinationFile = "dest_file"; IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource >(); localResources[destinationFile] = localResource; containerLaunchContext.SetLocalResources(localResources); IList <string> commands = Arrays.AsList(Shell.GetRunScriptCommand(scriptFile)); containerLaunchContext.SetCommands(commands); IPEndPoint containerManagerBindAddress = NetUtils.CreateSocketAddrForHost("127.0.0.1" , 12345); UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(cId.ToString ()); Org.Apache.Hadoop.Security.Token.Token <NMTokenIdentifier> nmToken = ConverterUtils .ConvertFromYarn(nm.GetNMContext().GetNMTokenSecretManager().CreateNMToken(cId.GetApplicationAttemptId (), nodeId, user), containerManagerBindAddress); currentUser.AddToken(nmToken); ContainerManagementProtocol containerManager = currentUser.DoAs(new _PrivilegedAction_229 ()); StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext , TestContainerManager.CreateContainerToken(cId, 0, nodeId, user, nm.GetNMContext ().GetContainerTokenSecretManager())); IList <StartContainerRequest> list = new AList <StartContainerRequest>(); list.AddItem(scRequest); StartContainersRequest allRequests = StartContainersRequest.NewInstance(list); containerManager.StartContainers(allRequests); IList <ContainerId> containerIds = new AList <ContainerId>(); containerIds.AddItem(cId); GetContainerStatusesRequest request = GetContainerStatusesRequest.NewInstance(containerIds ); ContainerStatus containerStatus = containerManager.GetContainerStatuses(request). GetContainerStatuses()[0]; NUnit.Framework.Assert.AreEqual(ContainerState.Running, containerStatus.GetState( )); }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> public virtual GetContainerStatusesResponse GetContainerStatuses(GetContainerStatusesRequest request) { GetContainerStatusesResponse response = TestRPC.recordFactory.NewRecordInstance <GetContainerStatusesResponse >(); response.SetContainerStatuses(this.statuses); return(response); }
/// <summary>This tests whether a containerId is serialized/deserialized with epoch.</summary> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> private void TestContainerTokenWithEpoch(Configuration conf) { Log.Info("Running test for serializing/deserializing containerIds"); NMTokenSecretManagerInRM nmTokenSecretManagerInRM = yarnCluster.GetResourceManager ().GetRMContext().GetNMTokenSecretManager(); ApplicationId appId = ApplicationId.NewInstance(1, 1); ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, 0); ContainerId cId = ContainerId.NewContainerId(appAttemptId, (5L << 40) | 3L); NodeManager nm = yarnCluster.GetNodeManager(0); NMTokenSecretManagerInNM nmTokenSecretManagerInNM = nm.GetNMContext().GetNMTokenSecretManager (); string user = "******"; WaitForNMToReceiveNMTokenKey(nmTokenSecretManagerInNM, nm); NodeId nodeId = nm.GetNMContext().GetNodeId(); // Both id should be equal. NUnit.Framework.Assert.AreEqual(nmTokenSecretManagerInNM.GetCurrentKey().GetKeyId (), nmTokenSecretManagerInRM.GetCurrentKey().GetKeyId()); // Creating a normal Container Token RMContainerTokenSecretManager containerTokenSecretManager = yarnCluster.GetResourceManager ().GetRMContext().GetContainerTokenSecretManager(); Resource r = Resource.NewInstance(1230, 2); Org.Apache.Hadoop.Yarn.Api.Records.Token containerToken = containerTokenSecretManager .CreateContainerToken(cId, nodeId, user, r, Priority.NewInstance(0), 0); ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier( ); byte[] tokenIdentifierContent = ((byte[])containerToken.GetIdentifier().Array()); DataInputBuffer dib = new DataInputBuffer(); dib.Reset(tokenIdentifierContent, tokenIdentifierContent.Length); containerTokenIdentifier.ReadFields(dib); NUnit.Framework.Assert.AreEqual(cId, containerTokenIdentifier.GetContainerID()); NUnit.Framework.Assert.AreEqual(cId.ToString(), containerTokenIdentifier.GetContainerID ().ToString()); Org.Apache.Hadoop.Yarn.Api.Records.Token nmToken = nmTokenSecretManagerInRM.CreateNMToken (appAttemptId, nodeId, user); YarnRPC rpc = YarnRPC.Create(conf); TestStartContainer(rpc, appAttemptId, nodeId, containerToken, nmToken, false); IList <ContainerId> containerIds = new List <ContainerId>(); containerIds.AddItem(cId); ContainerManagementProtocol proxy = GetContainerManagementProtocolProxy(rpc, nmToken , nodeId, user); GetContainerStatusesResponse res = proxy.GetContainerStatuses(GetContainerStatusesRequest .NewInstance(containerIds)); NUnit.Framework.Assert.IsNotNull(res.GetContainerStatuses()[0]); NUnit.Framework.Assert.AreEqual(cId, res.GetContainerStatuses()[0].GetContainerId ()); NUnit.Framework.Assert.AreEqual(cId.ToString(), res.GetContainerStatuses()[0].GetContainerId ().ToString()); }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> public override GetContainerStatusesResponse GetContainerStatuses(GetContainerStatusesRequest request) { if (this._enclosing.retryCount < 5) { this._enclosing.retryCount++; throw new ConnectException("get container status exception"); } return(base.GetContainerStatuses(request)); }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> public virtual GetContainerStatusesResponse GetContainerStatuses(GetContainerStatusesRequest request) { IList <ContainerStatus> list = new AList <ContainerStatus>(); list.AddItem(this.status); GetContainerStatusesResponse response = GetContainerStatusesResponse.NewInstance( list, null); return(null); }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> public virtual GetContainerStatusesResponse GetContainerStatuses(GetContainerStatusesRequest request) { YarnServiceProtos.GetContainerStatusesRequestProto requestProto = ((GetContainerStatusesRequestPBImpl )request).GetProto(); try { return(new GetContainerStatusesResponsePBImpl(proxy.GetContainerStatuses(null, requestProto ))); } catch (ServiceException e) { RPCUtil.UnwrapAndThrowException(e); return(null); } }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> public GetContainerStatusesResponse GetContainerStatuses(GetContainerStatusesRequest request) { return(GetContainerStatusesResponse.NewInstance(null, null)); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> private void TestContainerLaunchAndExit(int exitCode) { FilePath scriptFile = Shell.AppendScriptExtension(tmpDir, "scriptFile"); PrintWriter fileWriter = new PrintWriter(scriptFile); FilePath processStartFile = new FilePath(tmpDir, "start_file.txt").GetAbsoluteFile (); // ////// Construct the Container-id ContainerId cId = CreateContainerId(0); if (Shell.Windows) { fileWriter.WriteLine("@echo Hello World!> " + processStartFile); fileWriter.WriteLine("@echo " + cId + ">> " + processStartFile); if (exitCode != 0) { fileWriter.WriteLine("@exit " + exitCode); } } else { fileWriter.Write("\numask 0"); // So that start file is readable by the test fileWriter.Write("\necho Hello World! > " + processStartFile); fileWriter.Write("\necho $$ >> " + processStartFile); // Have script throw an exit code at the end if (exitCode != 0) { fileWriter.Write("\nexit " + exitCode); } } fileWriter.Close(); ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext >(); URL resource_alpha = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new Path(scriptFile.GetAbsolutePath()))); LocalResource rsrc_alpha = recordFactory.NewRecordInstance <LocalResource>(); rsrc_alpha.SetResource(resource_alpha); rsrc_alpha.SetSize(-1); rsrc_alpha.SetVisibility(LocalResourceVisibility.Application); rsrc_alpha.SetType(LocalResourceType.File); rsrc_alpha.SetTimestamp(scriptFile.LastModified()); string destinationFile = "dest_file"; IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource >(); localResources[destinationFile] = rsrc_alpha; containerLaunchContext.SetLocalResources(localResources); IList <string> commands = Arrays.AsList(Shell.GetRunScriptCommand(scriptFile)); containerLaunchContext.SetCommands(commands); StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext , CreateContainerToken(cId, DummyRmIdentifier, context.GetNodeId(), user, context .GetContainerTokenSecretManager())); IList <StartContainerRequest> list = new AList <StartContainerRequest>(); list.AddItem(scRequest); StartContainersRequest allRequests = StartContainersRequest.NewInstance(list); containerManager.StartContainers(allRequests); BaseContainerManagerTest.WaitForContainerState(containerManager, cId, ContainerState .Complete); IList <ContainerId> containerIds = new AList <ContainerId>(); containerIds.AddItem(cId); GetContainerStatusesRequest gcsRequest = GetContainerStatusesRequest.NewInstance( containerIds); ContainerStatus containerStatus = containerManager.GetContainerStatuses(gcsRequest ).GetContainerStatuses()[0]; // Verify exit status matches exit state of script NUnit.Framework.Assert.AreEqual(exitCode, containerStatus.GetExitStatus()); }
public virtual void TestMultipleContainersStopAndGetStatus() { containerManager.Start(); IList <StartContainerRequest> startRequest = new AList <StartContainerRequest>(); ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext >(); IList <ContainerId> containerIds = new AList <ContainerId>(); for (int i = 0; i < 10; i++) { ContainerId cId = CreateContainerId(i); string user = null; if ((i & 1) == 0) { // container with even id fail user = "******"; } else { user = "******"; } Token containerToken = CreateContainerToken(cId, DummyRmIdentifier, context.GetNodeId (), user, context.GetContainerTokenSecretManager()); StartContainerRequest request = StartContainerRequest.NewInstance(containerLaunchContext , containerToken); startRequest.AddItem(request); containerIds.AddItem(cId); } // start containers StartContainersRequest requestList = StartContainersRequest.NewInstance(startRequest ); containerManager.StartContainers(requestList); // Get container statuses GetContainerStatusesRequest statusRequest = GetContainerStatusesRequest.NewInstance (containerIds); GetContainerStatusesResponse statusResponse = containerManager.GetContainerStatuses (statusRequest); NUnit.Framework.Assert.AreEqual(5, statusResponse.GetContainerStatuses().Count); foreach (ContainerStatus status in statusResponse.GetContainerStatuses()) { // Containers with odd id should succeed NUnit.Framework.Assert.AreEqual(1, status.GetContainerId().GetContainerId() & 1); } NUnit.Framework.Assert.AreEqual(5, statusResponse.GetFailedRequests().Count); foreach (KeyValuePair <ContainerId, SerializedException> entry in statusResponse.GetFailedRequests ()) { // Containers with even id should fail. NUnit.Framework.Assert.AreEqual(0, entry.Key.GetContainerId() & 1); NUnit.Framework.Assert.IsTrue(entry.Value.GetMessage().Contains("Reject this container" )); } // stop containers StopContainersRequest stopRequest = StopContainersRequest.NewInstance(containerIds ); StopContainersResponse stopResponse = containerManager.StopContainers(stopRequest ); NUnit.Framework.Assert.AreEqual(5, stopResponse.GetSuccessfullyStoppedContainers( ).Count); foreach (ContainerId id in stopResponse.GetSuccessfullyStoppedContainers()) { // Containers with odd id should succeed. NUnit.Framework.Assert.AreEqual(1, id.GetContainerId() & 1); } NUnit.Framework.Assert.AreEqual(5, stopResponse.GetFailedRequests().Count); foreach (KeyValuePair <ContainerId, SerializedException> entry_1 in stopResponse.GetFailedRequests ()) { // Containers with even id should fail. NUnit.Framework.Assert.AreEqual(0, entry_1.Key.GetContainerId() & 1); NUnit.Framework.Assert.IsTrue(entry_1.Value.GetMessage().Contains("Reject this container" )); } }
public virtual void TestContainerLaunchAndStop() { containerManager.Start(); FilePath scriptFile = Shell.AppendScriptExtension(tmpDir, "scriptFile"); PrintWriter fileWriter = new PrintWriter(scriptFile); FilePath processStartFile = new FilePath(tmpDir, "start_file.txt").GetAbsoluteFile (); // ////// Construct the Container-id ContainerId cId = CreateContainerId(0); if (Shell.Windows) { fileWriter.WriteLine("@echo Hello World!> " + processStartFile); fileWriter.WriteLine("@echo " + cId + ">> " + processStartFile); fileWriter.WriteLine("@ping -n 100 127.0.0.1 >nul"); } else { fileWriter.Write("\numask 0"); // So that start file is readable by the test fileWriter.Write("\necho Hello World! > " + processStartFile); fileWriter.Write("\necho $$ >> " + processStartFile); fileWriter.Write("\nexec sleep 100"); } fileWriter.Close(); ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext >(); URL resource_alpha = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new Path(scriptFile.GetAbsolutePath()))); LocalResource rsrc_alpha = recordFactory.NewRecordInstance <LocalResource>(); rsrc_alpha.SetResource(resource_alpha); rsrc_alpha.SetSize(-1); rsrc_alpha.SetVisibility(LocalResourceVisibility.Application); rsrc_alpha.SetType(LocalResourceType.File); rsrc_alpha.SetTimestamp(scriptFile.LastModified()); string destinationFile = "dest_file"; IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource >(); localResources[destinationFile] = rsrc_alpha; containerLaunchContext.SetLocalResources(localResources); IList <string> commands = Arrays.AsList(Shell.GetRunScriptCommand(scriptFile)); containerLaunchContext.SetCommands(commands); StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext , CreateContainerToken(cId, DummyRmIdentifier, context.GetNodeId(), user, context .GetContainerTokenSecretManager())); IList <StartContainerRequest> list = new AList <StartContainerRequest>(); list.AddItem(scRequest); StartContainersRequest allRequests = StartContainersRequest.NewInstance(list); containerManager.StartContainers(allRequests); int timeoutSecs = 0; while (!processStartFile.Exists() && timeoutSecs++ < 20) { Sharpen.Thread.Sleep(1000); Log.Info("Waiting for process start-file to be created"); } NUnit.Framework.Assert.IsTrue("ProcessStartFile doesn't exist!", processStartFile .Exists()); // Now verify the contents of the file BufferedReader reader = new BufferedReader(new FileReader(processStartFile)); NUnit.Framework.Assert.AreEqual("Hello World!", reader.ReadLine()); // Get the pid of the process string pid = reader.ReadLine().Trim(); // No more lines NUnit.Framework.Assert.AreEqual(null, reader.ReadLine()); // Now test the stop functionality. // Assert that the process is alive NUnit.Framework.Assert.IsTrue("Process is not alive!", DefaultContainerExecutor.ContainerIsAlive (pid)); // Once more NUnit.Framework.Assert.IsTrue("Process is not alive!", DefaultContainerExecutor.ContainerIsAlive (pid)); IList <ContainerId> containerIds = new AList <ContainerId>(); containerIds.AddItem(cId); StopContainersRequest stopRequest = StopContainersRequest.NewInstance(containerIds ); containerManager.StopContainers(stopRequest); BaseContainerManagerTest.WaitForContainerState(containerManager, cId, ContainerState .Complete); GetContainerStatusesRequest gcsRequest = GetContainerStatusesRequest.NewInstance( containerIds); ContainerStatus containerStatus = containerManager.GetContainerStatuses(gcsRequest ).GetContainerStatuses()[0]; int expectedExitCode = ContainerExitStatus.KilledByAppmaster; NUnit.Framework.Assert.AreEqual(expectedExitCode, containerStatus.GetExitStatus() ); // Assert that the process is not alive anymore NUnit.Framework.Assert.IsFalse("Process is still alive!", DefaultContainerExecutor .ContainerIsAlive(pid)); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.Exception"/> public virtual void TestClearLocalDirWhenNodeReboot() { nm = new TestNodeManagerReboot.MyNodeManager(this); nm.Start(); ContainerManagementProtocol containerManager = nm.GetContainerManager(); // create files under fileCache CreateFiles(nmLocalDir.GetAbsolutePath(), ContainerLocalizer.Filecache, 100); localResourceDir.Mkdirs(); ContainerLaunchContext containerLaunchContext = Records.NewRecord <ContainerLaunchContext >(); // Construct the Container-id ContainerId cId = CreateContainerId(); URL localResourceUri = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new Path(localResourceDir.GetAbsolutePath()))); LocalResource localResource = LocalResource.NewInstance(localResourceUri, LocalResourceType .File, LocalResourceVisibility.Application, -1, localResourceDir.LastModified()); string destinationFile = "dest_file"; IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource >(); localResources[destinationFile] = localResource; containerLaunchContext.SetLocalResources(localResources); IList <string> commands = new AList <string>(); containerLaunchContext.SetCommands(commands); NodeId nodeId = nm.GetNMContext().GetNodeId(); StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext , TestContainerManager.CreateContainerToken(cId, 0, nodeId, destinationFile, nm. GetNMContext().GetContainerTokenSecretManager())); IList <StartContainerRequest> list = new AList <StartContainerRequest>(); list.AddItem(scRequest); StartContainersRequest allRequests = StartContainersRequest.NewInstance(list); UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(cId.GetApplicationAttemptId ().ToString()); NMTokenIdentifier nmIdentifier = new NMTokenIdentifier(cId.GetApplicationAttemptId (), nodeId, user, 123); currentUser.AddTokenIdentifier(nmIdentifier); currentUser.DoAs(new _PrivilegedExceptionAction_152(this, allRequests)); IList <ContainerId> containerIds = new AList <ContainerId>(); containerIds.AddItem(cId); GetContainerStatusesRequest request = GetContainerStatusesRequest.NewInstance(containerIds ); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container = nm.GetNMContext().GetContainers()[request.GetContainerIds()[0]]; int MaxTries = 20; int numTries = 0; while (!container.GetContainerState().Equals(ContainerState.Done) && numTries <= MaxTries) { try { Sharpen.Thread.Sleep(500); } catch (Exception) { } // Do nothing numTries++; } NUnit.Framework.Assert.AreEqual(ContainerState.Done, container.GetContainerState( )); NUnit.Framework.Assert.IsTrue("The container should create a subDir named currentUser: "******"under localDir/usercache", NumOfLocalDirs(nmLocalDir.GetAbsolutePath( ), ContainerLocalizer.Usercache) > 0); NUnit.Framework.Assert.IsTrue("There should be files or Dirs under nm_private when " + "container is launched", NumOfLocalDirs(nmLocalDir.GetAbsolutePath(), ResourceLocalizationService .NmPrivateDir) > 0); // restart the NodeManager RestartNM(MaxTries); CheckNumOfLocalDirs(); Org.Mockito.Mockito.Verify(delService, Org.Mockito.Mockito.Times(1)).Delete((string )Matchers.IsNull(), Matchers.ArgThat(new TestNodeManagerReboot.PathInclude(this, ResourceLocalizationService.NmPrivateDir + "_DEL_"))); Org.Mockito.Mockito.Verify(delService, Org.Mockito.Mockito.Times(1)).Delete((string )Matchers.IsNull(), Matchers.ArgThat(new TestNodeManagerReboot.PathInclude(this, ContainerLocalizer.Filecache + "_DEL_"))); Org.Mockito.Mockito.Verify(delService, Org.Mockito.Mockito.Times(1)).ScheduleFileDeletionTask (Matchers.ArgThat(new TestNodeManagerReboot.FileDeletionInclude(this, user, null , new string[] { destinationFile }))); Org.Mockito.Mockito.Verify(delService, Org.Mockito.Mockito.Times(1)).ScheduleFileDeletionTask (Matchers.ArgThat(new TestNodeManagerReboot.FileDeletionInclude(this, null, ContainerLocalizer .Usercache + "_DEL_", new string[] { }))); // restart the NodeManager again // this time usercache directory should be empty RestartNM(MaxTries); CheckNumOfLocalDirs(); }
/// <exception cref="System.IO.IOException"/> public virtual GetContainerStatusesResponse GetContainerStatuses(GetContainerStatusesRequest request) { return(null); }
/// <exception cref="System.Exception"/> private void Test(string rpcClass) { Configuration conf = new Configuration(); conf.Set(YarnConfiguration.IpcRpcImpl, rpcClass); YarnRPC rpc = YarnRPC.Create(conf); string bindAddr = "localhost:0"; IPEndPoint addr = NetUtils.CreateSocketAddr(bindAddr); Server server = rpc.GetServer(typeof(ContainerManagementProtocol), new TestRPC.DummyContainerManager (this), addr, conf, null, 1); server.Start(); RPC.SetProtocolEngine(conf, typeof(ContainerManagementProtocolPB), typeof(ProtobufRpcEngine )); ContainerManagementProtocol proxy = (ContainerManagementProtocol)rpc.GetProxy(typeof( ContainerManagementProtocol), NetUtils.GetConnectAddress(server), conf); ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext >(); ApplicationId applicationId = ApplicationId.NewInstance(0, 0); ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.NewInstance(applicationId , 0); ContainerId containerId = ContainerId.NewContainerId(applicationAttemptId, 100); NodeId nodeId = NodeId.NewInstance("localhost", 1234); Resource resource = Resource.NewInstance(1234, 2); ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier( containerId, "localhost", "user", resource, Runtime.CurrentTimeMillis() + 10000, 42, 42, Priority.NewInstance(0), 0); Token containerToken = NewContainerToken(nodeId, Sharpen.Runtime.GetBytesForString ("password"), containerTokenIdentifier); StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext , containerToken); IList <StartContainerRequest> list = new AList <StartContainerRequest>(); list.AddItem(scRequest); StartContainersRequest allRequests = StartContainersRequest.NewInstance(list); proxy.StartContainers(allRequests); IList <ContainerId> containerIds = new AList <ContainerId>(); containerIds.AddItem(containerId); GetContainerStatusesRequest gcsRequest = GetContainerStatusesRequest.NewInstance( containerIds); GetContainerStatusesResponse response = proxy.GetContainerStatuses(gcsRequest); IList <ContainerStatus> statuses = response.GetContainerStatuses(); //test remote exception bool exception = false; try { StopContainersRequest stopRequest = recordFactory.NewRecordInstance <StopContainersRequest >(); stopRequest.SetContainerIds(containerIds); proxy.StopContainers(stopRequest); } catch (YarnException e) { exception = true; NUnit.Framework.Assert.IsTrue(e.Message.Contains(ExceptionMsg)); NUnit.Framework.Assert.IsTrue(e.Message.Contains(ExceptionCause)); System.Console.Out.WriteLine("Test Exception is " + e.Message); } catch (Exception ex) { Sharpen.Runtime.PrintStackTrace(ex); } NUnit.Framework.Assert.IsTrue(exception); server.Stop(); NUnit.Framework.Assert.IsNotNull(statuses[0]); NUnit.Framework.Assert.AreEqual(ContainerState.Running, statuses[0].GetState()); }
public virtual void TestContainerKillOnMemoryOverflow() { if (!ProcfsBasedProcessTree.IsAvailable()) { return; } containerManager.Start(); FilePath scriptFile = new FilePath(tmpDir, "scriptFile.sh"); PrintWriter fileWriter = new PrintWriter(scriptFile); FilePath processStartFile = new FilePath(tmpDir, "start_file.txt").GetAbsoluteFile (); fileWriter.Write("\numask 0"); // So that start file is readable by the // test. fileWriter.Write("\necho Hello World! > " + processStartFile); fileWriter.Write("\necho $$ >> " + processStartFile); fileWriter.Write("\nsleep 15"); fileWriter.Close(); ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext >(); // ////// Construct the Container-id ApplicationId appId = ApplicationId.NewInstance(0, 0); ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, 1); ContainerId cId = ContainerId.NewContainerId(appAttemptId, 0); int port = 12345; URL resource_alpha = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new Path(scriptFile.GetAbsolutePath()))); LocalResource rsrc_alpha = recordFactory.NewRecordInstance <LocalResource>(); rsrc_alpha.SetResource(resource_alpha); rsrc_alpha.SetSize(-1); rsrc_alpha.SetVisibility(LocalResourceVisibility.Application); rsrc_alpha.SetType(LocalResourceType.File); rsrc_alpha.SetTimestamp(scriptFile.LastModified()); string destinationFile = "dest_file"; IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource >(); localResources[destinationFile] = rsrc_alpha; containerLaunchContext.SetLocalResources(localResources); IList <string> commands = new AList <string>(); commands.AddItem("/bin/bash"); commands.AddItem(scriptFile.GetAbsolutePath()); containerLaunchContext.SetCommands(commands); Resource r = BuilderUtils.NewResource(8 * 1024 * 1024, 1); ContainerTokenIdentifier containerIdentifier = new ContainerTokenIdentifier(cId, context.GetNodeId().ToString(), user, r, Runtime.CurrentTimeMillis() + 120000, 123 , DummyRmIdentifier, Priority.NewInstance(0), 0); Token containerToken = BuilderUtils.NewContainerToken(context.GetNodeId(), containerManager .GetContext().GetContainerTokenSecretManager().CreatePassword(containerIdentifier ), containerIdentifier); StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext , containerToken); IList <StartContainerRequest> list = new AList <StartContainerRequest>(); list.AddItem(scRequest); StartContainersRequest allRequests = StartContainersRequest.NewInstance(list); containerManager.StartContainers(allRequests); int timeoutSecs = 0; while (!processStartFile.Exists() && timeoutSecs++ < 20) { Sharpen.Thread.Sleep(1000); Log.Info("Waiting for process start-file to be created"); } NUnit.Framework.Assert.IsTrue("ProcessStartFile doesn't exist!", processStartFile .Exists()); // Now verify the contents of the file BufferedReader reader = new BufferedReader(new FileReader(processStartFile)); NUnit.Framework.Assert.AreEqual("Hello World!", reader.ReadLine()); // Get the pid of the process string pid = reader.ReadLine().Trim(); // No more lines NUnit.Framework.Assert.AreEqual(null, reader.ReadLine()); BaseContainerManagerTest.WaitForContainerState(containerManager, cId, ContainerState .Complete, 60); IList <ContainerId> containerIds = new AList <ContainerId>(); containerIds.AddItem(cId); GetContainerStatusesRequest gcsRequest = GetContainerStatusesRequest.NewInstance( containerIds); ContainerStatus containerStatus = containerManager.GetContainerStatuses(gcsRequest ).GetContainerStatuses()[0]; NUnit.Framework.Assert.AreEqual(ContainerExitStatus.KilledExceededVmem, containerStatus .GetExitStatus()); string expectedMsgPattern = "Container \\[pid=" + pid + ",containerID=" + cId + "\\] is running beyond virtual memory limits. Current usage: " + "[0-9.]+ ?[KMGTPE]?B of [0-9.]+ ?[KMGTPE]?B physical memory used; " + "[0-9.]+ ?[KMGTPE]?B of [0-9.]+ ?[KMGTPE]?B virtual memory used. " + "Killing container.\nDump of the process-tree for " + cId + " :\n"; Sharpen.Pattern pat = Sharpen.Pattern.Compile(expectedMsgPattern); NUnit.Framework.Assert.AreEqual("Expected message pattern is: " + expectedMsgPattern + "\n\nObserved message is: " + containerStatus.GetDiagnostics(), true, pat.Matcher (containerStatus.GetDiagnostics()).Find()); // Assert that the process is not alive anymore NUnit.Framework.Assert.IsFalse("Process is still alive!", exec.SignalContainer(user , pid, ContainerExecutor.Signal.Null)); }