/// <exception cref="System.Exception"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> public static void WaitForContainerState(ContainerManagementProtocol containerManager , ContainerId containerID, ContainerState finalState, int timeOutMax) { IList <ContainerId> list = new AList <ContainerId>(); list.AddItem(containerID); GetContainerStatusesRequest request = GetContainerStatusesRequest.NewInstance(list ); ContainerStatus containerStatus = containerManager.GetContainerStatuses(request). GetContainerStatuses()[0]; int timeoutSecs = 0; while (!containerStatus.GetState().Equals(finalState) && timeoutSecs++ < timeOutMax ) { Sharpen.Thread.Sleep(1000); Log.Info("Waiting for container to get into state " + finalState + ". Current state is " + containerStatus.GetState()); containerStatus = containerManager.GetContainerStatuses(request).GetContainerStatuses ()[0]; } Log.Info("Container state is " + containerStatus.GetState()); NUnit.Framework.Assert.AreEqual("ContainerState is not correct (timedout)", finalState , containerStatus.GetState()); }
/// <exception cref="System.Exception"/> private void GetContainerStatus(YarnRPC rpc, Token nmToken, ContainerId containerId , ApplicationAttemptId appAttemptId, NodeId nodeId, bool isExceptionExpected) { IList <ContainerId> containerIds = new AList <ContainerId>(); containerIds.AddItem(containerId); GetContainerStatusesRequest request = GetContainerStatusesRequest.NewInstance(containerIds ); ContainerManagementProtocol proxy = null; try { proxy = GetContainerManagementProtocolProxy(rpc, nmToken, nodeId, appAttemptId.ToString ()); GetContainerStatusesResponse statuses = proxy.GetContainerStatuses(request); if (statuses.GetFailedRequests() != null && statuses.GetFailedRequests().Contains (containerId)) { ParseAndThrowException(statuses.GetFailedRequests()[containerId].DeSerialize()); } } finally { if (proxy != null) { rpc.StopProxy(proxy, conf); } } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> public static void StartContainer(NodeManager nm, ContainerId cId, FileContext localFS , FilePath scriptFileDir, FilePath processStartFile) { FilePath scriptFile = CreateUnhaltingScriptFile(cId, scriptFileDir, processStartFile ); ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext >(); NodeId nodeId = BuilderUtils.NewNodeId(Sharpen.Extensions.GetAddressByName("localhost" ).ToString(), 12345); URL localResourceUri = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new Path(scriptFile.GetAbsolutePath()))); LocalResource localResource = recordFactory.NewRecordInstance <LocalResource>(); localResource.SetResource(localResourceUri); localResource.SetSize(-1); localResource.SetVisibility(LocalResourceVisibility.Application); localResource.SetType(LocalResourceType.File); localResource.SetTimestamp(scriptFile.LastModified()); string destinationFile = "dest_file"; IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource >(); localResources[destinationFile] = localResource; containerLaunchContext.SetLocalResources(localResources); IList <string> commands = Arrays.AsList(Shell.GetRunScriptCommand(scriptFile)); containerLaunchContext.SetCommands(commands); IPEndPoint containerManagerBindAddress = NetUtils.CreateSocketAddrForHost("127.0.0.1" , 12345); UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(cId.ToString ()); Org.Apache.Hadoop.Security.Token.Token <NMTokenIdentifier> nmToken = ConverterUtils .ConvertFromYarn(nm.GetNMContext().GetNMTokenSecretManager().CreateNMToken(cId.GetApplicationAttemptId (), nodeId, user), containerManagerBindAddress); currentUser.AddToken(nmToken); ContainerManagementProtocol containerManager = currentUser.DoAs(new _PrivilegedAction_229 ()); StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext , TestContainerManager.CreateContainerToken(cId, 0, nodeId, user, nm.GetNMContext ().GetContainerTokenSecretManager())); IList <StartContainerRequest> list = new AList <StartContainerRequest>(); list.AddItem(scRequest); StartContainersRequest allRequests = StartContainersRequest.NewInstance(list); containerManager.StartContainers(allRequests); IList <ContainerId> containerIds = new AList <ContainerId>(); containerIds.AddItem(cId); GetContainerStatusesRequest request = GetContainerStatusesRequest.NewInstance(containerIds ); ContainerStatus containerStatus = containerManager.GetContainerStatuses(request). GetContainerStatuses()[0]; NUnit.Framework.Assert.AreEqual(ContainerState.Running, containerStatus.GetState( )); }
/// <summary>This tests whether a containerId is serialized/deserialized with epoch.</summary> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> private void TestContainerTokenWithEpoch(Configuration conf) { Log.Info("Running test for serializing/deserializing containerIds"); NMTokenSecretManagerInRM nmTokenSecretManagerInRM = yarnCluster.GetResourceManager ().GetRMContext().GetNMTokenSecretManager(); ApplicationId appId = ApplicationId.NewInstance(1, 1); ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, 0); ContainerId cId = ContainerId.NewContainerId(appAttemptId, (5L << 40) | 3L); NodeManager nm = yarnCluster.GetNodeManager(0); NMTokenSecretManagerInNM nmTokenSecretManagerInNM = nm.GetNMContext().GetNMTokenSecretManager (); string user = "******"; WaitForNMToReceiveNMTokenKey(nmTokenSecretManagerInNM, nm); NodeId nodeId = nm.GetNMContext().GetNodeId(); // Both id should be equal. NUnit.Framework.Assert.AreEqual(nmTokenSecretManagerInNM.GetCurrentKey().GetKeyId (), nmTokenSecretManagerInRM.GetCurrentKey().GetKeyId()); // Creating a normal Container Token RMContainerTokenSecretManager containerTokenSecretManager = yarnCluster.GetResourceManager ().GetRMContext().GetContainerTokenSecretManager(); Resource r = Resource.NewInstance(1230, 2); Org.Apache.Hadoop.Yarn.Api.Records.Token containerToken = containerTokenSecretManager .CreateContainerToken(cId, nodeId, user, r, Priority.NewInstance(0), 0); ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier( ); byte[] tokenIdentifierContent = ((byte[])containerToken.GetIdentifier().Array()); DataInputBuffer dib = new DataInputBuffer(); dib.Reset(tokenIdentifierContent, tokenIdentifierContent.Length); containerTokenIdentifier.ReadFields(dib); NUnit.Framework.Assert.AreEqual(cId, containerTokenIdentifier.GetContainerID()); NUnit.Framework.Assert.AreEqual(cId.ToString(), containerTokenIdentifier.GetContainerID ().ToString()); Org.Apache.Hadoop.Yarn.Api.Records.Token nmToken = nmTokenSecretManagerInRM.CreateNMToken (appAttemptId, nodeId, user); YarnRPC rpc = YarnRPC.Create(conf); TestStartContainer(rpc, appAttemptId, nodeId, containerToken, nmToken, false); IList <ContainerId> containerIds = new List <ContainerId>(); containerIds.AddItem(cId); ContainerManagementProtocol proxy = GetContainerManagementProtocolProxy(rpc, nmToken , nodeId, user); GetContainerStatusesResponse res = proxy.GetContainerStatuses(GetContainerStatusesRequest .NewInstance(containerIds)); NUnit.Framework.Assert.IsNotNull(res.GetContainerStatuses()[0]); NUnit.Framework.Assert.AreEqual(cId, res.GetContainerStatuses()[0].GetContainerId ()); NUnit.Framework.Assert.AreEqual(cId.ToString(), res.GetContainerStatuses()[0].GetContainerId ().ToString()); }
/// <exception cref="Com.Google.Protobuf.ServiceException"/> public virtual YarnServiceProtos.GetContainerStatusesResponseProto GetContainerStatuses (RpcController arg0, YarnServiceProtos.GetContainerStatusesRequestProto proto) { GetContainerStatusesRequestPBImpl request = new GetContainerStatusesRequestPBImpl (proto); try { GetContainerStatusesResponse response = real.GetContainerStatuses(request); return(((GetContainerStatusesResponsePBImpl)response).GetProto()); } catch (YarnException e) { throw new ServiceException(e); } catch (IOException e) { throw new ServiceException(e); } }
/// <exception cref="System.Exception"/> public virtual void TestNMProxyRetry() { containerManager.Start(); containerManager.SetBlockNewContainerRequests(false); StartContainersRequest allRequests = Records.NewRecord <StartContainersRequest>(); ApplicationId appId = ApplicationId.NewInstance(1, 1); ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1); Token nmToken = context.GetNMTokenSecretManager().CreateNMToken(attemptId, context .GetNodeId(), user); IPEndPoint address = conf.GetSocketAddr(YarnConfiguration.NmBindHost, YarnConfiguration .NmAddress, YarnConfiguration.DefaultNmAddress, YarnConfiguration.DefaultNmPort); Org.Apache.Hadoop.Security.Token.Token <NMTokenIdentifier> token = ConverterUtils. ConvertFromYarn(nmToken, SecurityUtil.BuildTokenService(address)); UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser(user); ugi.AddToken(token); ContainerManagementProtocol proxy = NMProxy.CreateNMProxy <ContainerManagementProtocol >(conf, ugi, YarnRPC.Create(conf), address); retryCount = 0; shouldThrowNMNotYetReadyException = false; proxy.StartContainers(allRequests); NUnit.Framework.Assert.AreEqual(5, retryCount); retryCount = 0; shouldThrowNMNotYetReadyException = false; proxy.StopContainers(Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <StopContainersRequest >()); NUnit.Framework.Assert.AreEqual(5, retryCount); retryCount = 0; shouldThrowNMNotYetReadyException = false; proxy.GetContainerStatuses(Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <GetContainerStatusesRequest >()); NUnit.Framework.Assert.AreEqual(5, retryCount); retryCount = 0; shouldThrowNMNotYetReadyException = true; proxy.StartContainers(allRequests); NUnit.Framework.Assert.AreEqual(5, retryCount); }
/// <exception cref="System.Exception"/> private void Test(string rpcClass) { Configuration conf = new Configuration(); conf.Set(YarnConfiguration.IpcRpcImpl, rpcClass); YarnRPC rpc = YarnRPC.Create(conf); string bindAddr = "localhost:0"; IPEndPoint addr = NetUtils.CreateSocketAddr(bindAddr); Server server = rpc.GetServer(typeof(ContainerManagementProtocol), new TestRPC.DummyContainerManager (this), addr, conf, null, 1); server.Start(); RPC.SetProtocolEngine(conf, typeof(ContainerManagementProtocolPB), typeof(ProtobufRpcEngine )); ContainerManagementProtocol proxy = (ContainerManagementProtocol)rpc.GetProxy(typeof( ContainerManagementProtocol), NetUtils.GetConnectAddress(server), conf); ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext >(); ApplicationId applicationId = ApplicationId.NewInstance(0, 0); ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.NewInstance(applicationId , 0); ContainerId containerId = ContainerId.NewContainerId(applicationAttemptId, 100); NodeId nodeId = NodeId.NewInstance("localhost", 1234); Resource resource = Resource.NewInstance(1234, 2); ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier( containerId, "localhost", "user", resource, Runtime.CurrentTimeMillis() + 10000, 42, 42, Priority.NewInstance(0), 0); Token containerToken = NewContainerToken(nodeId, Sharpen.Runtime.GetBytesForString ("password"), containerTokenIdentifier); StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext , containerToken); IList <StartContainerRequest> list = new AList <StartContainerRequest>(); list.AddItem(scRequest); StartContainersRequest allRequests = StartContainersRequest.NewInstance(list); proxy.StartContainers(allRequests); IList <ContainerId> containerIds = new AList <ContainerId>(); containerIds.AddItem(containerId); GetContainerStatusesRequest gcsRequest = GetContainerStatusesRequest.NewInstance( containerIds); GetContainerStatusesResponse response = proxy.GetContainerStatuses(gcsRequest); IList <ContainerStatus> statuses = response.GetContainerStatuses(); //test remote exception bool exception = false; try { StopContainersRequest stopRequest = recordFactory.NewRecordInstance <StopContainersRequest >(); stopRequest.SetContainerIds(containerIds); proxy.StopContainers(stopRequest); } catch (YarnException e) { exception = true; NUnit.Framework.Assert.IsTrue(e.Message.Contains(ExceptionMsg)); NUnit.Framework.Assert.IsTrue(e.Message.Contains(ExceptionCause)); System.Console.Out.WriteLine("Test Exception is " + e.Message); } catch (Exception ex) { Sharpen.Runtime.PrintStackTrace(ex); } NUnit.Framework.Assert.IsTrue(exception); server.Stop(); NUnit.Framework.Assert.IsNotNull(statuses[0]); NUnit.Framework.Assert.AreEqual(ContainerState.Running, statuses[0].GetState()); }