public virtual void TestRetriesOnFailures() { ContainerManagementProtocol mockProxy = Org.Mockito.Mockito.Mock <ContainerManagementProtocol >(); StartContainersResponse mockResponse = Org.Mockito.Mockito.Mock <StartContainersResponse >(); Org.Mockito.Mockito.When(mockProxy.StartContainers(Matchers.Any <StartContainersRequest >())).ThenThrow(new NMNotYetReadyException("foo")).ThenReturn(mockResponse); Configuration conf = new Configuration(); conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1); conf.SetInt(YarnConfiguration.ClientNmConnectRetryIntervalMs, 1); DrainDispatcher dispatcher = new DrainDispatcher(); MockRM rm = new _MockRMWithCustomAMLauncher_206(dispatcher, mockProxy, conf, null ); rm.Start(); MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 5120); RMApp app = rm.SubmitApp(2000); ApplicationAttemptId appAttemptId = app.GetCurrentAppAttempt().GetAppAttemptId(); // kick the scheduling nm1.NodeHeartbeat(true); dispatcher.Await(); rm.WaitForState(appAttemptId, RMAppAttemptState.Launched, 500); }
public _MockRMWithCustomAMLauncher_206(DrainDispatcher dispatcher, ContainerManagementProtocol mockProxy, Configuration baseArg1, ContainerManagementProtocol baseArg2) : base(baseArg1, baseArg2) { this.dispatcher = dispatcher; this.mockProxy = mockProxy; }
/// <exception cref="System.Exception"/> private void GetContainerStatus(YarnRPC rpc, Token nmToken, ContainerId containerId , ApplicationAttemptId appAttemptId, NodeId nodeId, bool isExceptionExpected) { IList <ContainerId> containerIds = new AList <ContainerId>(); containerIds.AddItem(containerId); GetContainerStatusesRequest request = GetContainerStatusesRequest.NewInstance(containerIds ); ContainerManagementProtocol proxy = null; try { proxy = GetContainerManagementProtocolProxy(rpc, nmToken, nodeId, appAttemptId.ToString ()); GetContainerStatusesResponse statuses = proxy.GetContainerStatuses(request); if (statuses.GetFailedRequests() != null && statuses.GetFailedRequests().Contains (containerId)) { ParseAndThrowException(statuses.GetFailedRequests()[containerId].DeSerialize()); } } finally { if (proxy != null) { rpc.StopProxy(proxy, conf); } } }
/// <exception cref="System.Exception"/> private void StartContainer(YarnRPC rpc, Token nmToken, Token containerToken, NodeId nodeId, string user) { ContainerLaunchContext context = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <ContainerLaunchContext >(); StartContainerRequest scRequest = StartContainerRequest.NewInstance(context, containerToken ); IList <StartContainerRequest> list = new AList <StartContainerRequest>(); list.AddItem(scRequest); StartContainersRequest allRequests = StartContainersRequest.NewInstance(list); ContainerManagementProtocol proxy = null; try { proxy = GetContainerManagementProtocolProxy(rpc, nmToken, nodeId, user); StartContainersResponse response = proxy.StartContainers(allRequests); foreach (SerializedException ex in response.GetFailedRequests().Values) { ParseAndThrowException(ex.DeSerialize()); } } finally { if (proxy != null) { rpc.StopProxy(proxy, conf); } } }
/// <exception cref="System.Exception"/> private void StopContainer(YarnRPC rpc, Token nmToken, IList <ContainerId> containerId , ApplicationAttemptId appAttemptId, NodeId nodeId) { StopContainersRequest request = StopContainersRequest.NewInstance(containerId); ContainerManagementProtocol proxy = null; try { proxy = GetContainerManagementProtocolProxy(rpc, nmToken, nodeId, appAttemptId.ToString ()); StopContainersResponse response = proxy.StopContainers(request); if (response.GetFailedRequests() != null && response.GetFailedRequests().Contains (containerId)) { ParseAndThrowException(response.GetFailedRequests()[containerId].DeSerialize()); } } catch (Exception) { if (proxy != null) { rpc.StopProxy(proxy, conf); } } }
/// <exception cref="System.Exception"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> public static void WaitForContainerState(ContainerManagementProtocol containerManager , ContainerId containerID, ContainerState finalState, int timeOutMax) { IList <ContainerId> list = new AList <ContainerId>(); list.AddItem(containerID); GetContainerStatusesRequest request = GetContainerStatusesRequest.NewInstance(list ); ContainerStatus containerStatus = containerManager.GetContainerStatuses(request). GetContainerStatuses()[0]; int timeoutSecs = 0; while (!containerStatus.GetState().Equals(finalState) && timeoutSecs++ < timeOutMax ) { Sharpen.Thread.Sleep(1000); Log.Info("Waiting for container to get into state " + finalState + ". Current state is " + containerStatus.GetState()); containerStatus = containerManager.GetContainerStatuses(request).GetContainerStatuses ()[0]; } Log.Info("Container state is " + containerStatus.GetState()); NUnit.Framework.Assert.AreEqual("ContainerState is not correct (timedout)", finalState , containerStatus.GetState()); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> public static void StartContainer(NodeManager nm, ContainerId cId, FileContext localFS , FilePath scriptFileDir, FilePath processStartFile) { FilePath scriptFile = CreateUnhaltingScriptFile(cId, scriptFileDir, processStartFile ); ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext >(); NodeId nodeId = BuilderUtils.NewNodeId(Sharpen.Extensions.GetAddressByName("localhost" ).ToString(), 12345); URL localResourceUri = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new Path(scriptFile.GetAbsolutePath()))); LocalResource localResource = recordFactory.NewRecordInstance <LocalResource>(); localResource.SetResource(localResourceUri); localResource.SetSize(-1); localResource.SetVisibility(LocalResourceVisibility.Application); localResource.SetType(LocalResourceType.File); localResource.SetTimestamp(scriptFile.LastModified()); string destinationFile = "dest_file"; IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource >(); localResources[destinationFile] = localResource; containerLaunchContext.SetLocalResources(localResources); IList <string> commands = Arrays.AsList(Shell.GetRunScriptCommand(scriptFile)); containerLaunchContext.SetCommands(commands); IPEndPoint containerManagerBindAddress = NetUtils.CreateSocketAddrForHost("127.0.0.1" , 12345); UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(cId.ToString ()); Org.Apache.Hadoop.Security.Token.Token <NMTokenIdentifier> nmToken = ConverterUtils .ConvertFromYarn(nm.GetNMContext().GetNMTokenSecretManager().CreateNMToken(cId.GetApplicationAttemptId (), nodeId, user), containerManagerBindAddress); currentUser.AddToken(nmToken); ContainerManagementProtocol containerManager = currentUser.DoAs(new _PrivilegedAction_229 ()); StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext , TestContainerManager.CreateContainerToken(cId, 0, nodeId, user, nm.GetNMContext ().GetContainerTokenSecretManager())); IList <StartContainerRequest> list = new AList <StartContainerRequest>(); list.AddItem(scRequest); StartContainersRequest allRequests = StartContainersRequest.NewInstance(list); containerManager.StartContainers(allRequests); IList <ContainerId> containerIds = new AList <ContainerId>(); containerIds.AddItem(cId); GetContainerStatusesRequest request = GetContainerStatusesRequest.NewInstance(containerIds ); ContainerStatus containerStatus = containerManager.GetContainerStatuses(request). GetContainerStatuses()[0]; NUnit.Framework.Assert.AreEqual(ContainerState.Running, containerStatus.GetState( )); }
/// <summary>This tests whether a containerId is serialized/deserialized with epoch.</summary> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> private void TestContainerTokenWithEpoch(Configuration conf) { Log.Info("Running test for serializing/deserializing containerIds"); NMTokenSecretManagerInRM nmTokenSecretManagerInRM = yarnCluster.GetResourceManager ().GetRMContext().GetNMTokenSecretManager(); ApplicationId appId = ApplicationId.NewInstance(1, 1); ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, 0); ContainerId cId = ContainerId.NewContainerId(appAttemptId, (5L << 40) | 3L); NodeManager nm = yarnCluster.GetNodeManager(0); NMTokenSecretManagerInNM nmTokenSecretManagerInNM = nm.GetNMContext().GetNMTokenSecretManager (); string user = "******"; WaitForNMToReceiveNMTokenKey(nmTokenSecretManagerInNM, nm); NodeId nodeId = nm.GetNMContext().GetNodeId(); // Both id should be equal. NUnit.Framework.Assert.AreEqual(nmTokenSecretManagerInNM.GetCurrentKey().GetKeyId (), nmTokenSecretManagerInRM.GetCurrentKey().GetKeyId()); // Creating a normal Container Token RMContainerTokenSecretManager containerTokenSecretManager = yarnCluster.GetResourceManager ().GetRMContext().GetContainerTokenSecretManager(); Resource r = Resource.NewInstance(1230, 2); Org.Apache.Hadoop.Yarn.Api.Records.Token containerToken = containerTokenSecretManager .CreateContainerToken(cId, nodeId, user, r, Priority.NewInstance(0), 0); ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier( ); byte[] tokenIdentifierContent = ((byte[])containerToken.GetIdentifier().Array()); DataInputBuffer dib = new DataInputBuffer(); dib.Reset(tokenIdentifierContent, tokenIdentifierContent.Length); containerTokenIdentifier.ReadFields(dib); NUnit.Framework.Assert.AreEqual(cId, containerTokenIdentifier.GetContainerID()); NUnit.Framework.Assert.AreEqual(cId.ToString(), containerTokenIdentifier.GetContainerID ().ToString()); Org.Apache.Hadoop.Yarn.Api.Records.Token nmToken = nmTokenSecretManagerInRM.CreateNMToken (appAttemptId, nodeId, user); YarnRPC rpc = YarnRPC.Create(conf); TestStartContainer(rpc, appAttemptId, nodeId, containerToken, nmToken, false); IList <ContainerId> containerIds = new List <ContainerId>(); containerIds.AddItem(cId); ContainerManagementProtocol proxy = GetContainerManagementProtocolProxy(rpc, nmToken , nodeId, user); GetContainerStatusesResponse res = proxy.GetContainerStatuses(GetContainerStatusesRequest .NewInstance(containerIds)); NUnit.Framework.Assert.IsNotNull(res.GetContainerStatuses()[0]); NUnit.Framework.Assert.AreEqual(cId, res.GetContainerStatuses()[0].GetContainerId ()); NUnit.Framework.Assert.AreEqual(cId.ToString(), res.GetContainerStatuses()[0].GetContainerId ().ToString()); }
/// <exception cref="System.Exception"/> private void TestRPCTimeout(string rpcClass) { Configuration conf = new Configuration(); // set timeout low for the test conf.SetInt("yarn.rpc.nm-command-timeout", 3000); conf.Set(YarnConfiguration.IpcRpcImpl, rpcClass); YarnRPC rpc = YarnRPC.Create(conf); string bindAddr = "localhost:0"; IPEndPoint addr = NetUtils.CreateSocketAddr(bindAddr); Server server = rpc.GetServer(typeof(ContainerManagementProtocol), new TestContainerLaunchRPC.DummyContainerManager (this), addr, conf, null, 1); server.Start(); try { ContainerManagementProtocol proxy = (ContainerManagementProtocol)rpc.GetProxy(typeof( ContainerManagementProtocol), server.GetListenerAddress(), conf); ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext >(); ApplicationId applicationId = ApplicationId.NewInstance(0, 0); ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.NewInstance(applicationId , 0); ContainerId containerId = ContainerId.NewContainerId(applicationAttemptId, 100); NodeId nodeId = NodeId.NewInstance("localhost", 1234); Resource resource = Resource.NewInstance(1234, 2); ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier( containerId, "localhost", "user", resource, Runtime.CurrentTimeMillis() + 10000, 42, 42, Priority.NewInstance(0), 0); Token containerToken = TestRPC.NewContainerToken(nodeId, Sharpen.Runtime.GetBytesForString ("password"), containerTokenIdentifier); StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext , containerToken); IList <StartContainerRequest> list = new AList <StartContainerRequest>(); list.AddItem(scRequest); StartContainersRequest allRequests = StartContainersRequest.NewInstance(list); try { proxy.StartContainers(allRequests); } catch (Exception e) { Log.Info(StringUtils.StringifyException(e)); NUnit.Framework.Assert.AreEqual("Error, exception is not: " + typeof(SocketTimeoutException ).FullName, typeof(SocketTimeoutException).FullName, e.GetType().FullName); return; } } finally { server.Stop(); } NUnit.Framework.Assert.Fail("timeout exception should have occurred!"); }
public ContainerManagementProtocolProxyData(ContainerManagementProtocolProxy _enclosing , YarnRPC rpc, string containerManagerBindAddr, ContainerId containerId, Token token ) { this._enclosing = _enclosing; this.containerManagerBindAddr = containerManagerBindAddr; this.activeCallers = 0; this.scheduledForClose = false; this.token = token; this.proxy = this.NewProxy(rpc, containerManagerBindAddr, containerId, token); }
/// <exception cref="System.Exception"/> public virtual void TestNMProxyRetry() { containerManager.Start(); containerManager.SetBlockNewContainerRequests(false); StartContainersRequest allRequests = Records.NewRecord <StartContainersRequest>(); ApplicationId appId = ApplicationId.NewInstance(1, 1); ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1); Token nmToken = context.GetNMTokenSecretManager().CreateNMToken(attemptId, context .GetNodeId(), user); IPEndPoint address = conf.GetSocketAddr(YarnConfiguration.NmBindHost, YarnConfiguration .NmAddress, YarnConfiguration.DefaultNmAddress, YarnConfiguration.DefaultNmPort); Org.Apache.Hadoop.Security.Token.Token <NMTokenIdentifier> token = ConverterUtils. ConvertFromYarn(nmToken, SecurityUtil.BuildTokenService(address)); UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser(user); ugi.AddToken(token); ContainerManagementProtocol proxy = NMProxy.CreateNMProxy <ContainerManagementProtocol >(conf, ugi, YarnRPC.Create(conf), address); retryCount = 0; shouldThrowNMNotYetReadyException = false; proxy.StartContainers(allRequests); NUnit.Framework.Assert.AreEqual(5, retryCount); retryCount = 0; shouldThrowNMNotYetReadyException = false; proxy.StopContainers(Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <StopContainersRequest >()); NUnit.Framework.Assert.AreEqual(5, retryCount); retryCount = 0; shouldThrowNMNotYetReadyException = false; proxy.GetContainerStatuses(Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <GetContainerStatusesRequest >()); NUnit.Framework.Assert.AreEqual(5, retryCount); retryCount = 0; shouldThrowNMNotYetReadyException = true; proxy.StartContainers(allRequests); NUnit.Framework.Assert.AreEqual(5, retryCount); }
public MockRMWithCustomAMLauncher(ContainerManagementProtocol containerManager) : this(new Configuration(), containerManager) { }
public ContainerLauncherImplUnderTest(AppContext context, ContainerManagementProtocol containerManager) : base(context) { this.containerManager = containerManager; }
/// <exception cref="System.Exception"/> private void Test(string rpcClass) { Configuration conf = new Configuration(); conf.Set(YarnConfiguration.IpcRpcImpl, rpcClass); YarnRPC rpc = YarnRPC.Create(conf); string bindAddr = "localhost:0"; IPEndPoint addr = NetUtils.CreateSocketAddr(bindAddr); Server server = rpc.GetServer(typeof(ContainerManagementProtocol), new TestRPC.DummyContainerManager (this), addr, conf, null, 1); server.Start(); RPC.SetProtocolEngine(conf, typeof(ContainerManagementProtocolPB), typeof(ProtobufRpcEngine )); ContainerManagementProtocol proxy = (ContainerManagementProtocol)rpc.GetProxy(typeof( ContainerManagementProtocol), NetUtils.GetConnectAddress(server), conf); ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext >(); ApplicationId applicationId = ApplicationId.NewInstance(0, 0); ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.NewInstance(applicationId , 0); ContainerId containerId = ContainerId.NewContainerId(applicationAttemptId, 100); NodeId nodeId = NodeId.NewInstance("localhost", 1234); Resource resource = Resource.NewInstance(1234, 2); ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier( containerId, "localhost", "user", resource, Runtime.CurrentTimeMillis() + 10000, 42, 42, Priority.NewInstance(0), 0); Token containerToken = NewContainerToken(nodeId, Sharpen.Runtime.GetBytesForString ("password"), containerTokenIdentifier); StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext , containerToken); IList <StartContainerRequest> list = new AList <StartContainerRequest>(); list.AddItem(scRequest); StartContainersRequest allRequests = StartContainersRequest.NewInstance(list); proxy.StartContainers(allRequests); IList <ContainerId> containerIds = new AList <ContainerId>(); containerIds.AddItem(containerId); GetContainerStatusesRequest gcsRequest = GetContainerStatusesRequest.NewInstance( containerIds); GetContainerStatusesResponse response = proxy.GetContainerStatuses(gcsRequest); IList <ContainerStatus> statuses = response.GetContainerStatuses(); //test remote exception bool exception = false; try { StopContainersRequest stopRequest = recordFactory.NewRecordInstance <StopContainersRequest >(); stopRequest.SetContainerIds(containerIds); proxy.StopContainers(stopRequest); } catch (YarnException e) { exception = true; NUnit.Framework.Assert.IsTrue(e.Message.Contains(ExceptionMsg)); NUnit.Framework.Assert.IsTrue(e.Message.Contains(ExceptionCause)); System.Console.Out.WriteLine("Test Exception is " + e.Message); } catch (Exception ex) { Sharpen.Runtime.PrintStackTrace(ex); } NUnit.Framework.Assert.IsTrue(exception); server.Stop(); NUnit.Framework.Assert.IsNotNull(statuses[0]); NUnit.Framework.Assert.AreEqual(ContainerState.Running, statuses[0].GetState()); }
/// <exception cref="System.Exception"/> public virtual void TestClientTokenRace() { conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos"); UserGroupInformation.SetConfiguration(conf); ContainerManagementProtocol containerManager = Org.Mockito.Mockito.Mock <ContainerManagementProtocol >(); StartContainersResponse mockResponse = Org.Mockito.Mockito.Mock <StartContainersResponse >(); Org.Mockito.Mockito.When(containerManager.StartContainers((StartContainersRequest )Matchers.Any())).ThenReturn(mockResponse); DrainDispatcher dispatcher = new DrainDispatcher(); MockRM rm = new _MockRMWithCustomAMLauncher_433(dispatcher, conf, containerManager ); rm.Start(); // Submit an app RMApp app = rm.SubmitApp(1024); // Set up a node. MockNM nm1 = rm.RegisterNode("localhost:1234", 3072); nm1.NodeHeartbeat(true); dispatcher.Await(); nm1.NodeHeartbeat(true); dispatcher.Await(); ApplicationAttemptId appAttempt = app.GetCurrentAppAttempt().GetAppAttemptId(); MockAM mockAM = new MockAM(rm.GetRMContext(), rm.GetApplicationMasterService(), app .GetCurrentAppAttempt().GetAppAttemptId()); UserGroupInformation appUgi = UserGroupInformation.CreateRemoteUser(appAttempt.ToString ()); RegisterApplicationMasterResponse response = appUgi.DoAs(new _PrivilegedAction_469 (mockAM)); // Get the app-report. GetApplicationReportRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <GetApplicationReportRequest>(); request.SetApplicationId(app.GetApplicationId()); GetApplicationReportResponse reportResponse = rm.GetClientRMService().GetApplicationReport (request); ApplicationReport appReport = reportResponse.GetApplicationReport(); Org.Apache.Hadoop.Yarn.Api.Records.Token originalClientToAMToken = appReport.GetClientToAMToken (); // ClientToAMToken master key should have been received on register // application master response. ByteBuffer clientMasterKey = response.GetClientToAMTokenMasterKey(); NUnit.Framework.Assert.IsNotNull(clientMasterKey); NUnit.Framework.Assert.IsTrue(((byte[])clientMasterKey.Array()).Length > 0); // Start the AM with the correct shared-secret. ApplicationAttemptId appAttemptId = app.GetAppAttempts().Keys.GetEnumerator().Next (); NUnit.Framework.Assert.IsNotNull(appAttemptId); TestClientToAMTokens.CustomAM am = new TestClientToAMTokens.CustomAM(appAttemptId , null); am.Init(conf); am.Start(); // Now the real test! // Set up clients to be able to pick up correct tokens. SecurityUtil.SetSecurityInfoProviders(new TestClientToAMTokens.CustomSecurityInfo ()); Org.Apache.Hadoop.Security.Token.Token <ClientToAMTokenIdentifier> token = ConverterUtils .ConvertFromYarn(originalClientToAMToken, am.address); // Schedule the key to be set after a significant delay Timer timer = new Timer(); TimerTask timerTask = new _TimerTask_516(am, clientMasterKey); timer.Schedule(timerTask, 250); // connect should pause waiting for the master key to arrive VerifyValidToken(conf, am, token); am.Stop(); rm.Stop(); }
public virtual void TestClientToAMTokens() { conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos"); UserGroupInformation.SetConfiguration(conf); ContainerManagementProtocol containerManager = Org.Mockito.Mockito.Mock <ContainerManagementProtocol >(); StartContainersResponse mockResponse = Org.Mockito.Mockito.Mock <StartContainersResponse >(); Org.Mockito.Mockito.When(containerManager.StartContainers((StartContainersRequest )Matchers.Any())).ThenReturn(mockResponse); DrainDispatcher dispatcher = new DrainDispatcher(); MockRM rm = new _MockRMWithCustomAMLauncher_192(dispatcher, conf, containerManager ); rm.Start(); // Submit an app RMApp app = rm.SubmitApp(1024); // Set up a node. MockNM nm1 = rm.RegisterNode("localhost:1234", 3072); nm1.NodeHeartbeat(true); dispatcher.Await(); nm1.NodeHeartbeat(true); dispatcher.Await(); ApplicationAttemptId appAttempt = app.GetCurrentAppAttempt().GetAppAttemptId(); MockAM mockAM = new MockAM(rm.GetRMContext(), rm.GetApplicationMasterService(), app .GetCurrentAppAttempt().GetAppAttemptId()); UserGroupInformation appUgi = UserGroupInformation.CreateRemoteUser(appAttempt.ToString ()); RegisterApplicationMasterResponse response = appUgi.DoAs(new _PrivilegedAction_229 (mockAM)); // Get the app-report. GetApplicationReportRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <GetApplicationReportRequest>(); request.SetApplicationId(app.GetApplicationId()); GetApplicationReportResponse reportResponse = rm.GetClientRMService().GetApplicationReport (request); ApplicationReport appReport = reportResponse.GetApplicationReport(); Org.Apache.Hadoop.Yarn.Api.Records.Token originalClientToAMToken = appReport.GetClientToAMToken (); // ClientToAMToken master key should have been received on register // application master response. NUnit.Framework.Assert.IsNotNull(response.GetClientToAMTokenMasterKey()); NUnit.Framework.Assert.IsTrue(((byte[])response.GetClientToAMTokenMasterKey().Array ()).Length > 0); // Start the AM with the correct shared-secret. ApplicationAttemptId appAttemptId = app.GetAppAttempts().Keys.GetEnumerator().Next (); NUnit.Framework.Assert.IsNotNull(appAttemptId); TestClientToAMTokens.CustomAM am = new TestClientToAMTokens.CustomAM(appAttemptId , ((byte[])response.GetClientToAMTokenMasterKey().Array())); am.Init(conf); am.Start(); // Now the real test! // Set up clients to be able to pick up correct tokens. SecurityUtil.SetSecurityInfoProviders(new TestClientToAMTokens.CustomSecurityInfo ()); // Verify denial for unauthenticated user try { TestClientToAMTokens.CustomProtocol client = (TestClientToAMTokens.CustomProtocol )RPC.GetProxy <TestClientToAMTokens.CustomProtocol>(1L, am.address, conf); client.Ping(); NUnit.Framework.Assert.Fail("Access by unauthenticated user should fail!!"); } catch (Exception) { NUnit.Framework.Assert.IsFalse(am.pinged); } Org.Apache.Hadoop.Security.Token.Token <ClientToAMTokenIdentifier> token = ConverterUtils .ConvertFromYarn(originalClientToAMToken, am.address); // Verify denial for a malicious user with tampered ID VerifyTokenWithTamperedID(conf, am, token); // Verify denial for a malicious user with tampered user-name VerifyTokenWithTamperedUserName(conf, am, token); // Now for an authenticated user VerifyValidToken(conf, am, token); // Verify for a new version token VerifyNewVersionToken(conf, am, token, rm); am.Stop(); rm.Stop(); }
public _AMLauncher_213(ContainerManagementProtocol mockProxy, RMContext baseArg1, RMAppAttempt baseArg2, AMLauncherEventType baseArg3, Configuration baseArg4) : base(baseArg1, baseArg2, baseArg3, baseArg4) { this.mockProxy = mockProxy; }
public MockRMWithAMS(Configuration conf, ContainerManagementProtocol containerManager ) : base(conf, containerManager) { }
public MockRMWithCustomAMLauncher(Configuration conf, ContainerManagementProtocol containerManager) : base(conf) { this.containerManager = containerManager; }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.Exception"/> public virtual void TestClearLocalDirWhenNodeReboot() { nm = new TestNodeManagerReboot.MyNodeManager(this); nm.Start(); ContainerManagementProtocol containerManager = nm.GetContainerManager(); // create files under fileCache CreateFiles(nmLocalDir.GetAbsolutePath(), ContainerLocalizer.Filecache, 100); localResourceDir.Mkdirs(); ContainerLaunchContext containerLaunchContext = Records.NewRecord <ContainerLaunchContext >(); // Construct the Container-id ContainerId cId = CreateContainerId(); URL localResourceUri = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new Path(localResourceDir.GetAbsolutePath()))); LocalResource localResource = LocalResource.NewInstance(localResourceUri, LocalResourceType .File, LocalResourceVisibility.Application, -1, localResourceDir.LastModified()); string destinationFile = "dest_file"; IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource >(); localResources[destinationFile] = localResource; containerLaunchContext.SetLocalResources(localResources); IList <string> commands = new AList <string>(); containerLaunchContext.SetCommands(commands); NodeId nodeId = nm.GetNMContext().GetNodeId(); StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext , TestContainerManager.CreateContainerToken(cId, 0, nodeId, destinationFile, nm. GetNMContext().GetContainerTokenSecretManager())); IList <StartContainerRequest> list = new AList <StartContainerRequest>(); list.AddItem(scRequest); StartContainersRequest allRequests = StartContainersRequest.NewInstance(list); UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(cId.GetApplicationAttemptId ().ToString()); NMTokenIdentifier nmIdentifier = new NMTokenIdentifier(cId.GetApplicationAttemptId (), nodeId, user, 123); currentUser.AddTokenIdentifier(nmIdentifier); currentUser.DoAs(new _PrivilegedExceptionAction_152(this, allRequests)); IList <ContainerId> containerIds = new AList <ContainerId>(); containerIds.AddItem(cId); GetContainerStatusesRequest request = GetContainerStatusesRequest.NewInstance(containerIds ); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container = nm.GetNMContext().GetContainers()[request.GetContainerIds()[0]]; int MaxTries = 20; int numTries = 0; while (!container.GetContainerState().Equals(ContainerState.Done) && numTries <= MaxTries) { try { Sharpen.Thread.Sleep(500); } catch (Exception) { } // Do nothing numTries++; } NUnit.Framework.Assert.AreEqual(ContainerState.Done, container.GetContainerState( )); NUnit.Framework.Assert.IsTrue("The container should create a subDir named currentUser: "******"under localDir/usercache", NumOfLocalDirs(nmLocalDir.GetAbsolutePath( ), ContainerLocalizer.Usercache) > 0); NUnit.Framework.Assert.IsTrue("There should be files or Dirs under nm_private when " + "container is launched", NumOfLocalDirs(nmLocalDir.GetAbsolutePath(), ResourceLocalizationService .NmPrivateDir) > 0); // restart the NodeManager RestartNM(MaxTries); CheckNumOfLocalDirs(); Org.Mockito.Mockito.Verify(delService, Org.Mockito.Mockito.Times(1)).Delete((string )Matchers.IsNull(), Matchers.ArgThat(new TestNodeManagerReboot.PathInclude(this, ResourceLocalizationService.NmPrivateDir + "_DEL_"))); Org.Mockito.Mockito.Verify(delService, Org.Mockito.Mockito.Times(1)).Delete((string )Matchers.IsNull(), Matchers.ArgThat(new TestNodeManagerReboot.PathInclude(this, ContainerLocalizer.Filecache + "_DEL_"))); Org.Mockito.Mockito.Verify(delService, Org.Mockito.Mockito.Times(1)).ScheduleFileDeletionTask (Matchers.ArgThat(new TestNodeManagerReboot.FileDeletionInclude(this, user, null , new string[] { destinationFile }))); Org.Mockito.Mockito.Verify(delService, Org.Mockito.Mockito.Times(1)).ScheduleFileDeletionTask (Matchers.ArgThat(new TestNodeManagerReboot.FileDeletionInclude(this, null, ContainerLocalizer .Usercache + "_DEL_", new string[] { }))); // restart the NodeManager again // this time usercache directory should be empty RestartNM(MaxTries); CheckNumOfLocalDirs(); }
/// <exception cref="System.IO.IOException"/> private void Connect() { ContainerId masterContainerID = masterContainer.GetId(); containerMgrProxy = GetContainerMgrProxy(masterContainerID); }
/// <exception cref="System.Exception"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> public static void WaitForContainerState(ContainerManagementProtocol containerManager , ContainerId containerID, ContainerState finalState) { WaitForContainerState(containerManager, containerID, finalState, 20); }
public _ApplicationMasterLauncher_209(ContainerManagementProtocol mockProxy, RMContext baseArg1) : base(baseArg1) { this.mockProxy = mockProxy; }
public virtual void SetContainerManager(ContainerManagementProtocol containerManager ) { this.containerManager = containerManager; }
public ContainerManagementProtocolPBServiceImpl(ContainerManagementProtocol impl) { this.real = impl; }