public MockAM(RMContext context, ApplicationMasterProtocol amRMProtocol, ApplicationAttemptId attemptId) { this.context = context; this.amRMProtocol = amRMProtocol; this.attemptId = attemptId; }
public virtual void TestValidateResourceBlacklistRequest() { TestAMAuthorization.MyContainerManager containerManager = new TestAMAuthorization.MyContainerManager (); TestAMAuthorization.MockRMWithAMS rm = new TestAMAuthorization.MockRMWithAMS(new YarnConfiguration(), containerManager); rm.Start(); MockNM nm1 = rm.RegisterNode("localhost:1234", 5120); IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType , string>(2); acls[ApplicationAccessType.ViewApp] = "*"; RMApp app = rm.SubmitApp(1024, "appname", "appuser", acls); nm1.NodeHeartbeat(true); RMAppAttempt attempt = app.GetCurrentAppAttempt(); ApplicationAttemptId applicationAttemptId = attempt.GetAppAttemptId(); WaitForLaunchedState(attempt); // Create a client to the RM. Configuration conf = rm.GetConfig(); YarnRPC rpc = YarnRPC.Create(conf); UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(applicationAttemptId .ToString()); Credentials credentials = containerManager.GetContainerCredentials(); IPEndPoint rmBindAddress = rm.GetApplicationMasterService().GetBindAddress(); Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> amRMToken = TestAMAuthorization.MockRMWithAMS .SetupAndReturnAMRMToken(rmBindAddress, credentials.GetAllTokens()); currentUser.AddToken(amRMToken); ApplicationMasterProtocol client = currentUser.DoAs(new _PrivilegedAction_626(rpc , rmBindAddress, conf)); RegisterApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <RegisterApplicationMasterRequest>(); client.RegisterApplicationMaster(request); ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.NewInstance( Sharpen.Collections.SingletonList(ResourceRequest.Any), null); AllocateRequest allocateRequest = AllocateRequest.NewInstance(0, 0.0f, null, null , blacklistRequest); bool error = false; try { client.Allocate(allocateRequest); } catch (InvalidResourceBlacklistRequestException) { error = true; } rm.Stop(); NUnit.Framework.Assert.IsTrue("Didn't not catch InvalidResourceBlacklistRequestException" , error); }
/// <exception cref="System.Exception"/> protected override void ServiceStart() { scheduler = CreateSchedulerProxy(); JobID id = TypeConverter.FromYarn(this.applicationId); JobId jobId = TypeConverter.ToYarn(id); job = context.GetJob(jobId); Register(); StartAllocatorThread(); base.ServiceStart(); }
public virtual void TestAuthorizedAccess() { TestAMAuthorization.MyContainerManager containerManager = new TestAMAuthorization.MyContainerManager (); rm = new TestAMAuthorization.MockRMWithAMS(conf, containerManager); rm.Start(); MockNM nm1 = rm.RegisterNode("localhost:1234", 5120); IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType , string>(2); acls[ApplicationAccessType.ViewApp] = "*"; RMApp app = rm.SubmitApp(1024, "appname", "appuser", acls); nm1.NodeHeartbeat(true); int waitCount = 0; while (containerManager.containerTokens == null && waitCount++ < 20) { Log.Info("Waiting for AM Launch to happen.."); Sharpen.Thread.Sleep(1000); } NUnit.Framework.Assert.IsNotNull(containerManager.containerTokens); RMAppAttempt attempt = app.GetCurrentAppAttempt(); ApplicationAttemptId applicationAttemptId = attempt.GetAppAttemptId(); WaitForLaunchedState(attempt); // Create a client to the RM. Configuration conf = rm.GetConfig(); YarnRPC rpc = YarnRPC.Create(conf); UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(applicationAttemptId .ToString()); Credentials credentials = containerManager.GetContainerCredentials(); IPEndPoint rmBindAddress = rm.GetApplicationMasterService().GetBindAddress(); Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> amRMToken = TestAMAuthorization.MockRMWithAMS .SetupAndReturnAMRMToken(rmBindAddress, credentials.GetAllTokens()); currentUser.AddToken(amRMToken); ApplicationMasterProtocol client = currentUser.DoAs(new _PrivilegedAction_206(this , rpc, conf)); RegisterApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <RegisterApplicationMasterRequest>(); RegisterApplicationMasterResponse response = client.RegisterApplicationMaster(request ); NUnit.Framework.Assert.IsNotNull(response.GetClientToAMTokenMasterKey()); if (UserGroupInformation.IsSecurityEnabled()) { NUnit.Framework.Assert.IsTrue(((byte[])response.GetClientToAMTokenMasterKey().Array ()).Length > 0); } NUnit.Framework.Assert.AreEqual("Register response has bad ACLs", "*", response.GetApplicationACLs ()[ApplicationAccessType.ViewApp]); }
public virtual void Initialize() { StartHACluster(0, false, false, true); attemptId = this.cluster.CreateFakeApplicationAttemptId(); amClient = ClientRMProxy.CreateRMProxy <ApplicationMasterProtocol>(this.conf); Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> appToken = this.cluster .GetResourceManager().GetRMContext().GetAMRMTokenSecretManager().CreateAndGetAMRMToken (attemptId); appToken.SetService(ClientRMProxy.GetAMRMTokenService(conf)); UserGroupInformation.SetLoginUser(UserGroupInformation.CreateRemoteUser(UserGroupInformation .GetCurrentUser().GetUserName())); UserGroupInformation.GetCurrentUser().AddToken(appToken); SyncToken(appToken); }
/// <exception cref="System.Exception"/> protected override void ServiceStart() { YarnConfiguration conf = new YarnConfiguration(GetConfig()); try { rmClient = ClientRMProxy.CreateRMProxy <ApplicationMasterProtocol>(conf); } catch (IOException e) { throw new YarnRuntimeException(e); } base.ServiceStart(); }
public virtual void TestRMConnectionRetry() { // verify the connection exception is thrown // if we haven't exhausted the retry interval ApplicationMasterProtocol mockScheduler = Org.Mockito.Mockito.Mock <ApplicationMasterProtocol >(); Org.Mockito.Mockito.When(mockScheduler.Allocate(Matchers.IsA <AllocateRequest>())) .ThenThrow(RPCUtil.GetRemoteException(new IOException("forcefail"))); Configuration conf = new Configuration(); LocalContainerAllocator lca = new TestLocalContainerAllocator.StubbedLocalContainerAllocator (mockScheduler); lca.Init(conf); lca.Start(); try { lca.Heartbeat(); NUnit.Framework.Assert.Fail("heartbeat was supposed to throw"); } catch (YarnException) { } finally { // YarnException is expected lca.Stop(); } // verify YarnRuntimeException is thrown when the retry interval has expired conf.SetLong(MRJobConfig.MrAmToRmWaitIntervalMs, 0); lca = new TestLocalContainerAllocator.StubbedLocalContainerAllocator(mockScheduler ); lca.Init(conf); lca.Start(); try { lca.Heartbeat(); NUnit.Framework.Assert.Fail("heartbeat was supposed to throw"); } catch (YarnRuntimeException) { } finally { // YarnRuntimeException is expected lca.Stop(); } }
// Expected // provide main method so this class can act as AM /// <exception cref="System.Exception"/> public static void Main(string[] args) { if (args[0].Equals("success")) { ApplicationMasterProtocol client = ClientRMProxy.CreateRMProxy <ApplicationMasterProtocol >(conf); client.RegisterApplicationMaster(RegisterApplicationMasterRequest.NewInstance(NetUtils .GetHostname(), -1, string.Empty)); Sharpen.Thread.Sleep(1000); FinishApplicationMasterResponse resp = client.FinishApplicationMaster(FinishApplicationMasterRequest .NewInstance(FinalApplicationStatus.Succeeded, "success", null)); NUnit.Framework.Assert.IsTrue(resp.GetIsUnregistered()); System.Environment.Exit(0); } else { System.Environment.Exit(1); } }
private void TestPbClientFactory() { IPEndPoint addr = new IPEndPoint(0); System.Console.Error.WriteLine(addr.GetHostName() + addr.Port); Configuration conf = new Configuration(); ApplicationMasterProtocol instance = new TestRPCFactories.AMRMProtocolTestImpl(this ); Server server = null; try { server = RpcServerFactoryPBImpl.Get().GetServer(typeof(ApplicationMasterProtocol) , instance, addr, conf, null, 1); server.Start(); System.Console.Error.WriteLine(server.GetListenerAddress()); System.Console.Error.WriteLine(NetUtils.GetConnectAddress(server)); ApplicationMasterProtocol amrmClient = null; try { amrmClient = (ApplicationMasterProtocol)RpcClientFactoryPBImpl.Get().GetClient(typeof( ApplicationMasterProtocol), 1, NetUtils.GetConnectAddress(server), conf); } catch (YarnRuntimeException e) { Sharpen.Runtime.PrintStackTrace(e); NUnit.Framework.Assert.Fail("Failed to create client"); } } catch (YarnRuntimeException e) { Sharpen.Runtime.PrintStackTrace(e); NUnit.Framework.Assert.Fail("Failed to create server"); } finally { if (server != null) { server.Stop(); } } }
public virtual void TestTokenExpiry() { TestAMAuthorization.MyContainerManager containerManager = new TestAMAuthorization.MyContainerManager (); TestAMAuthorization.MockRMWithAMS rm = new TestAMAuthorization.MockRMWithAMS(conf , containerManager); rm.Start(); Configuration conf = rm.GetConfig(); YarnRPC rpc = YarnRPC.Create(conf); ApplicationMasterProtocol rmClient = null; try { MockNM nm1 = rm.RegisterNode("localhost:1234", 5120); RMApp app = rm.SubmitApp(1024); nm1.NodeHeartbeat(true); int waitCount = 0; while (containerManager.containerTokens == null && waitCount++ < 20) { Log.Info("Waiting for AM Launch to happen.."); Sharpen.Thread.Sleep(1000); } NUnit.Framework.Assert.IsNotNull(containerManager.containerTokens); RMAppAttempt attempt = app.GetCurrentAppAttempt(); ApplicationAttemptId applicationAttemptId = attempt.GetAppAttemptId(); // Create a client to the RM. UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(applicationAttemptId .ToString()); Credentials credentials = containerManager.GetContainerCredentials(); IPEndPoint rmBindAddress = rm.GetApplicationMasterService().GetBindAddress(); Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> amRMToken = TestAMAuthorization.MockRMWithAMS .SetupAndReturnAMRMToken(rmBindAddress, credentials.GetAllTokens()); currentUser.AddToken(amRMToken); rmClient = CreateRMClient(rm, conf, rpc, currentUser); RegisterApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <RegisterApplicationMasterRequest>(); rmClient.RegisterApplicationMaster(request); FinishApplicationMasterRequest finishAMRequest = Org.Apache.Hadoop.Yarn.Util.Records .NewRecord <FinishApplicationMasterRequest>(); finishAMRequest.SetFinalApplicationStatus(FinalApplicationStatus.Succeeded); finishAMRequest.SetDiagnostics("diagnostics"); finishAMRequest.SetTrackingUrl("url"); rmClient.FinishApplicationMaster(finishAMRequest); // Send RMAppAttemptEventType.CONTAINER_FINISHED to transit RMAppAttempt // from Finishing state to Finished State. Both AMRMToken and // ClientToAMToken will be removed. ContainerStatus containerStatus = BuilderUtils.NewContainerStatus(attempt.GetMasterContainer ().GetId(), ContainerState.Complete, "AM Container Finished", 0); rm.GetRMContext().GetDispatcher().GetEventHandler().Handle(new RMAppAttemptContainerFinishedEvent (applicationAttemptId, containerStatus, nm1.GetNodeId())); // Make sure the RMAppAttempt is at Finished State. // Both AMRMToken and ClientToAMToken have been removed. int count = 0; while (attempt.GetState() != RMAppAttemptState.Finished && count < maxWaitAttempts ) { Sharpen.Thread.Sleep(100); count++; } NUnit.Framework.Assert.IsTrue(attempt.GetState() == RMAppAttemptState.Finished); // Now simulate trying to allocate. RPC call itself should throw auth // exception. rpc.StopProxy(rmClient, conf); // To avoid using cached client rmClient = CreateRMClient(rm, conf, rpc, currentUser); AllocateRequest allocateRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest >(); try { rmClient.Allocate(allocateRequest); NUnit.Framework.Assert.Fail("You got to be kidding me! " + "Using App tokens after app-finish should fail!" ); } catch (Exception t) { Log.Info("Exception found is ", t); // The exception will still have the earlier appAttemptId as it picks it // up from the token. NUnit.Framework.Assert.IsTrue(t.InnerException.Message.Contains(applicationAttemptId .ToString() + " not found in AMRMTokenSecretManager.")); } } finally { rm.Stop(); if (rmClient != null) { rpc.StopProxy(rmClient, conf); } } }
public virtual void TestMasterKeyRollOver() { conf.SetLong(YarnConfiguration.RmAmrmTokenMasterKeyRollingIntervalSecs, rolling_interval_sec ); conf.SetLong(YarnConfiguration.RmAmExpiryIntervalMs, am_expire_ms); TestAMAuthorization.MyContainerManager containerManager = new TestAMAuthorization.MyContainerManager (); TestAMAuthorization.MockRMWithAMS rm = new TestAMAuthorization.MockRMWithAMS(conf , containerManager); rm.Start(); long startTime = Runtime.CurrentTimeMillis(); Configuration conf = rm.GetConfig(); YarnRPC rpc = YarnRPC.Create(conf); ApplicationMasterProtocol rmClient = null; AMRMTokenSecretManager appTokenSecretManager = rm.GetRMContext().GetAMRMTokenSecretManager (); MasterKeyData oldKey = appTokenSecretManager.GetMasterKey(); NUnit.Framework.Assert.IsNotNull(oldKey); try { MockNM nm1 = rm.RegisterNode("localhost:1234", 5120); RMApp app = rm.SubmitApp(1024); nm1.NodeHeartbeat(true); int waitCount = 0; while (containerManager.containerTokens == null && waitCount++ < maxWaitAttempts) { Log.Info("Waiting for AM Launch to happen.."); Sharpen.Thread.Sleep(1000); } NUnit.Framework.Assert.IsNotNull(containerManager.containerTokens); RMAppAttempt attempt = app.GetCurrentAppAttempt(); ApplicationAttemptId applicationAttemptId = attempt.GetAppAttemptId(); // Create a client to the RM. UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(applicationAttemptId .ToString()); Credentials credentials = containerManager.GetContainerCredentials(); IPEndPoint rmBindAddress = rm.GetApplicationMasterService().GetBindAddress(); Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> amRMToken = TestAMAuthorization.MockRMWithAMS .SetupAndReturnAMRMToken(rmBindAddress, credentials.GetAllTokens()); currentUser.AddToken(amRMToken); rmClient = CreateRMClient(rm, conf, rpc, currentUser); RegisterApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <RegisterApplicationMasterRequest>(); rmClient.RegisterApplicationMaster(request); // One allocate call. AllocateRequest allocateRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest >(); NUnit.Framework.Assert.IsTrue(rmClient.Allocate(allocateRequest).GetAMCommand() == null); // Wait for enough time and make sure the roll_over happens // At mean time, the old AMRMToken should continue to work while (Runtime.CurrentTimeMillis() - startTime < rolling_interval_sec * 1000) { rmClient.Allocate(allocateRequest); Sharpen.Thread.Sleep(500); } MasterKeyData newKey = appTokenSecretManager.GetMasterKey(); NUnit.Framework.Assert.IsNotNull(newKey); NUnit.Framework.Assert.IsFalse("Master key should have changed!", oldKey.Equals(newKey )); // Another allocate call with old AMRMToken. Should continue to work. rpc.StopProxy(rmClient, conf); // To avoid using cached client rmClient = CreateRMClient(rm, conf, rpc, currentUser); NUnit.Framework.Assert.IsTrue(rmClient.Allocate(allocateRequest).GetAMCommand() == null); waitCount = 0; while (waitCount++ <= maxWaitAttempts) { if (appTokenSecretManager.GetCurrnetMasterKeyData() != oldKey) { break; } try { rmClient.Allocate(allocateRequest); } catch (Exception) { break; } Sharpen.Thread.Sleep(200); } // active the nextMasterKey, and replace the currentMasterKey NUnit.Framework.Assert.IsTrue(appTokenSecretManager.GetCurrnetMasterKeyData().Equals (newKey)); NUnit.Framework.Assert.IsTrue(appTokenSecretManager.GetMasterKey().Equals(newKey) ); NUnit.Framework.Assert.IsTrue(appTokenSecretManager.GetNextMasterKeyData() == null ); // Create a new Token Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> newToken = appTokenSecretManager .CreateAndGetAMRMToken(applicationAttemptId); SecurityUtil.SetTokenService(newToken, rmBindAddress); currentUser.AddToken(newToken); // Another allocate call. Should continue to work. rpc.StopProxy(rmClient, conf); // To avoid using cached client rmClient = CreateRMClient(rm, conf, rpc, currentUser); allocateRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest>( ); NUnit.Framework.Assert.IsTrue(rmClient.Allocate(allocateRequest).GetAMCommand() == null); // Should not work by using the old AMRMToken. rpc.StopProxy(rmClient, conf); // To avoid using cached client try { currentUser.AddToken(amRMToken); rmClient = CreateRMClient(rm, conf, rpc, currentUser); allocateRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest>( ); NUnit.Framework.Assert.IsTrue(rmClient.Allocate(allocateRequest).GetAMCommand() == null); NUnit.Framework.Assert.Fail("The old Token should not work"); } catch (Exception) { } } finally { // expect exception rm.Stop(); if (rmClient != null) { rpc.StopProxy(rmClient, conf); } } }
public StubbedLocalContainerAllocator(ApplicationMasterProtocol scheduler) : base(Org.Mockito.Mockito.Mock <ClientService>(), CreateAppContext(), "nmhost", 1 , 2, null) { this.scheduler = scheduler; }
public ApplicationMasterProtocolPBServiceImpl(ApplicationMasterProtocol impl) { this.real = impl; }
public virtual void SetAMRMProtocol(ApplicationMasterProtocol amRMProtocol, RMContext context) { this.context = context; this.amRMProtocol = amRMProtocol; }
public virtual void TestUnauthorizedAccess() { TestAMAuthorization.MyContainerManager containerManager = new TestAMAuthorization.MyContainerManager (); rm = new TestAMAuthorization.MockRMWithAMS(conf, containerManager); rm.Start(); MockNM nm1 = rm.RegisterNode("localhost:1234", 5120); RMApp app = rm.SubmitApp(1024); nm1.NodeHeartbeat(true); int waitCount = 0; while (containerManager.containerTokens == null && waitCount++ < 40) { Log.Info("Waiting for AM Launch to happen.."); Sharpen.Thread.Sleep(1000); } NUnit.Framework.Assert.IsNotNull(containerManager.containerTokens); RMAppAttempt attempt = app.GetCurrentAppAttempt(); ApplicationAttemptId applicationAttemptId = attempt.GetAppAttemptId(); WaitForLaunchedState(attempt); Configuration conf = rm.GetConfig(); YarnRPC rpc = YarnRPC.Create(conf); IPEndPoint serviceAddr = conf.GetSocketAddr(YarnConfiguration.RmSchedulerAddress, YarnConfiguration.DefaultRmSchedulerAddress, YarnConfiguration.DefaultRmSchedulerPort ); UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(applicationAttemptId .ToString()); // First try contacting NM without tokens ApplicationMasterProtocol client = currentUser.DoAs(new _PrivilegedAction_262(rpc , serviceAddr, conf)); RegisterApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <RegisterApplicationMasterRequest>(); try { client.RegisterApplicationMaster(request); NUnit.Framework.Assert.Fail("Should fail with authorization error"); } catch (Exception e) { if (IsCause(typeof(AccessControlException), e)) { // Because there are no tokens, the request should be rejected as the // server side will assume we are trying simple auth. string expectedMessage = string.Empty; if (UserGroupInformation.IsSecurityEnabled()) { expectedMessage = "Client cannot authenticate via:[TOKEN]"; } else { expectedMessage = "SIMPLE authentication is not enabled. Available:[TOKEN]"; } NUnit.Framework.Assert.IsTrue(e.InnerException.Message.Contains(expectedMessage)); } else { throw; } } }