/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> public virtual void TestRegisterApplicationMasterOnHA() { RegisterApplicationMasterRequest request = RegisterApplicationMasterRequest.NewInstance ("localhost", 0, string.Empty); RegisterApplicationMasterResponse response = amClient.RegisterApplicationMaster(request ); NUnit.Framework.Assert.AreEqual(response, this.cluster.CreateFakeRegisterApplicationMasterResponse ()); }
public virtual void TestValidateResourceBlacklistRequest() { TestAMAuthorization.MyContainerManager containerManager = new TestAMAuthorization.MyContainerManager (); TestAMAuthorization.MockRMWithAMS rm = new TestAMAuthorization.MockRMWithAMS(new YarnConfiguration(), containerManager); rm.Start(); MockNM nm1 = rm.RegisterNode("localhost:1234", 5120); IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType , string>(2); acls[ApplicationAccessType.ViewApp] = "*"; RMApp app = rm.SubmitApp(1024, "appname", "appuser", acls); nm1.NodeHeartbeat(true); RMAppAttempt attempt = app.GetCurrentAppAttempt(); ApplicationAttemptId applicationAttemptId = attempt.GetAppAttemptId(); WaitForLaunchedState(attempt); // Create a client to the RM. Configuration conf = rm.GetConfig(); YarnRPC rpc = YarnRPC.Create(conf); UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(applicationAttemptId .ToString()); Credentials credentials = containerManager.GetContainerCredentials(); IPEndPoint rmBindAddress = rm.GetApplicationMasterService().GetBindAddress(); Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> amRMToken = TestAMAuthorization.MockRMWithAMS .SetupAndReturnAMRMToken(rmBindAddress, credentials.GetAllTokens()); currentUser.AddToken(amRMToken); ApplicationMasterProtocol client = currentUser.DoAs(new _PrivilegedAction_626(rpc , rmBindAddress, conf)); RegisterApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <RegisterApplicationMasterRequest>(); client.RegisterApplicationMaster(request); ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.NewInstance( Sharpen.Collections.SingletonList(ResourceRequest.Any), null); AllocateRequest allocateRequest = AllocateRequest.NewInstance(0, 0.0f, null, null , blacklistRequest); bool error = false; try { client.Allocate(allocateRequest); } catch (InvalidResourceBlacklistRequestException) { error = true; } rm.Stop(); NUnit.Framework.Assert.IsTrue("Didn't not catch InvalidResourceBlacklistRequestException" , error); }
public virtual void TestAuthorizedAccess() { TestAMAuthorization.MyContainerManager containerManager = new TestAMAuthorization.MyContainerManager (); rm = new TestAMAuthorization.MockRMWithAMS(conf, containerManager); rm.Start(); MockNM nm1 = rm.RegisterNode("localhost:1234", 5120); IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType , string>(2); acls[ApplicationAccessType.ViewApp] = "*"; RMApp app = rm.SubmitApp(1024, "appname", "appuser", acls); nm1.NodeHeartbeat(true); int waitCount = 0; while (containerManager.containerTokens == null && waitCount++ < 20) { Log.Info("Waiting for AM Launch to happen.."); Sharpen.Thread.Sleep(1000); } NUnit.Framework.Assert.IsNotNull(containerManager.containerTokens); RMAppAttempt attempt = app.GetCurrentAppAttempt(); ApplicationAttemptId applicationAttemptId = attempt.GetAppAttemptId(); WaitForLaunchedState(attempt); // Create a client to the RM. Configuration conf = rm.GetConfig(); YarnRPC rpc = YarnRPC.Create(conf); UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(applicationAttemptId .ToString()); Credentials credentials = containerManager.GetContainerCredentials(); IPEndPoint rmBindAddress = rm.GetApplicationMasterService().GetBindAddress(); Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> amRMToken = TestAMAuthorization.MockRMWithAMS .SetupAndReturnAMRMToken(rmBindAddress, credentials.GetAllTokens()); currentUser.AddToken(amRMToken); ApplicationMasterProtocol client = currentUser.DoAs(new _PrivilegedAction_206(this , rpc, conf)); RegisterApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <RegisterApplicationMasterRequest>(); RegisterApplicationMasterResponse response = client.RegisterApplicationMaster(request ); NUnit.Framework.Assert.IsNotNull(response.GetClientToAMTokenMasterKey()); if (UserGroupInformation.IsSecurityEnabled()) { NUnit.Framework.Assert.IsTrue(((byte[])response.GetClientToAMTokenMasterKey().Array ()).Length > 0); } NUnit.Framework.Assert.AreEqual("Register response has bad ACLs", "*", response.GetApplicationACLs ()[ApplicationAccessType.ViewApp]); }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> private RegisterApplicationMasterResponse RegisterApplicationMaster() { RegisterApplicationMasterRequest request = RegisterApplicationMasterRequest.NewInstance (this.appHostName, this.appHostPort, this.appTrackingUrl); RegisterApplicationMasterResponse response = rmClient.RegisterApplicationMaster(request ); lock (this) { lastResponseId = 0; if (!response.GetNMTokensFromPreviousAttempts().IsEmpty()) { PopulateNMTokens(response.GetNMTokensFromPreviousAttempts()); } } return(response); }
// Expected // provide main method so this class can act as AM /// <exception cref="System.Exception"/> public static void Main(string[] args) { if (args[0].Equals("success")) { ApplicationMasterProtocol client = ClientRMProxy.CreateRMProxy <ApplicationMasterProtocol >(conf); client.RegisterApplicationMaster(RegisterApplicationMasterRequest.NewInstance(NetUtils .GetHostname(), -1, string.Empty)); Sharpen.Thread.Sleep(1000); FinishApplicationMasterResponse resp = client.FinishApplicationMaster(FinishApplicationMasterRequest .NewInstance(FinalApplicationStatus.Succeeded, "success", null)); NUnit.Framework.Assert.IsTrue(resp.GetIsUnregistered()); System.Environment.Exit(0); } else { System.Environment.Exit(1); } }
protected internal virtual void Register() { //Register IPEndPoint serviceAddr = null; if (clientService != null) { serviceAddr = clientService.GetBindAddress(); } try { RegisterApplicationMasterRequest request = recordFactory.NewRecordInstance <RegisterApplicationMasterRequest >(); if (serviceAddr != null) { request.SetHost(serviceAddr.GetHostName()); request.SetRpcPort(serviceAddr.Port); request.SetTrackingUrl(MRWebAppUtil.GetAMWebappScheme(GetConfig()) + serviceAddr. GetHostName() + ":" + clientService.GetHttpPort()); } RegisterApplicationMasterResponse response = scheduler.RegisterApplicationMaster( request); isApplicationMasterRegistered = true; maxContainerCapability = response.GetMaximumResourceCapability(); this.context.GetClusterInfo().SetMaxContainerCapability(maxContainerCapability); if (UserGroupInformation.IsSecurityEnabled()) { SetClientToAMToken(response.GetClientToAMTokenMasterKey()); } this.applicationACLs = response.GetApplicationACLs(); Log.Info("maxContainerCapability: " + maxContainerCapability); string queue = response.GetQueue(); Log.Info("queue: " + queue); job.SetQueueName(queue); Sharpen.Collections.AddAll(this.schedulerResourceTypes, response.GetSchedulerResourceTypes ()); } catch (Exception are) { Log.Error("Exception while registering", are); throw new YarnRuntimeException(are); } }
/// <exception cref="Com.Google.Protobuf.ServiceException"/> public virtual YarnServiceProtos.RegisterApplicationMasterResponseProto RegisterApplicationMaster (RpcController arg0, YarnServiceProtos.RegisterApplicationMasterRequestProto proto ) { RegisterApplicationMasterRequestPBImpl request = new RegisterApplicationMasterRequestPBImpl (proto); try { RegisterApplicationMasterResponse response = real.RegisterApplicationMaster(request ); return(((RegisterApplicationMasterResponsePBImpl)response).GetProto()); } catch (YarnException e) { throw new ServiceException(e); } catch (IOException e) { throw new ServiceException(e); } }
public virtual void TestTokenExpiry() { TestAMAuthorization.MyContainerManager containerManager = new TestAMAuthorization.MyContainerManager (); TestAMAuthorization.MockRMWithAMS rm = new TestAMAuthorization.MockRMWithAMS(conf , containerManager); rm.Start(); Configuration conf = rm.GetConfig(); YarnRPC rpc = YarnRPC.Create(conf); ApplicationMasterProtocol rmClient = null; try { MockNM nm1 = rm.RegisterNode("localhost:1234", 5120); RMApp app = rm.SubmitApp(1024); nm1.NodeHeartbeat(true); int waitCount = 0; while (containerManager.containerTokens == null && waitCount++ < 20) { Log.Info("Waiting for AM Launch to happen.."); Sharpen.Thread.Sleep(1000); } NUnit.Framework.Assert.IsNotNull(containerManager.containerTokens); RMAppAttempt attempt = app.GetCurrentAppAttempt(); ApplicationAttemptId applicationAttemptId = attempt.GetAppAttemptId(); // Create a client to the RM. UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(applicationAttemptId .ToString()); Credentials credentials = containerManager.GetContainerCredentials(); IPEndPoint rmBindAddress = rm.GetApplicationMasterService().GetBindAddress(); Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> amRMToken = TestAMAuthorization.MockRMWithAMS .SetupAndReturnAMRMToken(rmBindAddress, credentials.GetAllTokens()); currentUser.AddToken(amRMToken); rmClient = CreateRMClient(rm, conf, rpc, currentUser); RegisterApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <RegisterApplicationMasterRequest>(); rmClient.RegisterApplicationMaster(request); FinishApplicationMasterRequest finishAMRequest = Org.Apache.Hadoop.Yarn.Util.Records .NewRecord <FinishApplicationMasterRequest>(); finishAMRequest.SetFinalApplicationStatus(FinalApplicationStatus.Succeeded); finishAMRequest.SetDiagnostics("diagnostics"); finishAMRequest.SetTrackingUrl("url"); rmClient.FinishApplicationMaster(finishAMRequest); // Send RMAppAttemptEventType.CONTAINER_FINISHED to transit RMAppAttempt // from Finishing state to Finished State. Both AMRMToken and // ClientToAMToken will be removed. ContainerStatus containerStatus = BuilderUtils.NewContainerStatus(attempt.GetMasterContainer ().GetId(), ContainerState.Complete, "AM Container Finished", 0); rm.GetRMContext().GetDispatcher().GetEventHandler().Handle(new RMAppAttemptContainerFinishedEvent (applicationAttemptId, containerStatus, nm1.GetNodeId())); // Make sure the RMAppAttempt is at Finished State. // Both AMRMToken and ClientToAMToken have been removed. int count = 0; while (attempt.GetState() != RMAppAttemptState.Finished && count < maxWaitAttempts ) { Sharpen.Thread.Sleep(100); count++; } NUnit.Framework.Assert.IsTrue(attempt.GetState() == RMAppAttemptState.Finished); // Now simulate trying to allocate. RPC call itself should throw auth // exception. rpc.StopProxy(rmClient, conf); // To avoid using cached client rmClient = CreateRMClient(rm, conf, rpc, currentUser); AllocateRequest allocateRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest >(); try { rmClient.Allocate(allocateRequest); NUnit.Framework.Assert.Fail("You got to be kidding me! " + "Using App tokens after app-finish should fail!" ); } catch (Exception t) { Log.Info("Exception found is ", t); // The exception will still have the earlier appAttemptId as it picks it // up from the token. NUnit.Framework.Assert.IsTrue(t.InnerException.Message.Contains(applicationAttemptId .ToString() + " not found in AMRMTokenSecretManager.")); } } finally { rm.Stop(); if (rmClient != null) { rpc.StopProxy(rmClient, conf); } } }
public virtual void TestMasterKeyRollOver() { conf.SetLong(YarnConfiguration.RmAmrmTokenMasterKeyRollingIntervalSecs, rolling_interval_sec ); conf.SetLong(YarnConfiguration.RmAmExpiryIntervalMs, am_expire_ms); TestAMAuthorization.MyContainerManager containerManager = new TestAMAuthorization.MyContainerManager (); TestAMAuthorization.MockRMWithAMS rm = new TestAMAuthorization.MockRMWithAMS(conf , containerManager); rm.Start(); long startTime = Runtime.CurrentTimeMillis(); Configuration conf = rm.GetConfig(); YarnRPC rpc = YarnRPC.Create(conf); ApplicationMasterProtocol rmClient = null; AMRMTokenSecretManager appTokenSecretManager = rm.GetRMContext().GetAMRMTokenSecretManager (); MasterKeyData oldKey = appTokenSecretManager.GetMasterKey(); NUnit.Framework.Assert.IsNotNull(oldKey); try { MockNM nm1 = rm.RegisterNode("localhost:1234", 5120); RMApp app = rm.SubmitApp(1024); nm1.NodeHeartbeat(true); int waitCount = 0; while (containerManager.containerTokens == null && waitCount++ < maxWaitAttempts) { Log.Info("Waiting for AM Launch to happen.."); Sharpen.Thread.Sleep(1000); } NUnit.Framework.Assert.IsNotNull(containerManager.containerTokens); RMAppAttempt attempt = app.GetCurrentAppAttempt(); ApplicationAttemptId applicationAttemptId = attempt.GetAppAttemptId(); // Create a client to the RM. UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(applicationAttemptId .ToString()); Credentials credentials = containerManager.GetContainerCredentials(); IPEndPoint rmBindAddress = rm.GetApplicationMasterService().GetBindAddress(); Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> amRMToken = TestAMAuthorization.MockRMWithAMS .SetupAndReturnAMRMToken(rmBindAddress, credentials.GetAllTokens()); currentUser.AddToken(amRMToken); rmClient = CreateRMClient(rm, conf, rpc, currentUser); RegisterApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <RegisterApplicationMasterRequest>(); rmClient.RegisterApplicationMaster(request); // One allocate call. AllocateRequest allocateRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest >(); NUnit.Framework.Assert.IsTrue(rmClient.Allocate(allocateRequest).GetAMCommand() == null); // Wait for enough time and make sure the roll_over happens // At mean time, the old AMRMToken should continue to work while (Runtime.CurrentTimeMillis() - startTime < rolling_interval_sec * 1000) { rmClient.Allocate(allocateRequest); Sharpen.Thread.Sleep(500); } MasterKeyData newKey = appTokenSecretManager.GetMasterKey(); NUnit.Framework.Assert.IsNotNull(newKey); NUnit.Framework.Assert.IsFalse("Master key should have changed!", oldKey.Equals(newKey )); // Another allocate call with old AMRMToken. Should continue to work. rpc.StopProxy(rmClient, conf); // To avoid using cached client rmClient = CreateRMClient(rm, conf, rpc, currentUser); NUnit.Framework.Assert.IsTrue(rmClient.Allocate(allocateRequest).GetAMCommand() == null); waitCount = 0; while (waitCount++ <= maxWaitAttempts) { if (appTokenSecretManager.GetCurrnetMasterKeyData() != oldKey) { break; } try { rmClient.Allocate(allocateRequest); } catch (Exception) { break; } Sharpen.Thread.Sleep(200); } // active the nextMasterKey, and replace the currentMasterKey NUnit.Framework.Assert.IsTrue(appTokenSecretManager.GetCurrnetMasterKeyData().Equals (newKey)); NUnit.Framework.Assert.IsTrue(appTokenSecretManager.GetMasterKey().Equals(newKey) ); NUnit.Framework.Assert.IsTrue(appTokenSecretManager.GetNextMasterKeyData() == null ); // Create a new Token Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> newToken = appTokenSecretManager .CreateAndGetAMRMToken(applicationAttemptId); SecurityUtil.SetTokenService(newToken, rmBindAddress); currentUser.AddToken(newToken); // Another allocate call. Should continue to work. rpc.StopProxy(rmClient, conf); // To avoid using cached client rmClient = CreateRMClient(rm, conf, rpc, currentUser); allocateRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest>( ); NUnit.Framework.Assert.IsTrue(rmClient.Allocate(allocateRequest).GetAMCommand() == null); // Should not work by using the old AMRMToken. rpc.StopProxy(rmClient, conf); // To avoid using cached client try { currentUser.AddToken(amRMToken); rmClient = CreateRMClient(rm, conf, rpc, currentUser); allocateRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest>( ); NUnit.Framework.Assert.IsTrue(rmClient.Allocate(allocateRequest).GetAMCommand() == null); NUnit.Framework.Assert.Fail("The old Token should not work"); } catch (Exception) { } } finally { // expect exception rm.Stop(); if (rmClient != null) { rpc.StopProxy(rmClient, conf); } } }
public virtual void TestUnauthorizedAccess() { TestAMAuthorization.MyContainerManager containerManager = new TestAMAuthorization.MyContainerManager (); rm = new TestAMAuthorization.MockRMWithAMS(conf, containerManager); rm.Start(); MockNM nm1 = rm.RegisterNode("localhost:1234", 5120); RMApp app = rm.SubmitApp(1024); nm1.NodeHeartbeat(true); int waitCount = 0; while (containerManager.containerTokens == null && waitCount++ < 40) { Log.Info("Waiting for AM Launch to happen.."); Sharpen.Thread.Sleep(1000); } NUnit.Framework.Assert.IsNotNull(containerManager.containerTokens); RMAppAttempt attempt = app.GetCurrentAppAttempt(); ApplicationAttemptId applicationAttemptId = attempt.GetAppAttemptId(); WaitForLaunchedState(attempt); Configuration conf = rm.GetConfig(); YarnRPC rpc = YarnRPC.Create(conf); IPEndPoint serviceAddr = conf.GetSocketAddr(YarnConfiguration.RmSchedulerAddress, YarnConfiguration.DefaultRmSchedulerAddress, YarnConfiguration.DefaultRmSchedulerPort ); UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(applicationAttemptId .ToString()); // First try contacting NM without tokens ApplicationMasterProtocol client = currentUser.DoAs(new _PrivilegedAction_262(rpc , serviceAddr, conf)); RegisterApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <RegisterApplicationMasterRequest>(); try { client.RegisterApplicationMaster(request); NUnit.Framework.Assert.Fail("Should fail with authorization error"); } catch (Exception e) { if (IsCause(typeof(AccessControlException), e)) { // Because there are no tokens, the request should be rejected as the // server side will assume we are trying simple auth. string expectedMessage = string.Empty; if (UserGroupInformation.IsSecurityEnabled()) { expectedMessage = "Client cannot authenticate via:[TOKEN]"; } else { expectedMessage = "SIMPLE authentication is not enabled. Available:[TOKEN]"; } NUnit.Framework.Assert.IsTrue(e.InnerException.Message.Contains(expectedMessage)); } else { throw; } } }