Exemplo n.º 1
0
        public virtual void TestValidateResourceBlacklistRequest()
        {
            TestAMAuthorization.MyContainerManager containerManager = new TestAMAuthorization.MyContainerManager
                                                                          ();
            TestAMAuthorization.MockRMWithAMS rm = new TestAMAuthorization.MockRMWithAMS(new
                                                                                         YarnConfiguration(), containerManager);
            rm.Start();
            MockNM nm1 = rm.RegisterNode("localhost:1234", 5120);
            IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
                                                                               , string>(2);

            acls[ApplicationAccessType.ViewApp] = "*";
            RMApp app = rm.SubmitApp(1024, "appname", "appuser", acls);

            nm1.NodeHeartbeat(true);
            RMAppAttempt         attempt = app.GetCurrentAppAttempt();
            ApplicationAttemptId applicationAttemptId = attempt.GetAppAttemptId();

            WaitForLaunchedState(attempt);
            // Create a client to the RM.
            Configuration        conf        = rm.GetConfig();
            YarnRPC              rpc         = YarnRPC.Create(conf);
            UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(applicationAttemptId
                                                                                     .ToString());
            Credentials credentials   = containerManager.GetContainerCredentials();
            IPEndPoint  rmBindAddress = rm.GetApplicationMasterService().GetBindAddress();

            Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> amRMToken = TestAMAuthorization.MockRMWithAMS
                                                                                 .SetupAndReturnAMRMToken(rmBindAddress, credentials.GetAllTokens());
            currentUser.AddToken(amRMToken);
            ApplicationMasterProtocol client = currentUser.DoAs(new _PrivilegedAction_626(rpc
                                                                                          , rmBindAddress, conf));
            RegisterApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
                                                       <RegisterApplicationMasterRequest>();

            client.RegisterApplicationMaster(request);
            ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.NewInstance(
                Sharpen.Collections.SingletonList(ResourceRequest.Any), null);
            AllocateRequest allocateRequest = AllocateRequest.NewInstance(0, 0.0f, null, null
                                                                          , blacklistRequest);
            bool error = false;

            try
            {
                client.Allocate(allocateRequest);
            }
            catch (InvalidResourceBlacklistRequestException)
            {
                error = true;
            }
            rm.Stop();
            NUnit.Framework.Assert.IsTrue("Didn't not catch InvalidResourceBlacklistRequestException"
                                          , error);
        }
Exemplo n.º 2
0
        public virtual void TestTokenExpiry()
        {
            TestAMAuthorization.MyContainerManager containerManager = new TestAMAuthorization.MyContainerManager
                                                                          ();
            TestAMAuthorization.MockRMWithAMS rm = new TestAMAuthorization.MockRMWithAMS(conf
                                                                                         , containerManager);
            rm.Start();
            Configuration             conf     = rm.GetConfig();
            YarnRPC                   rpc      = YarnRPC.Create(conf);
            ApplicationMasterProtocol rmClient = null;

            try
            {
                MockNM nm1 = rm.RegisterNode("localhost:1234", 5120);
                RMApp  app = rm.SubmitApp(1024);
                nm1.NodeHeartbeat(true);
                int waitCount = 0;
                while (containerManager.containerTokens == null && waitCount++ < 20)
                {
                    Log.Info("Waiting for AM Launch to happen..");
                    Sharpen.Thread.Sleep(1000);
                }
                NUnit.Framework.Assert.IsNotNull(containerManager.containerTokens);
                RMAppAttempt         attempt = app.GetCurrentAppAttempt();
                ApplicationAttemptId applicationAttemptId = attempt.GetAppAttemptId();
                // Create a client to the RM.
                UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(applicationAttemptId
                                                                                         .ToString());
                Credentials credentials   = containerManager.GetContainerCredentials();
                IPEndPoint  rmBindAddress = rm.GetApplicationMasterService().GetBindAddress();
                Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> amRMToken = TestAMAuthorization.MockRMWithAMS
                                                                                     .SetupAndReturnAMRMToken(rmBindAddress, credentials.GetAllTokens());
                currentUser.AddToken(amRMToken);
                rmClient = CreateRMClient(rm, conf, rpc, currentUser);
                RegisterApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
                                                           <RegisterApplicationMasterRequest>();
                rmClient.RegisterApplicationMaster(request);
                FinishApplicationMasterRequest finishAMRequest = Org.Apache.Hadoop.Yarn.Util.Records
                                                                 .NewRecord <FinishApplicationMasterRequest>();
                finishAMRequest.SetFinalApplicationStatus(FinalApplicationStatus.Succeeded);
                finishAMRequest.SetDiagnostics("diagnostics");
                finishAMRequest.SetTrackingUrl("url");
                rmClient.FinishApplicationMaster(finishAMRequest);
                // Send RMAppAttemptEventType.CONTAINER_FINISHED to transit RMAppAttempt
                // from Finishing state to Finished State. Both AMRMToken and
                // ClientToAMToken will be removed.
                ContainerStatus containerStatus = BuilderUtils.NewContainerStatus(attempt.GetMasterContainer
                                                                                      ().GetId(), ContainerState.Complete, "AM Container Finished", 0);
                rm.GetRMContext().GetDispatcher().GetEventHandler().Handle(new RMAppAttemptContainerFinishedEvent
                                                                               (applicationAttemptId, containerStatus, nm1.GetNodeId()));
                // Make sure the RMAppAttempt is at Finished State.
                // Both AMRMToken and ClientToAMToken have been removed.
                int count = 0;
                while (attempt.GetState() != RMAppAttemptState.Finished && count < maxWaitAttempts
                       )
                {
                    Sharpen.Thread.Sleep(100);
                    count++;
                }
                NUnit.Framework.Assert.IsTrue(attempt.GetState() == RMAppAttemptState.Finished);
                // Now simulate trying to allocate. RPC call itself should throw auth
                // exception.
                rpc.StopProxy(rmClient, conf);
                // To avoid using cached client
                rmClient = CreateRMClient(rm, conf, rpc, currentUser);
                AllocateRequest allocateRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest
                                                                                                 >();
                try
                {
                    rmClient.Allocate(allocateRequest);
                    NUnit.Framework.Assert.Fail("You got to be kidding me! " + "Using App tokens after app-finish should fail!"
                                                );
                }
                catch (Exception t)
                {
                    Log.Info("Exception found is ", t);
                    // The exception will still have the earlier appAttemptId as it picks it
                    // up from the token.
                    NUnit.Framework.Assert.IsTrue(t.InnerException.Message.Contains(applicationAttemptId
                                                                                    .ToString() + " not found in AMRMTokenSecretManager."));
                }
            }
            finally
            {
                rm.Stop();
                if (rmClient != null)
                {
                    rpc.StopProxy(rmClient, conf);
                }
            }
        }
Exemplo n.º 3
0
        public virtual void TestMasterKeyRollOver()
        {
            conf.SetLong(YarnConfiguration.RmAmrmTokenMasterKeyRollingIntervalSecs, rolling_interval_sec
                         );
            conf.SetLong(YarnConfiguration.RmAmExpiryIntervalMs, am_expire_ms);
            TestAMAuthorization.MyContainerManager containerManager = new TestAMAuthorization.MyContainerManager
                                                                          ();
            TestAMAuthorization.MockRMWithAMS rm = new TestAMAuthorization.MockRMWithAMS(conf
                                                                                         , containerManager);
            rm.Start();
            long                      startTime             = Runtime.CurrentTimeMillis();
            Configuration             conf                  = rm.GetConfig();
            YarnRPC                   rpc                   = YarnRPC.Create(conf);
            ApplicationMasterProtocol rmClient              = null;
            AMRMTokenSecretManager    appTokenSecretManager = rm.GetRMContext().GetAMRMTokenSecretManager
                                                                  ();
            MasterKeyData oldKey = appTokenSecretManager.GetMasterKey();

            NUnit.Framework.Assert.IsNotNull(oldKey);
            try
            {
                MockNM nm1 = rm.RegisterNode("localhost:1234", 5120);
                RMApp  app = rm.SubmitApp(1024);
                nm1.NodeHeartbeat(true);
                int waitCount = 0;
                while (containerManager.containerTokens == null && waitCount++ < maxWaitAttempts)
                {
                    Log.Info("Waiting for AM Launch to happen..");
                    Sharpen.Thread.Sleep(1000);
                }
                NUnit.Framework.Assert.IsNotNull(containerManager.containerTokens);
                RMAppAttempt         attempt = app.GetCurrentAppAttempt();
                ApplicationAttemptId applicationAttemptId = attempt.GetAppAttemptId();
                // Create a client to the RM.
                UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(applicationAttemptId
                                                                                         .ToString());
                Credentials credentials   = containerManager.GetContainerCredentials();
                IPEndPoint  rmBindAddress = rm.GetApplicationMasterService().GetBindAddress();
                Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> amRMToken = TestAMAuthorization.MockRMWithAMS
                                                                                     .SetupAndReturnAMRMToken(rmBindAddress, credentials.GetAllTokens());
                currentUser.AddToken(amRMToken);
                rmClient = CreateRMClient(rm, conf, rpc, currentUser);
                RegisterApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
                                                           <RegisterApplicationMasterRequest>();
                rmClient.RegisterApplicationMaster(request);
                // One allocate call.
                AllocateRequest allocateRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest
                                                                                                 >();
                NUnit.Framework.Assert.IsTrue(rmClient.Allocate(allocateRequest).GetAMCommand() ==
                                              null);
                // Wait for enough time and make sure the roll_over happens
                // At mean time, the old AMRMToken should continue to work
                while (Runtime.CurrentTimeMillis() - startTime < rolling_interval_sec * 1000)
                {
                    rmClient.Allocate(allocateRequest);
                    Sharpen.Thread.Sleep(500);
                }
                MasterKeyData newKey = appTokenSecretManager.GetMasterKey();
                NUnit.Framework.Assert.IsNotNull(newKey);
                NUnit.Framework.Assert.IsFalse("Master key should have changed!", oldKey.Equals(newKey
                                                                                                ));
                // Another allocate call with old AMRMToken. Should continue to work.
                rpc.StopProxy(rmClient, conf);
                // To avoid using cached client
                rmClient = CreateRMClient(rm, conf, rpc, currentUser);
                NUnit.Framework.Assert.IsTrue(rmClient.Allocate(allocateRequest).GetAMCommand() ==
                                              null);
                waitCount = 0;
                while (waitCount++ <= maxWaitAttempts)
                {
                    if (appTokenSecretManager.GetCurrnetMasterKeyData() != oldKey)
                    {
                        break;
                    }
                    try
                    {
                        rmClient.Allocate(allocateRequest);
                    }
                    catch (Exception)
                    {
                        break;
                    }
                    Sharpen.Thread.Sleep(200);
                }
                // active the nextMasterKey, and replace the currentMasterKey
                NUnit.Framework.Assert.IsTrue(appTokenSecretManager.GetCurrnetMasterKeyData().Equals
                                                  (newKey));
                NUnit.Framework.Assert.IsTrue(appTokenSecretManager.GetMasterKey().Equals(newKey)
                                              );
                NUnit.Framework.Assert.IsTrue(appTokenSecretManager.GetNextMasterKeyData() == null
                                              );
                // Create a new Token
                Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> newToken = appTokenSecretManager
                                                                                        .CreateAndGetAMRMToken(applicationAttemptId);
                SecurityUtil.SetTokenService(newToken, rmBindAddress);
                currentUser.AddToken(newToken);
                // Another allocate call. Should continue to work.
                rpc.StopProxy(rmClient, conf);
                // To avoid using cached client
                rmClient        = CreateRMClient(rm, conf, rpc, currentUser);
                allocateRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest>(
                    );
                NUnit.Framework.Assert.IsTrue(rmClient.Allocate(allocateRequest).GetAMCommand() ==
                                              null);
                // Should not work by using the old AMRMToken.
                rpc.StopProxy(rmClient, conf);
                // To avoid using cached client
                try
                {
                    currentUser.AddToken(amRMToken);
                    rmClient        = CreateRMClient(rm, conf, rpc, currentUser);
                    allocateRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest>(
                        );
                    NUnit.Framework.Assert.IsTrue(rmClient.Allocate(allocateRequest).GetAMCommand() ==
                                                  null);
                    NUnit.Framework.Assert.Fail("The old Token should not work");
                }
                catch (Exception)
                {
                }
            }
            finally
            {
                // expect exception
                rm.Stop();
                if (rmClient != null)
                {
                    rpc.StopProxy(rmClient, conf);
                }
            }
        }