public virtual void TestDelegationTokenAuth()
        {
            string token = GetDelegationToken("test");
            ApplicationSubmissionContextInfo app = new ApplicationSubmissionContextInfo();
            string appid = "application_123_0";

            app.SetApplicationId(appid);
            string            requestBody = GetMarshalledAppInfo(app);
            Uri               url         = new Uri("http://localhost:8088/ws/v1/cluster/apps");
            HttpURLConnection conn        = (HttpURLConnection)url.OpenConnection();

            SetupConn(conn, "POST", "application/xml", requestBody);
            // this should fail with unauthorized because only
            // auth is kerberos or delegation token
            try
            {
                conn.GetInputStream();
                NUnit.Framework.Assert.Fail("we should not be here");
            }
            catch (IOException)
            {
                NUnit.Framework.Assert.AreEqual(ClientResponse.Status.Unauthorized.GetStatusCode(
                                                    ), conn.GetResponseCode());
            }
            conn = (HttpURLConnection)url.OpenConnection();
            conn.SetRequestProperty(delegationTokenHeader, token);
            SetupConn(conn, "POST", MediaType.ApplicationXml, requestBody);
            // this should not fail
            try
            {
                conn.GetInputStream();
            }
            catch (IOException)
            {
                InputStream    errorStream = conn.GetErrorStream();
                string         error       = string.Empty;
                BufferedReader reader      = null;
                reader = new BufferedReader(new InputStreamReader(errorStream, "UTF8"));
                for (string line; (line = reader.ReadLine()) != null;)
                {
                    error += line;
                }
                reader.Close();
                errorStream.Close();
                NUnit.Framework.Assert.Fail("Response " + conn.GetResponseCode() + "; " + error);
            }
            bool appExists = rm.GetRMContext().GetRMApps().Contains(ConverterUtils.ToApplicationId
                                                                        (appid));

            NUnit.Framework.Assert.IsTrue(appExists);
            RMApp actualApp = rm.GetRMContext().GetRMApps()[ConverterUtils.ToApplicationId(appid
                                                                                           )];
            string owner = actualApp.GetUser();

            NUnit.Framework.Assert.AreEqual("client", owner);
        }
示例#2
0
        public virtual void TestZKRootPathAcls()
        {
            HAServiceProtocol.StateChangeRequestInfo req = new HAServiceProtocol.StateChangeRequestInfo
                                                               (HAServiceProtocol.RequestSource.RequestByUser);
            string rootPath = YarnConfiguration.DefaultZkRmStateStoreParentPath + "/" + ZKRMStateStore
                              .RootZnodeName;
            // Start RM with HA enabled
            Configuration conf = CreateHARMConf("rm1,rm2", "rm1", 1234);

            conf.SetBoolean(YarnConfiguration.AutoFailoverEnabled, false);
            ResourceManager rm = new MockRM(conf);

            rm.Start();
            rm.GetRMContext().GetRMAdminService().TransitionToActive(req);
            Stat        stat = new Stat();
            IList <ACL> acls = ((ZKRMStateStore)rm.GetRMContext().GetStateStore()).GetACLWithRetries
                                   (rootPath, stat);

            NUnit.Framework.Assert.AreEqual(acls.Count, 2);
            // CREATE and DELETE permissions for root node based on RM ID
            VerifyZKACL("digest", "localhost", ZooDefs.Perms.Create | ZooDefs.Perms.Delete, acls
                        );
            VerifyZKACL("world", "anyone", ZooDefs.Perms.All ^ (ZooDefs.Perms.Create | ZooDefs.Perms
                                                                .Delete), acls);
            rm.Close();
            // Now start RM with HA disabled. NoAuth Exception should not be thrown.
            conf.SetBoolean(YarnConfiguration.RmHaEnabled, false);
            rm = new MockRM(conf);
            rm.Start();
            rm.GetRMContext().GetRMAdminService().TransitionToActive(req);
            acls = ((ZKRMStateStore)rm.GetRMContext().GetStateStore()).GetACLWithRetries(rootPath
                                                                                         , stat);
            NUnit.Framework.Assert.AreEqual(acls.Count, 1);
            VerifyZKACL("world", "anyone", ZooDefs.Perms.All, acls);
            rm.Close();
            // Start RM with HA enabled.
            conf.SetBoolean(YarnConfiguration.RmHaEnabled, true);
            rm = new MockRM(conf);
            rm.Start();
            rm.GetRMContext().GetRMAdminService().TransitionToActive(req);
            acls = ((ZKRMStateStore)rm.GetRMContext().GetStateStore()).GetACLWithRetries(rootPath
                                                                                         , stat);
            NUnit.Framework.Assert.AreEqual(acls.Count, 2);
            VerifyZKACL("digest", "localhost", ZooDefs.Perms.Create | ZooDefs.Perms.Delete, acls
                        );
            VerifyZKACL("world", "anyone", ZooDefs.Perms.All ^ (ZooDefs.Perms.Create | ZooDefs.Perms
                                                                .Delete), acls);
            rm.Close();
        }
示例#3
0
        // Test regular RM restart/failover, new RM should not count
        // AM failure towards the max-retry-account and should be able to
        // re-launch the AM.
        /// <exception cref="System.Exception"/>
        public virtual void TestRMRestartOrFailoverNotCountedForAMFailures()
        {
            YarnConfiguration conf = new YarnConfiguration();

            conf.SetClass(YarnConfiguration.RmScheduler, typeof(CapacityScheduler), typeof(ResourceScheduler
                                                                                           ));
            conf.SetBoolean(YarnConfiguration.RecoveryEnabled, true);
            conf.SetBoolean(YarnConfiguration.RmWorkPreservingRecoveryEnabled, false);
            conf.Set(YarnConfiguration.RmStore, typeof(MemoryRMStateStore).FullName);
            // explicitly set max-am-retry count as 1.
            conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1);
            MemoryRMStateStore memStore = new MemoryRMStateStore();

            memStore.Init(conf);
            MockRM rm1 = new MockRM(conf, memStore);

            rm1.Start();
            MockNM nm1 = new MockNM("127.0.0.1:1234", 8000, rm1.GetResourceTrackerService());

            nm1.RegisterNode();
            RMApp app1 = rm1.SubmitApp(200);
            // AM should be restarted even though max-am-attempt is 1.
            MockAM       am1      = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
            RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();

            NUnit.Framework.Assert.IsTrue(((RMAppAttemptImpl)attempt1).MayBeLastAttempt());
            // Restart rm.
            MockRM rm2 = new MockRM(conf, memStore);

            rm2.Start();
            ApplicationStateData appState = memStore.GetState().GetApplicationState()[app1.GetApplicationId
                                                                                          ()];

            // re-register the NM
            nm1.SetResourceTrackerService(rm2.GetResourceTrackerService());
            NMContainerStatus status = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <NMContainerStatus
                                                                                      >();

            status.SetContainerExitStatus(ContainerExitStatus.KilledByResourcemanager);
            status.SetContainerId(attempt1.GetMasterContainer().GetId());
            status.SetContainerState(ContainerState.Complete);
            status.SetDiagnostics(string.Empty);
            nm1.RegisterNode(Sharpen.Collections.SingletonList(status), null);
            rm2.WaitForState(attempt1.GetAppAttemptId(), RMAppAttemptState.Failed);
            NUnit.Framework.Assert.AreEqual(ContainerExitStatus.KilledByResourcemanager, appState
                                            .GetAttempt(am1.GetApplicationAttemptId()).GetAMContainerExitStatus());
            // Will automatically start a new AppAttempt in rm2
            rm2.WaitForState(app1.GetApplicationId(), RMAppState.Accepted);
            MockAM am2 = rm2.WaitForNewAMToLaunchAndRegister(app1.GetApplicationId(), 2, nm1);

            MockRM.FinishAMAndVerifyAppState(app1, rm2, nm1, am2);
            RMAppAttempt attempt3 = rm2.GetRMContext().GetRMApps()[app1.GetApplicationId()].GetCurrentAppAttempt
                                        ();

            NUnit.Framework.Assert.IsTrue(attempt3.ShouldCountTowardsMaxAttemptRetry());
            NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Invalid, appState.GetAttempt(
                                                am2.GetApplicationAttemptId()).GetAMContainerExitStatus());
            rm1.Stop();
            rm2.Stop();
        }
示例#4
0
        // Test RM restarts after AM container is preempted, new RM should not count
        // AM preemption failure towards the max-retry-account and should be able to
        // re-launch the AM.
        /// <exception cref="System.Exception"/>
        public virtual void TestPreemptedAMRestartOnRMRestart()
        {
            YarnConfiguration conf = new YarnConfiguration();

            conf.SetClass(YarnConfiguration.RmScheduler, typeof(CapacityScheduler), typeof(ResourceScheduler
                                                                                           ));
            conf.SetBoolean(YarnConfiguration.RecoveryEnabled, true);
            conf.SetBoolean(YarnConfiguration.RmWorkPreservingRecoveryEnabled, false);
            conf.Set(YarnConfiguration.RmStore, typeof(MemoryRMStateStore).FullName);
            // explicitly set max-am-retry count as 1.
            conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1);
            MemoryRMStateStore memStore = new MemoryRMStateStore();

            memStore.Init(conf);
            MockRM rm1 = new MockRM(conf, memStore);

            rm1.Start();
            MockNM nm1 = new MockNM("127.0.0.1:1234", 8000, rm1.GetResourceTrackerService());

            nm1.RegisterNode();
            RMApp             app1        = rm1.SubmitApp(200);
            RMAppAttempt      attempt1    = app1.GetCurrentAppAttempt();
            MockAM            am1         = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
            CapacityScheduler scheduler   = (CapacityScheduler)rm1.GetResourceScheduler();
            ContainerId       amContainer = ContainerId.NewContainerId(am1.GetApplicationAttemptId(
                                                                           ), 1);

            // Forcibly preempt the am container;
            scheduler.KillContainer(scheduler.GetRMContainer(amContainer));
            am1.WaitForState(RMAppAttemptState.Failed);
            NUnit.Framework.Assert.IsTrue(!attempt1.ShouldCountTowardsMaxAttemptRetry());
            rm1.WaitForState(app1.GetApplicationId(), RMAppState.Accepted);
            // state store has 1 attempt stored.
            ApplicationStateData appState = memStore.GetState().GetApplicationState()[app1.GetApplicationId
                                                                                          ()];

            NUnit.Framework.Assert.AreEqual(1, appState.GetAttemptCount());
            // attempt stored has the preempted container exit status.
            NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Preempted, appState.GetAttempt
                                                (am1.GetApplicationAttemptId()).GetAMContainerExitStatus());
            // Restart rm.
            MockRM rm2 = new MockRM(conf, memStore);

            nm1.SetResourceTrackerService(rm2.GetResourceTrackerService());
            nm1.RegisterNode();
            rm2.Start();
            // Restarted RM should re-launch the am.
            MockAM am2 = rm2.WaitForNewAMToLaunchAndRegister(app1.GetApplicationId(), 2, nm1);

            MockRM.FinishAMAndVerifyAppState(app1, rm2, nm1, am2);
            RMAppAttempt attempt2 = rm2.GetRMContext().GetRMApps()[app1.GetApplicationId()].GetCurrentAppAttempt
                                        ();

            NUnit.Framework.Assert.IsTrue(attempt2.ShouldCountTowardsMaxAttemptRetry());
            NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Invalid, appState.GetAttempt(
                                                am2.GetApplicationAttemptId()).GetAMContainerExitStatus());
            rm1.Stop();
            rm2.Stop();
        }
示例#5
0
        /// <exception cref="System.Exception"/>
        private void TestAnonymousSimpleUser()
        {
            ApplicationSubmissionContextInfo app = new ApplicationSubmissionContextInfo();
            string appid = "application_123_0";

            app.SetApplicationId(appid);
            string requestBody = TestRMWebServicesDelegationTokenAuthentication.GetMarshalledAppInfo
                                     (app);
            Uri url = new Uri("http://localhost:8088/ws/v1/cluster/apps");
            HttpURLConnection conn = (HttpURLConnection)url.OpenConnection();

            TestRMWebServicesDelegationTokenAuthentication.SetupConn(conn, "POST", "application/xml"
                                                                     , requestBody);
            conn.GetInputStream();
            NUnit.Framework.Assert.AreEqual(ClientResponse.Status.Accepted.GetStatusCode(), conn
                                            .GetResponseCode());
            bool appExists = rm.GetRMContext().GetRMApps().Contains(ConverterUtils.ToApplicationId
                                                                        (appid));

            NUnit.Framework.Assert.IsTrue(appExists);
            RMApp actualApp = rm.GetRMContext().GetRMApps()[ConverterUtils.ToApplicationId(appid
                                                                                           )];
            string owner = actualApp.GetUser();

            NUnit.Framework.Assert.AreEqual(rm.GetConfig().Get(CommonConfigurationKeys.HadoopHttpStaticUser
                                                               , CommonConfigurationKeys.DefaultHadoopHttpStaticUser), owner);
            appid = "application_123_1";
            app.SetApplicationId(appid);
            requestBody = TestRMWebServicesDelegationTokenAuthentication.GetMarshalledAppInfo
                              (app);
            url  = new Uri("http://localhost:8088/ws/v1/cluster/apps?user.name=client");
            conn = (HttpURLConnection)url.OpenConnection();
            TestRMWebServicesDelegationTokenAuthentication.SetupConn(conn, "POST", MediaType.
                                                                     ApplicationXml, requestBody);
            conn.GetInputStream();
            appExists = rm.GetRMContext().GetRMApps().Contains(ConverterUtils.ToApplicationId
                                                                   (appid));
            NUnit.Framework.Assert.IsTrue(appExists);
            actualApp = rm.GetRMContext().GetRMApps()[ConverterUtils.ToApplicationId(appid)];
            owner     = actualApp.GetUser();
            NUnit.Framework.Assert.AreEqual("client", owner);
        }
示例#6
0
        public virtual void TestConcurrentAccess()
        {
            conf.Set(FairSchedulerConfiguration.AssignMultiple, "false");
            resourceManager = new MockRM(conf);
            resourceManager.Start();
            scheduler = (FairScheduler)resourceManager.GetResourceScheduler();
            string      queueName   = "root.queue1";
            FSLeafQueue schedulable = scheduler.GetQueueManager().GetLeafQueue(queueName, true
                                                                               );
            ApplicationAttemptId applicationAttemptId = CreateAppAttemptId(1, 1);
            RMContext            rmContext            = resourceManager.GetRMContext();
            FSAppAttempt         app = new FSAppAttempt(scheduler, applicationAttemptId, "user1", schedulable
                                                        , null, rmContext);
            // this needs to be in sync with the number of runnables declared below
            int testThreads            = 2;
            IList <Runnable> runnables = new AList <Runnable>();

            // add applications to modify the list
            runnables.AddItem(new _Runnable_257(schedulable, app));
            // iterate over the list a couple of times in a different thread
            runnables.AddItem(new _Runnable_267(schedulable));
            IList <Exception> exceptions = Sharpen.Collections.SynchronizedList(new AList <Exception
                                                                                           >());
            ExecutorService threadPool = Executors.NewFixedThreadPool(testThreads);

            try
            {
                CountDownLatch allExecutorThreadsReady = new CountDownLatch(testThreads);
                CountDownLatch startBlocker            = new CountDownLatch(1);
                CountDownLatch allDone = new CountDownLatch(testThreads);
                foreach (Runnable submittedTestRunnable in runnables)
                {
                    threadPool.Submit(new _Runnable_287(allExecutorThreadsReady, startBlocker, submittedTestRunnable
                                                        , exceptions, allDone));
                }
                // wait until all threads are ready
                allExecutorThreadsReady.Await();
                // start all test runners
                startBlocker.CountDown();
                int testTimeout = 2;
                NUnit.Framework.Assert.IsTrue("Timeout waiting for more than " + testTimeout + " seconds"
                                              , allDone.Await(testTimeout, TimeUnit.Seconds));
            }
            catch (Exception ie)
            {
                exceptions.AddItem(ie);
            }
            finally
            {
                threadPool.ShutdownNow();
            }
            NUnit.Framework.Assert.IsTrue("Test failed with exception(s)" + exceptions, exceptions
                                          .IsEmpty());
        }
        /// <exception cref="System.Exception"/>
        private AllocateResponse Allocate(ApplicationAttemptId attemptId, AllocateRequest
                                          req)
        {
            UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser(attemptId.ToString
                                                                                 ());

            Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> token = rm.GetRMContext
                                                                                     ().GetRMApps()[attemptId.GetApplicationId()].GetRMAppAttempt(attemptId).GetAMRMToken
                                                                                     ();
            ugi.AddTokenIdentifier(token.DecodeIdentifier());
            return(ugi.DoAs(new _PrivilegedExceptionAction_67(this, req)));
        }
        public virtual void TestNodesDefaultWithUnHealthyNode()
        {
            WebResource r   = Resource();
            MockNM      nm1 = rm.RegisterNode("h1:1234", 5120);
            MockNM      nm2 = rm.RegisterNode("h2:1235", 5121);

            rm.SendNodeStarted(nm1);
            rm.NMwaitForState(nm1.GetNodeId(), NodeState.Running);
            rm.NMwaitForState(nm2.GetNodeId(), NodeState.New);
            MockNM nm3 = rm.RegisterNode("h3:1236", 5122);

            rm.NMwaitForState(nm3.GetNodeId(), NodeState.New);
            rm.SendNodeStarted(nm3);
            rm.NMwaitForState(nm3.GetNodeId(), NodeState.Running);
            RMNodeImpl       node       = (RMNodeImpl)rm.GetRMContext().GetRMNodes()[nm3.GetNodeId()];
            NodeHealthStatus nodeHealth = NodeHealthStatus.NewInstance(false, "test health report"
                                                                       , Runtime.CurrentTimeMillis());

            node.Handle(new RMNodeStatusEvent(nm3.GetNodeId(), nodeHealth, new AList <ContainerStatus
                                                                                      >(), null, null));
            rm.NMwaitForState(nm3.GetNodeId(), NodeState.Unhealthy);
            ClientResponse response = r.Path("ws").Path("v1").Path("cluster").Path("nodes").Accept
                                          (MediaType.ApplicationJson).Get <ClientResponse>();

            NUnit.Framework.Assert.AreEqual(MediaType.ApplicationJsonType, response.GetType()
                                            );
            JSONObject json = response.GetEntity <JSONObject>();

            NUnit.Framework.Assert.AreEqual("incorrect number of elements", 1, json.Length());
            JSONObject nodes = json.GetJSONObject("nodes");

            NUnit.Framework.Assert.AreEqual("incorrect number of elements", 1, nodes.Length()
                                            );
            JSONArray nodeArray = nodes.GetJSONArray("node");

            // 3 nodes, including the unhealthy node and the new node.
            NUnit.Framework.Assert.AreEqual("incorrect number of elements", 3, nodeArray.Length
                                                ());
        }
示例#9
0
        /// <exception cref="System.Exception"/>
        public virtual void TestStoreOnlyAMContainerMetrics()
        {
            Configuration conf = new Configuration();

            conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1);
            conf.SetBoolean(YarnConfiguration.ApplicationHistorySaveNonAmContainerMetaInfo, false
                            );
            MockRM rm1 = new MockRM(conf);
            SystemMetricsPublisher publisher = Org.Mockito.Mockito.Mock <SystemMetricsPublisher
                                                                         >();

            rm1.GetRMContext().SetSystemMetricsPublisher(publisher);
            rm1.Start();
            MockNM nm1  = rm1.RegisterNode("unknownhost:1234", 8000);
            RMApp  app1 = rm1.SubmitApp(1024);
            MockAM am1  = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);

            nm1.NodeHeartbeat(am1.GetApplicationAttemptId(), 1, ContainerState.Running);
            // request a container.
            am1.Allocate("127.0.0.1", 1024, 1, new AList <ContainerId>());
            ContainerId containerId2 = ContainerId.NewContainerId(am1.GetApplicationAttemptId
                                                                      (), 2);

            rm1.WaitForState(nm1, containerId2, RMContainerState.Allocated);
            am1.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>()).GetAllocatedContainers
                ();
            rm1.WaitForState(nm1, containerId2, RMContainerState.Acquired);
            nm1.NodeHeartbeat(am1.GetApplicationAttemptId(), 2, ContainerState.Running);
            nm1.NodeHeartbeat(am1.GetApplicationAttemptId(), 2, ContainerState.Complete);
            nm1.NodeHeartbeat(am1.GetApplicationAttemptId(), 1, ContainerState.Complete);
            rm1.WaitForState(nm1, containerId2, RMContainerState.Completed);
            rm1.Stop();
            // RMContainer should be publishing system metrics only for AM container.
            Org.Mockito.Mockito.Verify(publisher, Org.Mockito.Mockito.Times(1)).ContainerCreated
                (Matchers.Any <RMContainer>(), Matchers.AnyLong());
            Org.Mockito.Mockito.Verify(publisher, Org.Mockito.Mockito.Times(1)).ContainerFinished
                (Matchers.Any <RMContainer>(), Matchers.AnyLong());
        }
示例#10
0
        public virtual void TestBlackListNodes()
        {
            Configuration conf = new Configuration();

            conf.SetClass(YarnConfiguration.RmScheduler, typeof(FifoScheduler), typeof(ResourceScheduler
                                                                                       ));
            MockRM rm = new MockRM(conf);

            rm.Start();
            FifoScheduler fs   = (FifoScheduler)rm.GetResourceScheduler();
            string        host = "127.0.0.1";
            RMNode        node = MockNodes.NewNodeInfo(0, MockNodes.NewResource(4 * Gb), 1, host);

            fs.Handle(new NodeAddedSchedulerEvent(node));
            ApplicationId        appId        = BuilderUtils.NewApplicationId(100, 1);
            ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
                                                                                     );

            CreateMockRMApp(appAttemptId, rm.GetRMContext());
            SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId, "default", "user");

            fs.Handle(appEvent);
            SchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId, false
                                                                            );

            fs.Handle(attemptEvent);
            // Verify the blacklist can be updated independent of requesting containers
            fs.Allocate(appAttemptId, Sharpen.Collections.EmptyList <ResourceRequest>(), Sharpen.Collections
                        .EmptyList <ContainerId>(), Sharpen.Collections.SingletonList(host), null);
            NUnit.Framework.Assert.IsTrue(fs.GetApplicationAttempt(appAttemptId).IsBlacklisted
                                              (host));
            fs.Allocate(appAttemptId, Sharpen.Collections.EmptyList <ResourceRequest>(), Sharpen.Collections
                        .EmptyList <ContainerId>(), null, Sharpen.Collections.SingletonList(host));
            NUnit.Framework.Assert.IsFalse(fs.GetApplicationAttempt(appAttemptId).IsBlacklisted
                                               (host));
            rm.Stop();
        }
示例#11
0
        public virtual void TestRefreshQueuesWithReservations()
        {
            CapacityScheduler cs = (CapacityScheduler)rm.GetResourceScheduler();
            // Test add one reservation dynamically and manually modify capacity
            ReservationQueue a1 = new ReservationQueue(cs, "a1", (PlanQueue)cs.GetQueue("a"));

            cs.AddQueue(a1);
            a1.SetEntitlement(new QueueEntitlement(A1Capacity / 100, 1f));
            // Test add another reservation queue and use setEntitlement to modify
            // capacity
            ReservationQueue a2 = new ReservationQueue(cs, "a2", (PlanQueue)cs.GetQueue("a"));

            cs.AddQueue(a2);
            cs.SetEntitlement("a2", new QueueEntitlement(A2Capacity / 100, 1.0f));
            // Verify all allocations match
            tcs.CheckQueueCapacities(cs, ACapacity, BCapacity);
            // Reinitialize and verify all dynamic queued survived
            CapacitySchedulerConfiguration conf = cs.GetConfiguration();

            conf.SetCapacity(A, 80f);
            conf.SetCapacity(B, 20f);
            cs.Reinitialize(conf, rm.GetRMContext());
            tcs.CheckQueueCapacities(cs, 80f, 20f);
        }
示例#12
0
        /// <exception cref="System.Exception"/>
        public virtual void TestExcessReservationThanNodeManagerCapacity()
        {
            MockRM rm = new MockRM(conf);

            rm.Start();
            // Register node1
            MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 2 * Gb, 4);
            MockNM nm2 = rm.RegisterNode("127.0.0.1:2234", 3 * Gb, 4);

            nm1.NodeHeartbeat(true);
            nm2.NodeHeartbeat(true);
            // wait..
            int waitCount = 20;
            int size      = rm.GetRMContext().GetRMNodes().Count;

            while ((size = rm.GetRMContext().GetRMNodes().Count) != 2 && waitCount-- > 0)
            {
                Log.Info("Waiting for node managers to register : " + size);
                Sharpen.Thread.Sleep(100);
            }
            NUnit.Framework.Assert.AreEqual(2, rm.GetRMContext().GetRMNodes().Count);
            // Submit an application
            RMApp app1 = rm.SubmitApp(128);

            // kick the scheduling
            nm1.NodeHeartbeat(true);
            RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
            MockAM       am1      = rm.SendAMLaunched(attempt1.GetAppAttemptId());

            am1.RegisterAppAttempt();
            Log.Info("sending container requests ");
            am1.AddRequests(new string[] { "*" }, 2 * Gb, 1, 1);
            AllocateResponse alloc1Response = am1.Schedule();

            // send the request
            // kick the scheduler
            nm1.NodeHeartbeat(true);
            int waitCounter = 20;

            Log.Info("heartbeating nm1");
            while (alloc1Response.GetAllocatedContainers().Count < 1 && waitCounter-- > 0)
            {
                Log.Info("Waiting for containers to be created for app 1...");
                Sharpen.Thread.Sleep(500);
                alloc1Response = am1.Schedule();
            }
            Log.Info("received container : " + alloc1Response.GetAllocatedContainers().Count);
            // No container should be allocated.
            // Internally it should not been reserved.
            NUnit.Framework.Assert.IsTrue(alloc1Response.GetAllocatedContainers().Count == 0);
            Log.Info("heartbeating nm2");
            waitCounter = 20;
            nm2.NodeHeartbeat(true);
            while (alloc1Response.GetAllocatedContainers().Count < 1 && waitCounter-- > 0)
            {
                Log.Info("Waiting for containers to be created for app 1...");
                Sharpen.Thread.Sleep(500);
                alloc1Response = am1.Schedule();
            }
            Log.Info("received container : " + alloc1Response.GetAllocatedContainers().Count);
            NUnit.Framework.Assert.IsTrue(alloc1Response.GetAllocatedContainers().Count == 1);
            rm.Stop();
        }