/// <exception cref="System.Exception"/>
        public virtual void TestAppCleanupWhenNMReconnects()
        {
            conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1);
            MemoryRMStateStore memStore = new MemoryRMStateStore();

            memStore.Init(conf);
            // start RM
            MockRM rm1 = new MockRM(conf, memStore);

            rm1.Start();
            MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.GetResourceTrackerService());

            nm1.RegisterNode();
            // create app and launch the AM
            RMApp  app0 = rm1.SubmitApp(200);
            MockAM am0  = LaunchAM(app0, rm1, nm1);

            nm1.NodeHeartbeat(am0.GetApplicationAttemptId(), 1, ContainerState.Complete);
            rm1.WaitForState(app0.GetApplicationId(), RMAppState.Failed);
            // wait for application cleanup message received
            WaitForAppCleanupMessageRecved(nm1, app0.GetApplicationId());
            // reconnect NM with application still active
            nm1.RegisterNode(Arrays.AsList(app0.GetApplicationId()));
            WaitForAppCleanupMessageRecved(nm1, app0.GetApplicationId());
            rm1.Stop();
        }
示例#2
0
        /// <exception cref="System.Exception"/>
        private void TestMinimumAllocation(YarnConfiguration conf, int testAlloc)
        {
            MockRM rm = new MockRM(conf);

            rm.Start();
            // Register node1
            MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb);
            // Submit an application
            RMApp app1 = rm.SubmitApp(testAlloc);

            // kick the scheduling
            nm1.NodeHeartbeat(true);
            RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
            MockAM       am1      = rm.SendAMLaunched(attempt1.GetAppAttemptId());

            am1.RegisterAppAttempt();
            SchedulerNodeReport report_nm1 = rm.GetResourceScheduler().GetNodeReport(nm1.GetNodeId
                                                                                         ());
            int checkAlloc = conf.GetInt(YarnConfiguration.RmSchedulerMinimumAllocationMb, YarnConfiguration
                                         .DefaultRmSchedulerMinimumAllocationMb);

            NUnit.Framework.Assert.AreEqual(checkAlloc, report_nm1.GetUsedResource().GetMemory
                                                ());
            rm.Stop();
        }
示例#3
0
        /// <exception cref="System.Exception"/>
        protected internal virtual void AllocateContainersAndValidateNMTokens(MockAM am,
                                                                              AList <Container> containersReceived, int totalContainerRequested, Dictionary <string
                                                                                                                                                             , Token> nmTokens, MockNM nm)
        {
            AList <ContainerId>     releaseContainerList = new AList <ContainerId>();
            AllocateResponse        response;
            AList <ResourceRequest> resourceRequest = new AList <ResourceRequest>();

            while (containersReceived.Count < totalContainerRequested)
            {
                nm.NodeHeartbeat(true);
                Log.Info("requesting containers..");
                response = am.Allocate(resourceRequest, releaseContainerList);
                Sharpen.Collections.AddAll(containersReceived, response.GetAllocatedContainers());
                if (!response.GetNMTokens().IsEmpty())
                {
                    foreach (NMToken nmToken in response.GetNMTokens())
                    {
                        string nodeId = nmToken.GetNodeId().ToString();
                        if (nmTokens.Contains(nodeId))
                        {
                            NUnit.Framework.Assert.Fail("Duplicate NMToken received for : " + nodeId);
                        }
                        nmTokens[nodeId] = nmToken.GetToken();
                    }
                }
                Log.Info("Got " + containersReceived.Count + " containers. Waiting to get " + totalContainerRequested
                         );
                Sharpen.Thread.Sleep(WaitSleepMs);
            }
        }
示例#4
0
        /// <exception cref="System.Exception"/>
        public virtual void TestInvalidatedAMHostPortOnAMRestart()
        {
            MockRM rm1 = new MockRM(conf);

            rm1.Start();
            MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.GetResourceTrackerService());

            nm1.RegisterNode();
            // a failed app
            RMApp  app2 = rm1.SubmitApp(200);
            MockAM am2  = MockRM.LaunchAndRegisterAM(app2, rm1, nm1);

            nm1.NodeHeartbeat(am2.GetApplicationAttemptId(), 1, ContainerState.Complete);
            am2.WaitForState(RMAppAttemptState.Failed);
            rm1.WaitForState(app2.GetApplicationId(), RMAppState.Accepted);
            // before new attempt is launched, the app report returns the invalid AM
            // host and port.
            GetApplicationReportRequest request1 = GetApplicationReportRequest.NewInstance(app2
                                                                                           .GetApplicationId());
            ApplicationReport report1 = rm1.GetClientRMService().GetApplicationReport(request1
                                                                                      ).GetApplicationReport();

            NUnit.Framework.Assert.AreEqual("N/A", report1.GetHost());
            NUnit.Framework.Assert.AreEqual(-1, report1.GetRpcPort());
        }
示例#5
0
        /// <exception cref="System.Exception"/>
        public virtual void SendAMLaunchFailed(ApplicationAttemptId appAttemptId)
        {
            MockAM am = new MockAM(GetRMContext(), masterService, appAttemptId);

            am.WaitForState(RMAppAttemptState.Allocated);
            GetRMContext().GetDispatcher().GetEventHandler().Handle(new RMAppAttemptEvent(appAttemptId
                                                                                          , RMAppAttemptEventType.LaunchFailed, "Failed"));
        }
示例#6
0
        /// <exception cref="System.Exception"/>
        public virtual void TestAppOnMultiNode()
        {
            Logger rootLogger = LogManager.GetRootLogger();

            rootLogger.SetLevel(Level.Debug);
            conf.Set("yarn.scheduler.capacity.node-locality-delay", "-1");
            MockRM rm = new MockRM(conf);

            rm.Start();
            MockNM nm1 = rm.RegisterNode("h1:1234", 5120);
            MockNM nm2 = rm.RegisterNode("h2:5678", 10240);
            RMApp  app = rm.SubmitApp(2000);

            //kick the scheduling
            nm1.NodeHeartbeat(true);
            RMAppAttempt attempt = app.GetCurrentAppAttempt();
            MockAM       am      = rm.SendAMLaunched(attempt.GetAppAttemptId());

            am.RegisterAppAttempt();
            //request for containers
            int request = 13;

            am.Allocate("h1", 1000, request, new AList <ContainerId>());
            //kick the scheduler
            IList <Container> conts = am.Allocate(new AList <ResourceRequest>(), new AList <ContainerId
                                                                                            >()).GetAllocatedContainers();
            int contReceived = conts.Count;

            while (contReceived < 3)
            {
                //only 3 containers are available on node1
                nm1.NodeHeartbeat(true);
                Sharpen.Collections.AddAll(conts, am.Allocate(new AList <ResourceRequest>(), new AList
                                                              <ContainerId>()).GetAllocatedContainers());
                contReceived = conts.Count;
                Log.Info("Got " + contReceived + " containers. Waiting to get " + 3);
                Sharpen.Thread.Sleep(WaitSleepMs);
            }
            NUnit.Framework.Assert.AreEqual(3, conts.Count);
            //send node2 heartbeat
            conts = am.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>()).GetAllocatedContainers
                        ();
            contReceived = conts.Count;
            while (contReceived < 10)
            {
                nm2.NodeHeartbeat(true);
                Sharpen.Collections.AddAll(conts, am.Allocate(new AList <ResourceRequest>(), new AList
                                                              <ContainerId>()).GetAllocatedContainers());
                contReceived = conts.Count;
                Log.Info("Got " + contReceived + " containers. Waiting to get " + 10);
                Sharpen.Thread.Sleep(WaitSleepMs);
            }
            NUnit.Framework.Assert.AreEqual(10, conts.Count);
            am.UnregisterAppAttempt();
            nm1.NodeHeartbeat(attempt.GetAppAttemptId(), 1, ContainerState.Complete);
            am.WaitForState(RMAppAttemptState.Finished);
            rm.Stop();
        }
示例#7
0
        /// <exception cref="System.Exception"/>
        public virtual void TestInvalidContainerReleaseRequest()
        {
            MockRM rm = new MockRM(conf);

            try
            {
                rm.Start();
                // Register node1
                MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb);
                // Submit an application
                RMApp app1 = rm.SubmitApp(1024);
                // kick the scheduling
                nm1.NodeHeartbeat(true);
                RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
                MockAM       am1      = rm.SendAMLaunched(attempt1.GetAppAttemptId());
                am1.RegisterAppAttempt();
                am1.AddRequests(new string[] { "127.0.0.1" }, Gb, 1, 1);
                AllocateResponse alloc1Response = am1.Schedule();
                // send the request
                // kick the scheduler
                nm1.NodeHeartbeat(true);
                while (alloc1Response.GetAllocatedContainers().Count < 1)
                {
                    Log.Info("Waiting for containers to be created for app 1...");
                    Sharpen.Thread.Sleep(1000);
                    alloc1Response = am1.Schedule();
                }
                NUnit.Framework.Assert.IsTrue(alloc1Response.GetAllocatedContainers().Count > 0);
                RMApp app2 = rm.SubmitApp(1024);
                nm1.NodeHeartbeat(true);
                RMAppAttempt attempt2 = app2.GetCurrentAppAttempt();
                MockAM       am2      = rm.SendAMLaunched(attempt2.GetAppAttemptId());
                am2.RegisterAppAttempt();
                // Now trying to release container allocated for app1 -> appAttempt1.
                ContainerId cId = alloc1Response.GetAllocatedContainers()[0].GetId();
                am2.AddContainerToBeReleased(cId);
                try
                {
                    am2.Schedule();
                    NUnit.Framework.Assert.Fail("Exception was expected!!");
                }
                catch (InvalidContainerReleaseException e)
                {
                    StringBuilder sb = new StringBuilder("Cannot release container : ");
                    sb.Append(cId.ToString());
                    sb.Append(" not belonging to this application attempt : ");
                    sb.Append(attempt2.GetAppAttemptId().ToString());
                    NUnit.Framework.Assert.IsTrue(e.Message.Contains(sb.ToString()));
                }
            }
            finally
            {
                if (rm != null)
                {
                    rm.Stop();
                }
            }
        }
示例#8
0
        /// <exception cref="System.Exception"/>
        public static MockAM LaunchAndRegisterAM(RMApp app, Org.Apache.Hadoop.Yarn.Server.Resourcemanager.MockRM
                                                 rm, MockNM nm)
        {
            MockAM am = LaunchAM(app, rm, nm);

            am.RegisterAppAttempt();
            rm.WaitForState(app.GetApplicationId(), RMAppState.Running);
            return(am);
        }
        public virtual void TestAMLaunchAndCleanup()
        {
            Logger rootLogger = LogManager.GetRootLogger();

            rootLogger.SetLevel(Level.Debug);
            TestApplicationMasterLauncher.MyContainerManagerImpl containerManager = new TestApplicationMasterLauncher.MyContainerManagerImpl
                                                                                        ();
            MockRMWithCustomAMLauncher rm = new MockRMWithCustomAMLauncher(containerManager);

            rm.Start();
            MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 5120);
            RMApp  app = rm.SubmitApp(2000);

            // kick the scheduling
            nm1.NodeHeartbeat(true);
            int waitCount = 0;

            while (containerManager.launched == false && waitCount++ < 20)
            {
                Log.Info("Waiting for AM Launch to happen..");
                Sharpen.Thread.Sleep(1000);
            }
            NUnit.Framework.Assert.IsTrue(containerManager.launched);
            RMAppAttempt         attempt      = app.GetCurrentAppAttempt();
            ApplicationAttemptId appAttemptId = attempt.GetAppAttemptId();

            NUnit.Framework.Assert.AreEqual(appAttemptId.ToString(), containerManager.attemptIdAtContainerManager
                                            );
            NUnit.Framework.Assert.AreEqual(app.GetSubmitTime(), containerManager.submitTimeAtContainerManager
                                            );
            NUnit.Framework.Assert.AreEqual(app.GetRMAppAttempt(appAttemptId).GetMasterContainer
                                                ().GetId().ToString(), containerManager.containerIdAtContainerManager);
            NUnit.Framework.Assert.AreEqual(nm1.GetNodeId().ToString(), containerManager.nmHostAtContainerManager
                                            );
            NUnit.Framework.Assert.AreEqual(YarnConfiguration.DefaultRmAmMaxAttempts, containerManager
                                            .maxAppAttempts);
            MockAM am = new MockAM(rm.GetRMContext(), rm.GetApplicationMasterService(), appAttemptId
                                   );

            am.RegisterAppAttempt();
            am.UnregisterAppAttempt();
            //complete the AM container to finish the app normally
            nm1.NodeHeartbeat(attempt.GetAppAttemptId(), 1, ContainerState.Complete);
            am.WaitForState(RMAppAttemptState.Finished);
            waitCount = 0;
            while (containerManager.cleanedup == false && waitCount++ < 20)
            {
                Log.Info("Waiting for AM Cleanup to happen..");
                Sharpen.Thread.Sleep(1000);
            }
            NUnit.Framework.Assert.IsTrue(containerManager.cleanedup);
            am.WaitForState(RMAppAttemptState.Finished);
            rm.Stop();
        }
        /// <exception cref="System.Exception"/>
        private MockAM LaunchAM(RMApp app, MockRM rm, MockNM nm)
        {
            RMAppAttempt attempt = app.GetCurrentAppAttempt();

            nm.NodeHeartbeat(true);
            MockAM am = rm.SendAMLaunched(attempt.GetAppAttemptId());

            am.RegisterAppAttempt();
            rm.WaitForState(app.GetApplicationId(), RMAppState.Running);
            return(am);
        }
        public static IList <NMContainerStatus> CreateNMContainerStatusForApp(MockAM am)
        {
            IList <NMContainerStatus> list        = new AList <NMContainerStatus>();
            NMContainerStatus         amContainer = CreateNMContainerStatus(am.GetApplicationAttemptId
                                                                                (), 1, ContainerState.Running, 1024);
            NMContainerStatus completedContainer = CreateNMContainerStatus(am.GetApplicationAttemptId
                                                                               (), 2, ContainerState.Complete, 2048);

            list.AddItem(amContainer);
            list.AddItem(completedContainer);
            return(list);
        }
示例#12
0
        // Disable webapp
        /// <exception cref="System.Exception"/>
        public static void FinishAMAndVerifyAppState(RMApp rmApp, Org.Apache.Hadoop.Yarn.Server.Resourcemanager.MockRM
                                                     rm, MockNM nm, MockAM am)
        {
            FinishApplicationMasterRequest req = FinishApplicationMasterRequest.NewInstance(FinalApplicationStatus
                                                                                            .Succeeded, string.Empty, string.Empty);

            am.UnregisterAppAttempt(req, true);
            am.WaitForState(RMAppAttemptState.Finishing);
            nm.NodeHeartbeat(am.GetApplicationAttemptId(), 1, ContainerState.Complete);
            am.WaitForState(RMAppAttemptState.Finished);
            rm.WaitForState(rmApp.GetApplicationId(), RMAppState.Finished);
        }
示例#13
0
        // This is to test AM Host and rpc port are invalidated after the am attempt
        // is killed or failed, so that client doesn't get the wrong information.
        /// <exception cref="System.Exception"/>
        public virtual void TestInvalidateAMHostPortWhenAMFailedOrKilled()
        {
            conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1);
            MockRM rm1 = new MockRM(conf);

            rm1.Start();
            // a succeeded app
            RMApp  app1 = rm1.SubmitApp(200);
            MockNM nm1  = new MockNM("127.0.0.1:1234", 15120, rm1.GetResourceTrackerService());

            nm1.RegisterNode();
            MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);

            MockRM.FinishAMAndVerifyAppState(app1, rm1, nm1, am1);
            // a failed app
            RMApp  app2 = rm1.SubmitApp(200);
            MockAM am2  = MockRM.LaunchAndRegisterAM(app2, rm1, nm1);

            nm1.NodeHeartbeat(am2.GetApplicationAttemptId(), 1, ContainerState.Complete);
            am2.WaitForState(RMAppAttemptState.Failed);
            rm1.WaitForState(app2.GetApplicationId(), RMAppState.Failed);
            // a killed app
            RMApp  app3 = rm1.SubmitApp(200);
            MockAM am3  = MockRM.LaunchAndRegisterAM(app3, rm1, nm1);

            rm1.KillApp(app3.GetApplicationId());
            rm1.WaitForState(app3.GetApplicationId(), RMAppState.Killed);
            rm1.WaitForState(am3.GetApplicationAttemptId(), RMAppAttemptState.Killed);
            GetApplicationsRequest request1 = GetApplicationsRequest.NewInstance(EnumSet.Of(YarnApplicationState
                                                                                            .Finished, YarnApplicationState.Killed, YarnApplicationState.Failed));
            GetApplicationsResponse response1 = rm1.GetClientRMService().GetApplications(request1
                                                                                         );
            IList <ApplicationReport> appList1 = response1.GetApplicationList();

            NUnit.Framework.Assert.AreEqual(3, appList1.Count);
            foreach (ApplicationReport report in appList1)
            {
                // killed/failed apps host and rpc port are invalidated.
                if (report.GetApplicationId().Equals(app2.GetApplicationId()) || report.GetApplicationId
                        ().Equals(app3.GetApplicationId()))
                {
                    NUnit.Framework.Assert.AreEqual("N/A", report.GetHost());
                    NUnit.Framework.Assert.AreEqual(-1, report.GetRpcPort());
                }
                // succeeded app's host and rpc port is not invalidated
                if (report.GetApplicationId().Equals(app1.GetApplicationId()))
                {
                    NUnit.Framework.Assert.IsFalse(report.GetHost().Equals("N/A"));
                    NUnit.Framework.Assert.IsTrue(report.GetRpcPort() != -1);
                }
            }
        }
示例#14
0
        // Test even if AM container is allocated with containerId not equal to 1, the
        // following allocate requests from AM should be able to retrieve the
        // corresponding NM Token.
        /// <exception cref="System.Exception"/>
        public virtual void TestNMTokenSentForNormalContainer()
        {
            conf.Set(YarnConfiguration.RmScheduler, typeof(CapacityScheduler).GetCanonicalName
                         ());
            MockRM rm = new MockRM(conf);

            rm.Start();
            MockNM       nm1     = rm.RegisterNode("h1:1234", 5120);
            RMApp        app     = rm.SubmitApp(2000);
            RMAppAttempt attempt = app.GetCurrentAppAttempt();
            // Call getNewContainerId to increase container Id so that the AM container
            // Id doesn't equal to one.
            CapacityScheduler cs = (CapacityScheduler)rm.GetResourceScheduler();

            cs.GetApplicationAttempt(attempt.GetAppAttemptId()).GetNewContainerId();
            // kick the scheduling
            nm1.NodeHeartbeat(true);
            MockAM am = MockRM.LaunchAM(app, rm, nm1);

            // am container Id not equal to 1.
            NUnit.Framework.Assert.IsTrue(attempt.GetMasterContainer().GetId().GetContainerId
                                              () != 1);
            // NMSecretManager doesn't record the node on which the am is allocated.
            NUnit.Framework.Assert.IsFalse(rm.GetRMContext().GetNMTokenSecretManager().IsApplicationAttemptNMTokenPresent
                                               (attempt.GetAppAttemptId(), nm1.GetNodeId()));
            am.RegisterAppAttempt();
            rm.WaitForState(app.GetApplicationId(), RMAppState.Running);
            int NumContainers            = 1;
            IList <Container> containers = new AList <Container>();
            // nmTokens keeps track of all the nmTokens issued in the allocate call.
            IList <NMToken> expectedNMTokens = new AList <NMToken>();

            // am1 allocate 1 container on nm1.
            while (true)
            {
                AllocateResponse response = am.Allocate("127.0.0.1", 2000, NumContainers, new AList
                                                        <ContainerId>());
                nm1.NodeHeartbeat(true);
                Sharpen.Collections.AddAll(containers, response.GetAllocatedContainers());
                Sharpen.Collections.AddAll(expectedNMTokens, response.GetNMTokens());
                if (containers.Count == NumContainers)
                {
                    break;
                }
                Sharpen.Thread.Sleep(200);
                System.Console.Out.WriteLine("Waiting for container to be allocated.");
            }
            NodeId nodeId = expectedNMTokens[0].GetNodeId();

            // NMToken is sent for the allocated container.
            NUnit.Framework.Assert.AreEqual(nm1.GetNodeId(), nodeId);
        }
示例#15
0
        /// <exception cref="System.Exception"/>
        public static MockAM LaunchAM(RMApp app, Org.Apache.Hadoop.Yarn.Server.Resourcemanager.MockRM
                                      rm, MockNM nm)
        {
            rm.WaitForState(app.GetApplicationId(), RMAppState.Accepted);
            RMAppAttempt attempt = app.GetCurrentAppAttempt();

            System.Console.Out.WriteLine("Launch AM " + attempt.GetAppAttemptId());
            nm.NodeHeartbeat(true);
            MockAM am = rm.SendAMLaunched(attempt.GetAppAttemptId());

            rm.WaitForState(attempt.GetAppAttemptId(), RMAppAttemptState.Launched);
            return(am);
        }
        /// <exception cref="System.Exception"/>
        public virtual void TestAppCleanupWhenRMRestartedBeforeAppFinished()
        {
            conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1);
            MemoryRMStateStore memStore = new MemoryRMStateStore();

            memStore.Init(conf);
            // start RM
            MockRM rm1 = new MockRM(conf, memStore);

            rm1.Start();
            MockNM nm1 = new MockNM("127.0.0.1:1234", 1024, rm1.GetResourceTrackerService());

            nm1.RegisterNode();
            MockNM nm2 = new MockNM("127.0.0.1:5678", 1024, rm1.GetResourceTrackerService());

            nm2.RegisterNode();
            // create app and launch the AM
            RMApp  app0 = rm1.SubmitApp(200);
            MockAM am0  = LaunchAM(app0, rm1, nm1);
            // alloc another container on nm2
            AllocateResponse allocResponse = am0.Allocate(Arrays.AsList(ResourceRequest.NewInstance
                                                                            (Priority.NewInstance(1), "*", Resource.NewInstance(1024, 0), 1)), null);

            while (null == allocResponse.GetAllocatedContainers() || allocResponse.GetAllocatedContainers
                       ().IsEmpty())
            {
                nm2.NodeHeartbeat(true);
                allocResponse = am0.Allocate(null, null);
                Sharpen.Thread.Sleep(1000);
            }
            // start new RM
            MockRM rm2 = new MockRM(conf, memStore);

            rm2.Start();
            // nm1/nm2 register to rm2, and do a heartbeat
            nm1.SetResourceTrackerService(rm2.GetResourceTrackerService());
            nm1.RegisterNode(Arrays.AsList(NMContainerStatus.NewInstance(ContainerId.NewContainerId
                                                                             (am0.GetApplicationAttemptId(), 1), ContainerState.Complete, Resource.NewInstance
                                                                             (1024, 1), string.Empty, 0, Priority.NewInstance(0), 1234)), Arrays.AsList(app0.
                                                                                                                                                        GetApplicationId()));
            nm2.SetResourceTrackerService(rm2.GetResourceTrackerService());
            nm2.RegisterNode(Arrays.AsList(app0.GetApplicationId()));
            // assert app state has been saved.
            rm2.WaitForState(app0.GetApplicationId(), RMAppState.Failed);
            // wait for application cleanup message received on NM1
            WaitForAppCleanupMessageRecved(nm1, app0.GetApplicationId());
            // wait for application cleanup message received on NM2
            WaitForAppCleanupMessageRecved(nm2, app0.GetApplicationId());
            rm1.Stop();
            rm2.Stop();
        }
示例#17
0
        // from AMLauncher
        /// <exception cref="System.Exception"/>
        public virtual MockAM SendAMLaunched(ApplicationAttemptId appAttemptId)
        {
            MockAM am = new MockAM(GetRMContext(), masterService, appAttemptId);

            am.WaitForState(RMAppAttemptState.Allocated);
            //create and set AMRMToken
            Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> amrmToken = this.rmContext
                                                                                     .GetAMRMTokenSecretManager().CreateAndGetAMRMToken(appAttemptId);
            ((RMAppAttemptImpl)this.rmContext.GetRMApps()[appAttemptId.GetApplicationId()].GetRMAppAttempt
                 (appAttemptId)).SetAMRMToken(amrmToken);
            GetRMContext().GetDispatcher().GetEventHandler().Handle(new RMAppAttemptEvent(appAttemptId
                                                                                          , RMAppAttemptEventType.Launched));
            return(am);
        }
        /// <exception cref="System.Exception"/>
        public virtual void TestUsageWithOneAttemptAndOneContainer()
        {
            MockRM rm = new MockRM(conf);

            rm.Start();
            MockNM nm = new MockNM("127.0.0.1:1234", 15120, rm.GetResourceTrackerService());

            nm.RegisterNode();
            RMApp        app0         = rm.SubmitApp(200);
            RMAppMetrics rmAppMetrics = app0.GetRMAppMetrics();

            NUnit.Framework.Assert.IsTrue("Before app submittion, memory seconds should have been 0 but was "
                                          + rmAppMetrics.GetMemorySeconds(), rmAppMetrics.GetMemorySeconds() == 0);
            NUnit.Framework.Assert.IsTrue("Before app submission, vcore seconds should have been 0 but was "
                                          + rmAppMetrics.GetVcoreSeconds(), rmAppMetrics.GetVcoreSeconds() == 0);
            RMAppAttempt attempt0 = app0.GetCurrentAppAttempt();

            nm.NodeHeartbeat(true);
            MockAM am0 = rm.SendAMLaunched(attempt0.GetAppAttemptId());

            am0.RegisterAppAttempt();
            RMContainer rmContainer = rm.GetResourceScheduler().GetRMContainer(attempt0.GetMasterContainer
                                                                                   ().GetId());
            // Allow metrics to accumulate.
            int sleepInterval       = 1000;
            int cumulativeSleepTime = 0;

            while (rmAppMetrics.GetMemorySeconds() <= 0 && cumulativeSleepTime < 5000)
            {
                Sharpen.Thread.Sleep(sleepInterval);
                cumulativeSleepTime += sleepInterval;
            }
            rmAppMetrics = app0.GetRMAppMetrics();
            NUnit.Framework.Assert.IsTrue("While app is running, memory seconds should be >0 but is "
                                          + rmAppMetrics.GetMemorySeconds(), rmAppMetrics.GetMemorySeconds() > 0);
            NUnit.Framework.Assert.IsTrue("While app is running, vcore seconds should be >0 but is "
                                          + rmAppMetrics.GetVcoreSeconds(), rmAppMetrics.GetVcoreSeconds() > 0);
            MockRM.FinishAMAndVerifyAppState(app0, rm, nm, am0);
            AggregateAppResourceUsage ru = CalculateContainerResourceMetrics(rmContainer);

            rmAppMetrics = app0.GetRMAppMetrics();
            NUnit.Framework.Assert.AreEqual("Unexcpected MemorySeconds value", ru.GetMemorySeconds
                                                (), rmAppMetrics.GetMemorySeconds());
            NUnit.Framework.Assert.AreEqual("Unexpected VcoreSeconds value", ru.GetVcoreSeconds
                                                (), rmAppMetrics.GetVcoreSeconds());
            rm.Stop();
        }
示例#19
0
        /// <exception cref="System.Exception"/>
        public virtual void TestResourceTypes()
        {
            Dictionary <YarnConfiguration, EnumSet <YarnServiceProtos.SchedulerResourceTypes> >
            driver = new Dictionary <YarnConfiguration, EnumSet <YarnServiceProtos.SchedulerResourceTypes
                                                                 > >();
            CapacitySchedulerConfiguration csconf = new CapacitySchedulerConfiguration();

            csconf.SetResourceComparator(typeof(DominantResourceCalculator));
            YarnConfiguration testCapacityDRConf = new YarnConfiguration(csconf);

            testCapacityDRConf.SetClass(YarnConfiguration.RmScheduler, typeof(CapacityScheduler
                                                                              ), typeof(ResourceScheduler));
            YarnConfiguration testCapacityDefConf = new YarnConfiguration();

            testCapacityDefConf.SetClass(YarnConfiguration.RmScheduler, typeof(CapacityScheduler
                                                                               ), typeof(ResourceScheduler));
            YarnConfiguration testFairDefConf = new YarnConfiguration();

            testFairDefConf.SetClass(YarnConfiguration.RmScheduler, typeof(FairScheduler), typeof(
                                         ResourceScheduler));
            driver[conf] = EnumSet.Of(YarnServiceProtos.SchedulerResourceTypes.Memory);
            driver[testCapacityDRConf] = EnumSet.Of(YarnServiceProtos.SchedulerResourceTypes.
                                                    Cpu, YarnServiceProtos.SchedulerResourceTypes.Memory);
            driver[testCapacityDefConf] = EnumSet.Of(YarnServiceProtos.SchedulerResourceTypes
                                                     .Memory);
            driver[testFairDefConf] = EnumSet.Of(YarnServiceProtos.SchedulerResourceTypes.Memory
                                                 , YarnServiceProtos.SchedulerResourceTypes.Cpu);
            foreach (KeyValuePair <YarnConfiguration, EnumSet <YarnServiceProtos.SchedulerResourceTypes
                                                               > > entry in driver)
            {
                EnumSet <YarnServiceProtos.SchedulerResourceTypes> expectedValue = entry.Value;
                MockRM rm = new MockRM(entry.Key);
                rm.Start();
                MockNM nm1  = rm.RegisterNode("127.0.0.1:1234", 6 * Gb);
                RMApp  app1 = rm.SubmitApp(2048);
                nm1.NodeHeartbeat(true);
                RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
                MockAM       am1      = rm.SendAMLaunched(attempt1.GetAppAttemptId());
                RegisterApplicationMasterResponse resp = am1.RegisterAppAttempt();
                EnumSet <YarnServiceProtos.SchedulerResourceTypes> types = resp.GetSchedulerResourceTypes
                                                                               ();
                Log.Info("types = " + types.ToString());
                NUnit.Framework.Assert.AreEqual(expectedValue, types);
                rm.Stop();
            }
        }
示例#20
0
        /// <exception cref="System.Exception"/>
        public virtual void TestFinishApplicationMasterBeforeRegistering()
        {
            MockRM rm = new MockRM(conf);

            try
            {
                rm.Start();
                // Register node1
                MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb);
                // Submit an application
                RMApp  app1 = rm.SubmitApp(2048);
                MockAM am1  = MockRM.LaunchAM(app1, rm, nm1);
                FinishApplicationMasterRequest req = FinishApplicationMasterRequest.NewInstance(FinalApplicationStatus
                                                                                                .Failed, string.Empty, string.Empty);
                try
                {
                    am1.UnregisterAppAttempt(req, false);
                    NUnit.Framework.Assert.Fail("ApplicationMasterNotRegisteredException should be thrown"
                                                );
                }
                catch (ApplicationMasterNotRegisteredException e)
                {
                    NUnit.Framework.Assert.IsNotNull(e);
                    NUnit.Framework.Assert.IsNotNull(e.Message);
                    NUnit.Framework.Assert.IsTrue(e.Message.Contains("Application Master is trying to unregister before registering for:"
                                                                     ));
                }
                catch (Exception)
                {
                    NUnit.Framework.Assert.Fail("ApplicationMasterNotRegisteredException should be thrown"
                                                );
                }
                am1.RegisterAppAttempt();
                am1.UnregisterAppAttempt(req, false);
                am1.WaitForState(RMAppAttemptState.Finishing);
            }
            finally
            {
                if (rm != null)
                {
                    rm.Stop();
                }
            }
        }
示例#21
0
        /// <summary>Validate killing an application when it is at accepted state.</summary>
        /// <exception cref="System.Exception">exception</exception>
        public virtual void TestApplicationKillAtAcceptedState()
        {
            Dispatcher dispatcher = new _AsyncDispatcher_573();
            MockRM     rm         = new _MockRM_596(dispatcher, conf);
            // test metrics
            QueueMetrics metrics       = rm.GetResourceScheduler().GetRootQueueMetrics();
            int          appsKilled    = metrics.GetAppsKilled();
            int          appsSubmitted = metrics.GetAppsSubmitted();

            rm.Start();
            MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm.GetResourceTrackerService());

            nm1.RegisterNode();
            // a failed app
            RMApp  application = rm.SubmitApp(200);
            MockAM am          = MockRM.LaunchAM(application, rm, nm1);

            am.WaitForState(RMAppAttemptState.Launched);
            nm1.NodeHeartbeat(am.GetApplicationAttemptId(), 1, ContainerState.Running);
            rm.WaitForState(application.GetApplicationId(), RMAppState.Accepted);
            // Now kill the application before new attempt is launched, the app report
            // returns the invalid AM host and port.
            KillApplicationRequest request = KillApplicationRequest.NewInstance(application.GetApplicationId
                                                                                    ());

            rm.GetClientRMService().ForceKillApplication(request);
            // Specific test for YARN-1689 follows
            // Now let's say a race causes AM to register now. This should not crash RM.
            am.RegisterAppAttempt(false);
            // We explicitly intercepted the kill-event to RMAppAttempt, so app should
            // still be in KILLING state.
            rm.WaitForState(application.GetApplicationId(), RMAppState.Killing);
            // AM should now be in running
            rm.WaitForState(am.GetApplicationAttemptId(), RMAppAttemptState.Running);
            // Simulate that appAttempt is killed.
            rm.GetRMContext().GetDispatcher().GetEventHandler().Handle(new RMAppEvent(application
                                                                                      .GetApplicationId(), RMAppEventType.AttemptKilled));
            rm.WaitForState(application.GetApplicationId(), RMAppState.Killed);
            // test metrics
            metrics = rm.GetResourceScheduler().GetRootQueueMetrics();
            NUnit.Framework.Assert.AreEqual(appsKilled + 1, metrics.GetAppsKilled());
            NUnit.Framework.Assert.AreEqual(appsSubmitted + 1, metrics.GetAppsSubmitted());
        }
示例#22
0
        /// <exception cref="System.Exception"/>
        public virtual void TestKillAppWhenFailoverHappensAtRunningState()
        {
            StartRMs();
            MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.GetResourceTrackerService());

            nm1.RegisterNode();
            // create app and launch the AM
            RMApp  app0 = rm1.SubmitApp(200);
            MockAM am0  = LaunchAM(app0, rm1, nm1);

            // failover and kill application
            // The application is at RUNNING State when failOver happens.
            // Since RMStateStore has already saved ApplicationState, the active RM
            // will load the ApplicationState. After that, the application will be at
            // ACCEPTED State. Because the application is not at Final State,
            // KillApplicationResponse.getIsKillCompleted is expected to return false.
            FailOverAndKillApp(app0.GetApplicationId(), am0.GetApplicationAttemptId(), RMAppState
                               .Running, RMAppAttemptState.Running, RMAppState.Accepted);
        }
示例#23
0
        // Test Kill an app while the app is failing
        /// <exception cref="System.Exception"/>
        public virtual void TestKillFailingApp()
        {
            // this dispatcher ignores RMAppAttemptEventType.KILL event
            Dispatcher dispatcher = new _AsyncDispatcher_708();
            MockRM     rm1        = new _MockRM_731(dispatcher, conf);

            rm1.Start();
            MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.GetResourceTrackerService());

            nm1.RegisterNode();
            RMApp  app1 = rm1.SubmitApp(200);
            MockAM am1  = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);

            rm1.KillApp(app1.GetApplicationId());
            // fail the app by sending container_finished event.
            nm1.NodeHeartbeat(am1.GetApplicationAttemptId(), 1, ContainerState.Complete);
            rm1.WaitForState(am1.GetApplicationAttemptId(), RMAppAttemptState.Failed);
            // app is killed, not launching a new attempt
            rm1.WaitForState(app1.GetApplicationId(), RMAppState.Killed);
        }
示例#24
0
        /// <exception cref="System.Exception"/>
        public virtual void TestAppWithNoContainers()
        {
            Logger rootLogger = LogManager.GetRootLogger();

            rootLogger.SetLevel(Level.Debug);
            MockRM rm = new MockRM(conf);

            rm.Start();
            MockNM nm1 = rm.RegisterNode("h1:1234", 5120);
            RMApp  app = rm.SubmitApp(2000);

            //kick the scheduling
            nm1.NodeHeartbeat(true);
            RMAppAttempt attempt = app.GetCurrentAppAttempt();
            MockAM       am      = rm.SendAMLaunched(attempt.GetAppAttemptId());

            am.RegisterAppAttempt();
            am.UnregisterAppAttempt();
            nm1.NodeHeartbeat(attempt.GetAppAttemptId(), 1, ContainerState.Complete);
            am.WaitForState(RMAppAttemptState.Finished);
            rm.Stop();
        }
示例#25
0
        /// <exception cref="System.Exception"/>
        public virtual void TestKillAppWhenFailOverHappensDuringApplicationKill()
        {
            // create a customized ClientRMService
            // When receives the killApplicationRequest, simply return the response
            // and make sure the application will not be KILLED State
            StartRMsWithCustomizedClientRMService();
            MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.GetResourceTrackerService());

            nm1.RegisterNode();
            // create app and launch the AM
            RMApp  app0 = rm1.SubmitApp(200);
            MockAM am0  = LaunchAM(app0, rm1, nm1);

            // ensure that the app is in running state
            NUnit.Framework.Assert.AreEqual(app0.GetState(), RMAppState.Running);
            // kill the app.
            rm1.KillApp(app0.GetApplicationId());
            // failover happens before this application goes to final state.
            // The RMAppState that will be loaded by the active rm
            // should be ACCEPTED.
            FailOverAndKillApp(app0.GetApplicationId(), am0.GetApplicationAttemptId(), RMAppState
                               .Running, RMAppAttemptState.Running, RMAppState.Accepted);
        }
示例#26
0
        /// <exception cref="System.Exception"/>
        public virtual void TestKillAppWhenFailoverHappensAtFinalState()
        {
            StartRMs();
            MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.GetResourceTrackerService());

            nm1.RegisterNode();
            // create app and launch the AM
            RMApp  app0 = rm1.SubmitApp(200);
            MockAM am0  = LaunchAM(app0, rm1, nm1);

            // kill the app.
            rm1.KillApp(app0.GetApplicationId());
            rm1.WaitForState(app0.GetApplicationId(), RMAppState.Killed);
            rm1.WaitForState(am0.GetApplicationAttemptId(), RMAppAttemptState.Killed);
            // failover and kill application
            // The application is at Killed State and RMStateStore has already
            // saved this applicationState. After failover happens, the current
            // active RM will load the ApplicationState whose RMAppState is killed.
            // Because this application is at Final State,
            // KillApplicationResponse.getIsKillCompleted is expected to return true.
            FailOverAndKillApp(app0.GetApplicationId(), am0.GetApplicationAttemptId(), RMAppState
                               .Killed, RMAppAttemptState.Killed, RMAppState.Killed);
        }
示例#27
0
        // Test Kill an app while the app is finishing in the meanwhile.
        /// <exception cref="System.Exception"/>
        public virtual void TestKillFinishingApp()
        {
            // this dispatcher ignores RMAppAttemptEventType.KILL event
            Dispatcher dispatcher = new _AsyncDispatcher_654();
            MockRM     rm1        = new _MockRM_677(dispatcher, conf);

            rm1.Start();
            MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.GetResourceTrackerService());

            nm1.RegisterNode();
            RMApp  app1 = rm1.SubmitApp(200);
            MockAM am1  = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);

            rm1.KillApp(app1.GetApplicationId());
            FinishApplicationMasterRequest req = FinishApplicationMasterRequest.NewInstance(FinalApplicationStatus
                                                                                            .Succeeded, string.Empty, string.Empty);

            am1.UnregisterAppAttempt(req, true);
            rm1.WaitForState(am1.GetApplicationAttemptId(), RMAppAttemptState.Finishing);
            nm1.NodeHeartbeat(am1.GetApplicationAttemptId(), 1, ContainerState.Complete);
            rm1.WaitForState(am1.GetApplicationAttemptId(), RMAppAttemptState.Finished);
            rm1.WaitForState(app1.GetApplicationId(), RMAppState.Finished);
        }
        /// <exception cref="System.Exception"/>
        public virtual void TestContainerCleanupWhenRMRestartedAppNotRegistered()
        {
            conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1);
            MemoryRMStateStore memStore = new MemoryRMStateStore();

            memStore.Init(conf);
            // start RM
            DrainDispatcher dispatcher = new DrainDispatcher();
            MockRM          rm1        = new _MockRM_413(dispatcher, conf, memStore);

            rm1.Start();
            MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.GetResourceTrackerService());

            nm1.RegisterNode();
            // create app and launch the AM
            RMApp  app0 = rm1.SubmitApp(200);
            MockAM am0  = LaunchAM(app0, rm1, nm1);

            nm1.NodeHeartbeat(am0.GetApplicationAttemptId(), 1, ContainerState.Running);
            rm1.WaitForState(app0.GetApplicationId(), RMAppState.Running);
            // start new RM
            DrainDispatcher dispatcher2 = new DrainDispatcher();
            MockRM          rm2         = new _MockRM_432(dispatcher2, conf, memStore);

            rm2.Start();
            // nm1 register to rm2, and do a heartbeat
            nm1.SetResourceTrackerService(rm2.GetResourceTrackerService());
            nm1.RegisterNode(Arrays.AsList(app0.GetApplicationId()));
            rm2.WaitForState(app0.GetApplicationId(), RMAppState.Accepted);
            // Add unknown container for application unknown to scheduler
            NodeHeartbeatResponse response = nm1.NodeHeartbeat(am0.GetApplicationAttemptId(),
                                                               2, ContainerState.Running);

            WaitForContainerCleanup(dispatcher2, nm1, response);
            rm1.Stop();
            rm2.Stop();
        }
示例#29
0
        /// <exception cref="System.Exception"/>
        public virtual void TestRMIdentifierOnContainerAllocation()
        {
            MockRM rm = new MockRM(conf);

            rm.Start();
            // Register node1
            MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb);
            // Submit an application
            RMApp app1 = rm.SubmitApp(2048);

            // kick the scheduling
            nm1.NodeHeartbeat(true);
            RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
            MockAM       am1      = rm.SendAMLaunched(attempt1.GetAppAttemptId());

            am1.RegisterAppAttempt();
            am1.AddRequests(new string[] { "127.0.0.1" }, Gb, 1, 1);
            AllocateResponse alloc1Response = am1.Schedule();

            // send the request
            // kick the scheduler
            nm1.NodeHeartbeat(true);
            while (alloc1Response.GetAllocatedContainers().Count < 1)
            {
                Log.Info("Waiting for containers to be created for app 1...");
                Sharpen.Thread.Sleep(1000);
                alloc1Response = am1.Schedule();
            }
            // assert RMIdentifer is set properly in allocated containers
            Container allocatedContainer     = alloc1Response.GetAllocatedContainers()[0];
            ContainerTokenIdentifier tokenId = BuilderUtils.NewContainerTokenIdentifier(allocatedContainer
                                                                                        .GetContainerToken());

            NUnit.Framework.Assert.AreEqual(MockRM.GetClusterTimeStamp(), tokenId.GetRMIdentifier
                                                ());
            rm.Stop();
        }
示例#30
0
        public virtual void TestAllocateContainerOnNodeWithoutOffSwitchSpecified()
        {
            Logger rootLogger = LogManager.GetRootLogger();

            rootLogger.SetLevel(Level.Debug);
            MockRM rm = new MockRM(conf);

            rm.Start();
            MockNM nm1  = rm.RegisterNode("127.0.0.1:1234", 6 * Gb);
            RMApp  app1 = rm.SubmitApp(2048);

            // kick the scheduling, 2 GB given to AM1, remaining 4GB on nm1
            nm1.NodeHeartbeat(true);
            RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
            MockAM       am1      = rm.SendAMLaunched(attempt1.GetAppAttemptId());

            am1.RegisterAppAttempt();
            // add request for containers
            IList <ResourceRequest> requests = new AList <ResourceRequest>();

            requests.AddItem(am1.CreateResourceReq("127.0.0.1", 1 * Gb, 1, 1));
            requests.AddItem(am1.CreateResourceReq("/default-rack", 1 * Gb, 1, 1));
            am1.Allocate(requests, null);
            // send the request
            try
            {
                // kick the schedule
                nm1.NodeHeartbeat(true);
            }
            catch (ArgumentNullException)
            {
                NUnit.Framework.Assert.Fail("NPE when allocating container on node but " + "forget to set off-switch request should be handled"
                                            );
            }
            rm.Stop();
        }