Пример #1
0
        /// <exception cref="System.Exception"/>
        public virtual void TestAppOnMultiNode()
        {
            Logger rootLogger = LogManager.GetRootLogger();

            rootLogger.SetLevel(Level.Debug);
            conf.Set("yarn.scheduler.capacity.node-locality-delay", "-1");
            MockRM rm = new MockRM(conf);

            rm.Start();
            MockNM nm1 = rm.RegisterNode("h1:1234", 5120);
            MockNM nm2 = rm.RegisterNode("h2:5678", 10240);
            RMApp  app = rm.SubmitApp(2000);

            //kick the scheduling
            nm1.NodeHeartbeat(true);
            RMAppAttempt attempt = app.GetCurrentAppAttempt();
            MockAM       am      = rm.SendAMLaunched(attempt.GetAppAttemptId());

            am.RegisterAppAttempt();
            //request for containers
            int request = 13;

            am.Allocate("h1", 1000, request, new AList <ContainerId>());
            //kick the scheduler
            IList <Container> conts = am.Allocate(new AList <ResourceRequest>(), new AList <ContainerId
                                                                                            >()).GetAllocatedContainers();
            int contReceived = conts.Count;

            while (contReceived < 3)
            {
                //only 3 containers are available on node1
                nm1.NodeHeartbeat(true);
                Sharpen.Collections.AddAll(conts, am.Allocate(new AList <ResourceRequest>(), new AList
                                                              <ContainerId>()).GetAllocatedContainers());
                contReceived = conts.Count;
                Log.Info("Got " + contReceived + " containers. Waiting to get " + 3);
                Sharpen.Thread.Sleep(WaitSleepMs);
            }
            NUnit.Framework.Assert.AreEqual(3, conts.Count);
            //send node2 heartbeat
            conts = am.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>()).GetAllocatedContainers
                        ();
            contReceived = conts.Count;
            while (contReceived < 10)
            {
                nm2.NodeHeartbeat(true);
                Sharpen.Collections.AddAll(conts, am.Allocate(new AList <ResourceRequest>(), new AList
                                                              <ContainerId>()).GetAllocatedContainers());
                contReceived = conts.Count;
                Log.Info("Got " + contReceived + " containers. Waiting to get " + 10);
                Sharpen.Thread.Sleep(WaitSleepMs);
            }
            NUnit.Framework.Assert.AreEqual(10, conts.Count);
            am.UnregisterAppAttempt();
            nm1.NodeHeartbeat(attempt.GetAppAttemptId(), 1, ContainerState.Complete);
            am.WaitForState(RMAppAttemptState.Finished);
            rm.Stop();
        }
Пример #2
0
        /// <exception cref="System.Exception"/>
        public virtual void TestInvalidContainerReleaseRequest()
        {
            MockRM rm = new MockRM(conf);

            try
            {
                rm.Start();
                // Register node1
                MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb);
                // Submit an application
                RMApp app1 = rm.SubmitApp(1024);
                // kick the scheduling
                nm1.NodeHeartbeat(true);
                RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
                MockAM       am1      = rm.SendAMLaunched(attempt1.GetAppAttemptId());
                am1.RegisterAppAttempt();
                am1.AddRequests(new string[] { "127.0.0.1" }, Gb, 1, 1);
                AllocateResponse alloc1Response = am1.Schedule();
                // send the request
                // kick the scheduler
                nm1.NodeHeartbeat(true);
                while (alloc1Response.GetAllocatedContainers().Count < 1)
                {
                    Log.Info("Waiting for containers to be created for app 1...");
                    Sharpen.Thread.Sleep(1000);
                    alloc1Response = am1.Schedule();
                }
                NUnit.Framework.Assert.IsTrue(alloc1Response.GetAllocatedContainers().Count > 0);
                RMApp app2 = rm.SubmitApp(1024);
                nm1.NodeHeartbeat(true);
                RMAppAttempt attempt2 = app2.GetCurrentAppAttempt();
                MockAM       am2      = rm.SendAMLaunched(attempt2.GetAppAttemptId());
                am2.RegisterAppAttempt();
                // Now trying to release container allocated for app1 -> appAttempt1.
                ContainerId cId = alloc1Response.GetAllocatedContainers()[0].GetId();
                am2.AddContainerToBeReleased(cId);
                try
                {
                    am2.Schedule();
                    NUnit.Framework.Assert.Fail("Exception was expected!!");
                }
                catch (InvalidContainerReleaseException e)
                {
                    StringBuilder sb = new StringBuilder("Cannot release container : ");
                    sb.Append(cId.ToString());
                    sb.Append(" not belonging to this application attempt : ");
                    sb.Append(attempt2.GetAppAttemptId().ToString());
                    NUnit.Framework.Assert.IsTrue(e.Message.Contains(sb.ToString()));
                }
            }
            finally
            {
                if (rm != null)
                {
                    rm.Stop();
                }
            }
        }
        public virtual void TestAMLaunchAndCleanup()
        {
            Logger rootLogger = LogManager.GetRootLogger();

            rootLogger.SetLevel(Level.Debug);
            TestApplicationMasterLauncher.MyContainerManagerImpl containerManager = new TestApplicationMasterLauncher.MyContainerManagerImpl
                                                                                        ();
            MockRMWithCustomAMLauncher rm = new MockRMWithCustomAMLauncher(containerManager);

            rm.Start();
            MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 5120);
            RMApp  app = rm.SubmitApp(2000);

            // kick the scheduling
            nm1.NodeHeartbeat(true);
            int waitCount = 0;

            while (containerManager.launched == false && waitCount++ < 20)
            {
                Log.Info("Waiting for AM Launch to happen..");
                Sharpen.Thread.Sleep(1000);
            }
            NUnit.Framework.Assert.IsTrue(containerManager.launched);
            RMAppAttempt         attempt      = app.GetCurrentAppAttempt();
            ApplicationAttemptId appAttemptId = attempt.GetAppAttemptId();

            NUnit.Framework.Assert.AreEqual(appAttemptId.ToString(), containerManager.attemptIdAtContainerManager
                                            );
            NUnit.Framework.Assert.AreEqual(app.GetSubmitTime(), containerManager.submitTimeAtContainerManager
                                            );
            NUnit.Framework.Assert.AreEqual(app.GetRMAppAttempt(appAttemptId).GetMasterContainer
                                                ().GetId().ToString(), containerManager.containerIdAtContainerManager);
            NUnit.Framework.Assert.AreEqual(nm1.GetNodeId().ToString(), containerManager.nmHostAtContainerManager
                                            );
            NUnit.Framework.Assert.AreEqual(YarnConfiguration.DefaultRmAmMaxAttempts, containerManager
                                            .maxAppAttempts);
            MockAM am = new MockAM(rm.GetRMContext(), rm.GetApplicationMasterService(), appAttemptId
                                   );

            am.RegisterAppAttempt();
            am.UnregisterAppAttempt();
            //complete the AM container to finish the app normally
            nm1.NodeHeartbeat(attempt.GetAppAttemptId(), 1, ContainerState.Complete);
            am.WaitForState(RMAppAttemptState.Finished);
            waitCount = 0;
            while (containerManager.cleanedup == false && waitCount++ < 20)
            {
                Log.Info("Waiting for AM Cleanup to happen..");
                Sharpen.Thread.Sleep(1000);
            }
            NUnit.Framework.Assert.IsTrue(containerManager.cleanedup);
            am.WaitForState(RMAppAttemptState.Finished);
            rm.Stop();
        }
Пример #4
0
        // Test even if AM container is allocated with containerId not equal to 1, the
        // following allocate requests from AM should be able to retrieve the
        // corresponding NM Token.
        /// <exception cref="System.Exception"/>
        public virtual void TestNMTokenSentForNormalContainer()
        {
            conf.Set(YarnConfiguration.RmScheduler, typeof(CapacityScheduler).GetCanonicalName
                         ());
            MockRM rm = new MockRM(conf);

            rm.Start();
            MockNM       nm1     = rm.RegisterNode("h1:1234", 5120);
            RMApp        app     = rm.SubmitApp(2000);
            RMAppAttempt attempt = app.GetCurrentAppAttempt();
            // Call getNewContainerId to increase container Id so that the AM container
            // Id doesn't equal to one.
            CapacityScheduler cs = (CapacityScheduler)rm.GetResourceScheduler();

            cs.GetApplicationAttempt(attempt.GetAppAttemptId()).GetNewContainerId();
            // kick the scheduling
            nm1.NodeHeartbeat(true);
            MockAM am = MockRM.LaunchAM(app, rm, nm1);

            // am container Id not equal to 1.
            NUnit.Framework.Assert.IsTrue(attempt.GetMasterContainer().GetId().GetContainerId
                                              () != 1);
            // NMSecretManager doesn't record the node on which the am is allocated.
            NUnit.Framework.Assert.IsFalse(rm.GetRMContext().GetNMTokenSecretManager().IsApplicationAttemptNMTokenPresent
                                               (attempt.GetAppAttemptId(), nm1.GetNodeId()));
            am.RegisterAppAttempt();
            rm.WaitForState(app.GetApplicationId(), RMAppState.Running);
            int NumContainers            = 1;
            IList <Container> containers = new AList <Container>();
            // nmTokens keeps track of all the nmTokens issued in the allocate call.
            IList <NMToken> expectedNMTokens = new AList <NMToken>();

            // am1 allocate 1 container on nm1.
            while (true)
            {
                AllocateResponse response = am.Allocate("127.0.0.1", 2000, NumContainers, new AList
                                                        <ContainerId>());
                nm1.NodeHeartbeat(true);
                Sharpen.Collections.AddAll(containers, response.GetAllocatedContainers());
                Sharpen.Collections.AddAll(expectedNMTokens, response.GetNMTokens());
                if (containers.Count == NumContainers)
                {
                    break;
                }
                Sharpen.Thread.Sleep(200);
                System.Console.Out.WriteLine("Waiting for container to be allocated.");
            }
            NodeId nodeId = expectedNMTokens[0].GetNodeId();

            // NMToken is sent for the allocated container.
            NUnit.Framework.Assert.AreEqual(nm1.GetNodeId(), nodeId);
        }
Пример #5
0
        /// <exception cref="System.Exception"/>
        public static MockAM LaunchAM(RMApp app, Org.Apache.Hadoop.Yarn.Server.Resourcemanager.MockRM
                                      rm, MockNM nm)
        {
            rm.WaitForState(app.GetApplicationId(), RMAppState.Accepted);
            RMAppAttempt attempt = app.GetCurrentAppAttempt();

            System.Console.Out.WriteLine("Launch AM " + attempt.GetAppAttemptId());
            nm.NodeHeartbeat(true);
            MockAM am = rm.SendAMLaunched(attempt.GetAppAttemptId());

            rm.WaitForState(attempt.GetAppAttemptId(), RMAppAttemptState.Launched);
            return(am);
        }
Пример #6
0
        /// <exception cref="System.Exception"/>
        private void TestMinimumAllocation(YarnConfiguration conf, int testAlloc)
        {
            MockRM rm = new MockRM(conf);

            rm.Start();
            // Register node1
            MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb);
            // Submit an application
            RMApp app1 = rm.SubmitApp(testAlloc);

            // kick the scheduling
            nm1.NodeHeartbeat(true);
            RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
            MockAM       am1      = rm.SendAMLaunched(attempt1.GetAppAttemptId());

            am1.RegisterAppAttempt();
            SchedulerNodeReport report_nm1 = rm.GetResourceScheduler().GetNodeReport(nm1.GetNodeId
                                                                                         ());
            int checkAlloc = conf.GetInt(YarnConfiguration.RmSchedulerMinimumAllocationMb, YarnConfiguration
                                         .DefaultRmSchedulerMinimumAllocationMb);

            NUnit.Framework.Assert.AreEqual(checkAlloc, report_nm1.GetUsedResource().GetMemory
                                                ());
            rm.Stop();
        }
Пример #7
0
        // This is to test fetching AM container will be retried, if AM container is
        // not fetchable since DNS is unavailable causing container token/NMtoken
        // creation failure.
        /// <exception cref="System.Exception"/>
        public virtual void TestAMContainerAllocationWhenDNSUnavailable()
        {
            MockRM rm1 = new _MockRM_303(this, conf);

            rm1.Start();
            MockNM nm1 = rm1.RegisterNode("unknownhost:1234", 8000);

            SecurityUtilTestHelper.SetTokenServiceUseIp(true);
            RMApp        app1    = rm1.SubmitApp(200);
            RMAppAttempt attempt = app1.GetCurrentAppAttempt();

            nm1.NodeHeartbeat(true);
            // fetching am container will fail, keep retrying 5 times.
            while (numRetries <= 5)
            {
                nm1.NodeHeartbeat(true);
                Sharpen.Thread.Sleep(1000);
                NUnit.Framework.Assert.AreEqual(RMAppAttemptState.Scheduled, attempt.GetAppAttemptState
                                                    ());
                System.Console.Out.WriteLine("Waiting for am container to be allocated.");
            }
            SecurityUtilTestHelper.SetTokenServiceUseIp(false);
            rm1.WaitForState(attempt.GetAppAttemptId(), RMAppAttemptState.Allocated);
            MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
        }
Пример #8
0
        private static RMApp CreateRMApp(ApplicationId appId)
        {
            RMApp app = Org.Mockito.Mockito.Mock <RMApp>();

            Org.Mockito.Mockito.When(app.GetApplicationId()).ThenReturn(appId);
            Org.Mockito.Mockito.When(app.GetName()).ThenReturn("test app");
            Org.Mockito.Mockito.When(app.GetApplicationType()).ThenReturn("test app type");
            Org.Mockito.Mockito.When(app.GetUser()).ThenReturn("test user");
            Org.Mockito.Mockito.When(app.GetQueue()).ThenReturn("test queue");
            Org.Mockito.Mockito.When(app.GetSubmitTime()).ThenReturn(int.MaxValue + 1L);
            Org.Mockito.Mockito.When(app.GetStartTime()).ThenReturn(int.MaxValue + 2L);
            Org.Mockito.Mockito.When(app.GetFinishTime()).ThenReturn(int.MaxValue + 3L);
            Org.Mockito.Mockito.When(app.GetDiagnostics()).ThenReturn(new StringBuilder("test diagnostics info"
                                                                                        ));
            RMAppAttempt appAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>();

            Org.Mockito.Mockito.When(appAttempt.GetAppAttemptId()).ThenReturn(ApplicationAttemptId
                                                                              .NewInstance(appId, 1));
            Org.Mockito.Mockito.When(app.GetCurrentAppAttempt()).ThenReturn(appAttempt);
            Org.Mockito.Mockito.When(app.GetFinalApplicationStatus()).ThenReturn(FinalApplicationStatus
                                                                                 .Undefined);
            Org.Mockito.Mockito.When(app.GetRMAppMetrics()).ThenReturn(new RMAppMetrics(null,
                                                                                        0, 0, int.MaxValue, long.MaxValue));
            return(app);
        }
Пример #9
0
 public AppAttemptInfo(ResourceManager rm, RMAppAttempt attempt, string user, string
                       schemePrefix)
 {
     this.startTime        = 0;
     this.containerId      = string.Empty;
     this.nodeHttpAddress  = string.Empty;
     this.nodeId           = string.Empty;
     this.logsLink         = string.Empty;
     this.blacklistedNodes = string.Empty;
     if (attempt != null)
     {
         this.id        = attempt.GetAppAttemptId().GetAttemptId();
         this.startTime = attempt.GetStartTime();
         Container masterContainer = attempt.GetMasterContainer();
         if (masterContainer != null)
         {
             this.containerId     = masterContainer.GetId().ToString();
             this.nodeHttpAddress = masterContainer.GetNodeHttpAddress();
             this.nodeId          = masterContainer.GetNodeId().ToString();
             this.logsLink        = WebAppUtils.GetRunningLogURL(schemePrefix + masterContainer.GetNodeHttpAddress
                                                                     (), ConverterUtils.ToString(masterContainer.GetId()), user);
             if (rm.GetResourceScheduler() is AbstractYarnScheduler)
             {
                 AbstractYarnScheduler ayScheduler = (AbstractYarnScheduler)rm.GetResourceScheduler
                                                         ();
                 SchedulerApplicationAttempt sattempt = ayScheduler.GetApplicationAttempt(attempt.
                                                                                          GetAppAttemptId());
                 if (sattempt != null)
                 {
                     blacklistedNodes = StringUtils.Join(sattempt.GetBlacklistedNodes(), ", ");
                 }
             }
         }
     }
 }
Пример #10
0
        // Test regular RM restart/failover, new RM should not count
        // AM failure towards the max-retry-account and should be able to
        // re-launch the AM.
        /// <exception cref="System.Exception"/>
        public virtual void TestRMRestartOrFailoverNotCountedForAMFailures()
        {
            YarnConfiguration conf = new YarnConfiguration();

            conf.SetClass(YarnConfiguration.RmScheduler, typeof(CapacityScheduler), typeof(ResourceScheduler
                                                                                           ));
            conf.SetBoolean(YarnConfiguration.RecoveryEnabled, true);
            conf.SetBoolean(YarnConfiguration.RmWorkPreservingRecoveryEnabled, false);
            conf.Set(YarnConfiguration.RmStore, typeof(MemoryRMStateStore).FullName);
            // explicitly set max-am-retry count as 1.
            conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1);
            MemoryRMStateStore memStore = new MemoryRMStateStore();

            memStore.Init(conf);
            MockRM rm1 = new MockRM(conf, memStore);

            rm1.Start();
            MockNM nm1 = new MockNM("127.0.0.1:1234", 8000, rm1.GetResourceTrackerService());

            nm1.RegisterNode();
            RMApp app1 = rm1.SubmitApp(200);
            // AM should be restarted even though max-am-attempt is 1.
            MockAM       am1      = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
            RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();

            NUnit.Framework.Assert.IsTrue(((RMAppAttemptImpl)attempt1).MayBeLastAttempt());
            // Restart rm.
            MockRM rm2 = new MockRM(conf, memStore);

            rm2.Start();
            ApplicationStateData appState = memStore.GetState().GetApplicationState()[app1.GetApplicationId
                                                                                          ()];

            // re-register the NM
            nm1.SetResourceTrackerService(rm2.GetResourceTrackerService());
            NMContainerStatus status = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <NMContainerStatus
                                                                                      >();

            status.SetContainerExitStatus(ContainerExitStatus.KilledByResourcemanager);
            status.SetContainerId(attempt1.GetMasterContainer().GetId());
            status.SetContainerState(ContainerState.Complete);
            status.SetDiagnostics(string.Empty);
            nm1.RegisterNode(Sharpen.Collections.SingletonList(status), null);
            rm2.WaitForState(attempt1.GetAppAttemptId(), RMAppAttemptState.Failed);
            NUnit.Framework.Assert.AreEqual(ContainerExitStatus.KilledByResourcemanager, appState
                                            .GetAttempt(am1.GetApplicationAttemptId()).GetAMContainerExitStatus());
            // Will automatically start a new AppAttempt in rm2
            rm2.WaitForState(app1.GetApplicationId(), RMAppState.Accepted);
            MockAM am2 = rm2.WaitForNewAMToLaunchAndRegister(app1.GetApplicationId(), 2, nm1);

            MockRM.FinishAMAndVerifyAppState(app1, rm2, nm1, am2);
            RMAppAttempt attempt3 = rm2.GetRMContext().GetRMApps()[app1.GetApplicationId()].GetCurrentAppAttempt
                                        ();

            NUnit.Framework.Assert.IsTrue(attempt3.ShouldCountTowardsMaxAttemptRetry());
            NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Invalid, appState.GetAttempt(
                                                am2.GetApplicationAttemptId()).GetAMContainerExitStatus());
            rm1.Stop();
            rm2.Stop();
        }
Пример #11
0
        /// <exception cref="System.Exception"/>
        protected internal virtual ContainerId StoreAttempt(RMStateStore store, ApplicationAttemptId
                                                            attemptId, string containerIdStr, Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier
                                                                                                                                      > appToken, SecretKey clientTokenMasterKey, RMStateStoreTestBase.TestDispatcher
                                                            dispatcher)
        {
            RMAppAttemptMetrics mockRmAppAttemptMetrics = Org.Mockito.Mockito.Mock <RMAppAttemptMetrics
                                                                                    >();
            Container container = new ContainerPBImpl();

            container.SetId(ConverterUtils.ToContainerId(containerIdStr));
            RMAppAttempt mockAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>();

            Org.Mockito.Mockito.When(mockAttempt.GetAppAttemptId()).ThenReturn(attemptId);
            Org.Mockito.Mockito.When(mockAttempt.GetMasterContainer()).ThenReturn(container);
            Org.Mockito.Mockito.When(mockAttempt.GetAMRMToken()).ThenReturn(appToken);
            Org.Mockito.Mockito.When(mockAttempt.GetClientTokenMasterKey()).ThenReturn(clientTokenMasterKey
                                                                                       );
            Org.Mockito.Mockito.When(mockAttempt.GetRMAppAttemptMetrics()).ThenReturn(mockRmAppAttemptMetrics
                                                                                      );
            Org.Mockito.Mockito.When(mockRmAppAttemptMetrics.GetAggregateAppResourceUsage()).
            ThenReturn(new AggregateAppResourceUsage(0, 0));
            dispatcher.attemptId = attemptId;
            store.StoreNewApplicationAttempt(mockAttempt);
            WaitNotify(dispatcher);
            return(container.GetId());
        }
Пример #12
0
        protected override void GenerateApplicationTable(HtmlBlock.Block html, UserGroupInformation
                                                         callerUGI, ICollection <ApplicationAttemptReport> attempts)
        {
            // Application Attempt Table
            Hamlet.TBODY <Hamlet.TABLE <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > tbody = html
                                                                                               .Table("#attempts").Thead().Tr().Th(".id", "Attempt ID").Th(".started", "Started"
                                                                                                                                                           ).Th(".node", "Node").Th(".logs", "Logs").Th(".blacklistednodes", "Blacklisted Nodes"
                                                                                                                                                                                                        ).().().Tbody();
            RMApp rmApp = this.rm.GetRMContext().GetRMApps()[this.appID];

            if (rmApp == null)
            {
                return;
            }
            StringBuilder attemptsTableData = new StringBuilder("[\n");

            foreach (ApplicationAttemptReport appAttemptReport in attempts)
            {
                RMAppAttempt rmAppAttempt = rmApp.GetRMAppAttempt(appAttemptReport.GetApplicationAttemptId
                                                                      ());
                if (rmAppAttempt == null)
                {
                    continue;
                }
                AppAttemptInfo attemptInfo = new AppAttemptInfo(this.rm, rmAppAttempt, rmApp.GetUser
                                                                    (), WebAppUtils.GetHttpSchemePrefix(conf));
                string blacklistedNodesCount = "N/A";
                ICollection <string> nodes   = RMAppAttemptBlock.GetBlacklistedNodes(rm, rmAppAttempt
                                                                                     .GetAppAttemptId());
                if (nodes != null)
                {
                    blacklistedNodesCount = nodes.Count.ToString();
                }
                string nodeLink = attemptInfo.GetNodeHttpAddress();
                if (nodeLink != null)
                {
                    nodeLink = WebAppUtils.GetHttpSchemePrefix(conf) + nodeLink;
                }
                string logsLink = attemptInfo.GetLogsLink();
                attemptsTableData.Append("[\"<a href='").Append(Url("appattempt", rmAppAttempt.GetAppAttemptId
                                                                        ().ToString())).Append("'>").Append(rmAppAttempt.GetAppAttemptId().ToString()).Append
                    ("</a>\",\"").Append(attemptInfo.GetStartTime()).Append("\",\"<a ").Append(nodeLink
                                                                                               == null ? "#" : "href='" + nodeLink).Append("'>").Append(nodeLink == null ? "N/A"
                                         : StringEscapeUtils.EscapeJavaScript(StringEscapeUtils.EscapeHtml(nodeLink))).Append
                    ("</a>\",\"<a ").Append(logsLink == null ? "#" : "href='" + logsLink).Append("'>"
                                                                                                 ).Append(logsLink == null ? "N/A" : "Logs").Append("</a>\",").Append("\"").Append
                    (blacklistedNodesCount).Append("\"],\n");
            }
            if (attemptsTableData[attemptsTableData.Length - 2] == ',')
            {
                attemptsTableData.Delete(attemptsTableData.Length - 2, attemptsTableData.Length -
                                         1);
            }
            attemptsTableData.Append("]");
            html.Script().$type("text/javascript").("var attemptsTableData=" + attemptsTableData
                                                    ).();
            tbody.().();
        }
Пример #13
0
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
        private void Launch()
        {
            Connect();
            ContainerId masterContainerID = masterContainer.GetId();
            ApplicationSubmissionContext applicationContext = application.GetSubmissionContext
                                                                  ();

            Log.Info("Setting up container " + masterContainer + " for AM " + application.GetAppAttemptId
                         ());
            ContainerLaunchContext launchContext = CreateAMContainerLaunchContext(applicationContext
                                                                                  , masterContainerID);
            StartContainerRequest scRequest = StartContainerRequest.NewInstance(launchContext
                                                                                , masterContainer.GetContainerToken());
            IList <StartContainerRequest> list = new AList <StartContainerRequest>();

            list.AddItem(scRequest);
            StartContainersRequest  allRequests = StartContainersRequest.NewInstance(list);
            StartContainersResponse response    = containerMgrProxy.StartContainers(allRequests);

            if (response.GetFailedRequests() != null && response.GetFailedRequests().Contains
                    (masterContainerID))
            {
                Exception t = response.GetFailedRequests()[masterContainerID].DeSerialize();
                ParseAndThrowException(t);
            }
            else
            {
                Log.Info("Done launching container " + masterContainer + " for AM " + application
                         .GetAppAttemptId());
            }
        }
Пример #14
0
        public virtual IDictionary <ApplicationAttemptId, RMAppAttempt> GetAppAttempts()
        {
            IDictionary <ApplicationAttemptId, RMAppAttempt> attempts = new LinkedHashMap <ApplicationAttemptId
                                                                                           , RMAppAttempt>();

            if (attempt != null)
            {
                attempts[attempt.GetAppAttemptId()] = attempt;
            }
            return(attempts);
        }
Пример #15
0
 public virtual void AppAttemptRegistered(RMAppAttempt appAttempt, long registeredTime
                                          )
 {
     if (publishSystemMetrics)
     {
         dispatcher.GetEventHandler().Handle(new AppAttemptRegisteredEvent(appAttempt.GetAppAttemptId
                                                                               (), appAttempt.GetHost(), appAttempt.GetRpcPort(), appAttempt.GetTrackingUrl(),
                                                                           appAttempt.GetOriginalTrackingUrl(), appAttempt.GetMasterContainer().GetId(), registeredTime
                                                                           ));
     }
 }
Пример #16
0
        /// <exception cref="System.Exception"/>
        private MockAM LaunchAM(RMApp app, MockRM rm, MockNM nm)
        {
            RMAppAttempt attempt = app.GetCurrentAppAttempt();

            nm.NodeHeartbeat(true);
            MockAM am = rm.SendAMLaunched(attempt.GetAppAttemptId());

            am.RegisterAppAttempt();
            rm.WaitForState(app.GetApplicationId(), RMAppState.Running);
            return(am);
        }
Пример #17
0
        public virtual void TestValidateResourceBlacklistRequest()
        {
            TestAMAuthorization.MyContainerManager containerManager = new TestAMAuthorization.MyContainerManager
                                                                          ();
            TestAMAuthorization.MockRMWithAMS rm = new TestAMAuthorization.MockRMWithAMS(new
                                                                                         YarnConfiguration(), containerManager);
            rm.Start();
            MockNM nm1 = rm.RegisterNode("localhost:1234", 5120);
            IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
                                                                               , string>(2);

            acls[ApplicationAccessType.ViewApp] = "*";
            RMApp app = rm.SubmitApp(1024, "appname", "appuser", acls);

            nm1.NodeHeartbeat(true);
            RMAppAttempt         attempt = app.GetCurrentAppAttempt();
            ApplicationAttemptId applicationAttemptId = attempt.GetAppAttemptId();

            WaitForLaunchedState(attempt);
            // Create a client to the RM.
            Configuration        conf        = rm.GetConfig();
            YarnRPC              rpc         = YarnRPC.Create(conf);
            UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(applicationAttemptId
                                                                                     .ToString());
            Credentials credentials   = containerManager.GetContainerCredentials();
            IPEndPoint  rmBindAddress = rm.GetApplicationMasterService().GetBindAddress();

            Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> amRMToken = TestAMAuthorization.MockRMWithAMS
                                                                                 .SetupAndReturnAMRMToken(rmBindAddress, credentials.GetAllTokens());
            currentUser.AddToken(amRMToken);
            ApplicationMasterProtocol client = currentUser.DoAs(new _PrivilegedAction_626(rpc
                                                                                          , rmBindAddress, conf));
            RegisterApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
                                                       <RegisterApplicationMasterRequest>();

            client.RegisterApplicationMaster(request);
            ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.NewInstance(
                Sharpen.Collections.SingletonList(ResourceRequest.Any), null);
            AllocateRequest allocateRequest = AllocateRequest.NewInstance(0, 0.0f, null, null
                                                                          , blacklistRequest);
            bool error = false;

            try
            {
                client.Allocate(allocateRequest);
            }
            catch (InvalidResourceBlacklistRequestException)
            {
                error = true;
            }
            rm.Stop();
            NUnit.Framework.Assert.IsTrue("Didn't not catch InvalidResourceBlacklistRequestException"
                                          , error);
        }
Пример #18
0
 public virtual void AppAttemptFinished(RMAppAttempt appAttempt, RMAppAttemptState
                                        appAttemtpState, RMApp app, long finishedTime)
 {
     if (publishSystemMetrics)
     {
         dispatcher.GetEventHandler().Handle(new AppAttemptFinishedEvent(appAttempt.GetAppAttemptId
                                                                             (), appAttempt.GetTrackingUrl(), appAttempt.GetOriginalTrackingUrl(), appAttempt
                                                                         .GetDiagnostics(), app.GetFinalApplicationStatus(), RMServerUtils.CreateApplicationAttemptState
                                                                             (appAttemtpState), finishedTime));
     }
 }
Пример #19
0
        public virtual void TestAuthorizedAccess()
        {
            TestAMAuthorization.MyContainerManager containerManager = new TestAMAuthorization.MyContainerManager
                                                                          ();
            rm = new TestAMAuthorization.MockRMWithAMS(conf, containerManager);
            rm.Start();
            MockNM nm1 = rm.RegisterNode("localhost:1234", 5120);
            IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
                                                                               , string>(2);

            acls[ApplicationAccessType.ViewApp] = "*";
            RMApp app = rm.SubmitApp(1024, "appname", "appuser", acls);

            nm1.NodeHeartbeat(true);
            int waitCount = 0;

            while (containerManager.containerTokens == null && waitCount++ < 20)
            {
                Log.Info("Waiting for AM Launch to happen..");
                Sharpen.Thread.Sleep(1000);
            }
            NUnit.Framework.Assert.IsNotNull(containerManager.containerTokens);
            RMAppAttempt         attempt = app.GetCurrentAppAttempt();
            ApplicationAttemptId applicationAttemptId = attempt.GetAppAttemptId();

            WaitForLaunchedState(attempt);
            // Create a client to the RM.
            Configuration        conf        = rm.GetConfig();
            YarnRPC              rpc         = YarnRPC.Create(conf);
            UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(applicationAttemptId
                                                                                     .ToString());
            Credentials credentials   = containerManager.GetContainerCredentials();
            IPEndPoint  rmBindAddress = rm.GetApplicationMasterService().GetBindAddress();

            Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> amRMToken = TestAMAuthorization.MockRMWithAMS
                                                                                 .SetupAndReturnAMRMToken(rmBindAddress, credentials.GetAllTokens());
            currentUser.AddToken(amRMToken);
            ApplicationMasterProtocol client = currentUser.DoAs(new _PrivilegedAction_206(this
                                                                                          , rpc, conf));
            RegisterApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
                                                       <RegisterApplicationMasterRequest>();
            RegisterApplicationMasterResponse response = client.RegisterApplicationMaster(request
                                                                                          );

            NUnit.Framework.Assert.IsNotNull(response.GetClientToAMTokenMasterKey());
            if (UserGroupInformation.IsSecurityEnabled())
            {
                NUnit.Framework.Assert.IsTrue(((byte[])response.GetClientToAMTokenMasterKey().Array
                                                   ()).Length > 0);
            }
            NUnit.Framework.Assert.AreEqual("Register response has bad ACLs", "*", response.GetApplicationACLs
                                                ()[ApplicationAccessType.ViewApp]);
        }
Пример #20
0
        /// <exception cref="System.Exception"/>
        public virtual void TestAppWithNoContainers()
        {
            Logger rootLogger = LogManager.GetRootLogger();

            rootLogger.SetLevel(Level.Debug);
            MockRM rm = new MockRM(conf);

            rm.Start();
            MockNM nm1 = rm.RegisterNode("h1:1234", 5120);
            RMApp  app = rm.SubmitApp(2000);

            //kick the scheduling
            nm1.NodeHeartbeat(true);
            RMAppAttempt attempt = app.GetCurrentAppAttempt();
            MockAM       am      = rm.SendAMLaunched(attempt.GetAppAttemptId());

            am.RegisterAppAttempt();
            am.UnregisterAppAttempt();
            nm1.NodeHeartbeat(attempt.GetAppAttemptId(), 1, ContainerState.Complete);
            am.WaitForState(RMAppAttemptState.Finished);
            rm.Stop();
        }
        /// <exception cref="System.Exception"/>
        private void TestRMWritingMassiveHistory(MockRM rm)
        {
            rm.Start();
            MockNM nm  = rm.RegisterNode("127.0.0.1:1234", 1024 * 10100);
            RMApp  app = rm.SubmitApp(1024);

            nm.NodeHeartbeat(true);
            RMAppAttempt attempt = app.GetCurrentAppAttempt();
            MockAM       am      = rm.SendAMLaunched(attempt.GetAppAttemptId());

            am.RegisterAppAttempt();
            int request = 10000;

            am.Allocate("127.0.0.1", 1024, request, new AList <ContainerId>());
            nm.NodeHeartbeat(true);
            IList <Container> allocated = am.Allocate(new AList <ResourceRequest>(), new AList <
                                                          ContainerId>()).GetAllocatedContainers();
            int waitCount     = 0;
            int allocatedSize = allocated.Count;

            while (allocatedSize < request && waitCount++ < 200)
            {
                Sharpen.Thread.Sleep(300);
                allocated = am.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>()).GetAllocatedContainers
                                ();
                allocatedSize += allocated.Count;
                nm.NodeHeartbeat(true);
            }
            NUnit.Framework.Assert.AreEqual(request, allocatedSize);
            am.UnregisterAppAttempt();
            am.WaitForState(RMAppAttemptState.Finishing);
            nm.NodeHeartbeat(am.GetApplicationAttemptId(), 1, ContainerState.Complete);
            am.WaitForState(RMAppAttemptState.Finished);
            NodeHeartbeatResponse resp    = nm.NodeHeartbeat(true);
            IList <ContainerId>   cleaned = resp.GetContainersToCleanup();
            int cleanedSize = cleaned.Count;

            waitCount = 0;
            while (cleanedSize < allocatedSize && waitCount++ < 200)
            {
                Sharpen.Thread.Sleep(300);
                resp         = nm.NodeHeartbeat(true);
                cleaned      = resp.GetContainersToCleanup();
                cleanedSize += cleaned.Count;
            }
            NUnit.Framework.Assert.AreEqual(allocatedSize, cleanedSize);
            rm.WaitForState(app.GetApplicationId(), RMAppState.Finished);
            rm.Stop();
        }
Пример #22
0
        public virtual void TestARRMResponseId()
        {
            MockNM nm1 = rm.RegisterNode("h1:1234", 5000);
            RMApp  app = rm.SubmitApp(2000);

            // Trigger the scheduling so the AM gets 'launched'
            nm1.NodeHeartbeat(true);
            RMAppAttempt attempt = app.GetCurrentAppAttempt();
            MockAM       am      = rm.SendAMLaunched(attempt.GetAppAttemptId());

            am.RegisterAppAttempt();
            AllocateRequest allocateRequest = AllocateRequest.NewInstance(0, 0F, null, null,
                                                                          null);
            AllocateResponse response = Allocate(attempt.GetAppAttemptId(), allocateRequest);

            NUnit.Framework.Assert.AreEqual(1, response.GetResponseId());
            NUnit.Framework.Assert.IsTrue(response.GetAMCommand() == null);
            allocateRequest = AllocateRequest.NewInstance(response.GetResponseId(), 0F, null,
                                                          null, null);
            response = Allocate(attempt.GetAppAttemptId(), allocateRequest);
            NUnit.Framework.Assert.AreEqual(2, response.GetResponseId());
            /* try resending */
            response = Allocate(attempt.GetAppAttemptId(), allocateRequest);
            NUnit.Framework.Assert.AreEqual(2, response.GetResponseId());
            allocateRequest = AllocateRequest.NewInstance(0, 0F, null, null, null);
            try
            {
                Allocate(attempt.GetAppAttemptId(), allocateRequest);
                NUnit.Framework.Assert.Fail();
            }
            catch (Exception e)
            {
                NUnit.Framework.Assert.IsTrue(e.InnerException is InvalidApplicationMasterRequestException
                                              );
            }
        }
        /// <exception cref="System.Exception"/>
        public virtual void TestUsageWithOneAttemptAndOneContainer()
        {
            MockRM rm = new MockRM(conf);

            rm.Start();
            MockNM nm = new MockNM("127.0.0.1:1234", 15120, rm.GetResourceTrackerService());

            nm.RegisterNode();
            RMApp        app0         = rm.SubmitApp(200);
            RMAppMetrics rmAppMetrics = app0.GetRMAppMetrics();

            NUnit.Framework.Assert.IsTrue("Before app submittion, memory seconds should have been 0 but was "
                                          + rmAppMetrics.GetMemorySeconds(), rmAppMetrics.GetMemorySeconds() == 0);
            NUnit.Framework.Assert.IsTrue("Before app submission, vcore seconds should have been 0 but was "
                                          + rmAppMetrics.GetVcoreSeconds(), rmAppMetrics.GetVcoreSeconds() == 0);
            RMAppAttempt attempt0 = app0.GetCurrentAppAttempt();

            nm.NodeHeartbeat(true);
            MockAM am0 = rm.SendAMLaunched(attempt0.GetAppAttemptId());

            am0.RegisterAppAttempt();
            RMContainer rmContainer = rm.GetResourceScheduler().GetRMContainer(attempt0.GetMasterContainer
                                                                                   ().GetId());
            // Allow metrics to accumulate.
            int sleepInterval       = 1000;
            int cumulativeSleepTime = 0;

            while (rmAppMetrics.GetMemorySeconds() <= 0 && cumulativeSleepTime < 5000)
            {
                Sharpen.Thread.Sleep(sleepInterval);
                cumulativeSleepTime += sleepInterval;
            }
            rmAppMetrics = app0.GetRMAppMetrics();
            NUnit.Framework.Assert.IsTrue("While app is running, memory seconds should be >0 but is "
                                          + rmAppMetrics.GetMemorySeconds(), rmAppMetrics.GetMemorySeconds() > 0);
            NUnit.Framework.Assert.IsTrue("While app is running, vcore seconds should be >0 but is "
                                          + rmAppMetrics.GetVcoreSeconds(), rmAppMetrics.GetVcoreSeconds() > 0);
            MockRM.FinishAMAndVerifyAppState(app0, rm, nm, am0);
            AggregateAppResourceUsage ru = CalculateContainerResourceMetrics(rmContainer);

            rmAppMetrics = app0.GetRMAppMetrics();
            NUnit.Framework.Assert.AreEqual("Unexcpected MemorySeconds value", ru.GetMemorySeconds
                                                (), rmAppMetrics.GetMemorySeconds());
            NUnit.Framework.Assert.AreEqual("Unexpected VcoreSeconds value", ru.GetVcoreSeconds
                                                (), rmAppMetrics.GetVcoreSeconds());
            rm.Stop();
        }
Пример #24
0
        /// <exception cref="System.Exception"/>
        public virtual void TestResourceTypes()
        {
            Dictionary <YarnConfiguration, EnumSet <YarnServiceProtos.SchedulerResourceTypes> >
            driver = new Dictionary <YarnConfiguration, EnumSet <YarnServiceProtos.SchedulerResourceTypes
                                                                 > >();
            CapacitySchedulerConfiguration csconf = new CapacitySchedulerConfiguration();

            csconf.SetResourceComparator(typeof(DominantResourceCalculator));
            YarnConfiguration testCapacityDRConf = new YarnConfiguration(csconf);

            testCapacityDRConf.SetClass(YarnConfiguration.RmScheduler, typeof(CapacityScheduler
                                                                              ), typeof(ResourceScheduler));
            YarnConfiguration testCapacityDefConf = new YarnConfiguration();

            testCapacityDefConf.SetClass(YarnConfiguration.RmScheduler, typeof(CapacityScheduler
                                                                               ), typeof(ResourceScheduler));
            YarnConfiguration testFairDefConf = new YarnConfiguration();

            testFairDefConf.SetClass(YarnConfiguration.RmScheduler, typeof(FairScheduler), typeof(
                                         ResourceScheduler));
            driver[conf] = EnumSet.Of(YarnServiceProtos.SchedulerResourceTypes.Memory);
            driver[testCapacityDRConf] = EnumSet.Of(YarnServiceProtos.SchedulerResourceTypes.
                                                    Cpu, YarnServiceProtos.SchedulerResourceTypes.Memory);
            driver[testCapacityDefConf] = EnumSet.Of(YarnServiceProtos.SchedulerResourceTypes
                                                     .Memory);
            driver[testFairDefConf] = EnumSet.Of(YarnServiceProtos.SchedulerResourceTypes.Memory
                                                 , YarnServiceProtos.SchedulerResourceTypes.Cpu);
            foreach (KeyValuePair <YarnConfiguration, EnumSet <YarnServiceProtos.SchedulerResourceTypes
                                                               > > entry in driver)
            {
                EnumSet <YarnServiceProtos.SchedulerResourceTypes> expectedValue = entry.Value;
                MockRM rm = new MockRM(entry.Key);
                rm.Start();
                MockNM nm1  = rm.RegisterNode("127.0.0.1:1234", 6 * Gb);
                RMApp  app1 = rm.SubmitApp(2048);
                nm1.NodeHeartbeat(true);
                RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
                MockAM       am1      = rm.SendAMLaunched(attempt1.GetAppAttemptId());
                RegisterApplicationMasterResponse resp = am1.RegisterAppAttempt();
                EnumSet <YarnServiceProtos.SchedulerResourceTypes> types = resp.GetSchedulerResourceTypes
                                                                               ();
                Log.Info("types = " + types.ToString());
                NUnit.Framework.Assert.AreEqual(expectedValue, types);
                rm.Stop();
            }
        }
        private static RMAppAttempt CreateRMAppAttempt(ApplicationAttemptId appAttemptId)
        {
            RMAppAttempt appAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>();

            Org.Mockito.Mockito.When(appAttempt.GetAppAttemptId()).ThenReturn(appAttemptId);
            Org.Mockito.Mockito.When(appAttempt.GetHost()).ThenReturn("test host");
            Org.Mockito.Mockito.When(appAttempt.GetRpcPort()).ThenReturn(-100);
            Container container = Org.Mockito.Mockito.Mock <Container>();

            Org.Mockito.Mockito.When(container.GetId()).ThenReturn(ContainerId.NewContainerId
                                                                       (appAttemptId, 1));
            Org.Mockito.Mockito.When(appAttempt.GetMasterContainer()).ThenReturn(container);
            Org.Mockito.Mockito.When(appAttempt.GetDiagnostics()).ThenReturn("test diagnostics info"
                                                                             );
            Org.Mockito.Mockito.When(appAttempt.GetTrackingUrl()).ThenReturn("test url");
            Org.Mockito.Mockito.When(appAttempt.GetFinalApplicationStatus()).ThenReturn(FinalApplicationStatus
                                                                                        .Undefined);
            return(appAttempt);
        }
Пример #26
0
        public virtual AggregateAppResourceUsage GetAggregateAppResourceUsage()
        {
            long memorySeconds = finishedMemorySeconds.Get();
            long vcoreSeconds  = finishedVcoreSeconds.Get();
            // Only add in the running containers if this is the active attempt.
            RMAppAttempt currentAttempt = rmContext.GetRMApps()[attemptId.GetApplicationId()]
                                          .GetCurrentAppAttempt();

            if (currentAttempt.GetAppAttemptId().Equals(attemptId))
            {
                ApplicationResourceUsageReport appResUsageReport = rmContext.GetScheduler().GetAppResourceUsageReport
                                                                       (attemptId);
                if (appResUsageReport != null)
                {
                    memorySeconds += appResUsageReport.GetMemorySeconds();
                    vcoreSeconds  += appResUsageReport.GetVcoreSeconds();
                }
            }
            return(new AggregateAppResourceUsage(memorySeconds, vcoreSeconds));
        }
Пример #27
0
 /// <exception cref="System.Exception"/>
 private void CheckActiveRMFunctionality()
 {
     NUnit.Framework.Assert.AreEqual(StateErr, HAServiceProtocol.HAServiceState.Active
                                     , rm.adminService.GetServiceStatus().GetState());
     NUnit.Framework.Assert.IsTrue("Active RM services aren't started", rm.AreActiveServicesRunning
                                       ());
     NUnit.Framework.Assert.IsTrue("RM is not ready to become active", rm.adminService
                                   .GetServiceStatus().IsReadyToBecomeActive());
     try
     {
         rm.GetNewAppId();
         rm.RegisterNode("127.0.0.1:1", 2048);
         app     = rm.SubmitApp(1024);
         attempt = app.GetCurrentAppAttempt();
         rm.WaitForState(attempt.GetAppAttemptId(), RMAppAttemptState.Scheduled);
     }
     catch (Exception e)
     {
         NUnit.Framework.Assert.Fail("Unable to perform Active RM functions");
         Log.Error("ActiveRM check failed", e);
     }
     CheckActiveRMWebServices();
 }
Пример #28
0
        /// <exception cref="System.Exception"/>
        public virtual void TestRMIdentifierOnContainerAllocation()
        {
            MockRM rm = new MockRM(conf);

            rm.Start();
            // Register node1
            MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb);
            // Submit an application
            RMApp app1 = rm.SubmitApp(2048);

            // kick the scheduling
            nm1.NodeHeartbeat(true);
            RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
            MockAM       am1      = rm.SendAMLaunched(attempt1.GetAppAttemptId());

            am1.RegisterAppAttempt();
            am1.AddRequests(new string[] { "127.0.0.1" }, Gb, 1, 1);
            AllocateResponse alloc1Response = am1.Schedule();

            // send the request
            // kick the scheduler
            nm1.NodeHeartbeat(true);
            while (alloc1Response.GetAllocatedContainers().Count < 1)
            {
                Log.Info("Waiting for containers to be created for app 1...");
                Sharpen.Thread.Sleep(1000);
                alloc1Response = am1.Schedule();
            }
            // assert RMIdentifer is set properly in allocated containers
            Container allocatedContainer     = alloc1Response.GetAllocatedContainers()[0];
            ContainerTokenIdentifier tokenId = BuilderUtils.NewContainerTokenIdentifier(allocatedContainer
                                                                                        .GetContainerToken());

            NUnit.Framework.Assert.AreEqual(MockRM.GetClusterTimeStamp(), tokenId.GetRMIdentifier
                                                ());
            rm.Stop();
        }
Пример #29
0
        public virtual void TestAllocateContainerOnNodeWithoutOffSwitchSpecified()
        {
            Logger rootLogger = LogManager.GetRootLogger();

            rootLogger.SetLevel(Level.Debug);
            MockRM rm = new MockRM(conf);

            rm.Start();
            MockNM nm1  = rm.RegisterNode("127.0.0.1:1234", 6 * Gb);
            RMApp  app1 = rm.SubmitApp(2048);

            // kick the scheduling, 2 GB given to AM1, remaining 4GB on nm1
            nm1.NodeHeartbeat(true);
            RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
            MockAM       am1      = rm.SendAMLaunched(attempt1.GetAppAttemptId());

            am1.RegisterAppAttempt();
            // add request for containers
            IList <ResourceRequest> requests = new AList <ResourceRequest>();

            requests.AddItem(am1.CreateResourceReq("127.0.0.1", 1 * Gb, 1, 1));
            requests.AddItem(am1.CreateResourceReq("/default-rack", 1 * Gb, 1, 1));
            am1.Allocate(requests, null);
            // send the request
            try
            {
                // kick the schedule
                nm1.NodeHeartbeat(true);
            }
            catch (ArgumentNullException)
            {
                NUnit.Framework.Assert.Fail("NPE when allocating container on node but " + "forget to set off-switch request should be handled"
                                            );
            }
            rm.Stop();
        }
Пример #30
0
        /// <exception cref="System.Exception"/>
        public virtual void TestActivatingApplicationAfterAddingNM()
        {
            MockRM rm1 = new MockRM(conf);

            // start like normal because state is empty
            rm1.Start();
            // app that gets launched
            RMApp app1 = rm1.SubmitApp(200);
            // app that does not get launched
            RMApp app2 = rm1.SubmitApp(200);
            // app1 and app2 should be scheduled, but because no resource is available,
            // they are not activated.
            RMAppAttempt         attempt1   = app1.GetCurrentAppAttempt();
            ApplicationAttemptId attemptId1 = attempt1.GetAppAttemptId();

            rm1.WaitForState(attemptId1, RMAppAttemptState.Scheduled);
            RMAppAttempt         attempt2   = app2.GetCurrentAppAttempt();
            ApplicationAttemptId attemptId2 = attempt2.GetAppAttemptId();

            rm1.WaitForState(attemptId2, RMAppAttemptState.Scheduled);
            MockNM nm1 = new MockNM("h1:1234", 15120, rm1.GetResourceTrackerService());
            MockNM nm2 = new MockNM("h2:5678", 15120, rm1.GetResourceTrackerService());

            nm1.RegisterNode();
            nm2.RegisterNode();
            //kick the scheduling
            nm1.NodeHeartbeat(true);
            // app1 should be allocated now
            rm1.WaitForState(attemptId1, RMAppAttemptState.Allocated);
            rm1.WaitForState(attemptId2, RMAppAttemptState.Scheduled);
            nm2.NodeHeartbeat(true);
            // app2 should be allocated now
            rm1.WaitForState(attemptId1, RMAppAttemptState.Allocated);
            rm1.WaitForState(attemptId2, RMAppAttemptState.Allocated);
            rm1.Stop();
        }