Example #1
0
        public RMContainerImpl(Container container, ApplicationAttemptId appAttemptId, NodeId
                               nodeId, string user, RMContext rmContext, long creationTime)
        {
            // Transitions from NEW state
            // Transitions from RESERVED state
            // nothing to do
            // nothing to do
            // Transitions from ALLOCATED state
            // Transitions from ACQUIRED state
            // Transitions from RUNNING state
            // Transitions from COMPLETED state
            // Transitions from EXPIRED state
            // Transitions from RELEASED state
            // Transitions from KILLED state
            // create the topology tables
            this.stateMachine = stateMachineFactory.Make(this);
            this.containerId  = container.GetId();
            this.nodeId       = nodeId;
            this.container    = container;
            this.appAttemptId = appAttemptId;
            this.user         = user;
            this.creationTime = creationTime;
            this.rmContext    = rmContext;
            this.eventHandler = rmContext.GetDispatcher().GetEventHandler();
            this.containerAllocationExpirer = rmContext.GetContainerAllocationExpirer();
            this.isAMContainer    = false;
            this.resourceRequests = null;
            ReentrantReadWriteLock Lock = new ReentrantReadWriteLock();

            this.readLock              = Lock.ReadLock();
            this.writeLock             = Lock.WriteLock();
            saveNonAMContainerMetaInfo = rmContext.GetYarnConfiguration().GetBoolean(YarnConfiguration
                                                                                     .ApplicationHistorySaveNonAmContainerMetaInfo, YarnConfiguration.DefaultApplicationHistorySaveNonAmContainerMetaInfo
                                                                                     );
            rmContext.GetRMApplicationHistoryWriter().ContainerStarted(this);
            // If saveNonAMContainerMetaInfo is true, store system metrics for all
            // containers. If false, and if this container is marked as the AM, metrics
            // will still be published for this container, but that calculation happens
            // later.
            if (saveNonAMContainerMetaInfo)
            {
                rmContext.GetSystemMetricsPublisher().ContainerCreated(this, this.creationTime);
            }
        }
Example #2
0
        public virtual void TestReleaseWhileRunning()
        {
            DrainDispatcher drainDispatcher = new DrainDispatcher();
            EventHandler <RMAppAttemptEvent> appAttemptEventHandler = Org.Mockito.Mockito.Mock
                                                                      <EventHandler>();
            EventHandler generic = Org.Mockito.Mockito.Mock <EventHandler>();

            drainDispatcher.Register(typeof(RMAppAttemptEventType), appAttemptEventHandler);
            drainDispatcher.Register(typeof(RMNodeEventType), generic);
            drainDispatcher.Init(new YarnConfiguration());
            drainDispatcher.Start();
            NodeId               nodeId       = BuilderUtils.NewNodeId("host", 3425);
            ApplicationId        appId        = BuilderUtils.NewApplicationId(1, 1);
            ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
                                                                                     );
            ContainerId containerId            = BuilderUtils.NewContainerId(appAttemptId, 1);
            ContainerAllocationExpirer expirer = Org.Mockito.Mockito.Mock <ContainerAllocationExpirer
                                                                           >();
            Resource  resource  = BuilderUtils.NewResource(512, 1);
            Priority  priority  = BuilderUtils.NewPriority(5);
            Container container = BuilderUtils.NewContainer(containerId, nodeId, "host:3465",
                                                            resource, priority, null);
            ConcurrentMap <ApplicationId, RMApp> rmApps = Org.Mockito.Mockito.Spy(new ConcurrentHashMap
                                                                                  <ApplicationId, RMApp>());
            RMApp rmApp = Org.Mockito.Mockito.Mock <RMApp>();

            Org.Mockito.Mockito.When(rmApp.GetRMAppAttempt((ApplicationAttemptId)Matchers.Any
                                                               ())).ThenReturn(null);
            Org.Mockito.Mockito.DoReturn(rmApp).When(rmApps)[(ApplicationId)Matchers.Any()];
            RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
                                                                          >();
            SystemMetricsPublisher publisher = Org.Mockito.Mockito.Mock <SystemMetricsPublisher
                                                                         >();
            RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>();

            Org.Mockito.Mockito.When(rmContext.GetDispatcher()).ThenReturn(drainDispatcher);
            Org.Mockito.Mockito.When(rmContext.GetContainerAllocationExpirer()).ThenReturn(expirer
                                                                                           );
            Org.Mockito.Mockito.When(rmContext.GetRMApplicationHistoryWriter()).ThenReturn(writer
                                                                                           );
            Org.Mockito.Mockito.When(rmContext.GetRMApps()).ThenReturn(rmApps);
            Org.Mockito.Mockito.When(rmContext.GetSystemMetricsPublisher()).ThenReturn(publisher
                                                                                       );
            Org.Mockito.Mockito.When(rmContext.GetYarnConfiguration()).ThenReturn(new YarnConfiguration
                                                                                      ());
            RMContainer rmContainer = new RMContainerImpl(container, appAttemptId, nodeId, "user"
                                                          , rmContext);

            NUnit.Framework.Assert.AreEqual(RMContainerState.New, rmContainer.GetState());
            NUnit.Framework.Assert.AreEqual(resource, rmContainer.GetAllocatedResource());
            NUnit.Framework.Assert.AreEqual(nodeId, rmContainer.GetAllocatedNode());
            NUnit.Framework.Assert.AreEqual(priority, rmContainer.GetAllocatedPriority());
            Org.Mockito.Mockito.Verify(writer).ContainerStarted(Matchers.Any <RMContainer>());
            Org.Mockito.Mockito.Verify(publisher).ContainerCreated(Matchers.Any <RMContainer>(
                                                                       ), Matchers.AnyLong());
            rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Start));
            drainDispatcher.Await();
            NUnit.Framework.Assert.AreEqual(RMContainerState.Allocated, rmContainer.GetState(
                                                ));
            rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Acquired
                                                    ));
            drainDispatcher.Await();
            NUnit.Framework.Assert.AreEqual(RMContainerState.Acquired, rmContainer.GetState()
                                            );
            rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Launched
                                                    ));
            drainDispatcher.Await();
            NUnit.Framework.Assert.AreEqual(RMContainerState.Running, rmContainer.GetState());
            NUnit.Framework.Assert.AreEqual("http://host:3465/node/containerlogs/container_1_0001_01_000001/user"
                                            , rmContainer.GetLogURL());
            // In RUNNING state. Verify RELEASED and associated actions.
            Org.Mockito.Mockito.Reset(appAttemptEventHandler);
            ContainerStatus containerStatus = SchedulerUtils.CreateAbnormalContainerStatus(containerId
                                                                                           , SchedulerUtils.ReleasedContainer);

            rmContainer.Handle(new RMContainerFinishedEvent(containerId, containerStatus, RMContainerEventType
                                                            .Released));
            drainDispatcher.Await();
            NUnit.Framework.Assert.AreEqual(RMContainerState.Released, rmContainer.GetState()
                                            );
            NUnit.Framework.Assert.AreEqual(SchedulerUtils.ReleasedContainer, rmContainer.GetDiagnosticsInfo
                                                ());
            NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Aborted, rmContainer.GetContainerExitStatus
                                                ());
            NUnit.Framework.Assert.AreEqual(ContainerState.Complete, rmContainer.GetContainerState
                                                ());
            Org.Mockito.Mockito.Verify(writer).ContainerFinished(Matchers.Any <RMContainer>());
            Org.Mockito.Mockito.Verify(publisher).ContainerFinished(Matchers.Any <RMContainer>
                                                                        (), Matchers.AnyLong());
            ArgumentCaptor <RMAppAttemptContainerFinishedEvent> captor = ArgumentCaptor.ForClass
                                                                         <RMAppAttemptContainerFinishedEvent>();

            Org.Mockito.Mockito.Verify(appAttemptEventHandler).Handle(captor.Capture());
            RMAppAttemptContainerFinishedEvent cfEvent = captor.GetValue();

            NUnit.Framework.Assert.AreEqual(appAttemptId, cfEvent.GetApplicationAttemptId());
            NUnit.Framework.Assert.AreEqual(containerStatus, cfEvent.GetContainerStatus());
            NUnit.Framework.Assert.AreEqual(RMAppAttemptEventType.ContainerFinished, cfEvent.
                                            GetType());
            // In RELEASED state. A FINIHSED event may come in.
            rmContainer.Handle(new RMContainerFinishedEvent(containerId, SchedulerUtils.CreateAbnormalContainerStatus
                                                                (containerId, "FinishedContainer"), RMContainerEventType.Finished));
            NUnit.Framework.Assert.AreEqual(RMContainerState.Released, rmContainer.GetState()
                                            );
        }
Example #3
0
        public virtual void TestExpireWhileRunning()
        {
            DrainDispatcher drainDispatcher = new DrainDispatcher();
            EventHandler <RMAppAttemptEvent> appAttemptEventHandler = Org.Mockito.Mockito.Mock
                                                                      <EventHandler>();
            EventHandler generic = Org.Mockito.Mockito.Mock <EventHandler>();

            drainDispatcher.Register(typeof(RMAppAttemptEventType), appAttemptEventHandler);
            drainDispatcher.Register(typeof(RMNodeEventType), generic);
            drainDispatcher.Init(new YarnConfiguration());
            drainDispatcher.Start();
            NodeId               nodeId       = BuilderUtils.NewNodeId("host", 3425);
            ApplicationId        appId        = BuilderUtils.NewApplicationId(1, 1);
            ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
                                                                                     );
            ContainerId containerId            = BuilderUtils.NewContainerId(appAttemptId, 1);
            ContainerAllocationExpirer expirer = Org.Mockito.Mockito.Mock <ContainerAllocationExpirer
                                                                           >();
            Resource  resource  = BuilderUtils.NewResource(512, 1);
            Priority  priority  = BuilderUtils.NewPriority(5);
            Container container = BuilderUtils.NewContainer(containerId, nodeId, "host:3465",
                                                            resource, priority, null);
            RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
                                                                          >();
            SystemMetricsPublisher publisher = Org.Mockito.Mockito.Mock <SystemMetricsPublisher
                                                                         >();
            RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>();

            Org.Mockito.Mockito.When(rmContext.GetDispatcher()).ThenReturn(drainDispatcher);
            Org.Mockito.Mockito.When(rmContext.GetContainerAllocationExpirer()).ThenReturn(expirer
                                                                                           );
            Org.Mockito.Mockito.When(rmContext.GetRMApplicationHistoryWriter()).ThenReturn(writer
                                                                                           );
            Org.Mockito.Mockito.When(rmContext.GetSystemMetricsPublisher()).ThenReturn(publisher
                                                                                       );
            Org.Mockito.Mockito.When(rmContext.GetYarnConfiguration()).ThenReturn(new YarnConfiguration
                                                                                      ());
            RMContainer rmContainer = new RMContainerImpl(container, appAttemptId, nodeId, "user"
                                                          , rmContext);

            NUnit.Framework.Assert.AreEqual(RMContainerState.New, rmContainer.GetState());
            NUnit.Framework.Assert.AreEqual(resource, rmContainer.GetAllocatedResource());
            NUnit.Framework.Assert.AreEqual(nodeId, rmContainer.GetAllocatedNode());
            NUnit.Framework.Assert.AreEqual(priority, rmContainer.GetAllocatedPriority());
            Org.Mockito.Mockito.Verify(writer).ContainerStarted(Matchers.Any <RMContainer>());
            Org.Mockito.Mockito.Verify(publisher).ContainerCreated(Matchers.Any <RMContainer>(
                                                                       ), Matchers.AnyLong());
            rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Start));
            drainDispatcher.Await();
            NUnit.Framework.Assert.AreEqual(RMContainerState.Allocated, rmContainer.GetState(
                                                ));
            rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Acquired
                                                    ));
            drainDispatcher.Await();
            NUnit.Framework.Assert.AreEqual(RMContainerState.Acquired, rmContainer.GetState()
                                            );
            rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Launched
                                                    ));
            drainDispatcher.Await();
            NUnit.Framework.Assert.AreEqual(RMContainerState.Running, rmContainer.GetState());
            NUnit.Framework.Assert.AreEqual("http://host:3465/node/containerlogs/container_1_0001_01_000001/user"
                                            , rmContainer.GetLogURL());
            // In RUNNING state. Verify EXPIRE and associated actions.
            Org.Mockito.Mockito.Reset(appAttemptEventHandler);
            ContainerStatus containerStatus = SchedulerUtils.CreateAbnormalContainerStatus(containerId
                                                                                           , SchedulerUtils.ExpiredContainer);

            rmContainer.Handle(new RMContainerFinishedEvent(containerId, containerStatus, RMContainerEventType
                                                            .Expire));
            drainDispatcher.Await();
            NUnit.Framework.Assert.AreEqual(RMContainerState.Running, rmContainer.GetState());
            Org.Mockito.Mockito.Verify(writer, Org.Mockito.Mockito.Never()).ContainerFinished
                (Matchers.Any <RMContainer>());
            Org.Mockito.Mockito.Verify(publisher, Org.Mockito.Mockito.Never()).ContainerFinished
                (Matchers.Any <RMContainer>(), Matchers.AnyLong());
        }