Exemplo n.º 1
0
 public override void Transition(RMContainerImpl container, RMContainerEvent @event
                                 )
 {
     // Tell scheduler to recover request of this container to app
     container.eventHandler.Handle(new ContainerRescheduledEvent(container));
     base.Transition(container, @event);
 }
Exemplo n.º 2
0
 public override void Transition(RMContainerImpl container, RMContainerEvent @event
                                 )
 {
     // Unregister from containerAllocationExpirer.
     container.containerAllocationExpirer.Unregister(container.GetContainerId());
     // Inform AppAttempt
     base.Transition(container, @event);
 }
Exemplo n.º 3
0
            public override void Transition(RMContainerImpl container, RMContainerEvent @event
                                            )
            {
                RMContainerReservedEvent e = (RMContainerReservedEvent)@event;

                container.reservedResource = e.GetReservedResource();
                container.reservedNode     = e.GetReservedNode();
                container.reservedPriority = e.GetReservedPriority();
            }
Exemplo n.º 4
0
 public override void Transition(RMContainerImpl container, RMContainerEvent @event
                                 )
 {
     // Unregister from containerAllocationExpirer.
     container.containerAllocationExpirer.Unregister(container.GetContainerId());
     // Inform node
     container.eventHandler.Handle(new RMNodeCleanContainerEvent(container.nodeId, container
                                                                 .containerId));
     // Inform appAttempt
     base.Transition(container, @event);
 }
Exemplo n.º 5
0
 public override void Transition(RMContainerImpl container, RMContainerEvent @event
                                 )
 {
     // Clear ResourceRequest stored in RMContainer
     container.SetResourceRequests(null);
     // Register with containerAllocationExpirer.
     container.containerAllocationExpirer.Register(container.GetContainerId());
     // Tell the app
     container.eventHandler.Handle(new RMAppRunningOnNodeEvent(container.GetApplicationAttemptId
                                                                   ().GetApplicationId(), container.nodeId));
 }
Exemplo n.º 6
0
            private static void UpdateAttemptMetrics(RMContainerImpl container)
            {
                // If this is a preempted container, update preemption metrics
                Resource     resource  = container.GetContainer().GetResource();
                RMAppAttempt rmAttempt = container.rmContext.GetRMApps()[container.GetApplicationAttemptId
                                                                             ().GetApplicationId()].GetCurrentAppAttempt();

                if (ContainerExitStatus.Preempted == container.finishedStatus.GetExitStatus())
                {
                    rmAttempt.GetRMAppAttemptMetrics().UpdatePreemptionInfo(resource, container);
                }
                if (rmAttempt != null)
                {
                    long usedMillis    = container.finishTime - container.creationTime;
                    long memorySeconds = resource.GetMemory() * usedMillis / DateUtils.MillisPerSecond;
                    long vcoreSeconds  = resource.GetVirtualCores() * usedMillis / DateUtils.MillisPerSecond;
                    rmAttempt.GetRMAppAttemptMetrics().UpdateAggregateAppResourceUsage(memorySeconds,
                                                                                       vcoreSeconds);
                }
            }
Exemplo n.º 7
0
            public override void Transition(RMContainerImpl container, RMContainerEvent @event
                                            )
            {
                RMContainerFinishedEvent finishedEvent = (RMContainerFinishedEvent)@event;

                container.finishTime     = Runtime.CurrentTimeMillis();
                container.finishedStatus = finishedEvent.GetRemoteContainerStatus();
                // Inform AppAttempt
                // container.getContainer() can return null when a RMContainer is a
                // reserved container
                UpdateAttemptMetrics(container);
                container.eventHandler.Handle(new RMAppAttemptContainerFinishedEvent(container.appAttemptId
                                                                                     , finishedEvent.GetRemoteContainerStatus(), container.GetAllocatedNode()));
                container.rmContext.GetRMApplicationHistoryWriter().ContainerFinished(container);
                bool saveNonAMContainerMetaInfo = container.rmContext.GetYarnConfiguration().GetBoolean
                                                      (YarnConfiguration.ApplicationHistorySaveNonAmContainerMetaInfo, YarnConfiguration
                                                      .DefaultApplicationHistorySaveNonAmContainerMetaInfo);

                if (saveNonAMContainerMetaInfo || container.IsAMContainer())
                {
                    container.rmContext.GetSystemMetricsPublisher().ContainerFinished(container, container
                                                                                      .finishTime);
                }
            }
Exemplo n.º 8
0
 public override void Transition(RMContainerImpl container, RMContainerEvent @event
                                 )
 {
     // Unregister from containerAllocationExpirer.
     container.containerAllocationExpirer.Unregister(container.GetContainerId());
 }
Exemplo n.º 9
0
 public override void Transition(RMContainerImpl container, RMContainerEvent @event
                                 )
 {
     container.eventHandler.Handle(new RMAppAttemptEvent(container.appAttemptId, RMAppAttemptEventType
                                                         .ContainerAllocated));
 }
Exemplo n.º 10
0
        public virtual void TestReleaseWhileRunning()
        {
            DrainDispatcher drainDispatcher = new DrainDispatcher();
            EventHandler <RMAppAttemptEvent> appAttemptEventHandler = Org.Mockito.Mockito.Mock
                                                                      <EventHandler>();
            EventHandler generic = Org.Mockito.Mockito.Mock <EventHandler>();

            drainDispatcher.Register(typeof(RMAppAttemptEventType), appAttemptEventHandler);
            drainDispatcher.Register(typeof(RMNodeEventType), generic);
            drainDispatcher.Init(new YarnConfiguration());
            drainDispatcher.Start();
            NodeId               nodeId       = BuilderUtils.NewNodeId("host", 3425);
            ApplicationId        appId        = BuilderUtils.NewApplicationId(1, 1);
            ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
                                                                                     );
            ContainerId containerId            = BuilderUtils.NewContainerId(appAttemptId, 1);
            ContainerAllocationExpirer expirer = Org.Mockito.Mockito.Mock <ContainerAllocationExpirer
                                                                           >();
            Resource  resource  = BuilderUtils.NewResource(512, 1);
            Priority  priority  = BuilderUtils.NewPriority(5);
            Container container = BuilderUtils.NewContainer(containerId, nodeId, "host:3465",
                                                            resource, priority, null);
            ConcurrentMap <ApplicationId, RMApp> rmApps = Org.Mockito.Mockito.Spy(new ConcurrentHashMap
                                                                                  <ApplicationId, RMApp>());
            RMApp rmApp = Org.Mockito.Mockito.Mock <RMApp>();

            Org.Mockito.Mockito.When(rmApp.GetRMAppAttempt((ApplicationAttemptId)Matchers.Any
                                                               ())).ThenReturn(null);
            Org.Mockito.Mockito.DoReturn(rmApp).When(rmApps)[(ApplicationId)Matchers.Any()];
            RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
                                                                          >();
            SystemMetricsPublisher publisher = Org.Mockito.Mockito.Mock <SystemMetricsPublisher
                                                                         >();
            RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>();

            Org.Mockito.Mockito.When(rmContext.GetDispatcher()).ThenReturn(drainDispatcher);
            Org.Mockito.Mockito.When(rmContext.GetContainerAllocationExpirer()).ThenReturn(expirer
                                                                                           );
            Org.Mockito.Mockito.When(rmContext.GetRMApplicationHistoryWriter()).ThenReturn(writer
                                                                                           );
            Org.Mockito.Mockito.When(rmContext.GetRMApps()).ThenReturn(rmApps);
            Org.Mockito.Mockito.When(rmContext.GetSystemMetricsPublisher()).ThenReturn(publisher
                                                                                       );
            Org.Mockito.Mockito.When(rmContext.GetYarnConfiguration()).ThenReturn(new YarnConfiguration
                                                                                      ());
            RMContainer rmContainer = new RMContainerImpl(container, appAttemptId, nodeId, "user"
                                                          , rmContext);

            NUnit.Framework.Assert.AreEqual(RMContainerState.New, rmContainer.GetState());
            NUnit.Framework.Assert.AreEqual(resource, rmContainer.GetAllocatedResource());
            NUnit.Framework.Assert.AreEqual(nodeId, rmContainer.GetAllocatedNode());
            NUnit.Framework.Assert.AreEqual(priority, rmContainer.GetAllocatedPriority());
            Org.Mockito.Mockito.Verify(writer).ContainerStarted(Matchers.Any <RMContainer>());
            Org.Mockito.Mockito.Verify(publisher).ContainerCreated(Matchers.Any <RMContainer>(
                                                                       ), Matchers.AnyLong());
            rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Start));
            drainDispatcher.Await();
            NUnit.Framework.Assert.AreEqual(RMContainerState.Allocated, rmContainer.GetState(
                                                ));
            rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Acquired
                                                    ));
            drainDispatcher.Await();
            NUnit.Framework.Assert.AreEqual(RMContainerState.Acquired, rmContainer.GetState()
                                            );
            rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Launched
                                                    ));
            drainDispatcher.Await();
            NUnit.Framework.Assert.AreEqual(RMContainerState.Running, rmContainer.GetState());
            NUnit.Framework.Assert.AreEqual("http://host:3465/node/containerlogs/container_1_0001_01_000001/user"
                                            , rmContainer.GetLogURL());
            // In RUNNING state. Verify RELEASED and associated actions.
            Org.Mockito.Mockito.Reset(appAttemptEventHandler);
            ContainerStatus containerStatus = SchedulerUtils.CreateAbnormalContainerStatus(containerId
                                                                                           , SchedulerUtils.ReleasedContainer);

            rmContainer.Handle(new RMContainerFinishedEvent(containerId, containerStatus, RMContainerEventType
                                                            .Released));
            drainDispatcher.Await();
            NUnit.Framework.Assert.AreEqual(RMContainerState.Released, rmContainer.GetState()
                                            );
            NUnit.Framework.Assert.AreEqual(SchedulerUtils.ReleasedContainer, rmContainer.GetDiagnosticsInfo
                                                ());
            NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Aborted, rmContainer.GetContainerExitStatus
                                                ());
            NUnit.Framework.Assert.AreEqual(ContainerState.Complete, rmContainer.GetContainerState
                                                ());
            Org.Mockito.Mockito.Verify(writer).ContainerFinished(Matchers.Any <RMContainer>());
            Org.Mockito.Mockito.Verify(publisher).ContainerFinished(Matchers.Any <RMContainer>
                                                                        (), Matchers.AnyLong());
            ArgumentCaptor <RMAppAttemptContainerFinishedEvent> captor = ArgumentCaptor.ForClass
                                                                         <RMAppAttemptContainerFinishedEvent>();

            Org.Mockito.Mockito.Verify(appAttemptEventHandler).Handle(captor.Capture());
            RMAppAttemptContainerFinishedEvent cfEvent = captor.GetValue();

            NUnit.Framework.Assert.AreEqual(appAttemptId, cfEvent.GetApplicationAttemptId());
            NUnit.Framework.Assert.AreEqual(containerStatus, cfEvent.GetContainerStatus());
            NUnit.Framework.Assert.AreEqual(RMAppAttemptEventType.ContainerFinished, cfEvent.
                                            GetType());
            // In RELEASED state. A FINIHSED event may come in.
            rmContainer.Handle(new RMContainerFinishedEvent(containerId, SchedulerUtils.CreateAbnormalContainerStatus
                                                                (containerId, "FinishedContainer"), RMContainerEventType.Finished));
            NUnit.Framework.Assert.AreEqual(RMContainerState.Released, rmContainer.GetState()
                                            );
        }
Exemplo n.º 11
0
        public virtual void TestExpireWhileRunning()
        {
            DrainDispatcher drainDispatcher = new DrainDispatcher();
            EventHandler <RMAppAttemptEvent> appAttemptEventHandler = Org.Mockito.Mockito.Mock
                                                                      <EventHandler>();
            EventHandler generic = Org.Mockito.Mockito.Mock <EventHandler>();

            drainDispatcher.Register(typeof(RMAppAttemptEventType), appAttemptEventHandler);
            drainDispatcher.Register(typeof(RMNodeEventType), generic);
            drainDispatcher.Init(new YarnConfiguration());
            drainDispatcher.Start();
            NodeId               nodeId       = BuilderUtils.NewNodeId("host", 3425);
            ApplicationId        appId        = BuilderUtils.NewApplicationId(1, 1);
            ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
                                                                                     );
            ContainerId containerId            = BuilderUtils.NewContainerId(appAttemptId, 1);
            ContainerAllocationExpirer expirer = Org.Mockito.Mockito.Mock <ContainerAllocationExpirer
                                                                           >();
            Resource  resource  = BuilderUtils.NewResource(512, 1);
            Priority  priority  = BuilderUtils.NewPriority(5);
            Container container = BuilderUtils.NewContainer(containerId, nodeId, "host:3465",
                                                            resource, priority, null);
            RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
                                                                          >();
            SystemMetricsPublisher publisher = Org.Mockito.Mockito.Mock <SystemMetricsPublisher
                                                                         >();
            RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>();

            Org.Mockito.Mockito.When(rmContext.GetDispatcher()).ThenReturn(drainDispatcher);
            Org.Mockito.Mockito.When(rmContext.GetContainerAllocationExpirer()).ThenReturn(expirer
                                                                                           );
            Org.Mockito.Mockito.When(rmContext.GetRMApplicationHistoryWriter()).ThenReturn(writer
                                                                                           );
            Org.Mockito.Mockito.When(rmContext.GetSystemMetricsPublisher()).ThenReturn(publisher
                                                                                       );
            Org.Mockito.Mockito.When(rmContext.GetYarnConfiguration()).ThenReturn(new YarnConfiguration
                                                                                      ());
            RMContainer rmContainer = new RMContainerImpl(container, appAttemptId, nodeId, "user"
                                                          , rmContext);

            NUnit.Framework.Assert.AreEqual(RMContainerState.New, rmContainer.GetState());
            NUnit.Framework.Assert.AreEqual(resource, rmContainer.GetAllocatedResource());
            NUnit.Framework.Assert.AreEqual(nodeId, rmContainer.GetAllocatedNode());
            NUnit.Framework.Assert.AreEqual(priority, rmContainer.GetAllocatedPriority());
            Org.Mockito.Mockito.Verify(writer).ContainerStarted(Matchers.Any <RMContainer>());
            Org.Mockito.Mockito.Verify(publisher).ContainerCreated(Matchers.Any <RMContainer>(
                                                                       ), Matchers.AnyLong());
            rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Start));
            drainDispatcher.Await();
            NUnit.Framework.Assert.AreEqual(RMContainerState.Allocated, rmContainer.GetState(
                                                ));
            rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Acquired
                                                    ));
            drainDispatcher.Await();
            NUnit.Framework.Assert.AreEqual(RMContainerState.Acquired, rmContainer.GetState()
                                            );
            rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Launched
                                                    ));
            drainDispatcher.Await();
            NUnit.Framework.Assert.AreEqual(RMContainerState.Running, rmContainer.GetState());
            NUnit.Framework.Assert.AreEqual("http://host:3465/node/containerlogs/container_1_0001_01_000001/user"
                                            , rmContainer.GetLogURL());
            // In RUNNING state. Verify EXPIRE and associated actions.
            Org.Mockito.Mockito.Reset(appAttemptEventHandler);
            ContainerStatus containerStatus = SchedulerUtils.CreateAbnormalContainerStatus(containerId
                                                                                           , SchedulerUtils.ExpiredContainer);

            rmContainer.Handle(new RMContainerFinishedEvent(containerId, containerStatus, RMContainerEventType
                                                            .Expire));
            drainDispatcher.Await();
            NUnit.Framework.Assert.AreEqual(RMContainerState.Running, rmContainer.GetState());
            Org.Mockito.Mockito.Verify(writer, Org.Mockito.Mockito.Never()).ContainerFinished
                (Matchers.Any <RMContainer>());
            Org.Mockito.Mockito.Verify(publisher, Org.Mockito.Mockito.Never()).ContainerFinished
                (Matchers.Any <RMContainer>(), Matchers.AnyLong());
        }