示例#1
0
        public virtual void TestUpdateMaxAllocationUsesTotal()
        {
            int configuredMaxVCores = 20;
            int configuredMaxMemory = 10 * 1024;

            Org.Apache.Hadoop.Yarn.Api.Records.Resource configuredMaximumResource = Org.Apache.Hadoop.Yarn.Api.Records.Resource
                                                                                    .NewInstance(configuredMaxMemory, configuredMaxVCores);
            ConfigureScheduler();
            YarnConfiguration conf = GetConf();

            conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationVcores, configuredMaxVCores
                        );
            conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationMb, configuredMaxMemory
                        );
            conf.SetLong(YarnConfiguration.RmWorkPreservingRecoverySchedulingWaitMs, 0);
            MockRM rm = new MockRM(conf);

            try
            {
                rm.Start();
                AbstractYarnScheduler scheduler = (AbstractYarnScheduler)rm.GetResourceScheduler(
                    );
                Org.Apache.Hadoop.Yarn.Api.Records.Resource emptyResource = Org.Apache.Hadoop.Yarn.Api.Records.Resource
                                                                            .NewInstance(0, 0);
                Org.Apache.Hadoop.Yarn.Api.Records.Resource fullResource1 = Org.Apache.Hadoop.Yarn.Api.Records.Resource
                                                                            .NewInstance(1024, 5);
                Org.Apache.Hadoop.Yarn.Api.Records.Resource fullResource2 = Org.Apache.Hadoop.Yarn.Api.Records.Resource
                                                                            .NewInstance(2048, 10);
                SchedulerNode mockNode1 = Org.Mockito.Mockito.Mock <SchedulerNode>();
                Org.Mockito.Mockito.When(mockNode1.GetNodeID()).ThenReturn(NodeId.NewInstance("foo"
                                                                                              , 8080));
                Org.Mockito.Mockito.When(mockNode1.GetAvailableResource()).ThenReturn(emptyResource
                                                                                      );
                Org.Mockito.Mockito.When(mockNode1.GetTotalResource()).ThenReturn(fullResource1);
                SchedulerNode mockNode2 = Org.Mockito.Mockito.Mock <SchedulerNode>();
                Org.Mockito.Mockito.When(mockNode1.GetNodeID()).ThenReturn(NodeId.NewInstance("bar"
                                                                                              , 8081));
                Org.Mockito.Mockito.When(mockNode2.GetAvailableResource()).ThenReturn(emptyResource
                                                                                      );
                Org.Mockito.Mockito.When(mockNode2.GetTotalResource()).ThenReturn(fullResource2);
                VerifyMaximumResourceCapability(configuredMaximumResource, scheduler);
                scheduler.nodes = new Dictionary <NodeId, SchedulerNode>();
                scheduler.nodes[mockNode1.GetNodeID()] = mockNode1;
                scheduler.UpdateMaximumAllocation(mockNode1, true);
                VerifyMaximumResourceCapability(fullResource1, scheduler);
                scheduler.nodes[mockNode2.GetNodeID()] = mockNode2;
                scheduler.UpdateMaximumAllocation(mockNode2, true);
                VerifyMaximumResourceCapability(fullResource2, scheduler);
                Sharpen.Collections.Remove(scheduler.nodes, mockNode2.GetNodeID());
                scheduler.UpdateMaximumAllocation(mockNode2, false);
                VerifyMaximumResourceCapability(fullResource1, scheduler);
                Sharpen.Collections.Remove(scheduler.nodes, mockNode1.GetNodeID());
                scheduler.UpdateMaximumAllocation(mockNode1, false);
                VerifyMaximumResourceCapability(configuredMaximumResource, scheduler);
            }
            finally
            {
                rm.Stop();
            }
        }
        public virtual void TestMove()
        {
            string               user          = "******";
            Queue                parentQueue   = CreateQueue("parent", null);
            Queue                oldQueue      = CreateQueue("old", parentQueue);
            Queue                newQueue      = CreateQueue("new", parentQueue);
            QueueMetrics         parentMetrics = parentQueue.GetMetrics();
            QueueMetrics         oldMetrics    = oldQueue.GetMetrics();
            QueueMetrics         newMetrics    = newQueue.GetMetrics();
            ApplicationAttemptId appAttId      = CreateAppAttemptId(0, 0);
            RMContext            rmContext     = Org.Mockito.Mockito.Mock <RMContext>();

            Org.Mockito.Mockito.When(rmContext.GetEpoch()).ThenReturn(3L);
            SchedulerApplicationAttempt app = new SchedulerApplicationAttempt(appAttId, user,
                                                                              oldQueue, oldQueue.GetActiveUsersManager(), rmContext);

            oldMetrics.SubmitApp(user);
            // confirm that containerId is calculated based on epoch.
            NUnit.Framework.Assert.AreEqual(unchecked ((long)(0x30000000001L)), app.GetNewContainerId
                                                ());
            // Resource request
            Resource        requestedResource = Resource.NewInstance(1536, 2);
            Priority        requestedPriority = Priority.NewInstance(2);
            ResourceRequest request           = ResourceRequest.NewInstance(requestedPriority, ResourceRequest
                                                                            .Any, requestedResource, 3);

            app.UpdateResourceRequests(Arrays.AsList(request));
            // Allocated container
            RMContainer container1 = CreateRMContainer(appAttId, 1, requestedResource);

            app.liveContainers[container1.GetContainerId()] = container1;
            SchedulerNode node = CreateNode();

            app.appSchedulingInfo.Allocate(NodeType.OffSwitch, node, requestedPriority, request
                                           , container1.GetContainer());
            // Reserved container
            Priority    prio1            = Priority.NewInstance(1);
            Resource    reservedResource = Resource.NewInstance(2048, 3);
            RMContainer container2       = CreateReservedRMContainer(appAttId, 1, reservedResource,
                                                                     node.GetNodeID(), prio1);
            IDictionary <NodeId, RMContainer> reservations = new Dictionary <NodeId, RMContainer
                                                                             >();

            reservations[node.GetNodeID()] = container2;
            app.reservedContainers[prio1]  = reservations;
            oldMetrics.ReserveResource(user, reservedResource);
            CheckQueueMetrics(oldMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4);
            CheckQueueMetrics(newMetrics, 0, 0, 0, 0, 0, 0, 0, 0);
            CheckQueueMetrics(parentMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4);
            app.Move(newQueue);
            CheckQueueMetrics(oldMetrics, 0, 0, 0, 0, 0, 0, 0, 0);
            CheckQueueMetrics(newMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4);
            CheckQueueMetrics(parentMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4);
        }
        private SchedulerNode CreateNode()
        {
            SchedulerNode node = Org.Mockito.Mockito.Mock <SchedulerNode>();

            Org.Mockito.Mockito.When(node.GetNodeName()).ThenReturn("somehost");
            Org.Mockito.Mockito.When(node.GetRackName()).ThenReturn("somerack");
            Org.Mockito.Mockito.When(node.GetNodeID()).ThenReturn(nodeId);
            return(node);
        }
 protected internal virtual void ContainerLaunchedOnNode(ContainerId containerId,
                                                         SchedulerNode node)
 {
     lock (this)
     {
         // Get the application for the finished container
         SchedulerApplicationAttempt application = GetCurrentAttemptForContainer(containerId
                                                                                 );
         if (application == null)
         {
             Log.Info("Unknown application " + containerId.GetApplicationAttemptId().GetApplicationId
                          () + " launched container " + containerId + " on node: " + node);
             this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMNodeCleanContainerEvent
                                                                         (node.GetNodeID(), containerId));
             return;
         }
         application.ContainerLaunchedOnNode(containerId, node.GetNodeID());
     }
 }
示例#5
0
 /// <summary>
 /// Has the application reserved the given <code>node</code> at the
 /// given <code>priority</code>?
 /// </summary>
 /// <param name="node">node to be checked</param>
 /// <param name="priority">priority of reserved container</param>
 /// <returns>true is reserved, false if not</returns>
 public virtual bool IsReserved(SchedulerNode node, Priority priority)
 {
     lock (this)
     {
         IDictionary <NodeId, RMContainer> reservedContainers = this.reservedContainers[priority
                                                                ];
         if (reservedContainers != null)
         {
             return(reservedContainers.Contains(node.GetNodeID()));
         }
         return(false);
     }
 }
示例#6
0
 public virtual RMContainer Reserve(SchedulerNode node, Priority priority, RMContainer
                                    rmContainer, Container container)
 {
     lock (this)
     {
         // Create RMContainer if necessary
         if (rmContainer == null)
         {
             rmContainer = new RMContainerImpl(container, GetApplicationAttemptId(), node.GetNodeID
                                                   (), appSchedulingInfo.GetUser(), rmContext);
             Resources.AddTo(currentReservation, container.GetResource());
             // Reset the re-reservation count
             ResetReReservations(priority);
         }
         else
         {
             // Note down the re-reservation
             AddReReservation(priority);
         }
         rmContainer.Handle(new RMContainerReservedEvent(container.GetId(), container.GetResource
                                                             (), node.GetNodeID(), priority));
         IDictionary <NodeId, RMContainer> reservedContainers = this.reservedContainers[priority
                                                                ];
         if (reservedContainers == null)
         {
             reservedContainers = new Dictionary <NodeId, RMContainer>();
             this.reservedContainers[priority] = reservedContainers;
         }
         reservedContainers[node.GetNodeID()] = rmContainer;
         if (Log.IsDebugEnabled())
         {
             Log.Debug("Application attempt " + GetApplicationAttemptId() + " reserved container "
                       + rmContainer + " on node " + node + ". This attempt currently has " + reservedContainers
                       .Count + " reserved containers at priority " + priority + "; currentReservation "
                       + currentReservation.GetMemory());
         }
         return(rmContainer);
     }
 }