/// <exception cref="System.IO.IOException"/> public PlanQueue(CapacitySchedulerContext cs, string queueName, CSQueue parent, CSQueue old) : base(cs, queueName, parent, old) { this.schedulerContext = cs; // Set the reservation queue attributes for the Plan CapacitySchedulerConfiguration conf = cs.GetConfiguration(); string queuePath = base.GetQueuePath(); int maxAppsForReservation = conf.GetMaximumApplicationsPerQueue(queuePath); showReservationsAsQueues = conf.GetShowReservationAsQueues(queuePath); if (maxAppsForReservation < 0) { maxAppsForReservation = (int)(CapacitySchedulerConfiguration.DefaultMaximumSystemApplicatiions * base.GetAbsoluteCapacity()); } int userLimit = conf.GetUserLimit(queuePath); float userLimitFactor = conf.GetUserLimitFactor(queuePath); int maxAppsPerUserForReservation = (int)(maxAppsForReservation * (userLimit / 100.0f ) * userLimitFactor); UpdateQuotas(userLimit, userLimitFactor, maxAppsForReservation, maxAppsPerUserForReservation ); StringBuilder queueInfo = new StringBuilder(); queueInfo.Append("Created Plan Queue: ").Append(queueName).Append("\nwith capacity: [" ).Append(base.GetCapacity()).Append("]\nwith max capacity: [").Append(base.GetMaximumCapacity ()).Append("\nwith max reservation apps: [").Append(maxAppsForReservation).Append ("]\nwith max reservation apps per user: [").Append(maxAppsPerUserForReservation ).Append("]\nwith user limit: [").Append(userLimit).Append("]\nwith user limit factor: [" ).Append(userLimitFactor).Append("]."); Log.Info(queueInfo.ToString()); }
public virtual void TestLimitsComputation() { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); SetupQueueConfiguration(csConf); YarnConfiguration conf = new YarnConfiguration(); CapacitySchedulerContext csContext = Org.Mockito.Mockito.Mock <CapacitySchedulerContext >(); Org.Mockito.Mockito.When(csContext.GetConfiguration()).ThenReturn(csConf); Org.Mockito.Mockito.When(csContext.GetConf()).ThenReturn(conf); Org.Mockito.Mockito.When(csContext.GetMinimumResourceCapability()).ThenReturn(Resources .CreateResource(Gb, 1)); Org.Mockito.Mockito.When(csContext.GetMaximumResourceCapability()).ThenReturn(Resources .CreateResource(16 * Gb, 16)); Org.Mockito.Mockito.When(csContext.GetApplicationComparator()).ThenReturn(CapacityScheduler .applicationComparator); Org.Mockito.Mockito.When(csContext.GetQueueComparator()).ThenReturn(CapacityScheduler .queueComparator); Org.Mockito.Mockito.When(csContext.GetResourceCalculator()).ThenReturn(resourceCalculator ); Org.Mockito.Mockito.When(csContext.GetRMContext()).ThenReturn(rmContext); // Say cluster has 100 nodes of 16G each Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = Resources.CreateResource (100 * 16 * Gb, 100 * 16); Org.Mockito.Mockito.When(csContext.GetClusterResource()).ThenReturn(clusterResource ); IDictionary <string, CSQueue> queues = new Dictionary <string, CSQueue>(); CSQueue root = CapacityScheduler.ParseQueue(csContext, csConf, null, "root", queues , queues, TestUtils.spyHook); LeafQueue queue = (LeafQueue)queues[A]; Log.Info("Queue 'A' -" + " AMResourceLimit=" + queue.GetAMResourceLimit() + " UserAMResourceLimit=" + queue.GetUserAMResourceLimit()); NUnit.Framework.Assert.AreEqual(queue.GetAMResourceLimit(), Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(160 * Gb, 1)); NUnit.Framework.Assert.AreEqual(queue.GetUserAMResourceLimit(), Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(80 * Gb, 1)); NUnit.Framework.Assert.AreEqual((int)(clusterResource.GetMemory() * queue.GetAbsoluteCapacity ()), queue.GetMetrics().GetAvailableMB()); // Add some nodes to the cluster & test new limits clusterResource = Resources.CreateResource(120 * 16 * Gb); root.UpdateClusterResource(clusterResource, new ResourceLimits(clusterResource)); NUnit.Framework.Assert.AreEqual(queue.GetAMResourceLimit(), Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(192 * Gb, 1)); NUnit.Framework.Assert.AreEqual(queue.GetUserAMResourceLimit(), Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(96 * Gb, 1)); NUnit.Framework.Assert.AreEqual((int)(clusterResource.GetMemory() * queue.GetAbsoluteCapacity ()), queue.GetMetrics().GetAvailableMB()); // should return -1 if per queue setting not set NUnit.Framework.Assert.AreEqual((int)CapacitySchedulerConfiguration.Undefined, csConf .GetMaximumApplicationsPerQueue(queue.GetQueuePath())); int expectedMaxApps = (int)(CapacitySchedulerConfiguration.DefaultMaximumSystemApplicatiions * queue.GetAbsoluteCapacity()); NUnit.Framework.Assert.AreEqual(expectedMaxApps, queue.GetMaxApplications()); int expectedMaxAppsPerUser = (int)(expectedMaxApps * (queue.GetUserLimit() / 100.0f ) * queue.GetUserLimitFactor()); NUnit.Framework.Assert.AreEqual(expectedMaxAppsPerUser, queue.GetMaxApplicationsPerUser ()); // should default to global setting if per queue setting not set NUnit.Framework.Assert.AreEqual((long)CapacitySchedulerConfiguration.DefaultMaximumApplicationmastersResourcePercent , (long)csConf.GetMaximumApplicationMasterResourcePerQueuePercent(queue.GetQueuePath ())); // Change the per-queue max AM resources percentage. csConf.SetFloat("yarn.scheduler.capacity." + queue.GetQueuePath() + ".maximum-am-resource-percent" , 0.5f); // Re-create queues to get new configs. queues = new Dictionary <string, CSQueue>(); root = CapacityScheduler.ParseQueue(csContext, csConf, null, "root", queues, queues , TestUtils.spyHook); clusterResource = Resources.CreateResource(100 * 16 * Gb); queue = (LeafQueue)queues[A]; NUnit.Framework.Assert.AreEqual((long)0.5, (long)csConf.GetMaximumApplicationMasterResourcePerQueuePercent (queue.GetQueuePath())); NUnit.Framework.Assert.AreEqual(queue.GetAMResourceLimit(), Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(800 * Gb, 1)); NUnit.Framework.Assert.AreEqual(queue.GetUserAMResourceLimit(), Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(400 * Gb, 1)); // Change the per-queue max applications. csConf.SetInt("yarn.scheduler.capacity." + queue.GetQueuePath() + ".maximum-applications" , 9999); // Re-create queues to get new configs. queues = new Dictionary <string, CSQueue>(); root = CapacityScheduler.ParseQueue(csContext, csConf, null, "root", queues, queues , TestUtils.spyHook); queue = (LeafQueue)queues[A]; NUnit.Framework.Assert.AreEqual(9999, (int)csConf.GetMaximumApplicationsPerQueue( queue.GetQueuePath())); NUnit.Framework.Assert.AreEqual(9999, queue.GetMaxApplications()); expectedMaxAppsPerUser = (int)(9999 * (queue.GetUserLimit() / 100.0f) * queue.GetUserLimitFactor ()); NUnit.Framework.Assert.AreEqual(expectedMaxAppsPerUser, queue.GetMaxApplicationsPerUser ()); }