public virtual void TestNormalizeRequest() { ResourceCalculator resourceCalculator = new DefaultResourceCalculator(); int minMemory = 1024; int maxMemory = 8192; Org.Apache.Hadoop.Yarn.Api.Records.Resource minResource = Resources.CreateResource (minMemory, 0); Org.Apache.Hadoop.Yarn.Api.Records.Resource maxResource = Resources.CreateResource (maxMemory, 0); ResourceRequest ask = new ResourceRequestPBImpl(); // case negative memory ask.SetCapability(Resources.CreateResource(-1024)); SchedulerUtils.NormalizeRequest(ask, resourceCalculator, null, minResource, maxResource ); NUnit.Framework.Assert.AreEqual(minMemory, ask.GetCapability().GetMemory()); // case zero memory ask.SetCapability(Resources.CreateResource(0)); SchedulerUtils.NormalizeRequest(ask, resourceCalculator, null, minResource, maxResource ); NUnit.Framework.Assert.AreEqual(minMemory, ask.GetCapability().GetMemory()); // case memory is a multiple of minMemory ask.SetCapability(Resources.CreateResource(2 * minMemory)); SchedulerUtils.NormalizeRequest(ask, resourceCalculator, null, minResource, maxResource ); NUnit.Framework.Assert.AreEqual(2 * minMemory, ask.GetCapability().GetMemory()); // case memory is not a multiple of minMemory ask.SetCapability(Resources.CreateResource(minMemory + 10)); SchedulerUtils.NormalizeRequest(ask, resourceCalculator, null, minResource, maxResource ); NUnit.Framework.Assert.AreEqual(2 * minMemory, ask.GetCapability().GetMemory()); // case memory is equal to max allowed ask.SetCapability(Resources.CreateResource(maxMemory)); SchedulerUtils.NormalizeRequest(ask, resourceCalculator, null, minResource, maxResource ); NUnit.Framework.Assert.AreEqual(maxMemory, ask.GetCapability().GetMemory()); // case memory is just less than max ask.SetCapability(Resources.CreateResource(maxMemory - 10)); SchedulerUtils.NormalizeRequest(ask, resourceCalculator, null, minResource, maxResource ); NUnit.Framework.Assert.AreEqual(maxMemory, ask.GetCapability().GetMemory()); // max is not a multiple of min maxResource = Resources.CreateResource(maxMemory - 10, 0); ask.SetCapability(Resources.CreateResource(maxMemory - 100)); // multiple of minMemory > maxMemory, then reduce to maxMemory SchedulerUtils.NormalizeRequest(ask, resourceCalculator, null, minResource, maxResource ); NUnit.Framework.Assert.AreEqual(maxResource.GetMemory(), ask.GetCapability().GetMemory ()); // ask is more than max maxResource = Resources.CreateResource(maxMemory, 0); ask.SetCapability(Resources.CreateResource(maxMemory + 100)); SchedulerUtils.NormalizeRequest(ask, resourceCalculator, null, minResource, maxResource ); NUnit.Framework.Assert.AreEqual(maxResource.GetMemory(), ask.GetCapability().GetMemory ()); }
private static ResourceScheduler MockResourceScheduler() { ResourceScheduler scheduler = Org.Mockito.Mockito.Mock <ResourceScheduler>(); Org.Mockito.Mockito.When(scheduler.GetMinimumResourceCapability()).ThenReturn(Resources .CreateResource(YarnConfiguration.DefaultRmSchedulerMinimumAllocationMb)); Org.Mockito.Mockito.When(scheduler.GetMaximumResourceCapability()).ThenReturn(Resources .CreateResource(YarnConfiguration.DefaultRmSchedulerMaximumAllocationMb)); ResourceCalculator rc = new DefaultResourceCalculator(); Org.Mockito.Mockito.When(scheduler.GetResourceCalculator()).ThenReturn(rc); return(scheduler); }
public virtual void TestZeroAlloaction() { ResourceCalculator resCalc = new DefaultResourceCalculator(); Org.Apache.Hadoop.Yarn.Api.Records.Resource minAlloc = Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(1, 1); RLESparseResourceAllocation rleSparseVector = new RLESparseResourceAllocation(resCalc , minAlloc); rleSparseVector.AddInterval(new ReservationInterval(0, long.MaxValue), ReservationRequest .NewInstance(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance(0, 0), (0)) ); Log.Info(rleSparseVector.ToString()); NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance (0, 0), rleSparseVector.GetCapacityAtTime(new Random().NextLong())); NUnit.Framework.Assert.IsTrue(rleSparseVector.IsEmpty()); }
public virtual void SetUp() { clock = Org.Mockito.Mockito.Mock <Clock>(); plan = Org.Mockito.Mockito.Mock <Plan>(); rSystem = Org.Mockito.Mockito.Mock <ReservationSystem>(); plans[PlanName] = plan; rrValidator = new ReservationInputValidator(clock); Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(1L); ResourceCalculator rCalc = new DefaultResourceCalculator(); Org.Apache.Hadoop.Yarn.Api.Records.Resource resource = Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(10240, 10); Org.Mockito.Mockito.When(plan.GetResourceCalculator()).ThenReturn(rCalc); Org.Mockito.Mockito.When(plan.GetTotalCapacity()).ThenReturn(resource); Org.Mockito.Mockito.When(rSystem.GetQueueForReservation(Matchers.Any <ReservationId >())).ThenReturn(PlanName); Org.Mockito.Mockito.When(rSystem.GetPlan(PlanName)).ThenReturn(plan); }
public virtual void TestSteps() { ResourceCalculator resCalc = new DefaultResourceCalculator(); Org.Apache.Hadoop.Yarn.Api.Records.Resource minAlloc = Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(1, 1); RLESparseResourceAllocation rleSparseVector = new RLESparseResourceAllocation(resCalc , minAlloc); int[] alloc = new int[] { 10, 10, 10, 10, 10, 10 }; int start = 100; ICollection <KeyValuePair <ReservationInterval, ReservationRequest> > inputs = GenerateAllocation (start, alloc, true); foreach (KeyValuePair <ReservationInterval, ReservationRequest> ip in inputs) { rleSparseVector.AddInterval(ip.Key, ip.Value); } Log.Info(rleSparseVector.ToString()); NUnit.Framework.Assert.IsFalse(rleSparseVector.IsEmpty()); NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance (0, 0), rleSparseVector.GetCapacityAtTime(99)); NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance (0, 0), rleSparseVector.GetCapacityAtTime(start + alloc.Length + 1)); for (int i = 0; i < alloc.Length; i++) { NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance (1024 * (alloc[i] + i), (alloc[i] + i)), rleSparseVector.GetCapacityAtTime(start + i)); } NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance (0, 0), rleSparseVector.GetCapacityAtTime(start + alloc.Length + 2)); foreach (KeyValuePair <ReservationInterval, ReservationRequest> ip_1 in inputs) { rleSparseVector.RemoveInterval(ip_1.Key, ip_1.Value); } Log.Info(rleSparseVector.ToString()); for (int i_1 = 0; i_1 < alloc.Length; i_1++) { NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance (0, 0), rleSparseVector.GetCapacityAtTime(start + i_1)); } NUnit.Framework.Assert.IsTrue(rleSparseVector.IsEmpty()); }
public virtual void TestReplanningPlanCapacityLoss() { Resource clusterCapacity = Resource.NewInstance(100 * 1024, 10); Resource minAlloc = Resource.NewInstance(1024, 1); Resource maxAlloc = Resource.NewInstance(1024 * 8, 8); ResourceCalculator res = new DefaultResourceCalculator(); long step = 1L; Clock clock = Org.Mockito.Mockito.Mock <Clock>(); ReservationAgent agent = Org.Mockito.Mockito.Mock <ReservationAgent>(); SharingPolicy policy = new NoOverCommitPolicy(); policy.Init("root.dedicated", null); QueueMetrics queueMetrics = Org.Mockito.Mockito.Mock <QueueMetrics>(); Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(0L); SimpleCapacityReplanner enf = new SimpleCapacityReplanner(clock); ReservationSchedulerConfiguration conf = Org.Mockito.Mockito.Mock <ReservationSchedulerConfiguration >(); Org.Mockito.Mockito.When(conf.GetEnforcementWindow(Matchers.Any <string>())).ThenReturn (6L); enf.Init("blah", conf); // Initialize the plan with more resources InMemoryPlan plan = new InMemoryPlan(queueMetrics, policy, agent, clusterCapacity , step, res, minAlloc, maxAlloc, "dedicated", enf, true, clock); // add reservation filling the plan (separating them 1ms, so we are sure // s2 follows s1 on acceptance long ts = Runtime.CurrentTimeMillis(); ReservationId r1 = ReservationId.NewInstance(ts, 1); int[] f5 = new int[] { 20, 20, 20, 20, 20 }; NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation (r1, null, "u3", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res, minAlloc))); Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(1L); ReservationId r2 = ReservationId.NewInstance(ts, 2); NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation (r2, null, "u4", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res, minAlloc))); Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(2L); ReservationId r3 = ReservationId.NewInstance(ts, 3); NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation (r3, null, "u5", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res, minAlloc))); Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(3L); ReservationId r4 = ReservationId.NewInstance(ts, 4); NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation (r4, null, "u6", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res, minAlloc))); Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(4L); ReservationId r5 = ReservationId.NewInstance(ts, 5); NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation (r5, null, "u7", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res, minAlloc))); int[] f6 = new int[] { 50, 50, 50, 50, 50 }; ReservationId r6 = ReservationId.NewInstance(ts, 6); NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation (r6, null, "u3", "dedicated", 10, 10 + f6.Length, GenerateAllocation(10, f6), res , minAlloc))); Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(6L); ReservationId r7 = ReservationId.NewInstance(ts, 7); NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation (r7, null, "u4", "dedicated", 10, 10 + f6.Length, GenerateAllocation(10, f6), res , minAlloc))); // remove some of the resources (requires replanning) plan.SetTotalCapacity(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance(70 * 1024, 70)); Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(0L); // run the replanner enf.Plan(plan, null); // check which reservation are still present NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r1)); NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r2)); NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r3)); NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r6)); NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r7)); // and which ones are removed NUnit.Framework.Assert.IsNull(plan.GetReservationById(r4)); NUnit.Framework.Assert.IsNull(plan.GetReservationById(r5)); // check resources at each moment in time no more exceed capacity for (int i = 0; i < 20; i++) { int tot = 0; foreach (ReservationAllocation r in plan.GetReservationsAtTime(i)) { tot = r.GetResourcesAtTime(i).GetMemory(); } NUnit.Framework.Assert.IsTrue(tot <= 70 * 1024); } }