public virtual void TestOrder() { PrepareBasicPlan(); // create a completely utilized segment around time 30 int[] f = new int[] { 100, 100 }; NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation (ReservationSystemTestUtil.GetNewReservationId(), null, "u1", "dedicated", 30 * step, 30 * step + f.Length * step, ReservationSystemTestUtil.GenerateAllocation( 30 * step, step, f), res, minAlloc))); // create a chain of 4 RR, mixing gang and non-gang ReservationDefinition rr = new ReservationDefinitionPBImpl(); rr.SetArrival(0 * step); rr.SetDeadline(70 * step); ReservationRequests reqs = new ReservationRequestsPBImpl(); reqs.SetInterpreter(ReservationRequestInterpreter.ROrder); ReservationRequest r = ReservationRequest.NewInstance(Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(2048, 2), 10, 1, 10 * step); ReservationRequest r2 = ReservationRequest.NewInstance(Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(1024, 1), 10, 10, 20 * step); IList <ReservationRequest> list = new AList <ReservationRequest>(); list.AddItem(r); list.AddItem(r2); list.AddItem(r); list.AddItem(r2); reqs.SetReservationResources(list); rr.SetReservationRequests(reqs); // submit to agent ReservationId reservationID = ReservationSystemTestUtil.GetNewReservationId(); agent.CreateReservation(reservationID, "u1", plan, rr); // validate NUnit.Framework.Assert.IsTrue("Agent-based allocation failed", reservationID != null ); NUnit.Framework.Assert.IsTrue("Agent-based allocation failed", plan.GetAllReservations ().Count == 4); ReservationAllocation cs = plan.GetReservationById(reservationID); NUnit.Framework.Assert.IsTrue(cs.ToString(), Check(cs, 0 * step, 10 * step, 20, 1024 , 1)); NUnit.Framework.Assert.IsTrue(cs.ToString(), Check(cs, 10 * step, 30 * step, 10, 1024, 1)); NUnit.Framework.Assert.IsTrue(cs.ToString(), Check(cs, 40 * step, 50 * step, 20, 1024, 1)); NUnit.Framework.Assert.IsTrue(cs.ToString(), Check(cs, 50 * step, 70 * step, 10, 1024, 1)); System.Console.Out.WriteLine("--------AFTER ORDER ALLOCATION (queue: " + reservationID + ")----------"); System.Console.Out.WriteLine(plan.ToString()); System.Console.Out.WriteLine(plan.ToCumulativeString()); }
public virtual void TestAddReservationAlreadyExists() { // First add a reservation Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, resCalc , minAlloc, maxAlloc, planName, replanner, true); ReservationId reservationID = ReservationSystemTestUtil.GetNewReservationId(); int[] alloc = new int[] { 10, 10, 10, 10, 10, 10 }; int start = 100; IDictionary <ReservationInterval, ReservationRequest> allocations = GenerateAllocation (start, alloc, false); ReservationDefinition rDef = CreateSimpleReservationDefinition(start, start + alloc .Length, alloc.Length, allocations.Values); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID , rDef, user, planName, start, start + alloc.Length, allocations, resCalc, minAlloc ); NUnit.Framework.Assert.IsNull(plan.GetReservationById(reservationID)); try { plan.AddReservation(rAllocation); } catch (PlanningException e) { NUnit.Framework.Assert.Fail(e.Message); } DoAssertions(plan, rAllocation); for (int i = 0; i < alloc.Length; i++) { NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance (1024 * (alloc[i]), (alloc[i])), plan.GetTotalCommittedResources(start + i)); NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance (1024 * (alloc[i]), (alloc[i])), plan.GetConsumptionForUser(user, start + i)); } // Try to add it again try { plan.AddReservation(rAllocation); NUnit.Framework.Assert.Fail("Add should fail as it already exists"); } catch (ArgumentException e) { NUnit.Framework.Assert.IsTrue(e.Message.EndsWith("already exists")); } catch (PlanningException e) { NUnit.Framework.Assert.Fail(e.Message); } DoAssertions(plan, rAllocation); }
public virtual void TestAddEmptyReservation() { Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, resCalc , minAlloc, maxAlloc, planName, replanner, true); ReservationId reservationID = ReservationSystemTestUtil.GetNewReservationId(); int[] alloc = new int[] { }; int start = 100; IDictionary <ReservationInterval, ReservationRequest> allocations = new Dictionary <ReservationInterval, ReservationRequest>(); ReservationDefinition rDef = CreateSimpleReservationDefinition(start, start + alloc .Length, alloc.Length, allocations.Values); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID , rDef, user, planName, start, start + alloc.Length, allocations, resCalc, minAlloc ); NUnit.Framework.Assert.IsNull(plan.GetReservationById(reservationID)); try { plan.AddReservation(rAllocation); } catch (PlanningException e) { NUnit.Framework.Assert.Fail(e.Message); } }
public virtual void TestSingleUserEasyFitPass() { // generate allocation that easily fit within resource constraints int[] f = GenerateData(3600, (int)Math.Ceil(0.2 * totCont)); NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation (ReservationSystemTestUtil.GetNewReservationId(), null, "u1", "dedicated", initTime , initTime + f.Length, ReservationSystemTestUtil.GenerateAllocation(initTime, step , f), res, minAlloc))); }
public virtual void TestArchiveCompletedReservations() { Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, resCalc , minAlloc, maxAlloc, planName, replanner, true); ReservationId reservationID1 = ReservationSystemTestUtil.GetNewReservationId(); // First add a reservation int[] alloc1 = new int[] { 10, 10, 10, 10, 10, 10 }; int start = 100; IDictionary <ReservationInterval, ReservationRequest> allocations1 = GenerateAllocation (start, alloc1, false); ReservationDefinition rDef1 = CreateSimpleReservationDefinition(start, start + alloc1 .Length, alloc1.Length, allocations1.Values); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID1 , rDef1, user, planName, start, start + alloc1.Length, allocations1, resCalc, minAlloc ); NUnit.Framework.Assert.IsNull(plan.GetReservationById(reservationID1)); try { plan.AddReservation(rAllocation); } catch (PlanningException e) { NUnit.Framework.Assert.Fail(e.Message); } DoAssertions(plan, rAllocation); for (int i = 0; i < alloc1.Length; i++) { NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance (1024 * (alloc1[i]), (alloc1[i])), plan.GetTotalCommittedResources(start + i)); NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance (1024 * (alloc1[i]), (alloc1[i])), plan.GetConsumptionForUser(user, start + i)); } // Now add another one ReservationId reservationID2 = ReservationSystemTestUtil.GetNewReservationId(); int[] alloc2 = new int[] { 0, 5, 10, 5, 0 }; IDictionary <ReservationInterval, ReservationRequest> allocations2 = GenerateAllocation (start, alloc2, true); ReservationDefinition rDef2 = CreateSimpleReservationDefinition(start, start + alloc2 .Length, alloc2.Length, allocations2.Values); rAllocation = new InMemoryReservationAllocation(reservationID2, rDef2, user, planName , start, start + alloc2.Length, allocations2, resCalc, minAlloc); NUnit.Framework.Assert.IsNull(plan.GetReservationById(reservationID2)); try { plan.AddReservation(rAllocation); } catch (PlanningException e) { NUnit.Framework.Assert.Fail(e.Message); } NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(reservationID2)); for (int i_1 = 0; i_1 < alloc2.Length; i_1++) { NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance (1024 * (alloc1[i_1] + alloc2[i_1] + i_1), alloc1[i_1] + alloc2[i_1] + i_1), plan .GetTotalCommittedResources(start + i_1)); NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance (1024 * (alloc1[i_1] + alloc2[i_1] + i_1), alloc1[i_1] + alloc2[i_1] + i_1), plan .GetConsumptionForUser(user, start + i_1)); } // Now archive completed reservations Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(106L); Org.Mockito.Mockito.When(policy.GetValidWindow()).ThenReturn(1L); try { // will only remove 2nd reservation as only that has fallen out of the // archival window plan.ArchiveCompletedReservations(clock.GetTime()); } catch (PlanningException e) { NUnit.Framework.Assert.Fail(e.Message); } NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(reservationID1)); NUnit.Framework.Assert.IsNull(plan.GetReservationById(reservationID2)); for (int i_2 = 0; i_2 < alloc1.Length; i_2++) { NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance (1024 * (alloc1[i_2]), (alloc1[i_2])), plan.GetTotalCommittedResources(start + i_2 )); NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance (1024 * (alloc1[i_2]), (alloc1[i_2])), plan.GetConsumptionForUser(user, start + i_2)); } Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(107L); try { // will remove 1st reservation also as it has fallen out of the archival // window plan.ArchiveCompletedReservations(clock.GetTime()); } catch (PlanningException e) { NUnit.Framework.Assert.Fail(e.Message); } NUnit.Framework.Assert.IsNull(plan.GetReservationById(reservationID1)); for (int i_3 = 0; i_3 < alloc1.Length; i_3++) { NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance (0, 0), plan.GetTotalCommittedResources(start + i_3)); NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance (0, 0), plan.GetConsumptionForUser(user, start + i_3)); } }
public virtual void TestUpdateReservation() { Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, resCalc , minAlloc, maxAlloc, planName, replanner, true); ReservationId reservationID = ReservationSystemTestUtil.GetNewReservationId(); // First add a reservation int[] alloc = new int[] { 10, 10, 10, 10, 10, 10 }; int start = 100; IDictionary <ReservationInterval, ReservationRequest> allocations = GenerateAllocation (start, alloc, false); ReservationDefinition rDef = CreateSimpleReservationDefinition(start, start + alloc .Length, alloc.Length, allocations.Values); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID , rDef, user, planName, start, start + alloc.Length, allocations, resCalc, minAlloc ); NUnit.Framework.Assert.IsNull(plan.GetReservationById(reservationID)); try { plan.AddReservation(rAllocation); } catch (PlanningException e) { NUnit.Framework.Assert.Fail(e.Message); } DoAssertions(plan, rAllocation); for (int i = 0; i < alloc.Length; i++) { NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance (1024 * (alloc[i]), (alloc[i])), plan.GetTotalCommittedResources(start + i)); NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance (1024 * (alloc[i]), (alloc[i])), plan.GetConsumptionForUser(user, start + i)); } // Now update it start = 110; int[] updatedAlloc = new int[] { 0, 5, 10, 10, 5, 0 }; allocations = GenerateAllocation(start, updatedAlloc, true); rDef = CreateSimpleReservationDefinition(start, start + updatedAlloc.Length, updatedAlloc .Length, allocations.Values); rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName , start, start + updatedAlloc.Length, allocations, resCalc, minAlloc); try { plan.UpdateReservation(rAllocation); } catch (PlanningException e) { NUnit.Framework.Assert.Fail(e.Message); } DoAssertions(plan, rAllocation); for (int i_1 = 0; i_1 < updatedAlloc.Length; i_1++) { NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance (1024 * (updatedAlloc[i_1] + i_1), updatedAlloc[i_1] + i_1), plan.GetTotalCommittedResources (start + i_1)); NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance (1024 * (updatedAlloc[i_1] + i_1), updatedAlloc[i_1] + i_1), plan.GetConsumptionForUser (user, start + i_1)); } }
public virtual void TestReplanningPlanCapacityLoss() { Resource clusterCapacity = Resource.NewInstance(100 * 1024, 10); Resource minAlloc = Resource.NewInstance(1024, 1); Resource maxAlloc = Resource.NewInstance(1024 * 8, 8); ResourceCalculator res = new DefaultResourceCalculator(); long step = 1L; Clock clock = Org.Mockito.Mockito.Mock <Clock>(); ReservationAgent agent = Org.Mockito.Mockito.Mock <ReservationAgent>(); SharingPolicy policy = new NoOverCommitPolicy(); policy.Init("root.dedicated", null); QueueMetrics queueMetrics = Org.Mockito.Mockito.Mock <QueueMetrics>(); Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(0L); SimpleCapacityReplanner enf = new SimpleCapacityReplanner(clock); ReservationSchedulerConfiguration conf = Org.Mockito.Mockito.Mock <ReservationSchedulerConfiguration >(); Org.Mockito.Mockito.When(conf.GetEnforcementWindow(Matchers.Any <string>())).ThenReturn (6L); enf.Init("blah", conf); // Initialize the plan with more resources InMemoryPlan plan = new InMemoryPlan(queueMetrics, policy, agent, clusterCapacity , step, res, minAlloc, maxAlloc, "dedicated", enf, true, clock); // add reservation filling the plan (separating them 1ms, so we are sure // s2 follows s1 on acceptance long ts = Runtime.CurrentTimeMillis(); ReservationId r1 = ReservationId.NewInstance(ts, 1); int[] f5 = new int[] { 20, 20, 20, 20, 20 }; NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation (r1, null, "u3", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res, minAlloc))); Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(1L); ReservationId r2 = ReservationId.NewInstance(ts, 2); NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation (r2, null, "u4", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res, minAlloc))); Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(2L); ReservationId r3 = ReservationId.NewInstance(ts, 3); NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation (r3, null, "u5", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res, minAlloc))); Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(3L); ReservationId r4 = ReservationId.NewInstance(ts, 4); NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation (r4, null, "u6", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res, minAlloc))); Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(4L); ReservationId r5 = ReservationId.NewInstance(ts, 5); NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation (r5, null, "u7", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res, minAlloc))); int[] f6 = new int[] { 50, 50, 50, 50, 50 }; ReservationId r6 = ReservationId.NewInstance(ts, 6); NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation (r6, null, "u3", "dedicated", 10, 10 + f6.Length, GenerateAllocation(10, f6), res , minAlloc))); Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(6L); ReservationId r7 = ReservationId.NewInstance(ts, 7); NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation (r7, null, "u4", "dedicated", 10, 10 + f6.Length, GenerateAllocation(10, f6), res , minAlloc))); // remove some of the resources (requires replanning) plan.SetTotalCapacity(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance(70 * 1024, 70)); Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(0L); // run the replanner enf.Plan(plan, null); // check which reservation are still present NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r1)); NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r2)); NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r3)); NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r6)); NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r7)); // and which ones are removed NUnit.Framework.Assert.IsNull(plan.GetReservationById(r4)); NUnit.Framework.Assert.IsNull(plan.GetReservationById(r5)); // check resources at each moment in time no more exceed capacity for (int i = 0; i < 20; i++) { int tot = 0; foreach (ReservationAllocation r in plan.GetReservationsAtTime(i)) { tot = r.GetResourcesAtTime(i).GetMemory(); } NUnit.Framework.Assert.IsTrue(tot <= 70 * 1024); } }