public virtual ReservationId GetNewReservationId() { writeLock.Lock(); try { ReservationId resId = ReservationId.NewInstance(ResourceManager.GetClusterTimeStamp (), resCounter.IncrementAndGet()); Log.Info("Allocated new reservationId: " + resId); return(resId); } finally { writeLock.Unlock(); } }
public virtual void TestZeroAlloaction() { ReservationId reservationID = ReservationId.NewInstance(rand.NextLong(), rand.NextLong ()); int[] alloc = new int[] { }; long start = 0; ReservationDefinition rDef = CreateSimpleReservationDefinition(start, start + alloc .Length + 1, alloc.Length); IDictionary <ReservationInterval, ReservationRequest> allocations = new Dictionary <ReservationInterval, ReservationRequest>(); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID , rDef, user, planName, start, start + alloc.Length + 1, allocations, resCalc, minAlloc ); DoAssertions(rAllocation, reservationID, rDef, allocations, (int)start, alloc); NUnit.Framework.Assert.IsFalse(rAllocation.ContainsGangs()); }
public virtual void TestBlocks() { ReservationId reservationID = ReservationId.NewInstance(rand.NextLong(), rand.NextLong ()); int[] alloc = new int[] { 10, 10, 10, 10, 10, 10 }; int start = 100; ReservationDefinition rDef = CreateSimpleReservationDefinition(start, start + alloc .Length + 1, alloc.Length); IDictionary <ReservationInterval, ReservationRequest> allocations = GenerateAllocation (start, alloc, false, false); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID , rDef, user, planName, start, start + alloc.Length + 1, allocations, resCalc, minAlloc ); DoAssertions(rAllocation, reservationID, rDef, allocations, start, alloc); NUnit.Framework.Assert.IsFalse(rAllocation.ContainsGangs()); for (int i = 0; i < alloc.Length; i++) { NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance (1024 * (alloc[i]), (alloc[i])), rAllocation.GetResourcesAtTime(start + i)); } }
public static ReservationId GetNewReservationId() { return(ReservationId.NewInstance(rand.NextLong(), rand.NextLong())); }
/// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Reservation.Exceptions.PlanningException /// "/> /// <exception cref="System.Exception"/> /// <exception cref="Org.Apache.Hadoop.Security.AccessControlException"/> protected internal virtual void TestPlanFollower(bool isMove) { // Initialize plan based on move flag plan = new InMemoryPlan(scheduler.GetRootQueueMetrics(), policy, mAgent, scheduler .GetClusterResource(), 1L, res, scheduler.GetMinimumResourceCapability(), maxAlloc , "dedicated", null, isMove); // add a few reservations to the plan long ts = Runtime.CurrentTimeMillis(); ReservationId r1 = ReservationId.NewInstance(ts, 1); int[] f1 = new int[] { 10, 10, 10, 10, 10 }; NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation (r1, null, "u3", "dedicated", 0, 0 + f1.Length, ReservationSystemTestUtil.GenerateAllocation (0L, 1L, f1), res, minAlloc))); ReservationId r2 = ReservationId.NewInstance(ts, 2); NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation (r2, null, "u3", "dedicated", 3, 3 + f1.Length, ReservationSystemTestUtil.GenerateAllocation (3L, 1L, f1), res, minAlloc))); ReservationId r3 = ReservationId.NewInstance(ts, 3); int[] f2 = new int[] { 0, 10, 20, 10, 0 }; NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation (r3, null, "u4", "dedicated", 10, 10 + f2.Length, ReservationSystemTestUtil.GenerateAllocation (10L, 1L, f2), res, minAlloc))); AbstractSchedulerPlanFollower planFollower = CreatePlanFollower(); Org.Mockito.Mockito.When(mClock.GetTime()).ThenReturn(0L); planFollower.Run(); Queue q = GetReservationQueue(r1.ToString()); AssertReservationQueueExists(r1); // submit an app to r1 string user_0 = "test-user"; ApplicationId appId = ApplicationId.NewInstance(0, 1); ApplicationAttemptId appAttemptId_0 = ApplicationAttemptId.NewInstance(appId, 0); AppAddedSchedulerEvent addAppEvent = new AppAddedSchedulerEvent(appId, q.GetQueueName (), user_0); scheduler.Handle(addAppEvent); AppAttemptAddedSchedulerEvent appAttemptAddedEvent = new AppAttemptAddedSchedulerEvent (appAttemptId_0, false); scheduler.Handle(appAttemptAddedEvent); // initial default reservation queue should have no apps Queue defQ = GetDefaultQueue(); NUnit.Framework.Assert.AreEqual(0, GetNumberOfApplications(defQ)); AssertReservationQueueExists(r1, 0.1, 0.1); NUnit.Framework.Assert.AreEqual(1, GetNumberOfApplications(q)); AssertReservationQueueDoesNotExist(r2); AssertReservationQueueDoesNotExist(r3); Org.Mockito.Mockito.When(mClock.GetTime()).ThenReturn(3L); planFollower.Run(); NUnit.Framework.Assert.AreEqual(0, GetNumberOfApplications(defQ)); AssertReservationQueueExists(r1, 0.1, 0.1); NUnit.Framework.Assert.AreEqual(1, GetNumberOfApplications(q)); AssertReservationQueueExists(r2, 0.1, 0.1); AssertReservationQueueDoesNotExist(r3); Org.Mockito.Mockito.When(mClock.GetTime()).ThenReturn(10L); planFollower.Run(); q = GetReservationQueue(r1.ToString()); if (isMove) { // app should have been moved to default reservation queue NUnit.Framework.Assert.AreEqual(1, GetNumberOfApplications(defQ)); NUnit.Framework.Assert.IsNull(q); } else { // app should be killed NUnit.Framework.Assert.AreEqual(0, GetNumberOfApplications(defQ)); NUnit.Framework.Assert.IsNotNull(q); AppAttemptRemovedSchedulerEvent appAttemptRemovedEvent = new AppAttemptRemovedSchedulerEvent (appAttemptId_0, RMAppAttemptState.Killed, false); scheduler.Handle(appAttemptRemovedEvent); } AssertReservationQueueDoesNotExist(r2); AssertReservationQueueExists(r3, 0, 1.0); Org.Mockito.Mockito.When(mClock.GetTime()).ThenReturn(11L); planFollower.Run(); if (isMove) { // app should have been moved to default reservation queue NUnit.Framework.Assert.AreEqual(1, GetNumberOfApplications(defQ)); } else { // app should be killed NUnit.Framework.Assert.AreEqual(0, GetNumberOfApplications(defQ)); } AssertReservationQueueDoesNotExist(r1); AssertReservationQueueDoesNotExist(r2); AssertReservationQueueExists(r3, 0.1, 0.1); Org.Mockito.Mockito.When(mClock.GetTime()).ThenReturn(12L); planFollower.Run(); AssertReservationQueueDoesNotExist(r1); AssertReservationQueueDoesNotExist(r2); AssertReservationQueueExists(r3, 0.2, 0.2); Org.Mockito.Mockito.When(mClock.GetTime()).ThenReturn(16L); planFollower.Run(); AssertReservationQueueDoesNotExist(r1); AssertReservationQueueDoesNotExist(r2); AssertReservationQueueDoesNotExist(r3); VerifyCapacity(defQ); }
public virtual void TestReplanningPlanCapacityLoss() { Resource clusterCapacity = Resource.NewInstance(100 * 1024, 10); Resource minAlloc = Resource.NewInstance(1024, 1); Resource maxAlloc = Resource.NewInstance(1024 * 8, 8); ResourceCalculator res = new DefaultResourceCalculator(); long step = 1L; Clock clock = Org.Mockito.Mockito.Mock <Clock>(); ReservationAgent agent = Org.Mockito.Mockito.Mock <ReservationAgent>(); SharingPolicy policy = new NoOverCommitPolicy(); policy.Init("root.dedicated", null); QueueMetrics queueMetrics = Org.Mockito.Mockito.Mock <QueueMetrics>(); Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(0L); SimpleCapacityReplanner enf = new SimpleCapacityReplanner(clock); ReservationSchedulerConfiguration conf = Org.Mockito.Mockito.Mock <ReservationSchedulerConfiguration >(); Org.Mockito.Mockito.When(conf.GetEnforcementWindow(Matchers.Any <string>())).ThenReturn (6L); enf.Init("blah", conf); // Initialize the plan with more resources InMemoryPlan plan = new InMemoryPlan(queueMetrics, policy, agent, clusterCapacity , step, res, minAlloc, maxAlloc, "dedicated", enf, true, clock); // add reservation filling the plan (separating them 1ms, so we are sure // s2 follows s1 on acceptance long ts = Runtime.CurrentTimeMillis(); ReservationId r1 = ReservationId.NewInstance(ts, 1); int[] f5 = new int[] { 20, 20, 20, 20, 20 }; NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation (r1, null, "u3", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res, minAlloc))); Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(1L); ReservationId r2 = ReservationId.NewInstance(ts, 2); NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation (r2, null, "u4", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res, minAlloc))); Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(2L); ReservationId r3 = ReservationId.NewInstance(ts, 3); NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation (r3, null, "u5", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res, minAlloc))); Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(3L); ReservationId r4 = ReservationId.NewInstance(ts, 4); NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation (r4, null, "u6", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res, minAlloc))); Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(4L); ReservationId r5 = ReservationId.NewInstance(ts, 5); NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation (r5, null, "u7", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res, minAlloc))); int[] f6 = new int[] { 50, 50, 50, 50, 50 }; ReservationId r6 = ReservationId.NewInstance(ts, 6); NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation (r6, null, "u3", "dedicated", 10, 10 + f6.Length, GenerateAllocation(10, f6), res , minAlloc))); Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(6L); ReservationId r7 = ReservationId.NewInstance(ts, 7); NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation (r7, null, "u4", "dedicated", 10, 10 + f6.Length, GenerateAllocation(10, f6), res , minAlloc))); // remove some of the resources (requires replanning) plan.SetTotalCapacity(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance(70 * 1024, 70)); Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(0L); // run the replanner enf.Plan(plan, null); // check which reservation are still present NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r1)); NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r2)); NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r3)); NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r6)); NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r7)); // and which ones are removed NUnit.Framework.Assert.IsNull(plan.GetReservationById(r4)); NUnit.Framework.Assert.IsNull(plan.GetReservationById(r5)); // check resources at each moment in time no more exceed capacity for (int i = 0; i < 20; i++) { int tot = 0; foreach (ReservationAllocation r in plan.GetReservationsAtTime(i)) { tot = r.GetResourcesAtTime(i).GetMemory(); } NUnit.Framework.Assert.IsTrue(tot <= 70 * 1024); } }