public virtual void Setup() { // 24h window timeWindow = 86400000L; // 1 sec step step = 1000L; // 25% avg cap on capacity avgConstraint = 25; // 70% instantaneous cap on capacity instConstraint = 70; initTime = Runtime.CurrentTimeMillis(); minAlloc = Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance(1024, 1); res = new DefaultResourceCalculator(); maxAlloc = Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance(1024 * 8, 8); mAgent = Org.Mockito.Mockito.Mock <ReservationAgent>(); ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil(); QueueMetrics rootQueueMetrics = Org.Mockito.Mockito.Mock <QueueMetrics>(); string reservationQ = testUtil.GetFullReservationQueueName(); Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = ReservationSystemTestUtil .CalculateClusterResource(totCont); ReservationSchedulerConfiguration conf = ReservationSystemTestUtil.CreateConf(reservationQ , timeWindow, instConstraint, avgConstraint); CapacityOverTimePolicy policy = new CapacityOverTimePolicy(); policy.Init(reservationQ, conf); plan = new InMemoryPlan(rootQueueMetrics, policy, mAgent, clusterResource, step, res, minAlloc, maxAlloc, "dedicated", null, true); }
protected internal virtual SharingPolicy GetAdmissionPolicy(string queueName) { ReservationSchedulerConfiguration reservationConfig = GetReservationSchedulerConfiguration (); string admissionPolicyClassName = reservationConfig.GetReservationAdmissionPolicy (queueName); Log.Info("Using AdmissionPolicy: " + admissionPolicyClassName + " for queue: " + queueName); try { Type admissionPolicyClazz = conf.GetClassByName(admissionPolicyClassName); if (typeof(SharingPolicy).IsAssignableFrom(admissionPolicyClazz)) { return((SharingPolicy)ReflectionUtils.NewInstance(admissionPolicyClazz, conf)); } else { throw new YarnRuntimeException("Class: " + admissionPolicyClassName + " not instance of " + typeof(SharingPolicy).GetCanonicalName()); } } catch (TypeLoadException e) { throw new YarnRuntimeException("Could not instantiate AdmissionPolicy: " + admissionPolicyClassName + " for queue: " + queueName, e); } }
public virtual void Setup() { long seed = rand.NextLong(); rand.SetSeed(seed); Org.Mortbay.Log.Log.Info("Running with seed: " + seed); // setting completely loose quotas long timeWindow = 1000000L; Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterCapacity = Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(100 * 1024, 100); step = 1000L; ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil(); string reservationQ = testUtil.GetFullReservationQueueName(); float instConstraint = 100; float avgConstraint = 100; ReservationSchedulerConfiguration conf = ReservationSystemTestUtil.CreateConf(reservationQ , timeWindow, instConstraint, avgConstraint); CapacityOverTimePolicy policy = new CapacityOverTimePolicy(); policy.Init(reservationQ, conf); agent = new GreedyReservationAgent(); QueueMetrics queueMetrics = Org.Mockito.Mockito.Mock <QueueMetrics>(); plan = new InMemoryPlan(queueMetrics, policy, agent, clusterCapacity, step, res, minAlloc, maxAlloc, "dedicated", null, true); }
protected internal virtual Planner GetReplanner(string planQueueName) { ReservationSchedulerConfiguration reservationConfig = GetReservationSchedulerConfiguration (); string plannerClassName = reservationConfig.GetReplanner(planQueueName); Log.Info("Using Replanner: " + plannerClassName + " for queue: " + planQueueName); try { Type plannerClazz = conf.GetClassByName(plannerClassName); if (typeof(Planner).IsAssignableFrom(plannerClazz)) { Planner planner = (Planner)ReflectionUtils.NewInstance(plannerClazz, conf); planner.Init(planQueueName, reservationConfig); return(planner); } else { throw new YarnRuntimeException("Class: " + plannerClazz + " not instance of " + typeof( Planner).GetCanonicalName()); } } catch (TypeLoadException e) { throw new YarnRuntimeException("Could not instantiate Planner: " + plannerClassName + " for queue: " + planQueueName, e); } }
// For now this is CapacityScheduler specific, but given a hierarchy in the // configuration structure of the schedulers (e.g., SchedulerConfiguration) // it should be easy to remove this limitation public virtual void Init(string reservationQueuePath, ReservationSchedulerConfiguration conf) { this.conf = conf; validWindow = this.conf.GetReservationWindow(reservationQueuePath); maxInst = this.conf.GetInstantaneousMaxCapacity(reservationQueuePath) / 100; maxAvg = this.conf.GetAverageCapacity(reservationQueuePath) / 100; }
public static ReservationSchedulerConfiguration CreateConf(string reservationQ, long timeWindow, float instConstraint, float avgConstraint) { ReservationSchedulerConfiguration conf = Org.Mockito.Mockito.Mock <ReservationSchedulerConfiguration >(); Org.Mockito.Mockito.When(conf.GetReservationWindow(reservationQ)).ThenReturn(timeWindow ); Org.Mockito.Mockito.When(conf.GetInstantaneousMaxCapacity(reservationQ)).ThenReturn (instConstraint); Org.Mockito.Mockito.When(conf.GetAverageCapacity(reservationQ)).ThenReturn(avgConstraint ); return(conf); }
/// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Reservation.Exceptions.PlanningException /// "/> /// <exception cref="System.IO.IOException"/> public virtual void TestStress(int numJobs) { long timeWindow = 1000000L; Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterCapacity = Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(500 * 100 * 1024, 500 * 32); step = 1000L; ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil(); CapacityScheduler scheduler = testUtil.MockCapacityScheduler(500 * 100); string reservationQ = testUtil.GetFullReservationQueueName(); float instConstraint = 100; float avgConstraint = 100; ReservationSchedulerConfiguration conf = ReservationSystemTestUtil.CreateConf(reservationQ , timeWindow, instConstraint, avgConstraint); CapacityOverTimePolicy policy = new CapacityOverTimePolicy(); policy.Init(reservationQ, conf); plan = new InMemoryPlan(scheduler.GetRootQueueMetrics(), policy, agent, clusterCapacity , step, res, minAlloc, maxAlloc, "dedicated", null, true); int acc = 0; IList <ReservationDefinition> list = new AList <ReservationDefinition>(); for (long i = 0; i < numJobs; i++) { list.AddItem(ReservationSystemTestUtil.GenerateRandomRR(rand, i)); } long start = Runtime.CurrentTimeMillis(); for (int i_1 = 0; i_1 < numJobs; i_1++) { try { if (agent.CreateReservation(ReservationSystemTestUtil.GetNewReservationId(), "u" + i_1 % 100, plan, list[i_1])) { acc++; } } catch (PlanningException) { } } // ignore exceptions long end = Runtime.CurrentTimeMillis(); System.Console.Out.WriteLine("Submitted " + numJobs + " jobs " + " accepted " + acc + " in " + (end - start) + "ms"); }
public virtual void Setup() { // 1 sec step step = 1000L; initTime = Runtime.CurrentTimeMillis(); minAlloc = Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance(1024, 1); res = new DefaultResourceCalculator(); maxAlloc = Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance(1024 * 8, 8); mAgent = Org.Mockito.Mockito.Mock <ReservationAgent>(); ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil(); string reservationQ = testUtil.GetFullReservationQueueName(); QueueMetrics rootQueueMetrics = Org.Mockito.Mockito.Mock <QueueMetrics>(); Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = ReservationSystemTestUtil .CalculateClusterResource(totCont); ReservationSchedulerConfiguration conf = Org.Mockito.Mockito.Mock <ReservationSchedulerConfiguration >(); NoOverCommitPolicy policy = new NoOverCommitPolicy(); policy.Init(reservationQ, conf); plan = new InMemoryPlan(rootQueueMetrics, policy, mAgent, clusterResource, step, res, minAlloc, maxAlloc, "dedicated", null, true); }
public virtual void Init(string planQueuePath, ReservationSchedulerConfiguration conf) { }
public virtual void Init(string planQueueName, ReservationSchedulerConfiguration conf) { this.lengthOfCheckZone = conf.GetEnforcementWindow(planQueueName); }
public virtual void TestReplanningPlanCapacityLoss() { Resource clusterCapacity = Resource.NewInstance(100 * 1024, 10); Resource minAlloc = Resource.NewInstance(1024, 1); Resource maxAlloc = Resource.NewInstance(1024 * 8, 8); ResourceCalculator res = new DefaultResourceCalculator(); long step = 1L; Clock clock = Org.Mockito.Mockito.Mock <Clock>(); ReservationAgent agent = Org.Mockito.Mockito.Mock <ReservationAgent>(); SharingPolicy policy = new NoOverCommitPolicy(); policy.Init("root.dedicated", null); QueueMetrics queueMetrics = Org.Mockito.Mockito.Mock <QueueMetrics>(); Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(0L); SimpleCapacityReplanner enf = new SimpleCapacityReplanner(clock); ReservationSchedulerConfiguration conf = Org.Mockito.Mockito.Mock <ReservationSchedulerConfiguration >(); Org.Mockito.Mockito.When(conf.GetEnforcementWindow(Matchers.Any <string>())).ThenReturn (6L); enf.Init("blah", conf); // Initialize the plan with more resources InMemoryPlan plan = new InMemoryPlan(queueMetrics, policy, agent, clusterCapacity , step, res, minAlloc, maxAlloc, "dedicated", enf, true, clock); // add reservation filling the plan (separating them 1ms, so we are sure // s2 follows s1 on acceptance long ts = Runtime.CurrentTimeMillis(); ReservationId r1 = ReservationId.NewInstance(ts, 1); int[] f5 = new int[] { 20, 20, 20, 20, 20 }; NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation (r1, null, "u3", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res, minAlloc))); Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(1L); ReservationId r2 = ReservationId.NewInstance(ts, 2); NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation (r2, null, "u4", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res, minAlloc))); Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(2L); ReservationId r3 = ReservationId.NewInstance(ts, 3); NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation (r3, null, "u5", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res, minAlloc))); Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(3L); ReservationId r4 = ReservationId.NewInstance(ts, 4); NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation (r4, null, "u6", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res, minAlloc))); Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(4L); ReservationId r5 = ReservationId.NewInstance(ts, 5); NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation (r5, null, "u7", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res, minAlloc))); int[] f6 = new int[] { 50, 50, 50, 50, 50 }; ReservationId r6 = ReservationId.NewInstance(ts, 6); NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation (r6, null, "u3", "dedicated", 10, 10 + f6.Length, GenerateAllocation(10, f6), res , minAlloc))); Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(6L); ReservationId r7 = ReservationId.NewInstance(ts, 7); NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation (r7, null, "u4", "dedicated", 10, 10 + f6.Length, GenerateAllocation(10, f6), res , minAlloc))); // remove some of the resources (requires replanning) plan.SetTotalCapacity(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance(70 * 1024, 70)); Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(0L); // run the replanner enf.Plan(plan, null); // check which reservation are still present NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r1)); NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r2)); NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r3)); NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r6)); NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r7)); // and which ones are removed NUnit.Framework.Assert.IsNull(plan.GetReservationById(r4)); NUnit.Framework.Assert.IsNull(plan.GetReservationById(r5)); // check resources at each moment in time no more exceed capacity for (int i = 0; i < 20; i++) { int tot = 0; foreach (ReservationAllocation r in plan.GetReservationsAtTime(i)) { tot = r.GetResourcesAtTime(i).GetMemory(); } NUnit.Framework.Assert.IsTrue(tot <= 70 * 1024); } }