public virtual void Setup()
{
// 24h window
timeWindow = 86400000L;
// 1 sec step
step = 1000L;
// 25% avg cap on capacity
avgConstraint = 25;
// 70% instantaneous cap on capacity
instConstraint = 70;
initTime = Runtime.CurrentTimeMillis();
minAlloc = Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance(1024, 1);
res = new DefaultResourceCalculator();
maxAlloc = Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance(1024 * 8, 8);
mAgent = Org.Mockito.Mockito.Mock <ReservationAgent>();
ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
QueueMetrics rootQueueMetrics = Org.Mockito.Mockito.Mock <QueueMetrics>();
string reservationQ = testUtil.GetFullReservationQueueName();
Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = ReservationSystemTestUtil
.CalculateClusterResource(totCont);
ReservationSchedulerConfiguration conf = ReservationSystemTestUtil.CreateConf(reservationQ
, timeWindow, instConstraint, avgConstraint);
CapacityOverTimePolicy policy = new CapacityOverTimePolicy();
policy.Init(reservationQ, conf);
plan = new InMemoryPlan(rootQueueMetrics, policy, mAgent, clusterResource, step,
res, minAlloc, maxAlloc, "dedicated", null, true);
}
protected internal virtual SharingPolicy GetAdmissionPolicy(string queueName)
{
ReservationSchedulerConfiguration reservationConfig = GetReservationSchedulerConfiguration
();
string admissionPolicyClassName = reservationConfig.GetReservationAdmissionPolicy
(queueName);
Log.Info("Using AdmissionPolicy: " + admissionPolicyClassName + " for queue: " +
queueName);
try
{
Type admissionPolicyClazz = conf.GetClassByName(admissionPolicyClassName);
if (typeof(SharingPolicy).IsAssignableFrom(admissionPolicyClazz))
{
return((SharingPolicy)ReflectionUtils.NewInstance(admissionPolicyClazz, conf));
}
else
{
throw new YarnRuntimeException("Class: " + admissionPolicyClassName + " not instance of "
+ typeof(SharingPolicy).GetCanonicalName());
}
}
catch (TypeLoadException e)
{
throw new YarnRuntimeException("Could not instantiate AdmissionPolicy: " + admissionPolicyClassName
+ " for queue: " + queueName, e);
}
}
public virtual void Setup()
{
long seed = rand.NextLong();
rand.SetSeed(seed);
Org.Mortbay.Log.Log.Info("Running with seed: " + seed);
// setting completely loose quotas
long timeWindow = 1000000L;
Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterCapacity = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(100 * 1024, 100);
step = 1000L;
ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
string reservationQ = testUtil.GetFullReservationQueueName();
float instConstraint = 100;
float avgConstraint = 100;
ReservationSchedulerConfiguration conf = ReservationSystemTestUtil.CreateConf(reservationQ
, timeWindow, instConstraint, avgConstraint);
CapacityOverTimePolicy policy = new CapacityOverTimePolicy();
policy.Init(reservationQ, conf);
agent = new GreedyReservationAgent();
QueueMetrics queueMetrics = Org.Mockito.Mockito.Mock <QueueMetrics>();
plan = new InMemoryPlan(queueMetrics, policy, agent, clusterCapacity, step, res,
minAlloc, maxAlloc, "dedicated", null, true);
}
protected internal virtual Planner GetReplanner(string planQueueName)
{
ReservationSchedulerConfiguration reservationConfig = GetReservationSchedulerConfiguration
();
string plannerClassName = reservationConfig.GetReplanner(planQueueName);
Log.Info("Using Replanner: " + plannerClassName + " for queue: " + planQueueName);
try
{
Type plannerClazz = conf.GetClassByName(plannerClassName);
if (typeof(Planner).IsAssignableFrom(plannerClazz))
{
Planner planner = (Planner)ReflectionUtils.NewInstance(plannerClazz, conf);
planner.Init(planQueueName, reservationConfig);
return(planner);
}
else
{
throw new YarnRuntimeException("Class: " + plannerClazz + " not instance of " + typeof(
Planner).GetCanonicalName());
}
}
catch (TypeLoadException e)
{
throw new YarnRuntimeException("Could not instantiate Planner: " + plannerClassName
+ " for queue: " + planQueueName, e);
}
}
// For now this is CapacityScheduler specific, but given a hierarchy in the
// configuration structure of the schedulers (e.g., SchedulerConfiguration)
// it should be easy to remove this limitation
public virtual void Init(string reservationQueuePath, ReservationSchedulerConfiguration
conf)
{
this.conf = conf;
validWindow = this.conf.GetReservationWindow(reservationQueuePath);
maxInst = this.conf.GetInstantaneousMaxCapacity(reservationQueuePath) / 100;
maxAvg = this.conf.GetAverageCapacity(reservationQueuePath) / 100;
}
public static ReservationSchedulerConfiguration CreateConf(string reservationQ, long
timeWindow, float instConstraint, float avgConstraint)
{
ReservationSchedulerConfiguration conf = Org.Mockito.Mockito.Mock <ReservationSchedulerConfiguration
>();
Org.Mockito.Mockito.When(conf.GetReservationWindow(reservationQ)).ThenReturn(timeWindow
);
Org.Mockito.Mockito.When(conf.GetInstantaneousMaxCapacity(reservationQ)).ThenReturn
(instConstraint);
Org.Mockito.Mockito.When(conf.GetAverageCapacity(reservationQ)).ThenReturn(avgConstraint
);
return(conf);
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Reservation.Exceptions.PlanningException
/// "/>
/// <exception cref="System.IO.IOException"/>
public virtual void TestStress(int numJobs)
{
long timeWindow = 1000000L;
Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterCapacity = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(500 * 100 * 1024, 500 * 32);
step = 1000L;
ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
CapacityScheduler scheduler = testUtil.MockCapacityScheduler(500 * 100);
string reservationQ = testUtil.GetFullReservationQueueName();
float instConstraint = 100;
float avgConstraint = 100;
ReservationSchedulerConfiguration conf = ReservationSystemTestUtil.CreateConf(reservationQ
, timeWindow, instConstraint, avgConstraint);
CapacityOverTimePolicy policy = new CapacityOverTimePolicy();
policy.Init(reservationQ, conf);
plan = new InMemoryPlan(scheduler.GetRootQueueMetrics(), policy, agent, clusterCapacity
, step, res, minAlloc, maxAlloc, "dedicated", null, true);
int acc = 0;
IList <ReservationDefinition> list = new AList <ReservationDefinition>();
for (long i = 0; i < numJobs; i++)
{
list.AddItem(ReservationSystemTestUtil.GenerateRandomRR(rand, i));
}
long start = Runtime.CurrentTimeMillis();
for (int i_1 = 0; i_1 < numJobs; i_1++)
{
try
{
if (agent.CreateReservation(ReservationSystemTestUtil.GetNewReservationId(), "u"
+ i_1 % 100, plan, list[i_1]))
{
acc++;
}
}
catch (PlanningException)
{
}
}
// ignore exceptions
long end = Runtime.CurrentTimeMillis();
System.Console.Out.WriteLine("Submitted " + numJobs + " jobs " + " accepted " + acc
+ " in " + (end - start) + "ms");
}
public virtual void Setup()
{
// 1 sec step
step = 1000L;
initTime = Runtime.CurrentTimeMillis();
minAlloc = Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance(1024, 1);
res = new DefaultResourceCalculator();
maxAlloc = Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance(1024 * 8, 8);
mAgent = Org.Mockito.Mockito.Mock <ReservationAgent>();
ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
string reservationQ = testUtil.GetFullReservationQueueName();
QueueMetrics rootQueueMetrics = Org.Mockito.Mockito.Mock <QueueMetrics>();
Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = ReservationSystemTestUtil
.CalculateClusterResource(totCont);
ReservationSchedulerConfiguration conf = Org.Mockito.Mockito.Mock <ReservationSchedulerConfiguration
>();
NoOverCommitPolicy policy = new NoOverCommitPolicy();
policy.Init(reservationQ, conf);
plan = new InMemoryPlan(rootQueueMetrics, policy, mAgent, clusterResource, step,
res, minAlloc, maxAlloc, "dedicated", null, true);
}
public virtual void Init(string planQueuePath, ReservationSchedulerConfiguration
conf)
{
}
public virtual void Init(string planQueueName, ReservationSchedulerConfiguration
conf)
{
this.lengthOfCheckZone = conf.GetEnforcementWindow(planQueueName);
}
public virtual void TestReplanningPlanCapacityLoss()
{
Resource clusterCapacity = Resource.NewInstance(100 * 1024, 10);
Resource minAlloc = Resource.NewInstance(1024, 1);
Resource maxAlloc = Resource.NewInstance(1024 * 8, 8);
ResourceCalculator res = new DefaultResourceCalculator();
long step = 1L;
Clock clock = Org.Mockito.Mockito.Mock <Clock>();
ReservationAgent agent = Org.Mockito.Mockito.Mock <ReservationAgent>();
SharingPolicy policy = new NoOverCommitPolicy();
policy.Init("root.dedicated", null);
QueueMetrics queueMetrics = Org.Mockito.Mockito.Mock <QueueMetrics>();
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(0L);
SimpleCapacityReplanner enf = new SimpleCapacityReplanner(clock);
ReservationSchedulerConfiguration conf = Org.Mockito.Mockito.Mock <ReservationSchedulerConfiguration
>();
Org.Mockito.Mockito.When(conf.GetEnforcementWindow(Matchers.Any <string>())).ThenReturn
(6L);
enf.Init("blah", conf);
// Initialize the plan with more resources
InMemoryPlan plan = new InMemoryPlan(queueMetrics, policy, agent, clusterCapacity
, step, res, minAlloc, maxAlloc, "dedicated", enf, true, clock);
// add reservation filling the plan (separating them 1ms, so we are sure
// s2 follows s1 on acceptance
long ts = Runtime.CurrentTimeMillis();
ReservationId r1 = ReservationId.NewInstance(ts, 1);
int[] f5 = new int[] { 20, 20, 20, 20, 20 };
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r1, null, "u3", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res,
minAlloc)));
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(1L);
ReservationId r2 = ReservationId.NewInstance(ts, 2);
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r2, null, "u4", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res,
minAlloc)));
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(2L);
ReservationId r3 = ReservationId.NewInstance(ts, 3);
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r3, null, "u5", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res,
minAlloc)));
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(3L);
ReservationId r4 = ReservationId.NewInstance(ts, 4);
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r4, null, "u6", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res,
minAlloc)));
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(4L);
ReservationId r5 = ReservationId.NewInstance(ts, 5);
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r5, null, "u7", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res,
minAlloc)));
int[] f6 = new int[] { 50, 50, 50, 50, 50 };
ReservationId r6 = ReservationId.NewInstance(ts, 6);
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r6, null, "u3", "dedicated", 10, 10 + f6.Length, GenerateAllocation(10, f6), res
, minAlloc)));
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(6L);
ReservationId r7 = ReservationId.NewInstance(ts, 7);
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r7, null, "u4", "dedicated", 10, 10 + f6.Length, GenerateAllocation(10, f6), res
, minAlloc)));
// remove some of the resources (requires replanning)
plan.SetTotalCapacity(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance(70
* 1024, 70));
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(0L);
// run the replanner
enf.Plan(plan, null);
// check which reservation are still present
NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r1));
NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r2));
NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r3));
NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r6));
NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r7));
// and which ones are removed
NUnit.Framework.Assert.IsNull(plan.GetReservationById(r4));
NUnit.Framework.Assert.IsNull(plan.GetReservationById(r5));
// check resources at each moment in time no more exceed capacity
for (int i = 0; i < 20; i++)
{
int tot = 0;
foreach (ReservationAllocation r in plan.GetReservationsAtTime(i))
{
tot = r.GetResourcesAtTime(i).GetMemory();
}
NUnit.Framework.Assert.IsTrue(tot <= 70 * 1024);
}
}
