public virtual void TestNormalizeRequest()
{
ResourceCalculator resourceCalculator = new DefaultResourceCalculator();
int minMemory = 1024;
int maxMemory = 8192;
Org.Apache.Hadoop.Yarn.Api.Records.Resource minResource = Resources.CreateResource
(minMemory, 0);
Org.Apache.Hadoop.Yarn.Api.Records.Resource maxResource = Resources.CreateResource
(maxMemory, 0);
ResourceRequest ask = new ResourceRequestPBImpl();
// case negative memory
ask.SetCapability(Resources.CreateResource(-1024));
SchedulerUtils.NormalizeRequest(ask, resourceCalculator, null, minResource, maxResource
);
NUnit.Framework.Assert.AreEqual(minMemory, ask.GetCapability().GetMemory());
// case zero memory
ask.SetCapability(Resources.CreateResource(0));
SchedulerUtils.NormalizeRequest(ask, resourceCalculator, null, minResource, maxResource
);
NUnit.Framework.Assert.AreEqual(minMemory, ask.GetCapability().GetMemory());
// case memory is a multiple of minMemory
ask.SetCapability(Resources.CreateResource(2 * minMemory));
SchedulerUtils.NormalizeRequest(ask, resourceCalculator, null, minResource, maxResource
);
NUnit.Framework.Assert.AreEqual(2 * minMemory, ask.GetCapability().GetMemory());
// case memory is not a multiple of minMemory
ask.SetCapability(Resources.CreateResource(minMemory + 10));
SchedulerUtils.NormalizeRequest(ask, resourceCalculator, null, minResource, maxResource
);
NUnit.Framework.Assert.AreEqual(2 * minMemory, ask.GetCapability().GetMemory());
// case memory is equal to max allowed
ask.SetCapability(Resources.CreateResource(maxMemory));
SchedulerUtils.NormalizeRequest(ask, resourceCalculator, null, minResource, maxResource
);
NUnit.Framework.Assert.AreEqual(maxMemory, ask.GetCapability().GetMemory());
// case memory is just less than max
ask.SetCapability(Resources.CreateResource(maxMemory - 10));
SchedulerUtils.NormalizeRequest(ask, resourceCalculator, null, minResource, maxResource
);
NUnit.Framework.Assert.AreEqual(maxMemory, ask.GetCapability().GetMemory());
// max is not a multiple of min
maxResource = Resources.CreateResource(maxMemory - 10, 0);
ask.SetCapability(Resources.CreateResource(maxMemory - 100));
// multiple of minMemory > maxMemory, then reduce to maxMemory
SchedulerUtils.NormalizeRequest(ask, resourceCalculator, null, minResource, maxResource
);
NUnit.Framework.Assert.AreEqual(maxResource.GetMemory(), ask.GetCapability().GetMemory
());
// ask is more than max
maxResource = Resources.CreateResource(maxMemory, 0);
ask.SetCapability(Resources.CreateResource(maxMemory + 100));
SchedulerUtils.NormalizeRequest(ask, resourceCalculator, null, minResource, maxResource
);
NUnit.Framework.Assert.AreEqual(maxResource.GetMemory(), ask.GetCapability().GetMemory
());
}
private static ResourceScheduler MockResourceScheduler()
{
ResourceScheduler scheduler = Org.Mockito.Mockito.Mock <ResourceScheduler>();
Org.Mockito.Mockito.When(scheduler.GetMinimumResourceCapability()).ThenReturn(Resources
.CreateResource(YarnConfiguration.DefaultRmSchedulerMinimumAllocationMb));
Org.Mockito.Mockito.When(scheduler.GetMaximumResourceCapability()).ThenReturn(Resources
.CreateResource(YarnConfiguration.DefaultRmSchedulerMaximumAllocationMb));
ResourceCalculator rc = new DefaultResourceCalculator();
Org.Mockito.Mockito.When(scheduler.GetResourceCalculator()).ThenReturn(rc);
return(scheduler);
}
public virtual void TestZeroAlloaction()
{
ResourceCalculator resCalc = new DefaultResourceCalculator();
Org.Apache.Hadoop.Yarn.Api.Records.Resource minAlloc = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(1, 1);
RLESparseResourceAllocation rleSparseVector = new RLESparseResourceAllocation(resCalc
, minAlloc);
rleSparseVector.AddInterval(new ReservationInterval(0, long.MaxValue), ReservationRequest
.NewInstance(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance(0, 0), (0))
);
Log.Info(rleSparseVector.ToString());
NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(0, 0), rleSparseVector.GetCapacityAtTime(new Random().NextLong()));
NUnit.Framework.Assert.IsTrue(rleSparseVector.IsEmpty());
}
public virtual void SetUp()
{
clock = Org.Mockito.Mockito.Mock <Clock>();
plan = Org.Mockito.Mockito.Mock <Plan>();
rSystem = Org.Mockito.Mockito.Mock <ReservationSystem>();
plans[PlanName] = plan;
rrValidator = new ReservationInputValidator(clock);
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(1L);
ResourceCalculator rCalc = new DefaultResourceCalculator();
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(10240, 10);
Org.Mockito.Mockito.When(plan.GetResourceCalculator()).ThenReturn(rCalc);
Org.Mockito.Mockito.When(plan.GetTotalCapacity()).ThenReturn(resource);
Org.Mockito.Mockito.When(rSystem.GetQueueForReservation(Matchers.Any <ReservationId
>())).ThenReturn(PlanName);
Org.Mockito.Mockito.When(rSystem.GetPlan(PlanName)).ThenReturn(plan);
}
public virtual void TestSteps()
{
ResourceCalculator resCalc = new DefaultResourceCalculator();
Org.Apache.Hadoop.Yarn.Api.Records.Resource minAlloc = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(1, 1);
RLESparseResourceAllocation rleSparseVector = new RLESparseResourceAllocation(resCalc
, minAlloc);
int[] alloc = new int[] { 10, 10, 10, 10, 10, 10 };
int start = 100;
ICollection <KeyValuePair <ReservationInterval, ReservationRequest> > inputs = GenerateAllocation
(start, alloc, true);
foreach (KeyValuePair <ReservationInterval, ReservationRequest> ip in inputs)
{
rleSparseVector.AddInterval(ip.Key, ip.Value);
}
Log.Info(rleSparseVector.ToString());
NUnit.Framework.Assert.IsFalse(rleSparseVector.IsEmpty());
NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(0, 0), rleSparseVector.GetCapacityAtTime(99));
NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(0, 0), rleSparseVector.GetCapacityAtTime(start + alloc.Length + 1));
for (int i = 0; i < alloc.Length; i++)
{
NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(1024 * (alloc[i] + i), (alloc[i] + i)), rleSparseVector.GetCapacityAtTime(start
+ i));
}
NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(0, 0), rleSparseVector.GetCapacityAtTime(start + alloc.Length + 2));
foreach (KeyValuePair <ReservationInterval, ReservationRequest> ip_1 in inputs)
{
rleSparseVector.RemoveInterval(ip_1.Key, ip_1.Value);
}
Log.Info(rleSparseVector.ToString());
for (int i_1 = 0; i_1 < alloc.Length; i_1++)
{
NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(0, 0), rleSparseVector.GetCapacityAtTime(start + i_1));
}
NUnit.Framework.Assert.IsTrue(rleSparseVector.IsEmpty());
}
public virtual void TestReplanningPlanCapacityLoss()
{
Resource clusterCapacity = Resource.NewInstance(100 * 1024, 10);
Resource minAlloc = Resource.NewInstance(1024, 1);
Resource maxAlloc = Resource.NewInstance(1024 * 8, 8);
ResourceCalculator res = new DefaultResourceCalculator();
long step = 1L;
Clock clock = Org.Mockito.Mockito.Mock <Clock>();
ReservationAgent agent = Org.Mockito.Mockito.Mock <ReservationAgent>();
SharingPolicy policy = new NoOverCommitPolicy();
policy.Init("root.dedicated", null);
QueueMetrics queueMetrics = Org.Mockito.Mockito.Mock <QueueMetrics>();
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(0L);
SimpleCapacityReplanner enf = new SimpleCapacityReplanner(clock);
ReservationSchedulerConfiguration conf = Org.Mockito.Mockito.Mock <ReservationSchedulerConfiguration
>();
Org.Mockito.Mockito.When(conf.GetEnforcementWindow(Matchers.Any <string>())).ThenReturn
(6L);
enf.Init("blah", conf);
// Initialize the plan with more resources
InMemoryPlan plan = new InMemoryPlan(queueMetrics, policy, agent, clusterCapacity
, step, res, minAlloc, maxAlloc, "dedicated", enf, true, clock);
// add reservation filling the plan (separating them 1ms, so we are sure
// s2 follows s1 on acceptance
long ts = Runtime.CurrentTimeMillis();
ReservationId r1 = ReservationId.NewInstance(ts, 1);
int[] f5 = new int[] { 20, 20, 20, 20, 20 };
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r1, null, "u3", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res,
minAlloc)));
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(1L);
ReservationId r2 = ReservationId.NewInstance(ts, 2);
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r2, null, "u4", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res,
minAlloc)));
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(2L);
ReservationId r3 = ReservationId.NewInstance(ts, 3);
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r3, null, "u5", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res,
minAlloc)));
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(3L);
ReservationId r4 = ReservationId.NewInstance(ts, 4);
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r4, null, "u6", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res,
minAlloc)));
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(4L);
ReservationId r5 = ReservationId.NewInstance(ts, 5);
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r5, null, "u7", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res,
minAlloc)));
int[] f6 = new int[] { 50, 50, 50, 50, 50 };
ReservationId r6 = ReservationId.NewInstance(ts, 6);
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r6, null, "u3", "dedicated", 10, 10 + f6.Length, GenerateAllocation(10, f6), res
, minAlloc)));
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(6L);
ReservationId r7 = ReservationId.NewInstance(ts, 7);
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r7, null, "u4", "dedicated", 10, 10 + f6.Length, GenerateAllocation(10, f6), res
, minAlloc)));
// remove some of the resources (requires replanning)
plan.SetTotalCapacity(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance(70
* 1024, 70));
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(0L);
// run the replanner
enf.Plan(plan, null);
// check which reservation are still present
NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r1));
NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r2));
NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r3));
NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r6));
NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r7));
// and which ones are removed
NUnit.Framework.Assert.IsNull(plan.GetReservationById(r4));
NUnit.Framework.Assert.IsNull(plan.GetReservationById(r5));
// check resources at each moment in time no more exceed capacity
for (int i = 0; i < 20; i++)
{
int tot = 0;
foreach (ReservationAllocation r in plan.GetReservationsAtTime(i))
{
tot = r.GetResourcesAtTime(i).GetMemory();
}
NUnit.Framework.Assert.IsTrue(tot <= 70 * 1024);
}
}
