public virtual void TestOrder()
{
PrepareBasicPlan();
// create a completely utilized segment around time 30
int[] f = new int[] { 100, 100 };
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(ReservationSystemTestUtil.GetNewReservationId(), null, "u1", "dedicated", 30 *
step, 30 * step + f.Length * step, ReservationSystemTestUtil.GenerateAllocation(
30 * step, step, f), res, minAlloc)));
// create a chain of 4 RR, mixing gang and non-gang
ReservationDefinition rr = new ReservationDefinitionPBImpl();
rr.SetArrival(0 * step);
rr.SetDeadline(70 * step);
ReservationRequests reqs = new ReservationRequestsPBImpl();
reqs.SetInterpreter(ReservationRequestInterpreter.ROrder);
ReservationRequest r = ReservationRequest.NewInstance(Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(2048, 2), 10, 1, 10 * step);
ReservationRequest r2 = ReservationRequest.NewInstance(Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(1024, 1), 10, 10, 20 * step);
IList <ReservationRequest> list = new AList <ReservationRequest>();
list.AddItem(r);
list.AddItem(r2);
list.AddItem(r);
list.AddItem(r2);
reqs.SetReservationResources(list);
rr.SetReservationRequests(reqs);
// submit to agent
ReservationId reservationID = ReservationSystemTestUtil.GetNewReservationId();
agent.CreateReservation(reservationID, "u1", plan, rr);
// validate
NUnit.Framework.Assert.IsTrue("Agent-based allocation failed", reservationID != null
);
NUnit.Framework.Assert.IsTrue("Agent-based allocation failed", plan.GetAllReservations
().Count == 4);
ReservationAllocation cs = plan.GetReservationById(reservationID);
NUnit.Framework.Assert.IsTrue(cs.ToString(), Check(cs, 0 * step, 10 * step, 20, 1024
, 1));
NUnit.Framework.Assert.IsTrue(cs.ToString(), Check(cs, 10 * step, 30 * step, 10,
1024, 1));
NUnit.Framework.Assert.IsTrue(cs.ToString(), Check(cs, 40 * step, 50 * step, 20,
1024, 1));
NUnit.Framework.Assert.IsTrue(cs.ToString(), Check(cs, 50 * step, 70 * step, 10,
1024, 1));
System.Console.Out.WriteLine("--------AFTER ORDER ALLOCATION (queue: " + reservationID
+ ")----------");
System.Console.Out.WriteLine(plan.ToString());
System.Console.Out.WriteLine(plan.ToCumulativeString());
}
public virtual void TestAddReservationAlreadyExists()
{
// First add a reservation
Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, resCalc
, minAlloc, maxAlloc, planName, replanner, true);
ReservationId reservationID = ReservationSystemTestUtil.GetNewReservationId();
int[] alloc = new int[] { 10, 10, 10, 10, 10, 10 };
int start = 100;
IDictionary <ReservationInterval, ReservationRequest> allocations = GenerateAllocation
(start, alloc, false);
ReservationDefinition rDef = CreateSimpleReservationDefinition(start, start + alloc
.Length, alloc.Length, allocations.Values);
ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID
, rDef, user, planName, start, start + alloc.Length, allocations, resCalc, minAlloc
);
NUnit.Framework.Assert.IsNull(plan.GetReservationById(reservationID));
try
{
plan.AddReservation(rAllocation);
}
catch (PlanningException e)
{
NUnit.Framework.Assert.Fail(e.Message);
}
DoAssertions(plan, rAllocation);
for (int i = 0; i < alloc.Length; i++)
{
NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(1024 * (alloc[i]), (alloc[i])), plan.GetTotalCommittedResources(start + i));
NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(1024 * (alloc[i]), (alloc[i])), plan.GetConsumptionForUser(user, start + i));
}
// Try to add it again
try
{
plan.AddReservation(rAllocation);
NUnit.Framework.Assert.Fail("Add should fail as it already exists");
}
catch (ArgumentException e)
{
NUnit.Framework.Assert.IsTrue(e.Message.EndsWith("already exists"));
}
catch (PlanningException e)
{
NUnit.Framework.Assert.Fail(e.Message);
}
DoAssertions(plan, rAllocation);
}
public virtual void TestAddEmptyReservation()
{
Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, resCalc
, minAlloc, maxAlloc, planName, replanner, true);
ReservationId reservationID = ReservationSystemTestUtil.GetNewReservationId();
int[] alloc = new int[] { };
int start = 100;
IDictionary <ReservationInterval, ReservationRequest> allocations = new Dictionary
<ReservationInterval, ReservationRequest>();
ReservationDefinition rDef = CreateSimpleReservationDefinition(start, start + alloc
.Length, alloc.Length, allocations.Values);
ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID
, rDef, user, planName, start, start + alloc.Length, allocations, resCalc, minAlloc
);
NUnit.Framework.Assert.IsNull(plan.GetReservationById(reservationID));
try
{
plan.AddReservation(rAllocation);
}
catch (PlanningException e)
{
NUnit.Framework.Assert.Fail(e.Message);
}
}
public virtual void TestSingleUserEasyFitPass()
{
// generate allocation that easily fit within resource constraints
int[] f = GenerateData(3600, (int)Math.Ceil(0.2 * totCont));
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(ReservationSystemTestUtil.GetNewReservationId(), null, "u1", "dedicated", initTime
, initTime + f.Length, ReservationSystemTestUtil.GenerateAllocation(initTime, step
, f), res, minAlloc)));
}
public virtual void TestArchiveCompletedReservations()
{
Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, resCalc
, minAlloc, maxAlloc, planName, replanner, true);
ReservationId reservationID1 = ReservationSystemTestUtil.GetNewReservationId();
// First add a reservation
int[] alloc1 = new int[] { 10, 10, 10, 10, 10, 10 };
int start = 100;
IDictionary <ReservationInterval, ReservationRequest> allocations1 = GenerateAllocation
(start, alloc1, false);
ReservationDefinition rDef1 = CreateSimpleReservationDefinition(start, start + alloc1
.Length, alloc1.Length, allocations1.Values);
ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID1
, rDef1, user, planName, start, start + alloc1.Length, allocations1, resCalc, minAlloc
);
NUnit.Framework.Assert.IsNull(plan.GetReservationById(reservationID1));
try
{
plan.AddReservation(rAllocation);
}
catch (PlanningException e)
{
NUnit.Framework.Assert.Fail(e.Message);
}
DoAssertions(plan, rAllocation);
for (int i = 0; i < alloc1.Length; i++)
{
NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(1024 * (alloc1[i]), (alloc1[i])), plan.GetTotalCommittedResources(start + i));
NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(1024 * (alloc1[i]), (alloc1[i])), plan.GetConsumptionForUser(user, start + i));
}
// Now add another one
ReservationId reservationID2 = ReservationSystemTestUtil.GetNewReservationId();
int[] alloc2 = new int[] { 0, 5, 10, 5, 0 };
IDictionary <ReservationInterval, ReservationRequest> allocations2 = GenerateAllocation
(start, alloc2, true);
ReservationDefinition rDef2 = CreateSimpleReservationDefinition(start, start + alloc2
.Length, alloc2.Length, allocations2.Values);
rAllocation = new InMemoryReservationAllocation(reservationID2, rDef2, user, planName
, start, start + alloc2.Length, allocations2, resCalc, minAlloc);
NUnit.Framework.Assert.IsNull(plan.GetReservationById(reservationID2));
try
{
plan.AddReservation(rAllocation);
}
catch (PlanningException e)
{
NUnit.Framework.Assert.Fail(e.Message);
}
NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(reservationID2));
for (int i_1 = 0; i_1 < alloc2.Length; i_1++)
{
NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(1024 * (alloc1[i_1] + alloc2[i_1] + i_1), alloc1[i_1] + alloc2[i_1] + i_1), plan
.GetTotalCommittedResources(start + i_1));
NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(1024 * (alloc1[i_1] + alloc2[i_1] + i_1), alloc1[i_1] + alloc2[i_1] + i_1), plan
.GetConsumptionForUser(user, start + i_1));
}
// Now archive completed reservations
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(106L);
Org.Mockito.Mockito.When(policy.GetValidWindow()).ThenReturn(1L);
try
{
// will only remove 2nd reservation as only that has fallen out of the
// archival window
plan.ArchiveCompletedReservations(clock.GetTime());
}
catch (PlanningException e)
{
NUnit.Framework.Assert.Fail(e.Message);
}
NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(reservationID1));
NUnit.Framework.Assert.IsNull(plan.GetReservationById(reservationID2));
for (int i_2 = 0; i_2 < alloc1.Length; i_2++)
{
NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(1024 * (alloc1[i_2]), (alloc1[i_2])), plan.GetTotalCommittedResources(start + i_2
));
NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(1024 * (alloc1[i_2]), (alloc1[i_2])), plan.GetConsumptionForUser(user, start +
i_2));
}
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(107L);
try
{
// will remove 1st reservation also as it has fallen out of the archival
// window
plan.ArchiveCompletedReservations(clock.GetTime());
}
catch (PlanningException e)
{
NUnit.Framework.Assert.Fail(e.Message);
}
NUnit.Framework.Assert.IsNull(plan.GetReservationById(reservationID1));
for (int i_3 = 0; i_3 < alloc1.Length; i_3++)
{
NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(0, 0), plan.GetTotalCommittedResources(start + i_3));
NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(0, 0), plan.GetConsumptionForUser(user, start + i_3));
}
}
public virtual void TestUpdateReservation()
{
Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, resCalc
, minAlloc, maxAlloc, planName, replanner, true);
ReservationId reservationID = ReservationSystemTestUtil.GetNewReservationId();
// First add a reservation
int[] alloc = new int[] { 10, 10, 10, 10, 10, 10 };
int start = 100;
IDictionary <ReservationInterval, ReservationRequest> allocations = GenerateAllocation
(start, alloc, false);
ReservationDefinition rDef = CreateSimpleReservationDefinition(start, start + alloc
.Length, alloc.Length, allocations.Values);
ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID
, rDef, user, planName, start, start + alloc.Length, allocations, resCalc, minAlloc
);
NUnit.Framework.Assert.IsNull(plan.GetReservationById(reservationID));
try
{
plan.AddReservation(rAllocation);
}
catch (PlanningException e)
{
NUnit.Framework.Assert.Fail(e.Message);
}
DoAssertions(plan, rAllocation);
for (int i = 0; i < alloc.Length; i++)
{
NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(1024 * (alloc[i]), (alloc[i])), plan.GetTotalCommittedResources(start + i));
NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(1024 * (alloc[i]), (alloc[i])), plan.GetConsumptionForUser(user, start + i));
}
// Now update it
start = 110;
int[] updatedAlloc = new int[] { 0, 5, 10, 10, 5, 0 };
allocations = GenerateAllocation(start, updatedAlloc, true);
rDef = CreateSimpleReservationDefinition(start, start + updatedAlloc.Length, updatedAlloc
.Length, allocations.Values);
rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName
, start, start + updatedAlloc.Length, allocations, resCalc, minAlloc);
try
{
plan.UpdateReservation(rAllocation);
}
catch (PlanningException e)
{
NUnit.Framework.Assert.Fail(e.Message);
}
DoAssertions(plan, rAllocation);
for (int i_1 = 0; i_1 < updatedAlloc.Length; i_1++)
{
NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(1024 * (updatedAlloc[i_1] + i_1), updatedAlloc[i_1] + i_1), plan.GetTotalCommittedResources
(start + i_1));
NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(1024 * (updatedAlloc[i_1] + i_1), updatedAlloc[i_1] + i_1), plan.GetConsumptionForUser
(user, start + i_1));
}
}
public virtual void TestReplanningPlanCapacityLoss()
{
Resource clusterCapacity = Resource.NewInstance(100 * 1024, 10);
Resource minAlloc = Resource.NewInstance(1024, 1);
Resource maxAlloc = Resource.NewInstance(1024 * 8, 8);
ResourceCalculator res = new DefaultResourceCalculator();
long step = 1L;
Clock clock = Org.Mockito.Mockito.Mock <Clock>();
ReservationAgent agent = Org.Mockito.Mockito.Mock <ReservationAgent>();
SharingPolicy policy = new NoOverCommitPolicy();
policy.Init("root.dedicated", null);
QueueMetrics queueMetrics = Org.Mockito.Mockito.Mock <QueueMetrics>();
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(0L);
SimpleCapacityReplanner enf = new SimpleCapacityReplanner(clock);
ReservationSchedulerConfiguration conf = Org.Mockito.Mockito.Mock <ReservationSchedulerConfiguration
>();
Org.Mockito.Mockito.When(conf.GetEnforcementWindow(Matchers.Any <string>())).ThenReturn
(6L);
enf.Init("blah", conf);
// Initialize the plan with more resources
InMemoryPlan plan = new InMemoryPlan(queueMetrics, policy, agent, clusterCapacity
, step, res, minAlloc, maxAlloc, "dedicated", enf, true, clock);
// add reservation filling the plan (separating them 1ms, so we are sure
// s2 follows s1 on acceptance
long ts = Runtime.CurrentTimeMillis();
ReservationId r1 = ReservationId.NewInstance(ts, 1);
int[] f5 = new int[] { 20, 20, 20, 20, 20 };
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r1, null, "u3", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res,
minAlloc)));
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(1L);
ReservationId r2 = ReservationId.NewInstance(ts, 2);
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r2, null, "u4", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res,
minAlloc)));
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(2L);
ReservationId r3 = ReservationId.NewInstance(ts, 3);
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r3, null, "u5", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res,
minAlloc)));
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(3L);
ReservationId r4 = ReservationId.NewInstance(ts, 4);
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r4, null, "u6", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res,
minAlloc)));
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(4L);
ReservationId r5 = ReservationId.NewInstance(ts, 5);
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r5, null, "u7", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res,
minAlloc)));
int[] f6 = new int[] { 50, 50, 50, 50, 50 };
ReservationId r6 = ReservationId.NewInstance(ts, 6);
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r6, null, "u3", "dedicated", 10, 10 + f6.Length, GenerateAllocation(10, f6), res
, minAlloc)));
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(6L);
ReservationId r7 = ReservationId.NewInstance(ts, 7);
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r7, null, "u4", "dedicated", 10, 10 + f6.Length, GenerateAllocation(10, f6), res
, minAlloc)));
// remove some of the resources (requires replanning)
plan.SetTotalCapacity(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance(70
* 1024, 70));
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(0L);
// run the replanner
enf.Plan(plan, null);
// check which reservation are still present
NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r1));
NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r2));
NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r3));
NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r6));
NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r7));
// and which ones are removed
NUnit.Framework.Assert.IsNull(plan.GetReservationById(r4));
NUnit.Framework.Assert.IsNull(plan.GetReservationById(r5));
// check resources at each moment in time no more exceed capacity
for (int i = 0; i < 20; i++)
{
int tot = 0;
foreach (ReservationAllocation r in plan.GetReservationsAtTime(i))
{
tot = r.GetResourcesAtTime(i).GetMemory();
}
NUnit.Framework.Assert.IsTrue(tot <= 70 * 1024);
}
}