public virtual void TestAddEmptyReservation()
{
Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, resCalc
, minAlloc, maxAlloc, planName, replanner, true);
ReservationId reservationID = ReservationSystemTestUtil.GetNewReservationId();
int[] alloc = new int[] { };
int start = 100;
IDictionary <ReservationInterval, ReservationRequest> allocations = new Dictionary
<ReservationInterval, ReservationRequest>();
ReservationDefinition rDef = CreateSimpleReservationDefinition(start, start + alloc
.Length, alloc.Length, allocations.Values);
ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID
, rDef, user, planName, start, start + alloc.Length, allocations, resCalc, minAlloc
);
NUnit.Framework.Assert.IsNull(plan.GetReservationById(reservationID));
try
{
plan.AddReservation(rAllocation);
}
catch (PlanningException e)
{
NUnit.Framework.Assert.Fail(e.Message);
}
}
public virtual void TestAddReservation()
{
Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, resCalc
, minAlloc, maxAlloc, planName, replanner, true);
ReservationId reservationID = ReservationSystemTestUtil.GetNewReservationId();
int[] alloc = new int[] { 10, 10, 10, 10, 10, 10 };
int start = 100;
IDictionary <ReservationInterval, ReservationRequest> allocations = GenerateAllocation
(start, alloc, false);
ReservationDefinition rDef = CreateSimpleReservationDefinition(start, start + alloc
.Length, alloc.Length, allocations.Values);
ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID
, rDef, user, planName, start, start + alloc.Length, allocations, resCalc, minAlloc
);
NUnit.Framework.Assert.IsNull(plan.GetReservationById(reservationID));
try
{
plan.AddReservation(rAllocation);
}
catch (PlanningException e)
{
NUnit.Framework.Assert.Fail(e.Message);
}
DoAssertions(plan, rAllocation);
for (int i = 0; i < alloc.Length; i++)
{
NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(1024 * (alloc[i]), (alloc[i])), plan.GetTotalCommittedResources(start + i));
NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(1024 * (alloc[i]), (alloc[i])), plan.GetConsumptionForUser(user, start + i));
}
}
public virtual void TestSimple()
{
PrepareBasicPlan();
// create a request with a single atomic ask
ReservationDefinition rr = new ReservationDefinitionPBImpl();
rr.SetArrival(5 * step);
rr.SetDeadline(20 * step);
ReservationRequest r = ReservationRequest.NewInstance(Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(2048, 2), 10, 5, 10 * step);
ReservationRequests reqs = new ReservationRequestsPBImpl();
reqs.SetReservationResources(Collections.SingletonList(r));
rr.SetReservationRequests(reqs);
ReservationId reservationID = ReservationSystemTestUtil.GetNewReservationId();
agent.CreateReservation(reservationID, "u1", plan, rr);
NUnit.Framework.Assert.IsTrue("Agent-based allocation failed", reservationID != null
);
NUnit.Framework.Assert.IsTrue("Agent-based allocation failed", plan.GetAllReservations
().Count == 3);
ReservationAllocation cs = plan.GetReservationById(reservationID);
System.Console.Out.WriteLine("--------AFTER SIMPLE ALLOCATION (queue: " + reservationID
+ ")----------");
System.Console.Out.WriteLine(plan.ToString());
System.Console.Out.WriteLine(plan.ToCumulativeString());
for (long i = 10 * step; i < 20 * step; i++)
{
NUnit.Framework.Assert.IsTrue("Agent-based allocation unexpected", Resources.Equals
(cs.GetResourcesAtTime(i), Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(2048 * 10, 2 * 10)));
}
}
public virtual void Setup()
{
// 24h window
timeWindow = 86400000L;
// 1 sec step
step = 1000L;
// 25% avg cap on capacity
avgConstraint = 25;
// 70% instantaneous cap on capacity
instConstraint = 70;
initTime = Runtime.CurrentTimeMillis();
minAlloc = Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance(1024, 1);
res = new DefaultResourceCalculator();
maxAlloc = Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance(1024 * 8, 8);
mAgent = Org.Mockito.Mockito.Mock <ReservationAgent>();
ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
QueueMetrics rootQueueMetrics = Org.Mockito.Mockito.Mock <QueueMetrics>();
string reservationQ = testUtil.GetFullReservationQueueName();
Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = ReservationSystemTestUtil
.CalculateClusterResource(totCont);
ReservationSchedulerConfiguration conf = ReservationSystemTestUtil.CreateConf(reservationQ
, timeWindow, instConstraint, avgConstraint);
CapacityOverTimePolicy policy = new CapacityOverTimePolicy();
policy.Init(reservationQ, conf);
plan = new InMemoryPlan(rootQueueMetrics, policy, mAgent, clusterResource, step,
res, minAlloc, maxAlloc, "dedicated", null, true);
}
public virtual void TestInitialize()
{
ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
CapacityScheduler capScheduler = null;
try
{
capScheduler = testUtil.MockCapacityScheduler(10);
}
catch (IOException e)
{
NUnit.Framework.Assert.Fail(e.Message);
}
CapacityReservationSystem reservationSystem = new CapacityReservationSystem();
reservationSystem.SetRMContext(capScheduler.GetRMContext());
try
{
reservationSystem.Reinitialize(capScheduler.GetConf(), capScheduler.GetRMContext(
));
}
catch (YarnException e)
{
NUnit.Framework.Assert.Fail(e.Message);
}
string planQName = testUtil.GetreservationQueueName();
ReservationSystemTestUtil.ValidateReservationQueue(reservationSystem, planQName);
}
public virtual void Setup()
{
long seed = rand.NextLong();
rand.SetSeed(seed);
Org.Mortbay.Log.Log.Info("Running with seed: " + seed);
// setting completely loose quotas
long timeWindow = 1000000L;
Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterCapacity = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(100 * 1024, 100);
step = 1000L;
ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
string reservationQ = testUtil.GetFullReservationQueueName();
float instConstraint = 100;
float avgConstraint = 100;
ReservationSchedulerConfiguration conf = ReservationSystemTestUtil.CreateConf(reservationQ
, timeWindow, instConstraint, avgConstraint);
CapacityOverTimePolicy policy = new CapacityOverTimePolicy();
policy.Init(reservationQ, conf);
agent = new GreedyReservationAgent();
QueueMetrics queueMetrics = Org.Mockito.Mockito.Mock <QueueMetrics>();
plan = new InMemoryPlan(queueMetrics, policy, agent, clusterCapacity, step, res,
minAlloc, maxAlloc, "dedicated", null, true);
}
private ReservationUpdateRequest CreateSimpleReservationUpdateRequest(int numRequests
, int numContainers, long arrival, long deadline, long duration)
{
// create a request with a single atomic ask
ReservationUpdateRequest request = new ReservationUpdateRequestPBImpl();
ReservationDefinition rDef = new ReservationDefinitionPBImpl();
rDef.SetArrival(arrival);
rDef.SetDeadline(deadline);
if (numRequests > 0)
{
ReservationRequests reqs = new ReservationRequestsPBImpl();
rDef.SetReservationRequests(reqs);
if (numContainers > 0)
{
ReservationRequest r = ReservationRequest.NewInstance(Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(1024, 1), numContainers, 1, duration);
reqs.SetReservationResources(Sharpen.Collections.SingletonList(r));
reqs.SetInterpreter(ReservationRequestInterpreter.RAll);
}
}
request.SetReservationDefinition(rDef);
request.SetReservationId(ReservationSystemTestUtil.GetNewReservationId());
return(request);
}
public virtual void TestUpdateNonExistingReservation()
{
Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, resCalc
, minAlloc, maxAlloc, planName, replanner, true);
ReservationId reservationID = ReservationSystemTestUtil.GetNewReservationId();
// Try to update a reservation without adding
int[] alloc = new int[] { 10, 10, 10, 10, 10, 10 };
int start = 100;
IDictionary <ReservationInterval, ReservationRequest> allocations = GenerateAllocation
(start, alloc, false);
ReservationDefinition rDef = CreateSimpleReservationDefinition(start, start + alloc
.Length, alloc.Length, allocations.Values);
ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID
, rDef, user, planName, start, start + alloc.Length, allocations, resCalc, minAlloc
);
NUnit.Framework.Assert.IsNull(plan.GetReservationById(reservationID));
try
{
plan.UpdateReservation(rAllocation);
NUnit.Framework.Assert.Fail("Update should fail as it does not exist in the plan"
);
}
catch (ArgumentException e)
{
NUnit.Framework.Assert.IsTrue(e.Message.EndsWith("does not exist in the plan"));
}
catch (PlanningException e)
{
NUnit.Framework.Assert.Fail(e.Message);
}
NUnit.Framework.Assert.IsNull(plan.GetReservationById(reservationID));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Reservation.Exceptions.PlanningException
/// "/>
public virtual void TestSingleFail()
{
// generate allocation from single tenant that exceed capacity
int[] f = GenerateData(3600, (int)(1.1 * totCont));
plan.AddReservation(new InMemoryReservationAllocation(ReservationSystemTestUtil.GetNewReservationId
(), null, "u1", "dedicated", initTime, initTime + f.Length, ReservationSystemTestUtil
.GenerateAllocation(initTime, step, f), res, minAlloc));
}
public virtual void TestSingleUserBarelyFitPass()
{
// generate allocation from single tenant that barely fit
int[] f = GenerateData(3600, totCont);
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(ReservationSystemTestUtil.GetNewReservationId(), null, "u1", "dedicated", initTime
, initTime + f.Length, ReservationSystemTestUtil.GenerateAllocation(initTime, step
, f), res, minAlloc)));
}
public virtual void TestSimplePass()
{
// generate allocation that simply fit within all constraints
int[] f = GenerateData(3600, (int)Math.Ceil(0.2 * totCont));
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(ReservationSystemTestUtil.GetNewReservationId(), null, "u1", "dedicated", initTime
, initTime + f.Length, ReservationSystemTestUtil.GenerateAllocation(initTime, step
, f), res, minAlloc)));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Reservation.Exceptions.PlanningException
/// "/>
public virtual void TestInstFail()
{
// generate allocation that exceed the instantaneous cap single-show
int[] f = GenerateData(3600, (int)Math.Ceil(0.71 * totCont));
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(ReservationSystemTestUtil.GetNewReservationId(), null, "u1", "dedicated", initTime
, initTime + f.Length, ReservationSystemTestUtil.GenerateAllocation(initTime, step
, f), res, minAlloc)));
NUnit.Framework.Assert.Fail("should not have accepted this");
}
public virtual void TestSimplePass2()
{
// generate allocation from single tenant that exceed avg momentarily but
// fit within
// max instantanesou
int[] f = GenerateData(3600, (int)Math.Ceil(0.69 * totCont));
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(ReservationSystemTestUtil.GetNewReservationId(), null, "u1", "dedicated", initTime
, initTime + f.Length, ReservationSystemTestUtil.GenerateAllocation(initTime, step
, f), res, minAlloc)));
}
public virtual void TestReinitialize()
{
ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
CapacityScheduler capScheduler = null;
try
{
capScheduler = testUtil.MockCapacityScheduler(10);
}
catch (IOException e)
{
NUnit.Framework.Assert.Fail(e.Message);
}
CapacityReservationSystem reservationSystem = new CapacityReservationSystem();
CapacitySchedulerConfiguration conf = capScheduler.GetConfiguration();
RMContext mockContext = capScheduler.GetRMContext();
reservationSystem.SetRMContext(mockContext);
try
{
reservationSystem.Reinitialize(capScheduler.GetConfiguration(), mockContext);
}
catch (YarnException e)
{
NUnit.Framework.Assert.Fail(e.Message);
}
// Assert queue in original config
string planQName = testUtil.GetreservationQueueName();
ReservationSystemTestUtil.ValidateReservationQueue(reservationSystem, planQName);
// Dynamically add a plan
string newQ = "reservation";
NUnit.Framework.Assert.IsNull(reservationSystem.GetPlan(newQ));
testUtil.UpdateQueueConfiguration(conf, newQ);
try
{
capScheduler.Reinitialize(conf, mockContext);
}
catch (IOException e)
{
NUnit.Framework.Assert.Fail(e.Message);
}
try
{
reservationSystem.Reinitialize(conf, mockContext);
}
catch (YarnException e)
{
NUnit.Framework.Assert.Fail(e.Message);
}
ReservationSystemTestUtil.ValidateNewReservationQueue(reservationSystem, newQ);
}
public virtual void TestOrder()
{
PrepareBasicPlan();
// create a completely utilized segment around time 30
int[] f = new int[] { 100, 100 };
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(ReservationSystemTestUtil.GetNewReservationId(), null, "u1", "dedicated", 30 *
step, 30 * step + f.Length * step, ReservationSystemTestUtil.GenerateAllocation(
30 * step, step, f), res, minAlloc)));
// create a chain of 4 RR, mixing gang and non-gang
ReservationDefinition rr = new ReservationDefinitionPBImpl();
rr.SetArrival(0 * step);
rr.SetDeadline(70 * step);
ReservationRequests reqs = new ReservationRequestsPBImpl();
reqs.SetInterpreter(ReservationRequestInterpreter.ROrder);
ReservationRequest r = ReservationRequest.NewInstance(Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(2048, 2), 10, 1, 10 * step);
ReservationRequest r2 = ReservationRequest.NewInstance(Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(1024, 1), 10, 10, 20 * step);
IList <ReservationRequest> list = new AList <ReservationRequest>();
list.AddItem(r);
list.AddItem(r2);
list.AddItem(r);
list.AddItem(r2);
reqs.SetReservationResources(list);
rr.SetReservationRequests(reqs);
// submit to agent
ReservationId reservationID = ReservationSystemTestUtil.GetNewReservationId();
agent.CreateReservation(reservationID, "u1", plan, rr);
// validate
NUnit.Framework.Assert.IsTrue("Agent-based allocation failed", reservationID != null
);
NUnit.Framework.Assert.IsTrue("Agent-based allocation failed", plan.GetAllReservations
().Count == 4);
ReservationAllocation cs = plan.GetReservationById(reservationID);
NUnit.Framework.Assert.IsTrue(cs.ToString(), Check(cs, 0 * step, 10 * step, 20, 1024
, 1));
NUnit.Framework.Assert.IsTrue(cs.ToString(), Check(cs, 10 * step, 30 * step, 10,
1024, 1));
NUnit.Framework.Assert.IsTrue(cs.ToString(), Check(cs, 40 * step, 50 * step, 20,
1024, 1));
NUnit.Framework.Assert.IsTrue(cs.ToString(), Check(cs, 50 * step, 70 * step, 10,
1024, 1));
System.Console.Out.WriteLine("--------AFTER ORDER ALLOCATION (queue: " + reservationID
+ ")----------");
System.Console.Out.WriteLine(plan.ToString());
System.Console.Out.WriteLine(plan.ToCumulativeString());
}
public virtual void TestOrderNoGapImpossible()
{
PrepareBasicPlan();
// create a completely utilized segment at time 30
int[] f = new int[] { 100, 100 };
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(ReservationSystemTestUtil.GetNewReservationId(), null, "u1", "dedicated", 30 *
step, 30 * step + f.Length * step, ReservationSystemTestUtil.GenerateAllocation(
30 * step, step, f), res, minAlloc)));
// create a chain of 4 RR, mixing gang and non-gang
ReservationDefinition rr = new ReservationDefinitionPBImpl();
rr.SetArrival(0L);
rr.SetDeadline(70L);
ReservationRequests reqs = new ReservationRequestsPBImpl();
reqs.SetInterpreter(ReservationRequestInterpreter.ROrderNoGap);
ReservationRequest r = ReservationRequest.NewInstance(Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(2048, 2), 10, 1, 10);
ReservationRequest r2 = ReservationRequest.NewInstance(Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(1024, 1), 10, 10, 20);
IList <ReservationRequest> list = new AList <ReservationRequest>();
list.AddItem(r);
list.AddItem(r2);
list.AddItem(r);
list.AddItem(r2);
reqs.SetReservationResources(list);
rr.SetReservationRequests(reqs);
ReservationId reservationID = ReservationSystemTestUtil.GetNewReservationId();
bool result = false;
try
{
// submit to agent
result = agent.CreateReservation(reservationID, "u1", plan, rr);
NUnit.Framework.Assert.Fail();
}
catch (PlanningException)
{
}
// expected
// validate
NUnit.Framework.Assert.IsFalse("Agent-based allocation should have failed", result
);
NUnit.Framework.Assert.IsTrue("Agent-based allocation should have failed", plan.GetAllReservations
().Count == 3);
System.Console.Out.WriteLine("--------AFTER ORDER_NO_GAP IMPOSSIBLE ALLOCATION (queue: "
+ reservationID + ")----------");
System.Console.Out.WriteLine(plan.ToString());
System.Console.Out.WriteLine(plan.ToCumulativeString());
}
/// <exception cref="System.Exception"/>
private void SetupPlanFollower()
{
ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
mClock = Org.Mockito.Mockito.Mock <Clock>();
mAgent = Org.Mockito.Mockito.Mock <ReservationAgent>();
string reservationQ = testUtil.GetFullReservationQueueName();
AllocationConfiguration allocConf = fs.GetAllocationConfiguration();
allocConf.SetReservationWindow(20L);
allocConf.SetAverageCapacity(20);
policy.Init(reservationQ, allocConf);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Reservation.Exceptions.PlanningException
/// "/>
public virtual void TestUserMismatch()
{
// generate allocation from single tenant that exceed capacity
int[] f = GenerateData(3600, (int)(0.5 * totCont));
ReservationId rid = ReservationSystemTestUtil.GetNewReservationId();
plan.AddReservation(new InMemoryReservationAllocation(rid, null, "u1", "dedicated"
, initTime, initTime + f.Length, ReservationSystemTestUtil.GenerateAllocation(initTime
, step, f), res, minAlloc));
// trying to update a reservation with a mismatching user
plan.UpdateReservation(new InMemoryReservationAllocation(rid, null, "u2", "dedicated"
, initTime, initTime + f.Length, ReservationSystemTestUtil.GenerateAllocation(initTime
, step, f), res, minAlloc));
}
// expected
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Reservation.Exceptions.PlanningException
/// "/>
public virtual void TestFailAvg()
{
// generate an allocation which violates the 25% average single-shot
IDictionary <ReservationInterval, ReservationRequest> req = new SortedDictionary <ReservationInterval
, ReservationRequest>();
long win = timeWindow / 2 + 100;
int cont = (int)Math.Ceil(0.5 * totCont);
req[new ReservationInterval(initTime, initTime + win)] = ReservationRequest.NewInstance
(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance(1024, 1), cont);
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(ReservationSystemTestUtil.GetNewReservationId(), null, "u1", "dedicated", initTime
, initTime + win, req, res, minAlloc)));
}
/// <exception cref="System.Exception"/>
private void SetupPlanFollower()
{
ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
mClock = Org.Mockito.Mockito.Mock <Clock>();
mAgent = Org.Mockito.Mockito.Mock <ReservationAgent>();
string reservationQ = testUtil.GetFullReservationQueueName();
CapacitySchedulerConfiguration csConf = cs.GetConfiguration();
csConf.SetReservationWindow(reservationQ, 20L);
csConf.SetMaximumCapacity(reservationQ, 40);
csConf.SetAverageCapacity(reservationQ, 20);
policy.Init(reservationQ, csConf);
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Reservation.Exceptions.PlanningException
/// "/>
/// <exception cref="System.IO.IOException"/>
public virtual void TestStress(int numJobs)
{
long timeWindow = 1000000L;
Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterCapacity = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(500 * 100 * 1024, 500 * 32);
step = 1000L;
ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
CapacityScheduler scheduler = testUtil.MockCapacityScheduler(500 * 100);
string reservationQ = testUtil.GetFullReservationQueueName();
float instConstraint = 100;
float avgConstraint = 100;
ReservationSchedulerConfiguration conf = ReservationSystemTestUtil.CreateConf(reservationQ
, timeWindow, instConstraint, avgConstraint);
CapacityOverTimePolicy policy = new CapacityOverTimePolicy();
policy.Init(reservationQ, conf);
plan = new InMemoryPlan(scheduler.GetRootQueueMetrics(), policy, agent, clusterCapacity
, step, res, minAlloc, maxAlloc, "dedicated", null, true);
int acc = 0;
IList <ReservationDefinition> list = new AList <ReservationDefinition>();
for (long i = 0; i < numJobs; i++)
{
list.AddItem(ReservationSystemTestUtil.GenerateRandomRR(rand, i));
}
long start = Runtime.CurrentTimeMillis();
for (int i_1 = 0; i_1 < numJobs; i_1++)
{
try
{
if (agent.CreateReservation(ReservationSystemTestUtil.GetNewReservationId(), "u"
+ i_1 % 100, plan, list[i_1]))
{
acc++;
}
}
catch (PlanningException)
{
}
}
// ignore exceptions
long end = Runtime.CurrentTimeMillis();
System.Console.Out.WriteLine("Submitted " + numJobs + " jobs " + " accepted " + acc
+ " in " + (end - start) + "ms");
}
public virtual void TestAnyImpossible()
{
PrepareBasicPlan();
// create an ANY request, with all impossible alternatives
ReservationDefinition rr = new ReservationDefinitionPBImpl();
rr.SetArrival(100L);
rr.SetDeadline(120L);
ReservationRequests reqs = new ReservationRequestsPBImpl();
reqs.SetInterpreter(ReservationRequestInterpreter.RAny);
// longer than arrival-deadline
ReservationRequest r1 = ReservationRequest.NewInstance(Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(1024, 1), 35, 5, 30);
// above max cluster size
ReservationRequest r2 = ReservationRequest.NewInstance(Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(1024, 1), 110, 110, 10);
IList <ReservationRequest> list = new AList <ReservationRequest>();
list.AddItem(r1);
list.AddItem(r2);
reqs.SetReservationResources(list);
rr.SetReservationRequests(reqs);
ReservationId reservationID = ReservationSystemTestUtil.GetNewReservationId();
bool result = false;
try
{
// submit to agent
result = agent.CreateReservation(reservationID, "u1", plan, rr);
NUnit.Framework.Assert.Fail();
}
catch (PlanningException)
{
}
// expected
// validate results, we expect the second one to be accepted
NUnit.Framework.Assert.IsFalse("Agent-based allocation should have failed", result
);
NUnit.Framework.Assert.IsTrue("Agent-based allocation should have failed", plan.GetAllReservations
().Count == 2);
System.Console.Out.WriteLine("--------AFTER ANY IMPOSSIBLE ALLOCATION (queue: " +
reservationID + ")----------");
System.Console.Out.WriteLine(plan.ToString());
System.Console.Out.WriteLine(plan.ToCumulativeString());
}
public virtual void SetUp()
{
CapacityScheduler spyCs = new CapacityScheduler();
cs = Org.Mockito.Mockito.Spy(spyCs);
scheduler = cs;
rmContext = TestUtils.GetMockRMContext();
spyRMContext = Org.Mockito.Mockito.Spy(rmContext);
ConcurrentMap <ApplicationId, RMApp> spyApps = Org.Mockito.Mockito.Spy(new ConcurrentHashMap
<ApplicationId, RMApp>());
RMApp rmApp = Org.Mockito.Mockito.Mock <RMApp>();
Org.Mockito.Mockito.When(rmApp.GetRMAppAttempt((ApplicationAttemptId)Matchers.Any
())).ThenReturn(null);
Org.Mockito.Mockito.DoReturn(rmApp).When(spyApps)[(ApplicationId)Matchers.Any()];
Org.Mockito.Mockito.When(spyRMContext.GetRMApps()).ThenReturn(spyApps);
Org.Mockito.Mockito.When(spyRMContext.GetScheduler()).ThenReturn(scheduler);
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
ReservationSystemTestUtil.SetupQueueConfiguration(csConf);
cs.SetConf(csConf);
csContext = Org.Mockito.Mockito.Mock <CapacitySchedulerContext>();
Org.Mockito.Mockito.When(csContext.GetConfiguration()).ThenReturn(csConf);
Org.Mockito.Mockito.When(csContext.GetConf()).ThenReturn(csConf);
Org.Mockito.Mockito.When(csContext.GetMinimumResourceCapability()).ThenReturn(minAlloc
);
Org.Mockito.Mockito.When(csContext.GetMaximumResourceCapability()).ThenReturn(maxAlloc
);
Org.Mockito.Mockito.When(csContext.GetClusterResource()).ThenReturn(Resources.CreateResource
(100 * 16 * Gb, 100 * 32));
Org.Mockito.Mockito.When(scheduler.GetClusterResource()).ThenReturn(Resources.CreateResource
(125 * Gb, 125));
Org.Mockito.Mockito.When(csContext.GetResourceCalculator()).ThenReturn(new DefaultResourceCalculator
());
RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager
(csConf);
containerTokenSecretManager.RollMasterKey();
Org.Mockito.Mockito.When(csContext.GetContainerTokenSecretManager()).ThenReturn(containerTokenSecretManager
);
cs.SetRMContext(spyRMContext);
cs.Init(csConf);
cs.Start();
SetupPlanFollower();
}
/// <exception cref="System.IO.IOException"/>
public static FairScheduler SetupFairScheduler(ReservationSystemTestUtil testUtil
, RMContext rmContext, Configuration conf, int numContainers)
{
FairScheduler scheduler = new FairScheduler();
scheduler.SetRMContext(rmContext);
Org.Mockito.Mockito.When(rmContext.GetScheduler()).ThenReturn(scheduler);
scheduler.Init(conf);
scheduler.Start();
scheduler.Reinitialize(conf, rmContext);
Resource resource = ReservationSystemTestUtil.CalculateClusterResource(numContainers
);
RMNode node1 = MockNodes.NewNodeInfo(1, resource, 1, "127.0.0.1");
NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
scheduler.Handle(nodeEvent1);
return(scheduler);
}
public virtual void TestAny()
{
PrepareBasicPlan();
// create an ANY request, with an impossible step (last in list, first
// considered),
// and two satisfiable ones. We expect the second one to be returned.
ReservationDefinition rr = new ReservationDefinitionPBImpl();
rr.SetArrival(100 * step);
rr.SetDeadline(120 * step);
ReservationRequests reqs = new ReservationRequestsPBImpl();
reqs.SetInterpreter(ReservationRequestInterpreter.RAny);
ReservationRequest r = ReservationRequest.NewInstance(Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(1024, 1), 5, 5, 10 * step);
ReservationRequest r2 = ReservationRequest.NewInstance(Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(2048, 2), 10, 5, 10 * step);
ReservationRequest r3 = ReservationRequest.NewInstance(Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(1024, 1), 110, 110, 10 * step);
IList <ReservationRequest> list = new AList <ReservationRequest>();
list.AddItem(r);
list.AddItem(r2);
list.AddItem(r3);
reqs.SetReservationResources(list);
rr.SetReservationRequests(reqs);
// submit to agent
ReservationId reservationID = ReservationSystemTestUtil.GetNewReservationId();
bool res = agent.CreateReservation(reservationID, "u1", plan, rr);
// validate results, we expect the second one to be accepted
NUnit.Framework.Assert.IsTrue("Agent-based allocation failed", res);
NUnit.Framework.Assert.IsTrue("Agent-based allocation failed", plan.GetAllReservations
().Count == 3);
ReservationAllocation cs = plan.GetReservationById(reservationID);
NUnit.Framework.Assert.IsTrue(cs.ToString(), Check(cs, 110 * step, 120 * step, 20
, 1024, 1));
System.Console.Out.WriteLine("--------AFTER ANY ALLOCATION (queue: " + reservationID
+ ")----------");
System.Console.Out.WriteLine(plan.ToString());
System.Console.Out.WriteLine(plan.ToCumulativeString());
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Reservation.Exceptions.PlanningException
/// "/>
private void PrepareBasicPlan()
{
// insert in the reservation a couple of controlled reservations, to create
// conditions for assignment that are non-empty
int[] f = new int[] { 10, 10, 20, 20, 20, 10, 10 };
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(ReservationSystemTestUtil.GetNewReservationId(), null, "u1", "dedicated", 0L, 0L
+ f.Length * step, ReservationSystemTestUtil.GenerateAllocation(0, step, f), res
, minAlloc)));
int[] f2 = new int[] { 5, 5, 5, 5, 5, 5, 5 };
IDictionary <ReservationInterval, ReservationRequest> alloc = ReservationSystemTestUtil
.GenerateAllocation(5000, step, f2);
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(ReservationSystemTestUtil.GetNewReservationId(), null, "u1", "dedicated", 5000,
5000 + f2.Length * step, alloc, res, minAlloc)));
System.Console.Out.WriteLine("--------BEFORE AGENT----------");
System.Console.Out.WriteLine(plan.ToString());
System.Console.Out.WriteLine(plan.ToCumulativeString());
}
public virtual void TestUpdateReservationNoDefinition()
{
ReservationUpdateRequest request = new ReservationUpdateRequestPBImpl();
request.SetReservationId(ReservationSystemTestUtil.GetNewReservationId());
Plan plan = null;
try
{
plan = rrValidator.ValidateReservationUpdateRequest(rSystem, request);
NUnit.Framework.Assert.Fail();
}
catch (YarnException e)
{
NUnit.Framework.Assert.IsNull(plan);
string message = e.Message;
NUnit.Framework.Assert.IsTrue(message.StartsWith("Missing reservation definition. Please try again by specifying a reservation definition."
));
Log.Info(message);
}
}
public virtual void TestFairReservationSystemReinitialize()
{
ReservationSystemTestUtil.SetupFSAllocationFile(AllocFile);
ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
// Setup
RMContext mockRMContext = ReservationSystemTestUtil.CreateRMContext(conf);
scheduler = ReservationSystemTestUtil.SetupFairScheduler(testUtil, mockRMContext,
conf, 10);
FairReservationSystem reservationSystem = new FairReservationSystem();
reservationSystem.SetRMContext(mockRMContext);
try
{
reservationSystem.Reinitialize(scheduler.GetConf(), mockRMContext);
}
catch (YarnException e)
{
NUnit.Framework.Assert.Fail(e.Message);
}
// Assert queue in original config
string planQNam = testUtil.GetFullReservationQueueName();
ReservationSystemTestUtil.ValidateReservationQueue(reservationSystem, planQNam);
// Dynamically add a plan
ReservationSystemTestUtil.UpdateFSAllocationFile(AllocFile);
scheduler.Reinitialize(conf, mockRMContext);
try
{
reservationSystem.Reinitialize(conf, mockRMContext);
}
catch (YarnException e)
{
NUnit.Framework.Assert.Fail(e.Message);
}
string newQueue = "root.reservation";
ReservationSystemTestUtil.ValidateNewReservationQueue(reservationSystem, newQueue
);
}
public virtual void TestDeleteReservationDoesnotExist()
{
ReservationDeleteRequest request = new ReservationDeleteRequestPBImpl();
ReservationId rId = ReservationSystemTestUtil.GetNewReservationId();
request.SetReservationId(rId);
Org.Mockito.Mockito.When(rSystem.GetQueueForReservation(rId)).ThenReturn(null);
Plan plan = null;
try
{
plan = rrValidator.ValidateReservationDeleteRequest(rSystem, request);
NUnit.Framework.Assert.Fail();
}
catch (YarnException e)
{
NUnit.Framework.Assert.IsNull(plan);
string message = e.Message;
NUnit.Framework.Assert.IsTrue(message.Equals(MessageFormat.Format("The specified reservation with ID: {0} is unknown. Please try again with a valid reservation."
, rId)));
Log.Info(message);
}
}
public virtual void TestDeleteReservationInvalidPlan()
{
ReservationDeleteRequest request = new ReservationDeleteRequestPBImpl();
ReservationId reservationID = ReservationSystemTestUtil.GetNewReservationId();
request.SetReservationId(reservationID);
Org.Mockito.Mockito.When(rSystem.GetPlan(PlanName)).ThenReturn(null);
Plan plan = null;
try
{
plan = rrValidator.ValidateReservationDeleteRequest(rSystem, request);
NUnit.Framework.Assert.Fail();
}
catch (YarnException e)
{
NUnit.Framework.Assert.IsNull(plan);
string message = e.Message;
NUnit.Framework.Assert.IsTrue(message.EndsWith(" is not associated with any valid plan. Please try again with a valid reservation."
));
Log.Info(message);
}
}