public virtual void TestAddReservation()
{
Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, resCalc
, minAlloc, maxAlloc, planName, replanner, true);
ReservationId reservationID = ReservationSystemTestUtil.GetNewReservationId();
int[] alloc = new int[] { 10, 10, 10, 10, 10, 10 };
int start = 100;
IDictionary <ReservationInterval, ReservationRequest> allocations = GenerateAllocation
(start, alloc, false);
ReservationDefinition rDef = CreateSimpleReservationDefinition(start, start + alloc
.Length, alloc.Length, allocations.Values);
ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID
, rDef, user, planName, start, start + alloc.Length, allocations, resCalc, minAlloc
);
NUnit.Framework.Assert.IsNull(plan.GetReservationById(reservationID));
try
{
plan.AddReservation(rAllocation);
}
catch (PlanningException e)
{
NUnit.Framework.Assert.Fail(e.Message);
}
DoAssertions(plan, rAllocation);
for (int i = 0; i < alloc.Length; i++)
{
NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(1024 * (alloc[i]), (alloc[i])), plan.GetTotalCommittedResources(start + i));
NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(1024 * (alloc[i]), (alloc[i])), plan.GetConsumptionForUser(user, start + i));
}
}
public virtual void TestUpdateNonExistingReservation()
{
Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, resCalc
, minAlloc, maxAlloc, planName, replanner, true);
ReservationId reservationID = ReservationSystemTestUtil.GetNewReservationId();
// Try to update a reservation without adding
int[] alloc = new int[] { 10, 10, 10, 10, 10, 10 };
int start = 100;
IDictionary <ReservationInterval, ReservationRequest> allocations = GenerateAllocation
(start, alloc, false);
ReservationDefinition rDef = CreateSimpleReservationDefinition(start, start + alloc
.Length, alloc.Length, allocations.Values);
ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID
, rDef, user, planName, start, start + alloc.Length, allocations, resCalc, minAlloc
);
NUnit.Framework.Assert.IsNull(plan.GetReservationById(reservationID));
try
{
plan.UpdateReservation(rAllocation);
NUnit.Framework.Assert.Fail("Update should fail as it does not exist in the plan"
);
}
catch (ArgumentException e)
{
NUnit.Framework.Assert.IsTrue(e.Message.EndsWith("does not exist in the plan"));
}
catch (PlanningException e)
{
NUnit.Framework.Assert.Fail(e.Message);
}
NUnit.Framework.Assert.IsNull(plan.GetReservationById(reservationID));
}
public virtual void TestAddEmptyReservation()
{
Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, resCalc
, minAlloc, maxAlloc, planName, replanner, true);
ReservationId reservationID = ReservationSystemTestUtil.GetNewReservationId();
int[] alloc = new int[] { };
int start = 100;
IDictionary <ReservationInterval, ReservationRequest> allocations = new Dictionary
<ReservationInterval, ReservationRequest>();
ReservationDefinition rDef = CreateSimpleReservationDefinition(start, start + alloc
.Length, alloc.Length, allocations.Values);
ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID
, rDef, user, planName, start, start + alloc.Length, allocations, resCalc, minAlloc
);
NUnit.Framework.Assert.IsNull(plan.GetReservationById(reservationID));
try
{
plan.AddReservation(rAllocation);
}
catch (PlanningException e)
{
NUnit.Framework.Assert.Fail(e.Message);
}
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Reservation.Exceptions.PlanningException
/// "/>
public virtual bool AddReservation(ReservationAllocation reservation)
{
// Verify the allocation is memory based otherwise it is not supported
InMemoryReservationAllocation inMemReservation = (InMemoryReservationAllocation)reservation;
if (inMemReservation.GetUser() == null)
{
string errMsg = "The specified Reservation with ID " + inMemReservation.GetReservationId
() + " is not mapped to any user";
Log.Error(errMsg);
throw new ArgumentException(errMsg);
}
writeLock.Lock();
try
{
if (reservationTable.Contains(inMemReservation.GetReservationId()))
{
string errMsg = "The specified Reservation with ID " + inMemReservation.GetReservationId
() + " already exists";
Log.Error(errMsg);
throw new ArgumentException(errMsg);
}
// Validate if we can accept this reservation, throws exception if
// validation fails
policy.Validate(this, inMemReservation);
// we record here the time in which the allocation has been accepted
reservation.SetAcceptanceTimestamp(clock.GetTime());
ReservationInterval searchInterval = new ReservationInterval(inMemReservation.GetStartTime
(), inMemReservation.GetEndTime());
ICollection <InMemoryReservationAllocation> reservations = currentReservations[searchInterval
];
if (reservations == null)
{
reservations = new HashSet <InMemoryReservationAllocation>();
}
if (!reservations.AddItem(inMemReservation))
{
Log.Error("Unable to add reservation: {} to plan.", inMemReservation.GetReservationId
());
return(false);
}
currentReservations[searchInterval] = reservations;
reservationTable[inMemReservation.GetReservationId()] = inMemReservation;
IncrementAllocation(inMemReservation);
Log.Info("Sucessfully added reservation: {} to plan.", inMemReservation.GetReservationId
());
return(true);
}
finally
{
writeLock.Unlock();
}
}
public virtual void TestZeroAlloaction()
{
ReservationId reservationID = ReservationId.NewInstance(rand.NextLong(), rand.NextLong
());
int[] alloc = new int[] { };
long start = 0;
ReservationDefinition rDef = CreateSimpleReservationDefinition(start, start + alloc
.Length + 1, alloc.Length);
IDictionary <ReservationInterval, ReservationRequest> allocations = new Dictionary
<ReservationInterval, ReservationRequest>();
ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID
, rDef, user, planName, start, start + alloc.Length + 1, allocations, resCalc, minAlloc
);
DoAssertions(rAllocation, reservationID, rDef, allocations, (int)start, alloc);
NUnit.Framework.Assert.IsFalse(rAllocation.ContainsGangs());
}
public virtual void TestBlocks()
{
ReservationId reservationID = ReservationId.NewInstance(rand.NextLong(), rand.NextLong
());
int[] alloc = new int[] { 10, 10, 10, 10, 10, 10 };
int start = 100;
ReservationDefinition rDef = CreateSimpleReservationDefinition(start, start + alloc
.Length + 1, alloc.Length);
IDictionary <ReservationInterval, ReservationRequest> allocations = GenerateAllocation
(start, alloc, false, false);
ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID
, rDef, user, planName, start, start + alloc.Length + 1, allocations, resCalc, minAlloc
);
DoAssertions(rAllocation, reservationID, rDef, allocations, start, alloc);
NUnit.Framework.Assert.IsFalse(rAllocation.ContainsGangs());
for (int i = 0; i < alloc.Length; i++)
{
NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(1024 * (alloc[i]), (alloc[i])), rAllocation.GetResourcesAtTime(start + i));
}
}
public virtual void TestArchiveCompletedReservations()
{
Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, resCalc
, minAlloc, maxAlloc, planName, replanner, true);
ReservationId reservationID1 = ReservationSystemTestUtil.GetNewReservationId();
// First add a reservation
int[] alloc1 = new int[] { 10, 10, 10, 10, 10, 10 };
int start = 100;
IDictionary <ReservationInterval, ReservationRequest> allocations1 = GenerateAllocation
(start, alloc1, false);
ReservationDefinition rDef1 = CreateSimpleReservationDefinition(start, start + alloc1
.Length, alloc1.Length, allocations1.Values);
ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID1
, rDef1, user, planName, start, start + alloc1.Length, allocations1, resCalc, minAlloc
);
NUnit.Framework.Assert.IsNull(plan.GetReservationById(reservationID1));
try
{
plan.AddReservation(rAllocation);
}
catch (PlanningException e)
{
NUnit.Framework.Assert.Fail(e.Message);
}
DoAssertions(plan, rAllocation);
for (int i = 0; i < alloc1.Length; i++)
{
NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(1024 * (alloc1[i]), (alloc1[i])), plan.GetTotalCommittedResources(start + i));
NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(1024 * (alloc1[i]), (alloc1[i])), plan.GetConsumptionForUser(user, start + i));
}
// Now add another one
ReservationId reservationID2 = ReservationSystemTestUtil.GetNewReservationId();
int[] alloc2 = new int[] { 0, 5, 10, 5, 0 };
IDictionary <ReservationInterval, ReservationRequest> allocations2 = GenerateAllocation
(start, alloc2, true);
ReservationDefinition rDef2 = CreateSimpleReservationDefinition(start, start + alloc2
.Length, alloc2.Length, allocations2.Values);
rAllocation = new InMemoryReservationAllocation(reservationID2, rDef2, user, planName
, start, start + alloc2.Length, allocations2, resCalc, minAlloc);
NUnit.Framework.Assert.IsNull(plan.GetReservationById(reservationID2));
try
{
plan.AddReservation(rAllocation);
}
catch (PlanningException e)
{
NUnit.Framework.Assert.Fail(e.Message);
}
NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(reservationID2));
for (int i_1 = 0; i_1 < alloc2.Length; i_1++)
{
NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(1024 * (alloc1[i_1] + alloc2[i_1] + i_1), alloc1[i_1] + alloc2[i_1] + i_1), plan
.GetTotalCommittedResources(start + i_1));
NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(1024 * (alloc1[i_1] + alloc2[i_1] + i_1), alloc1[i_1] + alloc2[i_1] + i_1), plan
.GetConsumptionForUser(user, start + i_1));
}
// Now archive completed reservations
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(106L);
Org.Mockito.Mockito.When(policy.GetValidWindow()).ThenReturn(1L);
try
{
// will only remove 2nd reservation as only that has fallen out of the
// archival window
plan.ArchiveCompletedReservations(clock.GetTime());
}
catch (PlanningException e)
{
NUnit.Framework.Assert.Fail(e.Message);
}
NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(reservationID1));
NUnit.Framework.Assert.IsNull(plan.GetReservationById(reservationID2));
for (int i_2 = 0; i_2 < alloc1.Length; i_2++)
{
NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(1024 * (alloc1[i_2]), (alloc1[i_2])), plan.GetTotalCommittedResources(start + i_2
));
NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(1024 * (alloc1[i_2]), (alloc1[i_2])), plan.GetConsumptionForUser(user, start +
i_2));
}
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(107L);
try
{
// will remove 1st reservation also as it has fallen out of the archival
// window
plan.ArchiveCompletedReservations(clock.GetTime());
}
catch (PlanningException e)
{
NUnit.Framework.Assert.Fail(e.Message);
}
NUnit.Framework.Assert.IsNull(plan.GetReservationById(reservationID1));
for (int i_3 = 0; i_3 < alloc1.Length; i_3++)
{
NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(0, 0), plan.GetTotalCommittedResources(start + i_3));
NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(0, 0), plan.GetConsumptionForUser(user, start + i_3));
}
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Reservation.Exceptions.PlanningException
/// "/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Reservation.Exceptions.ContractValidationException
/// "/>
private bool ComputeAllocation(ReservationId reservationId, string user, Plan plan
, ReservationDefinition contract, ReservationAllocation oldReservation)
{
Log.Info("placing the following ReservationRequest: " + contract);
Resource totalCapacity = plan.GetTotalCapacity();
// Here we can addd logic to adjust the ResourceDefinition to account for
// system "imperfections" (e.g., scheduling delays for large containers).
// Align with plan step conservatively (i.e., ceil arrival, and floor
// deadline)
long earliestStart = contract.GetArrival();
long step = plan.GetStep();
if (earliestStart % step != 0)
{
earliestStart = earliestStart + (step - (earliestStart % step));
}
long deadline = contract.GetDeadline() - contract.GetDeadline() % plan.GetStep();
// setup temporary variables to handle time-relations between stages and
// intermediate answers
long curDeadline = deadline;
long oldDeadline = -1;
IDictionary <ReservationInterval, ReservationRequest> allocations = new Dictionary
<ReservationInterval, ReservationRequest>();
RLESparseResourceAllocation tempAssigned = new RLESparseResourceAllocation(plan.GetResourceCalculator
(), plan.GetMinimumAllocation());
IList <ReservationRequest> stages = contract.GetReservationRequests().GetReservationResources
();
ReservationRequestInterpreter type = contract.GetReservationRequests().GetInterpreter
();
// Iterate the stages in backward from deadline
for (ListIterator <ReservationRequest> li = stages.ListIterator(stages.Count); li.
HasPrevious();)
{
ReservationRequest currentReservationStage = li.Previous();
// validate the RR respect basic constraints
ValidateInput(plan, currentReservationStage, totalCapacity);
// run allocation for a single stage
IDictionary <ReservationInterval, ReservationRequest> curAlloc = PlaceSingleStage(
plan, tempAssigned, currentReservationStage, earliestStart, curDeadline, oldReservation
, totalCapacity);
if (curAlloc == null)
{
// if we did not find an allocation for the currentReservationStage
// return null, unless the ReservationDefinition we are placing is of
// type ANY
if (type != ReservationRequestInterpreter.RAny)
{
throw new PlanningException("The GreedyAgent" + " couldn't find a valid allocation for your request"
);
}
else
{
continue;
}
}
else
{
// if we did find an allocation add it to the set of allocations
allocations.PutAll(curAlloc);
// if this request is of type ANY we are done searching (greedy)
// and can return the current allocation (break-out of the search)
if (type == ReservationRequestInterpreter.RAny)
{
break;
}
// if the request is of ORDER or ORDER_NO_GAP we constraint the next
// round of allocation to precede the current allocation, by setting
// curDeadline
if (type == ReservationRequestInterpreter.ROrder || type == ReservationRequestInterpreter
.ROrderNoGap)
{
curDeadline = FindEarliestTime(curAlloc.Keys);
// for ORDER_NO_GAP verify that the allocation found so far has no
// gap, return null otherwise (the greedy procedure failed to find a
// no-gap
// allocation)
if (type == ReservationRequestInterpreter.ROrderNoGap && oldDeadline > 0)
{
if (oldDeadline - FindLatestTime(curAlloc.Keys) > plan.GetStep())
{
throw new PlanningException("The GreedyAgent" + " couldn't find a valid allocation for your request"
);
}
}
// keep the variable oldDeadline pointing to the last deadline we
// found
oldDeadline = curDeadline;
}
}
}
// / If we got here is because we failed to find an allocation for the
// ReservationDefinition give-up and report failure to the user
if (allocations.IsEmpty())
{
throw new PlanningException("The GreedyAgent" + " couldn't find a valid allocation for your request"
);
}
// create reservation with above allocations if not null/empty
ReservationRequest ZeroRes = ReservationRequest.NewInstance(Resource.NewInstance(
0, 0), 0);
long firstStartTime = FindEarliestTime(allocations.Keys);
// add zero-padding from arrival up to the first non-null allocation
// to guarantee that the reservation exists starting at arrival
if (firstStartTime > earliestStart)
{
allocations[new ReservationInterval(earliestStart, firstStartTime)] = ZeroRes;
firstStartTime = earliestStart;
}
// consider to add trailing zeros at the end for simmetry
// Actually add/update the reservation in the plan.
// This is subject to validation as other agents might be placing
// in parallel and there might be sharing policies the agent is not
// aware off.
ReservationAllocation capReservation = new InMemoryReservationAllocation(reservationId
, contract, user, plan.GetQueueName(), firstStartTime, FindLatestTime(allocations
.Keys), allocations, plan.GetResourceCalculator(), plan.GetMinimumAllocation());
if (oldReservation != null)
{
return(plan.UpdateReservation(capReservation));
}
else
{
return(plan.AddReservation(capReservation));
}
}
