public virtual void SynchronizePlan(Plan plan)
{
lock (this)
{
string planQueueName = plan.GetQueueName();
if (Log.IsDebugEnabled())
{
Log.Debug("Running plan follower edit policy for plan: " + planQueueName);
}
// align with plan step
long step = plan.GetStep();
long now = clock.GetTime();
if (now % step != 0)
{
now += step - (now % step);
}
Queue planQueue = GetPlanQueue(planQueueName);
if (planQueue == null)
{
return;
}
// first we publish to the plan the current availability of resources
Resource clusterResources = scheduler.GetClusterResource();
Resource planResources = GetPlanResources(plan, planQueue, clusterResources);
ICollection <ReservationAllocation> currentReservations = plan.GetReservationsAtTime
(now);
ICollection <string> curReservationNames = new HashSet <string>();
Resource reservedResources = Resource.NewInstance(0, 0);
int numRes = GetReservedResources(now, currentReservations, curReservationNames,
reservedResources);
// create the default reservation queue if it doesnt exist
string defReservationId = GetReservationIdFromQueueName(planQueueName) + ReservationConstants
.DefaultQueueSuffix;
string defReservationQueue = GetReservationQueueName(planQueueName, defReservationId
);
CreateDefaultReservationQueue(planQueueName, planQueue, defReservationId);
curReservationNames.AddItem(defReservationId);
// if the resources dedicated to this plan has shrunk invoke replanner
if (ArePlanResourcesLessThanReservations(clusterResources, planResources, reservedResources
))
{
try
{
plan.GetReplanner().Plan(plan, null);
}
catch (PlanningException e)
{
Log.Warn("Exception while trying to replan: {}", planQueueName, e);
}
}
// identify the reservations that have expired and new reservations that
// have to be activated
IList <Queue> resQueues = GetChildReservationQueues(planQueue);
ICollection <string> expired = new HashSet <string>();
foreach (Queue resQueue in resQueues)
{
string resQueueName = resQueue.GetQueueName();
string reservationId = GetReservationIdFromQueueName(resQueueName);
if (curReservationNames.Contains(reservationId))
{
// it is already existing reservation, so needed not create new
// reservation queue
curReservationNames.Remove(reservationId);
}
else
{
// the reservation has termination, mark for cleanup
expired.AddItem(reservationId);
}
}
// garbage collect expired reservations
CleanupExpiredQueues(planQueueName, plan.GetMoveOnExpiry(), expired, defReservationQueue
);
// Add new reservations and update existing ones
float totalAssignedCapacity = 0f;
if (currentReservations != null)
{
// first release all excess capacity in default queue
try
{
SetQueueEntitlement(planQueueName, defReservationQueue, 0f, 1.0f);
}
catch (YarnException e)
{
Log.Warn("Exception while trying to release default queue capacity for plan: {}",
planQueueName, e);
}
// sort allocations from the one giving up the most resources, to the
// one asking for the most
// avoid order-of-operation errors that temporarily violate 100%
// capacity bound
IList <ReservationAllocation> sortedAllocations = SortByDelta(new AList <ReservationAllocation
>(currentReservations), now, plan);
foreach (ReservationAllocation res in sortedAllocations)
{
string currResId = res.GetReservationId().ToString();
if (curReservationNames.Contains(currResId))
{
AddReservationQueue(planQueueName, planQueue, currResId);
}
Resource capToAssign = res.GetResourcesAtTime(now);
float targetCapacity = 0f;
if (planResources.GetMemory() > 0 && planResources.GetVirtualCores() > 0)
{
targetCapacity = CalculateReservationToPlanRatio(clusterResources, planResources,
capToAssign);
}
if (Log.IsDebugEnabled())
{
Log.Debug("Assigning capacity of {} to queue {} with target capacity {}", capToAssign
, currResId, targetCapacity);
}
// set maxCapacity to 100% unless the job requires gang, in which
// case we stick to capacity (as running early/before is likely a
// waste of resources)
float maxCapacity = 1.0f;
if (res.ContainsGangs())
{
maxCapacity = targetCapacity;
}
try
{
SetQueueEntitlement(planQueueName, currResId, targetCapacity, maxCapacity);
}
catch (YarnException e)
{
Log.Warn("Exception while trying to size reservation for plan: {}", currResId, planQueueName
, e);
}
totalAssignedCapacity += targetCapacity;
}
}
// compute the default queue capacity
float defQCap = 1.0f - totalAssignedCapacity;
if (Log.IsDebugEnabled())
{
Log.Debug("PlanFollowerEditPolicyTask: total Plan Capacity: {} " + "currReservation: {} default-queue capacity: {}"
, planResources, numRes, defQCap);
}
// set the default queue to eat-up all remaining capacity
try
{
SetQueueEntitlement(planQueueName, defReservationQueue, defQCap, 1.0f);
}
catch (YarnException e)
{
Log.Warn("Exception while trying to reclaim default queue capacity for plan: {}",
planQueueName, e);
}
// garbage collect finished reservations from plan
try
{
plan.ArchiveCompletedReservations(now);
}
catch (PlanningException e)
{
Log.Error("Exception in archiving completed reservations: ", e);
}
Log.Info("Finished iteration of plan follower edit policy for plan: " + planQueueName
);
}
}
