public virtual void SynchronizePlan(Plan plan) { lock (this) { string planQueueName = plan.GetQueueName(); if (Log.IsDebugEnabled()) { Log.Debug("Running plan follower edit policy for plan: " + planQueueName); } // align with plan step long step = plan.GetStep(); long now = clock.GetTime(); if (now % step != 0) { now += step - (now % step); } Queue planQueue = GetPlanQueue(planQueueName); if (planQueue == null) { return; } // first we publish to the plan the current availability of resources Resource clusterResources = scheduler.GetClusterResource(); Resource planResources = GetPlanResources(plan, planQueue, clusterResources); ICollection <ReservationAllocation> currentReservations = plan.GetReservationsAtTime (now); ICollection <string> curReservationNames = new HashSet <string>(); Resource reservedResources = Resource.NewInstance(0, 0); int numRes = GetReservedResources(now, currentReservations, curReservationNames, reservedResources); // create the default reservation queue if it doesnt exist string defReservationId = GetReservationIdFromQueueName(planQueueName) + ReservationConstants .DefaultQueueSuffix; string defReservationQueue = GetReservationQueueName(planQueueName, defReservationId ); CreateDefaultReservationQueue(planQueueName, planQueue, defReservationId); curReservationNames.AddItem(defReservationId); // if the resources dedicated to this plan has shrunk invoke replanner if (ArePlanResourcesLessThanReservations(clusterResources, planResources, reservedResources )) { try { plan.GetReplanner().Plan(plan, null); } catch (PlanningException e) { Log.Warn("Exception while trying to replan: {}", planQueueName, e); } } // identify the reservations that have expired and new reservations that // have to be activated IList <Queue> resQueues = GetChildReservationQueues(planQueue); ICollection <string> expired = new HashSet <string>(); foreach (Queue resQueue in resQueues) { string resQueueName = resQueue.GetQueueName(); string reservationId = GetReservationIdFromQueueName(resQueueName); if (curReservationNames.Contains(reservationId)) { // it is already existing reservation, so needed not create new // reservation queue curReservationNames.Remove(reservationId); } else { // the reservation has termination, mark for cleanup expired.AddItem(reservationId); } } // garbage collect expired reservations CleanupExpiredQueues(planQueueName, plan.GetMoveOnExpiry(), expired, defReservationQueue ); // Add new reservations and update existing ones float totalAssignedCapacity = 0f; if (currentReservations != null) { // first release all excess capacity in default queue try { SetQueueEntitlement(planQueueName, defReservationQueue, 0f, 1.0f); } catch (YarnException e) { Log.Warn("Exception while trying to release default queue capacity for plan: {}", planQueueName, e); } // sort allocations from the one giving up the most resources, to the // one asking for the most // avoid order-of-operation errors that temporarily violate 100% // capacity bound IList <ReservationAllocation> sortedAllocations = SortByDelta(new AList <ReservationAllocation >(currentReservations), now, plan); foreach (ReservationAllocation res in sortedAllocations) { string currResId = res.GetReservationId().ToString(); if (curReservationNames.Contains(currResId)) { AddReservationQueue(planQueueName, planQueue, currResId); } Resource capToAssign = res.GetResourcesAtTime(now); float targetCapacity = 0f; if (planResources.GetMemory() > 0 && planResources.GetVirtualCores() > 0) { targetCapacity = CalculateReservationToPlanRatio(clusterResources, planResources, capToAssign); } if (Log.IsDebugEnabled()) { Log.Debug("Assigning capacity of {} to queue {} with target capacity {}", capToAssign , currResId, targetCapacity); } // set maxCapacity to 100% unless the job requires gang, in which // case we stick to capacity (as running early/before is likely a // waste of resources) float maxCapacity = 1.0f; if (res.ContainsGangs()) { maxCapacity = targetCapacity; } try { SetQueueEntitlement(planQueueName, currResId, targetCapacity, maxCapacity); } catch (YarnException e) { Log.Warn("Exception while trying to size reservation for plan: {}", currResId, planQueueName , e); } totalAssignedCapacity += targetCapacity; } } // compute the default queue capacity float defQCap = 1.0f - totalAssignedCapacity; if (Log.IsDebugEnabled()) { Log.Debug("PlanFollowerEditPolicyTask: total Plan Capacity: {} " + "currReservation: {} default-queue capacity: {}" , planResources, numRes, defQCap); } // set the default queue to eat-up all remaining capacity try { SetQueueEntitlement(planQueueName, defReservationQueue, defQCap, 1.0f); } catch (YarnException e) { Log.Warn("Exception while trying to reclaim default queue capacity for plan: {}", planQueueName, e); } // garbage collect finished reservations from plan try { plan.ArchiveCompletedReservations(now); } catch (PlanningException e) { Log.Error("Exception in archiving completed reservations: ", e); } Log.Info("Finished iteration of plan follower edit policy for plan: " + planQueueName ); } }