protected internal virtual void ReleaseContainers(IList <ContainerId> containers,
SchedulerApplicationAttempt attempt)
{
foreach (ContainerId containerId in containers)
{
RMContainer rmContainer = GetRMContainer(containerId);
if (rmContainer == null)
{
if (Runtime.CurrentTimeMillis() - ResourceManager.GetClusterTimeStamp() < nmExpireInterval)
{
Log.Info(containerId + " doesn't exist. Add the container" + " to the release request cache as it maybe on recovery."
);
lock (attempt)
{
attempt.GetPendingRelease().AddItem(containerId);
}
}
else
{
RMAuditLogger.LogFailure(attempt.GetUser(), RMAuditLogger.AuditConstants.ReleaseContainer
, "Unauthorized access or invalid container", "Scheduler", "Trying to release container not owned by app or with invalid id."
, attempt.GetApplicationId(), containerId);
}
}
CompletedContainer(rmContainer, SchedulerUtils.CreateAbnormalContainerStatus(containerId
, SchedulerUtils.ReleasedContainer), RMContainerEventType.Released);
}
}
public virtual void ContainerCompleted(RMContainer rmContainer, ContainerStatus containerStatus
, RMContainerEventType @event)
{
lock (this)
{
Container container = rmContainer.GetContainer();
ContainerId containerId = container.GetId();
// Remove from the list of newly allocated containers if found
newlyAllocatedContainers.Remove(rmContainer);
// Inform the container
rmContainer.Handle(new RMContainerFinishedEvent(containerId, containerStatus, @event
));
Log.Info("Completed container: " + rmContainer.GetContainerId() + " in state: " +
rmContainer.GetState() + " event:" + @event);
// Remove from the list of containers
Sharpen.Collections.Remove(liveContainers, rmContainer.GetContainerId());
RMAuditLogger.LogSuccess(GetUser(), RMAuditLogger.AuditConstants.ReleaseContainer
, "SchedulerApp", GetApplicationId(), containerId);
// Update usage metrics
Org.Apache.Hadoop.Yarn.Api.Records.Resource containerResource = rmContainer.GetContainer
().GetResource();
queue.GetMetrics().ReleaseResources(GetUser(), 1, containerResource);
Resources.SubtractFrom(currentConsumption, containerResource);
// remove from preemption map if it is completed
Sharpen.Collections.Remove(preemptionMap, rmContainer);
// Clear resource utilization metrics cache.
lastMemoryAggregateAllocationUpdateTime = -1;
}
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
private Plan ValidateReservation(ReservationSystem reservationSystem, ReservationId
reservationId, string auditConstant)
{
string message = string.Empty;
// check if the reservation id is valid
if (reservationId == null)
{
message = "Missing reservation id." + " Please try again by specifying a reservation id.";
RMAuditLogger.LogFailure("UNKNOWN", auditConstant, "validate reservation input",
"ClientRMService", message);
throw RPCUtil.GetRemoteException(message);
}
string queueName = reservationSystem.GetQueueForReservation(reservationId);
if (queueName == null)
{
message = "The specified reservation with ID: " + reservationId + " is unknown. Please try again with a valid reservation.";
RMAuditLogger.LogFailure("UNKNOWN", auditConstant, "validate reservation input",
"ClientRMService", message);
throw RPCUtil.GetRemoteException(message);
}
// check if the associated plan is valid
Plan plan = reservationSystem.GetPlan(queueName);
if (plan == null)
{
message = "The specified reservation: " + reservationId + " is not associated with any valid plan."
+ " Please try again with a valid reservation.";
RMAuditLogger.LogFailure("UNKNOWN", auditConstant, "validate reservation input",
"ClientRMService", message);
throw RPCUtil.GetRemoteException(message);
}
return(plan);
}
/// <summary>
/// Quick validation on the input to check some obvious fail conditions (fail
/// fast) the input and returns the appropriate
/// <see cref="Plan"/>
/// associated with
/// the specified
/// <see cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.Queue"/>
/// or throws an exception message illustrating the
/// details of any validation check failures
/// </summary>
/// <param name="reservationSystem">
/// the
/// <see cref="ReservationSystem"/>
/// to validate against
/// </param>
/// <param name="request">
/// the
/// <see cref="Org.Apache.Hadoop.Yarn.Api.Protocolrecords.ReservationSubmissionRequest
/// "/>
/// defining the
/// resources required over time for the request
/// </param>
/// <param name="reservationId">
/// the
/// <see cref="Org.Apache.Hadoop.Yarn.Api.Records.ReservationId"/>
/// associated with the current
/// request
/// </param>
/// <returns>
/// the
/// <see cref="Plan"/>
/// to submit the request to
/// </returns>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public virtual Plan ValidateReservationSubmissionRequest(ReservationSystem reservationSystem
, ReservationSubmissionRequest request, ReservationId reservationId)
{
// Check if it is a managed queue
string queueName = request.GetQueue();
if (queueName == null || queueName.IsEmpty())
{
string errMsg = "The queue to submit is not specified." + " Please try again with a valid reservable queue.";
RMAuditLogger.LogFailure("UNKNOWN", RMAuditLogger.AuditConstants.SubmitReservationRequest
, "validate reservation input", "ClientRMService", errMsg);
throw RPCUtil.GetRemoteException(errMsg);
}
Plan plan = reservationSystem.GetPlan(queueName);
if (plan == null)
{
string errMsg = "The specified queue: " + queueName + " is not managed by reservation system."
+ " Please try again with a valid reservable queue.";
RMAuditLogger.LogFailure("UNKNOWN", RMAuditLogger.AuditConstants.SubmitReservationRequest
, "validate reservation input", "ClientRMService", errMsg);
throw RPCUtil.GetRemoteException(errMsg);
}
ValidateReservationDefinition(reservationId, request.GetReservationDefinition(),
plan, RMAuditLogger.AuditConstants.SubmitReservationRequest);
return(plan);
}
public virtual RMContainer Allocate(NodeType type, FSSchedulerNode node, Priority
priority, ResourceRequest request, Container container)
{
lock (this)
{
// Update allowed locality level
NodeType allowed = allowedLocalityLevel[priority];
if (allowed != null)
{
if (allowed.Equals(NodeType.OffSwitch) && (type.Equals(NodeType.NodeLocal) || type
.Equals(NodeType.RackLocal)))
{
this.ResetAllowedLocalityLevel(priority, type);
}
else
{
if (allowed.Equals(NodeType.RackLocal) && type.Equals(NodeType.NodeLocal))
{
this.ResetAllowedLocalityLevel(priority, type);
}
}
}
// Required sanity check - AM can call 'allocate' to update resource
// request without locking the scheduler, hence we need to check
if (GetTotalRequiredResources(priority) <= 0)
{
return(null);
}
// Create RMContainer
RMContainer rmContainer = new RMContainerImpl(container, GetApplicationAttemptId(
), node.GetNodeID(), appSchedulingInfo.GetUser(), rmContext);
// Add it to allContainers list.
newlyAllocatedContainers.AddItem(rmContainer);
liveContainers[container.GetId()] = rmContainer;
// Update consumption and track allocations
IList <ResourceRequest> resourceRequestList = appSchedulingInfo.Allocate(type, node
, priority, request, container);
Resources.AddTo(currentConsumption, container.GetResource());
// Update resource requests related to "request" and store in RMContainer
((RMContainerImpl)rmContainer).SetResourceRequests(resourceRequestList);
// Inform the container
rmContainer.Handle(new RMContainerEvent(container.GetId(), RMContainerEventType.Start
));
if (Log.IsDebugEnabled())
{
Log.Debug("allocate: applicationAttemptId=" + container.GetId().GetApplicationAttemptId
() + " container=" + container.GetId() + " host=" + container.GetNodeId().GetHost
() + " type=" + type);
}
RMAuditLogger.LogSuccess(GetUser(), RMAuditLogger.AuditConstants.AllocContainer,
"SchedulerApp", GetApplicationId(), container.GetId());
return(rmContainer);
}
}
public override void Run()
{
foreach (SchedulerApplication <T> app in this._enclosing.applications.Values)
{
T attempt = app.GetCurrentAppAttempt();
lock (attempt)
{
foreach (ContainerId containerId in attempt.GetPendingRelease())
{
RMAuditLogger.LogFailure(app.GetUser(), RMAuditLogger.AuditConstants.ReleaseContainer
, "Unauthorized access or invalid container", "Scheduler", "Trying to release container not owned by app or with invalid id."
, attempt.GetApplicationId(), containerId);
}
attempt.GetPendingRelease().Clear();
}
}
Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.AbstractYarnScheduler.Log
.Info("Release request cache is cleaned up");
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
private void ValidateReservationDefinition(ReservationId reservationId, ReservationDefinition
contract, Plan plan, string auditConstant)
{
string message = string.Empty;
// check if deadline is in the past
if (contract == null)
{
message = "Missing reservation definition." + " Please try again by specifying a reservation definition.";
RMAuditLogger.LogFailure("UNKNOWN", auditConstant, "validate reservation input definition"
, "ClientRMService", message);
throw RPCUtil.GetRemoteException(message);
}
if (contract.GetDeadline() <= clock.GetTime())
{
message = "The specified deadline: " + contract.GetDeadline() + " is the past. Please try again with deadline in the future.";
RMAuditLogger.LogFailure("UNKNOWN", auditConstant, "validate reservation input definition"
, "ClientRMService", message);
throw RPCUtil.GetRemoteException(message);
}
// Check if at least one RR has been specified
ReservationRequests resReqs = contract.GetReservationRequests();
if (resReqs == null)
{
message = "No resources have been specified to reserve." + "Please try again by specifying the resources to reserve.";
RMAuditLogger.LogFailure("UNKNOWN", auditConstant, "validate reservation input definition"
, "ClientRMService", message);
throw RPCUtil.GetRemoteException(message);
}
IList <ReservationRequest> resReq = resReqs.GetReservationResources();
if (resReq == null || resReq.IsEmpty())
{
message = "No resources have been specified to reserve." + " Please try again by specifying the resources to reserve.";
RMAuditLogger.LogFailure("UNKNOWN", auditConstant, "validate reservation input definition"
, "ClientRMService", message);
throw RPCUtil.GetRemoteException(message);
}
// compute minimum duration and max gang size
long minDuration = 0;
Resource maxGangSize = Resource.NewInstance(0, 0);
ReservationRequestInterpreter type = contract.GetReservationRequests().GetInterpreter
();
foreach (ReservationRequest rr in resReq)
{
if (type == ReservationRequestInterpreter.RAll || type == ReservationRequestInterpreter
.RAny)
{
minDuration = Math.Max(minDuration, rr.GetDuration());
}
else
{
minDuration += rr.GetDuration();
}
maxGangSize = Resources.Max(plan.GetResourceCalculator(), plan.GetTotalCapacity()
, maxGangSize, Resources.Multiply(rr.GetCapability(), rr.GetConcurrency()));
}
// verify the allocation is possible (skip for ANY)
if (contract.GetDeadline() - contract.GetArrival() < minDuration && type != ReservationRequestInterpreter
.RAny)
{
message = "The time difference (" + (contract.GetDeadline() - contract.GetArrival
()) + ") between arrival (" + contract.GetArrival() + ") " + "and deadline (" +
contract.GetDeadline() + ") must " + " be greater or equal to the minimum resource duration ("
+ minDuration + ")";
RMAuditLogger.LogFailure("UNKNOWN", auditConstant, "validate reservation input definition"
, "ClientRMService", message);
throw RPCUtil.GetRemoteException(message);
}
// check that the largest gang does not exceed the inventory available
// capacity (skip for ANY)
if (Resources.GreaterThan(plan.GetResourceCalculator(), plan.GetTotalCapacity(),
maxGangSize, plan.GetTotalCapacity()) && type != ReservationRequestInterpreter.RAny)
{
message = "The size of the largest gang in the reservation refinition (" + maxGangSize
+ ") exceed the capacity available (" + plan.GetTotalCapacity() + " )";
RMAuditLogger.LogFailure("UNKNOWN", auditConstant, "validate reservation input definition"
, "ClientRMService", message);
throw RPCUtil.GetRemoteException(message);
}
}
