/// <summary>Process resource update on a node.</summary>
public virtual void UpdateNodeResource(RMNode nm, ResourceOption resourceOption)
{
lock (this)
{
SchedulerNode node = GetSchedulerNode(nm.GetNodeID());
Org.Apache.Hadoop.Yarn.Api.Records.Resource newResource = resourceOption.GetResource
();
Org.Apache.Hadoop.Yarn.Api.Records.Resource oldResource = node.GetTotalResource();
if (!oldResource.Equals(newResource))
{
// Log resource change
Log.Info("Update resource on node: " + node.GetNodeName() + " from: " + oldResource
+ ", to: " + newResource);
Sharpen.Collections.Remove(nodes, nm.GetNodeID());
UpdateMaximumAllocation(node, false);
// update resource to node
node.SetTotalResource(newResource);
nodes[nm.GetNodeID()] = (N)node;
UpdateMaximumAllocation(node, true);
// update resource to clusterResource
Resources.SubtractFrom(clusterResource, oldResource);
Resources.AddTo(clusterResource, newResource);
}
else
{
// Log resource change
Log.Warn("Update resource on node: " + node.GetNodeName() + " with the same resource: "
+ newResource);
}
}
}
/// <summary>Removes a resource for the specified interval</summary>
/// <param name="reservationInterval">
/// the interval for which the resource is to be
/// removed
/// </param>
/// <param name="capacity">the resource to be removed</param>
/// <returns>true if removal is successful, false otherwise</returns>
public virtual bool RemoveInterval(ReservationInterval reservationInterval, ReservationRequest
capacity)
{
Org.Apache.Hadoop.Yarn.Api.Records.Resource totCap = Resources.Multiply(capacity.
GetCapability(), (float)capacity.GetNumContainers());
if (totCap.Equals(ZeroResource))
{
return(true);
}
writeLock.Lock();
try
{
long startKey = reservationInterval.GetStartTime();
long endKey = reservationInterval.GetEndTime();
// update the start key
NavigableMap <long, Org.Apache.Hadoop.Yarn.Api.Records.Resource> ticks = cumulativeCapacity
.HeadMap(endKey, false);
// Decrease all the capacities of overlapping intervals
SortedDictionary <long, Org.Apache.Hadoop.Yarn.Api.Records.Resource> overlapSet =
ticks.TailMap(startKey);
if (overlapSet != null && !overlapSet.IsEmpty())
{
Org.Apache.Hadoop.Yarn.Api.Records.Resource updatedCapacity = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(0, 0);
long currentKey = -1;
for (IEnumerator <KeyValuePair <long, Org.Apache.Hadoop.Yarn.Api.Records.Resource> >
overlapEntries = overlapSet.GetEnumerator(); overlapEntries.HasNext();)
{
KeyValuePair <long, Org.Apache.Hadoop.Yarn.Api.Records.Resource> entry = overlapEntries
.Next();
currentKey = entry.Key;
updatedCapacity = Resources.Subtract(entry.Value, totCap);
// update each entry between start and end key
cumulativeCapacity[currentKey] = updatedCapacity;
}
// Remove the first overlap entry if it is same as previous after
// updation
long firstKey = overlapSet.FirstKey();
if (IsSameAsPrevious(firstKey, overlapSet[firstKey]))
{
Sharpen.Collections.Remove(cumulativeCapacity, firstKey);
}
// Remove the next entry if it is same as end entry after updation
if ((currentKey != -1) && (IsSameAsNext(currentKey, updatedCapacity)))
{
Sharpen.Collections.Remove(cumulativeCapacity, cumulativeCapacity.HigherKey(currentKey
));
}
}
return(true);
}
finally
{
writeLock.Unlock();
}
}
public override Org.Apache.Hadoop.Yarn.Api.Records.Resource AssignContainer(FSSchedulerNode
node)
{
Org.Apache.Hadoop.Yarn.Api.Records.Resource assigned = Resources.None();
if (Log.IsDebugEnabled())
{
Log.Debug("Node " + node.GetNodeName() + " offered to queue: " + GetName());
}
if (!AssignContainerPreCheck(node))
{
return(assigned);
}
IComparer <Schedulable> comparator = policy.GetComparator();
writeLock.Lock();
try
{
runnableApps.Sort(comparator);
}
finally
{
writeLock.Unlock();
}
// Release write lock here for better performance and avoiding deadlocks.
// runnableApps can be in unsorted state because of this section,
// but we can accept it in practice since the probability is low.
readLock.Lock();
try
{
foreach (FSAppAttempt sched in runnableApps)
{
if (SchedulerAppUtils.IsBlacklisted(sched, node, Log))
{
continue;
}
assigned = sched.AssignContainer(node);
if (!assigned.Equals(Resources.None()))
{
break;
}
}
}
finally
{
readLock.Unlock();
}
return(assigned);
}
public override bool Equals(object obj)
{
if (this == obj)
{
return(true);
}
if (obj == null)
{
return(false);
}
if (GetType() != obj.GetType())
{
return(false);
}
ReservationRequest other = (ReservationRequest)obj;
Resource capability = GetCapability();
if (capability == null)
{
if (other.GetCapability() != null)
{
return(false);
}
}
else
{
if (!capability.Equals(other.GetCapability()))
{
return(false);
}
}
if (GetNumContainers() != other.GetNumContainers())
{
return(false);
}
if (GetConcurrency() != other.GetConcurrency())
{
return(false);
}
return(true);
}
public override bool Equals(object obj)
{
if (this == obj)
{
return(true);
}
if (obj == null)
{
return(false);
}
if (GetType() != obj.GetType())
{
return(false);
}
ResourceRequest other = (ResourceRequest)obj;
Resource capability = GetCapability();
if (capability == null)
{
if (other.GetCapability() != null)
{
return(false);
}
}
else
{
if (!capability.Equals(other.GetCapability()))
{
return(false);
}
}
string hostName = GetResourceName();
if (hostName == null)
{
if (other.GetResourceName() != null)
{
return(false);
}
}
else
{
if (!hostName.Equals(other.GetResourceName()))
{
return(false);
}
}
if (GetNumContainers() != other.GetNumContainers())
{
return(false);
}
Priority priority = GetPriority();
if (priority == null)
{
if (other.GetPriority() != null)
{
return(false);
}
}
else
{
if (!priority.Equals(other.GetPriority()))
{
return(false);
}
}
if (GetNodeLabelExpression() == null)
{
if (other.GetNodeLabelExpression() != null)
{
return(false);
}
}
else
{
// do normalize on label expression before compare
string label1 = GetNodeLabelExpression().ReplaceAll("[\\t ]", string.Empty);
string label2 = other.GetNodeLabelExpression() == null ? null : other.GetNodeLabelExpression
().ReplaceAll("[\\t ]", string.Empty);
if (!label1.Equals(label2))
{
return(false);
}
}
return(true);
}
/// <summary>Add a resource for the specified interval</summary>
/// <param name="reservationInterval">
/// the interval for which the resource is to be
/// added
/// </param>
/// <param name="capacity">the resource to be added</param>
/// <returns>true if addition is successful, false otherwise</returns>
public virtual bool AddInterval(ReservationInterval reservationInterval, ReservationRequest
capacity)
{
Org.Apache.Hadoop.Yarn.Api.Records.Resource totCap = Resources.Multiply(capacity.
GetCapability(), (float)capacity.GetNumContainers());
if (totCap.Equals(ZeroResource))
{
return(true);
}
writeLock.Lock();
try
{
long startKey = reservationInterval.GetStartTime();
long endKey = reservationInterval.GetEndTime();
NavigableMap <long, Org.Apache.Hadoop.Yarn.Api.Records.Resource> ticks = cumulativeCapacity
.HeadMap(endKey, false);
if (ticks != null && !ticks.IsEmpty())
{
Org.Apache.Hadoop.Yarn.Api.Records.Resource updatedCapacity = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(0, 0);
KeyValuePair <long, Org.Apache.Hadoop.Yarn.Api.Records.Resource> lowEntry = ticks.
FloorEntry(startKey);
if (lowEntry == null)
{
// This is the earliest starting interval
cumulativeCapacity[startKey] = totCap;
}
else
{
updatedCapacity = Resources.Add(lowEntry.Value, totCap);
// Add a new tick only if the updated value is different
// from the previous tick
if ((startKey == lowEntry.Key) && (IsSameAsPrevious(lowEntry.Key, updatedCapacity
)))
{
Sharpen.Collections.Remove(cumulativeCapacity, lowEntry.Key);
}
else
{
cumulativeCapacity[startKey] = updatedCapacity;
}
}
// Increase all the capacities of overlapping intervals
ICollection <KeyValuePair <long, Org.Apache.Hadoop.Yarn.Api.Records.Resource> > overlapSet
= ticks.TailMap(startKey, false);
foreach (KeyValuePair <long, Org.Apache.Hadoop.Yarn.Api.Records.Resource> entry in
overlapSet)
{
updatedCapacity = Resources.Add(entry.Value, totCap);
entry.SetValue(updatedCapacity);
}
}
else
{
// This is the first interval to be added
cumulativeCapacity[startKey] = totCap;
}
Org.Apache.Hadoop.Yarn.Api.Records.Resource nextTick = cumulativeCapacity[endKey];
if (nextTick != null)
{
// If there is overlap, remove the duplicate entry
if (IsSameAsPrevious(endKey, nextTick))
{
Sharpen.Collections.Remove(cumulativeCapacity, endKey);
}
}
else
{
// Decrease capacity as this is end of the interval
cumulativeCapacity[endKey] = Resources.Subtract(cumulativeCapacity.FloorEntry(endKey
).Value, totCap);
}
return(true);
}
finally
{
writeLock.Unlock();
}
}