public virtual int Compare(Schedulable s1, Schedulable s2)
{
ResourceWeights sharesOfCluster1 = new ResourceWeights();
ResourceWeights sharesOfCluster2 = new ResourceWeights();
ResourceWeights sharesOfMinShare1 = new ResourceWeights();
ResourceWeights sharesOfMinShare2 = new ResourceWeights();
ResourceType[] resourceOrder1 = new ResourceType[NumResources];
ResourceType[] resourceOrder2 = new ResourceType[NumResources];
// Calculate shares of the cluster for each resource both schedulables.
CalculateShares(s1.GetResourceUsage(), clusterCapacity, sharesOfCluster1, resourceOrder1
, s1.GetWeights());
CalculateShares(s1.GetResourceUsage(), s1.GetMinShare(), sharesOfMinShare1, null,
ResourceWeights.Neutral);
CalculateShares(s2.GetResourceUsage(), clusterCapacity, sharesOfCluster2, resourceOrder2
, s2.GetWeights());
CalculateShares(s2.GetResourceUsage(), s2.GetMinShare(), sharesOfMinShare2, null,
ResourceWeights.Neutral);
// A queue is needy for its min share if its dominant resource
// (with respect to the cluster capacity) is below its configured min share
// for that resource
bool s1Needy = sharesOfMinShare1.GetWeight(resourceOrder1[0]) < 1.0f;
bool s2Needy = sharesOfMinShare2.GetWeight(resourceOrder2[0]) < 1.0f;
int res = 0;
if (!s2Needy && !s1Needy)
{
res = CompareShares(sharesOfCluster1, sharesOfCluster2, resourceOrder1, resourceOrder2
);
}
else
{
if (s1Needy && !s2Needy)
{
res = -1;
}
else
{
if (s2Needy && !s1Needy)
{
res = 1;
}
else
{
// both are needy below min share
res = CompareShares(sharesOfMinShare1, sharesOfMinShare2, resourceOrder1, resourceOrder2
);
}
}
}
if (res == 0)
{
// Apps are tied in fairness ratio. Break the tie by submit time.
res = (int)(s1.GetStartTime() - s2.GetStartTime());
}
return(res);
}
public FSAppAttempt(FairScheduler scheduler, ApplicationAttemptId applicationAttemptId
, string user, FSLeafQueue queue, ActiveUsersManager activeUsersManager, RMContext
rmContext)
: base(applicationAttemptId, user, queue, activeUsersManager, rmContext)
{
this.scheduler = scheduler;
this.startTime = scheduler.GetClock().GetTime();
this.priority = Priority.NewInstance(1);
this.resourceWeights = new ResourceWeights();
}
private Schedulable CreateSchedulable(int memUsage, int cpuUsage, ResourceWeights
weights, int minMemShare, int minCpuShare)
{
Org.Apache.Hadoop.Yarn.Api.Records.Resource usage = BuilderUtils.NewResource(memUsage
, cpuUsage);
Org.Apache.Hadoop.Yarn.Api.Records.Resource minShare = BuilderUtils.NewResource(minMemShare
, minCpuShare);
return(new FakeSchedulable(minShare, Resources.CreateResource(int.MaxValue, int.MaxValue
), weights, Resources.None(), usage, 0l));
}
public FakeSchedulable(Org.Apache.Hadoop.Yarn.Api.Records.Resource minShare, Org.Apache.Hadoop.Yarn.Api.Records.Resource
maxShare, ResourceWeights weight, Org.Apache.Hadoop.Yarn.Api.Records.Resource fairShare
, Org.Apache.Hadoop.Yarn.Api.Records.Resource usage, long startTime)
{
this.minShare = minShare;
this.maxShare = maxShare;
this.weights = weight;
SetFairShare(fairShare);
this.usage = usage;
this.priority = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <Priority>();
this.startTime = startTime;
}
private int CompareShares(ResourceWeights shares1, ResourceWeights shares2, ResourceType
[] resourceOrder1, ResourceType[] resourceOrder2)
{
for (int i = 0; i < resourceOrder1.Length; i++)
{
int ret = (int)Math.Signum(shares1.GetWeight(resourceOrder1[i]) - shares2.GetWeight
(resourceOrder2[i]));
if (ret != 0)
{
return(ret);
}
}
return(0);
}
public virtual void TestCalculateShares()
{
Org.Apache.Hadoop.Yarn.Api.Records.Resource used = Resources.CreateResource(10, 5
);
Org.Apache.Hadoop.Yarn.Api.Records.Resource capacity = Resources.CreateResource(100
, 10);
ResourceType[] resourceOrder = new ResourceType[2];
ResourceWeights shares = new ResourceWeights();
DominantResourceFairnessPolicy.DominantResourceFairnessComparator comparator = new
DominantResourceFairnessPolicy.DominantResourceFairnessComparator();
comparator.CalculateShares(used, capacity, shares, resourceOrder, ResourceWeights
.Neutral);
NUnit.Framework.Assert.AreEqual(.1, shares.GetWeight(ResourceType.Memory), .00001
);
NUnit.Framework.Assert.AreEqual(.5, shares.GetWeight(ResourceType.Cpu), .00001);
NUnit.Framework.Assert.AreEqual(ResourceType.Cpu, resourceOrder[0]);
NUnit.Framework.Assert.AreEqual(ResourceType.Memory, resourceOrder[1]);
}
/// <summary>Calculates and orders a resource's share of a pool in terms of two vectors.
/// </summary>
/// <remarks>
/// Calculates and orders a resource's share of a pool in terms of two vectors.
/// The shares vector contains, for each resource, the fraction of the pool that
/// it takes up. The resourceOrder vector contains an ordering of resources
/// by largest share. So if resource=<10 MB, 5 CPU>, and pool=<100 MB, 10 CPU>,
/// shares will be [.1, .5] and resourceOrder will be [CPU, MEMORY].
/// </remarks>
internal virtual void CalculateShares(Org.Apache.Hadoop.Yarn.Api.Records.Resource
resource, Org.Apache.Hadoop.Yarn.Api.Records.Resource pool, ResourceWeights shares
, ResourceType[] resourceOrder, ResourceWeights weights)
{
shares.SetWeight(ResourceType.Memory, (float)resource.GetMemory() / (pool.GetMemory
() * weights.GetWeight(ResourceType.Memory)));
shares.SetWeight(ResourceType.Cpu, (float)resource.GetVirtualCores() / (pool.GetVirtualCores
() * weights.GetWeight(ResourceType.Cpu)));
// sort order vector by resource share
if (resourceOrder != null)
{
if (shares.GetWeight(ResourceType.Memory) > shares.GetWeight(ResourceType.Cpu))
{
resourceOrder[0] = ResourceType.Memory;
resourceOrder[1] = ResourceType.Cpu;
}
else
{
resourceOrder[0] = ResourceType.Cpu;
resourceOrder[1] = ResourceType.Memory;
}
}
}
/// <summary>Loads a queue from a queue element in the configuration file</summary>
/// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.Fair.AllocationConfigurationException
/// "/>
private void LoadQueue(string parentName, Element element, IDictionary <string, Org.Apache.Hadoop.Yarn.Api.Records.Resource
> minQueueResources, IDictionary <string, Org.Apache.Hadoop.Yarn.Api.Records.Resource
> maxQueueResources, IDictionary <string, int> queueMaxApps, IDictionary <string,
int> userMaxApps, IDictionary <string, float> queueMaxAMShares, IDictionary <string
, ResourceWeights> queueWeights, IDictionary <string, SchedulingPolicy> queuePolicies
, IDictionary <string, long> minSharePreemptionTimeouts, IDictionary <string, long
> fairSharePreemptionTimeouts, IDictionary <string, float> fairSharePreemptionThresholds
, IDictionary <string, IDictionary <QueueACL, AccessControlList> > queueAcls, IDictionary
<FSQueueType, ICollection <string> > configuredQueues, ICollection <string> reservableQueues
)
{
string queueName = element.GetAttribute("name");
if (queueName.Contains("."))
{
throw new AllocationConfigurationException("Bad fair scheduler config " + "file: queue name ("
+ queueName + ") shouldn't contain period.");
}
if (parentName != null)
{
queueName = parentName + "." + queueName;
}
IDictionary <QueueACL, AccessControlList> acls = new Dictionary <QueueACL, AccessControlList
>();
NodeList fields = element.GetChildNodes();
bool isLeaf = true;
for (int j = 0; j < fields.GetLength(); j++)
{
Node fieldNode = fields.Item(j);
if (!(fieldNode is Element))
{
continue;
}
Element field = (Element)fieldNode;
if ("minResources".Equals(field.GetTagName()))
{
string text = ((Text)field.GetFirstChild()).GetData().Trim();
Org.Apache.Hadoop.Yarn.Api.Records.Resource val = FairSchedulerConfiguration.ParseResourceConfigValue
(text);
minQueueResources[queueName] = val;
}
else
{
if ("maxResources".Equals(field.GetTagName()))
{
string text = ((Text)field.GetFirstChild()).GetData().Trim();
Org.Apache.Hadoop.Yarn.Api.Records.Resource val = FairSchedulerConfiguration.ParseResourceConfigValue
(text);
maxQueueResources[queueName] = val;
}
else
{
if ("maxRunningApps".Equals(field.GetTagName()))
{
string text = ((Text)field.GetFirstChild()).GetData().Trim();
int val = System.Convert.ToInt32(text);
queueMaxApps[queueName] = val;
}
else
{
if ("maxAMShare".Equals(field.GetTagName()))
{
string text = ((Text)field.GetFirstChild()).GetData().Trim();
float val = float.ParseFloat(text);
val = Math.Min(val, 1.0f);
queueMaxAMShares[queueName] = val;
}
else
{
if ("weight".Equals(field.GetTagName()))
{
string text = ((Text)field.GetFirstChild()).GetData().Trim();
double val = double.ParseDouble(text);
queueWeights[queueName] = new ResourceWeights((float)val);
}
else
{
if ("minSharePreemptionTimeout".Equals(field.GetTagName()))
{
string text = ((Text)field.GetFirstChild()).GetData().Trim();
long val = long.Parse(text) * 1000L;
minSharePreemptionTimeouts[queueName] = val;
}
else
{
if ("fairSharePreemptionTimeout".Equals(field.GetTagName()))
{
string text = ((Text)field.GetFirstChild()).GetData().Trim();
long val = long.Parse(text) * 1000L;
fairSharePreemptionTimeouts[queueName] = val;
}
else
{
if ("fairSharePreemptionThreshold".Equals(field.GetTagName()))
{
string text = ((Text)field.GetFirstChild()).GetData().Trim();
float val = float.ParseFloat(text);
val = Math.Max(Math.Min(val, 1.0f), 0.0f);
fairSharePreemptionThresholds[queueName] = val;
}
else
{
if ("schedulingPolicy".Equals(field.GetTagName()) || "schedulingMode".Equals(field
.GetTagName()))
{
string text = ((Text)field.GetFirstChild()).GetData().Trim();
SchedulingPolicy policy = SchedulingPolicy.Parse(text);
queuePolicies[queueName] = policy;
}
else
{
if ("aclSubmitApps".Equals(field.GetTagName()))
{
string text = ((Text)field.GetFirstChild()).GetData();
acls[QueueACL.SubmitApplications] = new AccessControlList(text);
}
else
{
if ("aclAdministerApps".Equals(field.GetTagName()))
{
string text = ((Text)field.GetFirstChild()).GetData();
acls[QueueACL.AdministerQueue] = new AccessControlList(text);
}
else
{
if ("reservation".Equals(field.GetTagName()))
{
isLeaf = false;
reservableQueues.AddItem(queueName);
configuredQueues[FSQueueType.Parent].AddItem(queueName);
}
else
{
if ("queue".EndsWith(field.GetTagName()) || "pool".Equals(field.GetTagName()))
{
LoadQueue(queueName, field, minQueueResources, maxQueueResources, queueMaxApps, userMaxApps
, queueMaxAMShares, queueWeights, queuePolicies, minSharePreemptionTimeouts, fairSharePreemptionTimeouts
, fairSharePreemptionThresholds, queueAcls, configuredQueues, reservableQueues);
isLeaf = false;
}
}
}
}
}
}
}
}
}
}
}
}
}
}
if (isLeaf)
{
// if a leaf in the alloc file is marked as type='parent'
// then store it under 'parent'
if ("parent".Equals(element.GetAttribute("type")))
{
configuredQueues[FSQueueType.Parent].AddItem(queueName);
}
else
{
configuredQueues[FSQueueType.Leaf].AddItem(queueName);
}
}
else
{
if ("parent".Equals(element.GetAttribute("type")))
{
throw new AllocationConfigurationException("Both <reservation> and " + "type=\"parent\" found for queue "
+ queueName + " which is " + "unsupported");
}
configuredQueues[FSQueueType.Parent].AddItem(queueName);
}
queueAcls[queueName] = acls;
if (maxQueueResources.Contains(queueName) && minQueueResources.Contains(queueName
) && !Resources.FitsIn(minQueueResources[queueName], maxQueueResources[queueName
]))
{
Log.Warn(string.Format("Queue %s has max resources %s less than min resources %s"
, queueName, maxQueueResources[queueName], minQueueResources[queueName]));
}
}
public FakeSchedulable(Org.Apache.Hadoop.Yarn.Api.Records.Resource minShare, ResourceWeights
weights)
: this(minShare, Resources.CreateResource(int.MaxValue, int.MaxValue), weights, Resources
.CreateResource(0, 0), Resources.CreateResource(0, 0), 0)
{
}
public virtual void SetQueueWeight(string queue, ResourceWeights weight)
{
queueWeights[queue] = weight;
}
public virtual ResourceWeights GetQueueWeight(string queue)
{
ResourceWeights weight = queueWeights[queue];
return((weight == null) ? ResourceWeights.Neutral : weight);
}
private Schedulable CreateSchedulable(int memUsage, int cpuUsage, ResourceWeights
weights)
{
return(CreateSchedulable(memUsage, cpuUsage, weights, 0, 0));
}
