/// <exception cref="System.IO.IOException"/>
public override void Reinitialize(CSQueue newlyParsedQueue, Resource clusterResource
)
{
lock (this)
{
// Sanity check
if (!(newlyParsedQueue is Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.Capacity.PlanQueue
) || !newlyParsedQueue.GetQueuePath().Equals(GetQueuePath()))
{
throw new IOException("Trying to reinitialize " + GetQueuePath() + " from " + newlyParsedQueue
.GetQueuePath());
}
Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.Capacity.PlanQueue newlyParsedParentQueue
= (Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.Capacity.PlanQueue)newlyParsedQueue;
if (newlyParsedParentQueue.GetChildQueues().Count > 0)
{
throw new IOException("Reservable Queue should not have sub-queues in the" + "configuration"
);
}
// Set new configs
SetupQueueConfigs(clusterResource);
UpdateQuotas(newlyParsedParentQueue.userLimit, newlyParsedParentQueue.userLimitFactor
, newlyParsedParentQueue.maxAppsForReservation, newlyParsedParentQueue.maxAppsPerUserForReservation
);
// run reinitialize on each existing queue, to trigger absolute cap
// recomputations
foreach (CSQueue res in this.GetChildQueues())
{
res.Reinitialize(res, clusterResource);
}
showReservationsAsQueues = newlyParsedParentQueue.showReservationsAsQueues;
}
}
public virtual void TestOffSwitchSchedulingMultiLevelQueues()
{
// Setup queue configs
SetupMultiLevelQueues(csConf);
//B3
IDictionary <string, CSQueue> queues = new Dictionary <string, CSQueue>();
CSQueue root = CapacityScheduler.ParseQueue(csContext, csConf, null, CapacitySchedulerConfiguration
.Root, queues, queues, TestUtils.spyHook);
// Setup some nodes
int memoryPerNode = 10;
int coresPerNode = 10;
int numNodes = 2;
FiCaSchedulerNode node_0 = TestUtils.GetMockNode("host_0", DefaultRack, 0, memoryPerNode
* Gb);
FiCaSchedulerNode node_1 = TestUtils.GetMockNode("host_1", DefaultRack, 0, memoryPerNode
* Gb);
Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = Resources.CreateResource
(numNodes * (memoryPerNode * Gb), numNodes * coresPerNode);
Org.Mockito.Mockito.When(csContext.GetNumClusterNodes()).ThenReturn(numNodes);
// Start testing
LeafQueue b3 = (LeafQueue)queues[B3];
LeafQueue b2 = (LeafQueue)queues[B2];
// Simulate B3 returning a container on node_0
StubQueueAllocation(b2, clusterResource, node_0, 0 * Gb, NodeType.OffSwitch);
StubQueueAllocation(b3, clusterResource, node_0, 1 * Gb, NodeType.OffSwitch);
root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
VerifyQueueMetrics(b2, 0 * Gb, clusterResource);
VerifyQueueMetrics(b3, 1 * Gb, clusterResource);
// Now, B2 should get the scheduling opportunity since B2=0G/2G, B3=1G/7G
// also, B3 gets a scheduling opportunity since B2 allocates RACK_LOCAL
StubQueueAllocation(b2, clusterResource, node_1, 1 * Gb, NodeType.RackLocal);
StubQueueAllocation(b3, clusterResource, node_1, 1 * Gb, NodeType.OffSwitch);
root.AssignContainers(clusterResource, node_1, new ResourceLimits(clusterResource
));
InOrder allocationOrder = Org.Mockito.Mockito.InOrder(b2, b3);
allocationOrder.Verify(b2).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any
<FiCaSchedulerNode>(), AnyResourceLimits());
allocationOrder.Verify(b3).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any
<FiCaSchedulerNode>(), AnyResourceLimits());
VerifyQueueMetrics(b2, 1 * Gb, clusterResource);
VerifyQueueMetrics(b3, 2 * Gb, clusterResource);
// Now, B3 should get the scheduling opportunity
// since B2 has 1/2G while B3 has 2/7G,
// However, since B3 returns off-switch, B2 won't get an opportunity
StubQueueAllocation(b2, clusterResource, node_0, 1 * Gb, NodeType.NodeLocal);
StubQueueAllocation(b3, clusterResource, node_0, 1 * Gb, NodeType.OffSwitch);
root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
allocationOrder = Org.Mockito.Mockito.InOrder(b3, b2);
allocationOrder.Verify(b3).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any
<FiCaSchedulerNode>(), AnyResourceLimits());
allocationOrder.Verify(b2).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any
<FiCaSchedulerNode>(), AnyResourceLimits());
VerifyQueueMetrics(b2, 1 * Gb, clusterResource);
VerifyQueueMetrics(b3, 3 * Gb, clusterResource);
}
/// <exception cref="System.IO.IOException"/>
public PlanQueue(CapacitySchedulerContext cs, string queueName, CSQueue parent, CSQueue
old)
: base(cs, queueName, parent, old)
{
this.schedulerContext = cs;
// Set the reservation queue attributes for the Plan
CapacitySchedulerConfiguration conf = cs.GetConfiguration();
string queuePath = base.GetQueuePath();
int maxAppsForReservation = conf.GetMaximumApplicationsPerQueue(queuePath);
showReservationsAsQueues = conf.GetShowReservationAsQueues(queuePath);
if (maxAppsForReservation < 0)
{
maxAppsForReservation = (int)(CapacitySchedulerConfiguration.DefaultMaximumSystemApplicatiions
* base.GetAbsoluteCapacity());
}
int userLimit = conf.GetUserLimit(queuePath);
float userLimitFactor = conf.GetUserLimitFactor(queuePath);
int maxAppsPerUserForReservation = (int)(maxAppsForReservation * (userLimit / 100.0f
) * userLimitFactor);
UpdateQuotas(userLimit, userLimitFactor, maxAppsForReservation, maxAppsPerUserForReservation
);
StringBuilder queueInfo = new StringBuilder();
queueInfo.Append("Created Plan Queue: ").Append(queueName).Append("\nwith capacity: ["
).Append(base.GetCapacity()).Append("]\nwith max capacity: [").Append(base.GetMaximumCapacity
()).Append("\nwith max reservation apps: [").Append(maxAppsForReservation).Append
("]\nwith max reservation apps per user: [").Append(maxAppsPerUserForReservation
).Append("]\nwith user limit: [").Append(userLimit).Append("]\nwith user limit factor: ["
).Append(userLimitFactor).Append("].");
Log.Info(queueInfo.ToString());
}
public virtual void TestNestedQueueParsingShouldTrimSpaces()
{
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
SetupNestedQueueConfigurationWithSpacesShouldBeTrimmed(csConf);
YarnConfiguration conf = new YarnConfiguration(csConf);
CapacityScheduler capacityScheduler = new CapacityScheduler();
capacityScheduler.SetConf(conf);
capacityScheduler.SetRMContext(TestUtils.GetMockRMContext());
capacityScheduler.Init(conf);
capacityScheduler.Start();
capacityScheduler.Reinitialize(conf, TestUtils.GetMockRMContext());
CSQueue a = capacityScheduler.GetQueue("a");
NUnit.Framework.Assert.IsNotNull(a);
NUnit.Framework.Assert.AreEqual(0.10, a.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(0.15, a.GetAbsoluteMaximumCapacity(), Delta);
CSQueue c = capacityScheduler.GetQueue("c");
NUnit.Framework.Assert.IsNotNull(c);
NUnit.Framework.Assert.AreEqual(0.70, c.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(0.70, c.GetAbsoluteMaximumCapacity(), Delta);
CSQueue a1 = capacityScheduler.GetQueue("a1");
NUnit.Framework.Assert.IsNotNull(a1);
NUnit.Framework.Assert.AreEqual(0.10 * 0.6, a1.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(0.15, a1.GetAbsoluteMaximumCapacity(), Delta);
CSQueue a2 = capacityScheduler.GetQueue("a2");
NUnit.Framework.Assert.IsNotNull(a2);
NUnit.Framework.Assert.AreEqual(0.10 * 0.4, a2.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(0.15, a2.GetAbsoluteMaximumCapacity(), Delta);
}
public virtual void TestQueueParsing()
{
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
SetupQueueConfiguration(csConf);
YarnConfiguration conf = new YarnConfiguration(csConf);
CapacityScheduler capacityScheduler = new CapacityScheduler();
capacityScheduler.SetConf(conf);
capacityScheduler.SetRMContext(TestUtils.GetMockRMContext());
capacityScheduler.Init(conf);
capacityScheduler.Start();
capacityScheduler.Reinitialize(conf, TestUtils.GetMockRMContext());
CSQueue a = capacityScheduler.GetQueue("a");
NUnit.Framework.Assert.AreEqual(0.10, a.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(0.15, a.GetAbsoluteMaximumCapacity(), Delta);
CSQueue b1 = capacityScheduler.GetQueue("b1");
NUnit.Framework.Assert.AreEqual(0.2 * 0.5, b1.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual("Parent B has no MAX_CAP", 0.85, b1.GetAbsoluteMaximumCapacity
(), Delta);
CSQueue c12 = capacityScheduler.GetQueue("c12");
NUnit.Framework.Assert.AreEqual(0.7 * 0.5 * 0.45, c12.GetAbsoluteCapacity(), Delta
);
NUnit.Framework.Assert.AreEqual(0.7 * 0.55 * 0.7, c12.GetAbsoluteMaximumCapacity(
), Delta);
ServiceOperations.StopQuietly(capacityScheduler);
}
protected override void Render(HtmlBlock.Block html)
{
html.(typeof(MetricsOverviewTable));
Hamlet.UL <Hamlet.DIV <Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > > ul
= html.Div("#cs-wrapper.ui-widget").Div(".ui-widget-header.ui-corner-top").("Application Queues"
).().Div("#cs.ui-widget-content.ui-corner-bottom").Ul();
if (cs == null)
{
ul.Li().A(Q).$style(Width(QMaxWidth)).Span().$style(QEnd).("100% ").().Span(".q",
"default").().();
}
else
{
CSQueue root = cs.GetRootQueue();
CapacitySchedulerInfo sinfo = new CapacitySchedulerInfo(root);
csqinfo.csinfo = sinfo;
csqinfo.qinfo = null;
float used = sinfo.GetUsedCapacity() / 100;
ul.Li().$style("margin-bottom: 1em").Span().$style("font-weight: bold").("Legend:"
).().Span().$class("qlegend ui-corner-all").$style(QGiven).("Capacity").().Span(
).$class("qlegend ui-corner-all").$style(QUnder).("Used").().Span().$class("qlegend ui-corner-all"
).$style(QOver).("Used (over capacity)").().Span().$class("qlegend ui-corner-all ui-state-default"
).("Max Capacity").().().Li().A(Q).$style(Width(QMaxWidth)).Span().$style(StringHelper.Join
(Width(used), ";left:0%;", used > 1 ? QOver : QUnder)).(".").().Span(".q", "root"
).().Span().$class("qstats").$style(Left(QStatsPos)).(StringHelper.Join(Percent(
used), " used")).().(typeof(CapacitySchedulerPage.QueueBlock)).();
}
ul.().().Script().$type("text/javascript").("$('#cs').hide();").().().(typeof(RMAppsBlock
));
}
internal CapacitySchedulerQueueInfo(CSQueue q)
{
queuePath = q.GetQueuePath();
capacity = q.GetCapacity() * 100;
usedCapacity = q.GetUsedCapacity() * 100;
maxCapacity = q.GetMaximumCapacity();
if (maxCapacity < Epsilon || maxCapacity > 1f)
{
maxCapacity = 1f;
}
maxCapacity *= 100;
absoluteCapacity = Cap(q.GetAbsoluteCapacity(), 0f, 1f) * 100;
absoluteMaxCapacity = Cap(q.GetAbsoluteMaximumCapacity(), 0f, 1f) * 100;
absoluteUsedCapacity = Cap(q.GetAbsoluteUsedCapacity(), 0f, 1f) * 100;
numApplications = q.GetNumApplications();
queueName = q.GetQueueName();
state = q.GetState();
resourcesUsed = new ResourceInfo(q.GetUsedResources());
if (q is PlanQueue && !((PlanQueue)q).ShowReservationsAsQueues())
{
hideReservationQueues = true;
}
// add labels
ICollection <string> labelSet = q.GetAccessibleNodeLabels();
if (labelSet != null)
{
Sharpen.Collections.AddAll(nodeLabels, labelSet);
nodeLabels.Sort();
}
}
protected internal override void AssertReservationQueueDoesNotExist(ReservationId
r2)
{
CSQueue q2 = cs.GetQueue(r2.ToString());
NUnit.Framework.Assert.IsNull(q2);
}
protected internal override int GetNumberOfApplications(Queue queue)
{
CSQueue csQueue = (CSQueue)queue;
int numberOfApplications = csQueue.GetNumApplications();
return(numberOfApplications);
}
public static void UpdateQueueStatistics(ResourceCalculator calculator, CSQueue childQueue
, CSQueue parentQueue, Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource
, Org.Apache.Hadoop.Yarn.Api.Records.Resource minimumAllocation)
{
Org.Apache.Hadoop.Yarn.Api.Records.Resource queueLimit = Resources.None();
Org.Apache.Hadoop.Yarn.Api.Records.Resource usedResources = childQueue.GetUsedResources
();
float absoluteUsedCapacity = 0.0f;
float usedCapacity = 0.0f;
if (Resources.GreaterThan(calculator, clusterResource, clusterResource, Resources
.None()))
{
queueLimit = Resources.Multiply(clusterResource, childQueue.GetAbsoluteCapacity()
);
absoluteUsedCapacity = Resources.Divide(calculator, clusterResource, usedResources
, clusterResource);
usedCapacity = Resources.Equals(queueLimit, Resources.None()) ? 0 : Resources.Divide
(calculator, clusterResource, usedResources, queueLimit);
}
childQueue.SetUsedCapacity(usedCapacity);
childQueue.SetAbsoluteUsedCapacity(absoluteUsedCapacity);
Org.Apache.Hadoop.Yarn.Api.Records.Resource available = Resources.Subtract(queueLimit
, usedResources);
childQueue.GetMetrics().SetAvailableResourcesToQueue(Resources.Max(calculator, clusterResource
, available, Resources.None()));
}
/// <summary>
/// The specified queue is preemptable if system-wide preemption is turned on
/// unless any queue in the <em>qPath</em> hierarchy has explicitly turned
/// preemption off.
/// </summary>
/// <remarks>
/// The specified queue is preemptable if system-wide preemption is turned on
/// unless any queue in the <em>qPath</em> hierarchy has explicitly turned
/// preemption off.
/// NOTE: Preemptability is inherited from a queue's parent.
/// </remarks>
/// <returns>true if queue has preemption disabled, false otherwise</returns>
private bool IsQueueHierarchyPreemptionDisabled(CSQueue q)
{
CapacitySchedulerConfiguration csConf = csContext.GetConfiguration();
bool systemWidePreemption = csConf.GetBoolean(YarnConfiguration.RmSchedulerEnableMonitors
, YarnConfiguration.DefaultRmSchedulerEnableMonitors);
CSQueue parentQ = q.GetParent();
// If the system-wide preemption switch is turned off, all of the queues in
// the qPath hierarchy have preemption disabled, so return true.
if (!systemWidePreemption)
{
return(true);
}
// If q is the root queue and the system-wide preemption switch is turned
// on, then q does not have preemption disabled (default=false, below)
// unless the preemption_disabled property is explicitly set.
if (parentQ == null)
{
return(csConf.GetPreemptionDisabled(q.GetQueuePath(), false));
}
// If this is not the root queue, inherit the default value for the
// preemption_disabled property from the parent. Preemptability will be
// inherited from the parent's hierarchy unless explicitly overridden at
// this level.
return(csConf.GetPreemptionDisabled(q.GetQueuePath(), parentQ.GetPreemptionDisabled
()));
}
public virtual void SetParent(CSQueue newParentQueue)
{
lock (this)
{
this.parent = (ParentQueue)newParentQueue;
}
}
public CSRate(Employee emp)
{
this.csq = CSQueue.getInstance();
this.connect = ConnectDatabase.getInstance();
this.employee = emp;
InitializeComponent();
employeename.Content = emp.name;
}
private void VerifyQueueMetrics(CSQueue queue, int expectedMemory, Org.Apache.Hadoop.Yarn.Api.Records.Resource
clusterResource)
{
NUnit.Framework.Assert.AreEqual(ComputeQueueAbsoluteUsedCapacity(queue, expectedMemory
, clusterResource), queue.GetAbsoluteUsedCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(ComputeQueueUsedCapacity(queue, expectedMemory, clusterResource
), queue.GetUsedCapacity(), Delta);
}
public static float ComputeAbsoluteMaximumCapacity(float maximumCapacity, CSQueue
parent)
{
float parentAbsMaxCapacity = (parent == null) ? 1.0f : parent.GetAbsoluteMaximumCapacity
();
return(parentAbsMaxCapacity * maximumCapacity);
}
private void StubQueueAllocation(CSQueue queue, Org.Apache.Hadoop.Yarn.Api.Records.Resource
clusterResource, FiCaSchedulerNode node, int allocation, NodeType type)
{
// Simulate the queue allocation
Org.Mockito.Mockito.DoAnswer(new _Answer_137(this, queue, allocation, node, clusterResource
, type)).When(queue).AssignContainers(Matchers.Eq(clusterResource), Matchers.Eq(
node), Matchers.Any <ResourceLimits>());
}
protected internal override void AssertReservationQueueExists(ReservationId r2, double
expectedCapacity, double expectedMaxCapacity)
{
CSQueue q = cs.GetQueue(r2.ToString());
NUnit.Framework.Assert.IsNotNull(q);
NUnit.Framework.Assert.AreEqual(expectedCapacity, q.GetCapacity(), 0.01);
NUnit.Framework.Assert.AreEqual(expectedMaxCapacity, q.GetMaximumCapacity(), 1.0);
}
private void CheckUsedResource(MockRM rm, string queueName, int memory, string label
)
{
CapacityScheduler scheduler = (CapacityScheduler)rm.GetResourceScheduler();
CSQueue queue = scheduler.GetQueue(queueName);
NUnit.Framework.Assert.AreEqual(memory, queue.GetQueueResourceUsage().GetUsed(label
).GetMemory());
}
public QMShowQueue()
{
tq = TellerQueue.getInstance();
csq = CSQueue.getInstance();
this.connect = ConnectDatabase.getInstance();
InitializeComponent();
tellerqueue.Content = "";
csqueue.Content = "";
init();
}
protected internal override Org.Apache.Hadoop.Yarn.Api.Records.Resource GetPlanQueueCapacity
(string planQueueName)
{
Org.Apache.Hadoop.Yarn.Api.Records.Resource minAllocation = GetMinAllocation();
ResourceCalculator rescCalc = GetResourceCalculator();
CSQueue planQueue = capScheduler.GetQueue(planQueueName);
return(rescCalc.MultiplyAndNormalizeDown(capScheduler.GetClusterResource(), planQueue
.GetAbsoluteCapacity(), minAllocation));
}
protected internal override Queue GetPlanQueue(string planQueueName)
{
CSQueue queue = cs.GetQueue(planQueueName);
if (!(queue is PlanQueue))
{
Log.Error("The Plan is not an PlanQueue!");
return(null);
}
return(queue);
}
public _Answer_137(TestParentQueue _enclosing, CSQueue queue, int allocation, FiCaSchedulerNode
node, Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource, NodeType type
)
{
this._enclosing = _enclosing;
this.queue = queue;
this.allocation = allocation;
this.node = node;
this.clusterResource = clusterResource;
this.type = type;
}
private void StubQueueAllocation(CSQueue queue, Org.Apache.Hadoop.Yarn.Api.Records.Resource
clusterResource, FiCaSchedulerNode node, int allocation, NodeType type)
{
// Simulate the queue allocation
Org.Mockito.Mockito.DoAnswer(new _Answer_124(this, queue, allocation, node, clusterResource
, type)).When(queue).AssignContainers(Matchers.Eq(clusterResource), Matchers.Eq(
node), Matchers.Any <ResourceLimits>());
// Next call - nothing
// Mock the node's resource availability
Org.Mockito.Mockito.DoNothing().When(node).ReleaseContainer(Matchers.Any <Container
>());
}
public CSQRCode(Employee emp, Customer cust)
{
this.csq = CSQueue.getInstance();
this.employee = emp;
this.customer = cust;
InitializeComponent();
QRCodeGenerator qrGenerator = new QRCodeGenerator();
QRCodeData qrCodeData = qrGenerator.CreateQrCode(csq.uniquequeue, QRCodeGenerator.ECCLevel.H);
XamlQRCode qrCode = new XamlQRCode(qrCodeData);
DrawingImage qrCodeAsXaml = qrCode.GetGraphic(20);
qrcode.Source = qrCodeAsXaml;
}
protected internal override Org.Apache.Hadoop.Yarn.Api.Records.Resource GetReservationQueueResourceIfExists
(Plan plan, ReservationId reservationId)
{
CSQueue resQueue = cs.GetQueue(reservationId.ToString());
Org.Apache.Hadoop.Yarn.Api.Records.Resource reservationResource = null;
if (resQueue != null)
{
reservationResource = Resources.Multiply(cs.GetClusterResource(), resQueue.GetAbsoluteCapacity
());
}
return(reservationResource);
}
/// <exception cref="System.IO.IOException"/>
public override void Reinitialize(CSQueue newlyParsedQueue, Resource clusterResource
)
{
lock (this)
{
// Sanity check
if (!(newlyParsedQueue is Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.Capacity.ParentQueue
) || !newlyParsedQueue.GetQueuePath().Equals(GetQueuePath()))
{
throw new IOException("Trying to reinitialize " + GetQueuePath() + " from " + newlyParsedQueue
.GetQueuePath());
}
Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.Capacity.ParentQueue newlyParsedParentQueue
= (Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.Capacity.ParentQueue
)newlyParsedQueue;
// Set new configs
SetupQueueConfigs(clusterResource);
// Re-configure existing child queues and add new ones
// The CS has already checked to ensure all existing child queues are present!
IDictionary <string, CSQueue> currentChildQueues = GetQueues(childQueues);
IDictionary <string, CSQueue> newChildQueues = GetQueues(newlyParsedParentQueue.childQueues
);
foreach (KeyValuePair <string, CSQueue> e in newChildQueues)
{
string newChildQueueName = e.Key;
CSQueue newChildQueue = e.Value;
CSQueue childQueue = currentChildQueues[newChildQueueName];
// Check if the child-queue already exists
if (childQueue != null)
{
// Re-init existing child queues
childQueue.Reinitialize(newChildQueue, clusterResource);
Log.Info(GetQueueName() + ": re-configured queue: " + childQueue);
}
else
{
// New child queue, do not re-init
// Set parent to 'this'
newChildQueue.SetParent(this);
// Save in list of current child queues
currentChildQueues[newChildQueueName] = newChildQueue;
Log.Info(GetQueueName() + ": added new child queue: " + newChildQueue);
}
}
// Re-sort all queues
childQueues.Clear();
Sharpen.Collections.AddAll(childQueues, currentChildQueues.Values);
}
}
public CapacitySchedulerInfo(CSQueue parent)
{
// JAXB needs this
this.queueName = parent.GetQueueName();
this.usedCapacity = parent.GetUsedCapacity() * 100;
this.capacity = parent.GetCapacity() * 100;
float max = parent.GetMaximumCapacity();
if (max < Epsilon || max > 1f)
{
max = 1f;
}
this.maxCapacity = max * 100;
queues = GetQueues(parent);
}
public virtual void TestQueueParsingWithUnusedLabels()
{
ImmutableSet <string> labels = ImmutableSet.Of("red", "blue");
// Initialize a cluster with labels, but doesn't use them, reinitialize
// shouldn't fail
nodeLabelManager.AddToCluserNodeLabels(labels);
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
SetupQueueConfiguration(csConf);
csConf.SetAccessibleNodeLabels(CapacitySchedulerConfiguration.Root, labels);
YarnConfiguration conf = new YarnConfiguration(csConf);
CapacityScheduler capacityScheduler = new CapacityScheduler();
capacityScheduler.SetConf(conf);
RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new
RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new
ClientToAMTokenSecretManagerInRM(), null);
rmContext.SetNodeLabelManager(nodeLabelManager);
capacityScheduler.SetRMContext(rmContext);
capacityScheduler.Init(conf);
capacityScheduler.Start();
capacityScheduler.Reinitialize(conf, rmContext);
// check root queue's capacity by label -- they should be all zero
CSQueue root = capacityScheduler.GetQueue(CapacitySchedulerConfiguration.Root);
NUnit.Framework.Assert.AreEqual(0, root.GetQueueCapacities().GetCapacity("red"),
Delta);
NUnit.Framework.Assert.AreEqual(0, root.GetQueueCapacities().GetCapacity("blue"),
Delta);
CSQueue a = capacityScheduler.GetQueue("a");
NUnit.Framework.Assert.AreEqual(0.10, a.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(0.15, a.GetAbsoluteMaximumCapacity(), Delta);
CSQueue b1 = capacityScheduler.GetQueue("b1");
NUnit.Framework.Assert.AreEqual(0.2 * 0.5, b1.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual("Parent B has no MAX_CAP", 0.85, b1.GetAbsoluteMaximumCapacity
(), Delta);
CSQueue c12 = capacityScheduler.GetQueue("c12");
NUnit.Framework.Assert.AreEqual(0.7 * 0.5 * 0.45, c12.GetAbsoluteCapacity(), Delta
);
NUnit.Framework.Assert.AreEqual(0.7 * 0.55 * 0.7, c12.GetAbsoluteMaximumCapacity(
), Delta);
capacityScheduler.Stop();
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.SchedulerDynamicEditException
/// "/>
internal virtual void AddChildQueue(CSQueue newQueue)
{
lock (this)
{
if (newQueue.GetCapacity() > 0)
{
throw new SchedulerDynamicEditException("Queue " + newQueue + " being added has non zero capacity."
);
}
bool added = this.childQueues.AddItem(newQueue);
if (Log.IsDebugEnabled())
{
Log.Debug("updateChildQueues (action: add queue): " + added + " " + GetChildQueuesToPrint
());
}
}
}
public override void CompletedContainer(Org.Apache.Hadoop.Yarn.Api.Records.Resource
clusterResource, FiCaSchedulerApp application, FiCaSchedulerNode node, RMContainer
rmContainer, ContainerStatus containerStatus, RMContainerEventType @event, CSQueue
completedChildQueue, bool sortQueues)
{
if (application != null)
{
// Careful! Locking order is important!
// Book keeping
lock (this)
{
base.ReleaseResource(clusterResource, rmContainer.GetContainer().GetResource(), node
.GetLabels());
Log.Info("completedContainer" + " queue=" + GetQueueName() + " usedCapacity=" + GetUsedCapacity
() + " absoluteUsedCapacity=" + GetAbsoluteUsedCapacity() + " used=" + queueUsage
.GetUsed() + " cluster=" + clusterResource);
// Note that this is using an iterator on the childQueues so this can't
// be called if already within an iterator for the childQueues. Like
// from assignContainersToChildQueues.
if (sortQueues)
{
// reinsert the updated queue
for (IEnumerator <CSQueue> iter = childQueues.GetEnumerator(); iter.HasNext();)
{
CSQueue csqueue = iter.Next();
if (csqueue.Equals(completedChildQueue))
{
iter.Remove();
Log.Info("Re-sorting completed queue: " + csqueue.GetQueuePath() + " stats: " + csqueue
);
childQueues.AddItem(csqueue);
break;
}
}
}
}
// Inform the parent
if (parent != null)
{
// complete my parent
parent.CompletedContainer(clusterResource, application, node, rmContainer, null,
@event, this, sortQueues);
}
}
}