/// <summary>
/// Get a mock
/// <see cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.RMContext"/>
/// for use in test cases.
/// </summary>
/// <returns>
/// a mock
/// <see cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.RMContext"/>
/// for use in test cases
/// </returns>
public static RMContext GetMockRMContext()
{
// Null dispatcher
Dispatcher nullDispatcher = new _Dispatcher_73();
// No op
ContainerAllocationExpirer cae = new ContainerAllocationExpirer(nullDispatcher);
Configuration conf = new Configuration();
RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
>();
RMContextImpl rmContext = new RMContextImpl(nullDispatcher, cae, null, null, null
, new AMRMTokenSecretManager(conf, null), new RMContainerTokenSecretManager(conf
), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM());
RMNodeLabelsManager nlm = Org.Mockito.Mockito.Mock <RMNodeLabelsManager>();
Org.Mockito.Mockito.When(nlm.GetQueueResource(Matchers.Any <string>(), Matchers.Any
<Set>(), Matchers.Any <Resource>())).ThenAnswer(new _Answer_105());
Org.Mockito.Mockito.When(nlm.GetResourceByLabel(Matchers.Any <string>(), Matchers.Any
<Resource>())).ThenAnswer(new _Answer_114());
rmContext.SetNodeLabelManager(nlm);
rmContext.SetSystemMetricsPublisher(Org.Mockito.Mockito.Mock <SystemMetricsPublisher
>());
rmContext.SetRMApplicationHistoryWriter(Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
>());
return(rmContext);
}
/// <exception cref="System.IO.IOException"/>
private void InitializeNodeLabels()
{
RMContext rmContext = distShellTest.yarnCluster.GetResourceManager(0).GetRMContext
();
// Setup node labels
RMNodeLabelsManager labelsMgr = rmContext.GetNodeLabelManager();
ICollection <string> labels = new HashSet <string>();
labels.AddItem("x");
labelsMgr.AddToCluserNodeLabels(labels);
// Setup queue access to node labels
distShellTest.conf.Set("yarn.scheduler.capacity.root.accessible-node-labels", "x"
);
distShellTest.conf.Set("yarn.scheduler.capacity.root.accessible-node-labels.x.capacity"
, "100");
distShellTest.conf.Set("yarn.scheduler.capacity.root.default.accessible-node-labels"
, "x");
distShellTest.conf.Set("yarn.scheduler.capacity.root.default.accessible-node-labels.x.capacity"
, "100");
rmContext.GetScheduler().Reinitialize(distShellTest.conf, rmContext);
// Fetch node-ids from yarn cluster
NodeId[] nodeIds = new NodeId[NumNms];
for (int i = 0; i < NumNms; i++)
{
NodeManager mgr = distShellTest.yarnCluster.GetNodeManager(i);
nodeIds[i] = mgr.GetNMContext().GetNodeId();
}
// Set label x to NM[1]
labelsMgr.AddLabelsToNode(ImmutableMap.Of(nodeIds[1], labels));
}
protected override void Render(HtmlBlock.Block html)
{
Hamlet.TBODY <Hamlet.TABLE <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > tbody = html
.Table("#nodelabels").Thead().Tr().Th(".name", "Label Name").Th(".numOfActiveNMs"
, "Num Of Active NMs").Th(".totalResource", "Total Resource").().().Tbody();
RMNodeLabelsManager nlm = rm.GetRMContext().GetNodeLabelManager();
foreach (NodeLabel info in nlm.PullRMNodeLabelsInfo())
{
Hamlet.TR <Hamlet.TBODY <Hamlet.TABLE <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> >
> row = tbody.Tr().Td(info.GetLabelName().IsEmpty() ? "<NO_LABEL>" : info.GetLabelName
());
int nActiveNMs = info.GetNumActiveNMs();
if (nActiveNMs > 0)
{
row = row.Td().A(Url("nodes", "?" + YarnWebParams.NodeLabel + "=" + info.GetLabelName
()), nActiveNMs.ToString()).();
}
else
{
row = row.Td(nActiveNMs.ToString());
}
row.Td(info.GetResource().ToString()).();
}
tbody.().();
}
public virtual void SetUp()
{
conf = new YarnConfiguration();
conf.SetClass(YarnConfiguration.RmScheduler, typeof(CapacityScheduler), typeof(ResourceScheduler
));
mgr = new NullRMNodeLabelsManager();
mgr.Init(conf);
}
public virtual void TestCompareRMNodeAfterReconnect()
{
Configuration yarnConf = new YarnConfiguration();
CapacityScheduler scheduler = new CapacityScheduler();
scheduler.SetConf(yarnConf);
ConfigurationProvider configurationProvider = ConfigurationProviderFactory.GetConfigurationProvider
(yarnConf);
configurationProvider.Init(yarnConf);
context.SetConfigurationProvider(configurationProvider);
RMNodeLabelsManager nlm = new RMNodeLabelsManager();
nlm.Init(yarnConf);
nlm.Start();
context.SetNodeLabelManager(nlm);
scheduler.SetRMContext(context);
scheduler.Init(yarnConf);
scheduler.Start();
dispatcher.Register(typeof(SchedulerEventType), scheduler);
string hostname1 = "localhost1";
Resource capability = BuilderUtils.NewResource(4096, 4);
RegisterNodeManagerRequest request1 = recordFactory.NewRecordInstance <RegisterNodeManagerRequest
>();
NodeId nodeId1 = NodeId.NewInstance(hostname1, 0);
request1.SetNodeId(nodeId1);
request1.SetHttpPort(0);
request1.SetResource(capability);
resourceTrackerService.RegisterNodeManager(request1);
NUnit.Framework.Assert.IsNotNull(context.GetRMNodes()[nodeId1]);
// verify Scheduler and RMContext use same RMNode reference.
NUnit.Framework.Assert.IsTrue(scheduler.GetSchedulerNode(nodeId1).GetRMNode() ==
context.GetRMNodes()[nodeId1]);
NUnit.Framework.Assert.AreEqual(context.GetRMNodes()[nodeId1].GetTotalCapability(
), capability);
Resource capability1 = BuilderUtils.NewResource(2048, 2);
request1.SetResource(capability1);
resourceTrackerService.RegisterNodeManager(request1);
NUnit.Framework.Assert.IsNotNull(context.GetRMNodes()[nodeId1]);
// verify Scheduler and RMContext use same RMNode reference
// after reconnect.
NUnit.Framework.Assert.IsTrue(scheduler.GetSchedulerNode(nodeId1).GetRMNode() ==
context.GetRMNodes()[nodeId1]);
// verify RMNode's capability is changed.
NUnit.Framework.Assert.AreEqual(context.GetRMNodes()[nodeId1].GetTotalCapability(
), capability1);
nlm.Stop();
scheduler.Stop();
}
public static RMContext CreateRMContext(Configuration conf)
{
RMContext mockRmContext = Org.Mockito.Mockito.Spy(new RMContextImpl(null, null, null
, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM
(conf), new ClientToAMTokenSecretManagerInRM(), null));
RMNodeLabelsManager nlm = Org.Mockito.Mockito.Mock <RMNodeLabelsManager>();
Org.Mockito.Mockito.When(nlm.GetQueueResource(Matchers.Any <string>(), Matchers.AnySetOf
<string>(), Matchers.Any <Resource>())).ThenAnswer(new _Answer_228());
Org.Mockito.Mockito.When(nlm.GetResourceByLabel(Matchers.Any <string>(), Matchers.Any
<Resource>())).ThenAnswer(new _Answer_237());
mockRmContext.SetNodeLabelManager(nlm);
return(mockRmContext);
}
/// <summary>
/// Check queue label expression, check if node label in queue's
/// node-label-expression existed in clusterNodeLabels if rmContext != null
/// </summary>
public static bool CheckQueueLabelExpression(ICollection <string> queueLabels, string
labelExpression, RMContext rmContext)
{
// if label expression is empty, we can allocate container on any node
if (labelExpression == null)
{
return(true);
}
foreach (string str in labelExpression.Split("&&"))
{
str = str.Trim();
if (!str.Trim().IsEmpty())
{
// check queue label
if (queueLabels == null)
{
return(false);
}
else
{
if (!queueLabels.Contains(str) && !queueLabels.Contains(RMNodeLabelsManager.Any))
{
return(false);
}
}
// check node label manager contains this label
if (null != rmContext)
{
RMNodeLabelsManager nlm = rmContext.GetNodeLabelManager();
if (nlm != null && !nlm.ContainsNodeLabel(str))
{
return(false);
}
}
}
}
return(true);
}
/// <exception cref="System.IO.IOException"/>
public static void CheckIfLabelInClusterNodeLabels(RMNodeLabelsManager mgr, ICollection
<string> labels)
{
if (mgr == null)
{
if (labels != null && !labels.IsEmpty())
{
throw new IOException("NodeLabelManager is null, please check");
}
return;
}
if (labels != null)
{
foreach (string label in labels)
{
if (!label.Equals(RMNodeLabelsManager.Any) && !mgr.ContainsNodeLabel(label))
{
throw new IOException("NodeLabelManager doesn't include label = " + label + ", please check."
);
}
}
}
}
/// <exception cref="System.IO.IOException"/>
public AbstractCSQueue(CapacitySchedulerContext cs, string queueName, CSQueue parent
, CSQueue old)
{
// Track resource usage-by-label like used-resource/pending-resource, etc.
// Track capacities like used-capcity/abs-used-capacity/capacity/abs-capacity,
// etc.
this.labelManager = cs.GetRMContext().GetNodeLabelManager();
this.parent = parent;
this.queueName = queueName;
this.resourceCalculator = cs.GetResourceCalculator();
// must be called after parent and queueName is set
this.metrics = old != null?old.GetMetrics() : QueueMetrics.ForQueue(GetQueuePath
(), parent, cs.GetConfiguration().GetEnableUserMetrics(), cs.GetConf());
this.csContext = cs;
this.minimumAllocation = csContext.GetMinimumResourceCapability();
// initialize ResourceUsage
queueUsage = new ResourceUsage();
queueEntity = new PrivilegedEntity(PrivilegedEntity.EntityType.Queue, GetQueuePath
());
// initialize QueueCapacities
queueCapacities = new QueueCapacities(parent == null);
}
public virtual void SetNodeLabelManager(RMNodeLabelsManager mgr)
{
activeServiceContext.SetNodeLabelManager(mgr);
}
public virtual void Setup()
{
nodeLabelManager = new NullRMNodeLabelsManager();
nodeLabelManager.Init(new YarnConfiguration());
nodeLabelManager.Start();
}
public _MockRM_471(RMNodeLabelsManager mgr, Configuration baseArg1)
: base(baseArg1)
{
this.mgr = mgr;
}
public virtual void SetNodeLabelManager(RMNodeLabelsManager mgr)
{
nodeLabelManager = mgr;
}