public RMActiveServiceContext(Dispatcher rmDispatcher, ContainerAllocationExpirer
containerAllocationExpirer, AMLivelinessMonitor amLivelinessMonitor, AMLivelinessMonitor
amFinishingMonitor, DelegationTokenRenewer delegationTokenRenewer, AMRMTokenSecretManager
appTokenSecretManager, RMContainerTokenSecretManager containerTokenSecretManager
, NMTokenSecretManagerInRM nmTokenSecretManager, ClientToAMTokenSecretManagerInRM
clientToAMTokenSecretManager, ResourceScheduler scheduler)
: this()
{
this.SetContainerAllocationExpirer(containerAllocationExpirer);
this.SetAMLivelinessMonitor(amLivelinessMonitor);
this.SetAMFinishingMonitor(amFinishingMonitor);
this.SetDelegationTokenRenewer(delegationTokenRenewer);
this.SetAMRMTokenSecretManager(appTokenSecretManager);
this.SetContainerTokenSecretManager(containerTokenSecretManager);
this.SetNMTokenSecretManager(nmTokenSecretManager);
this.SetClientToAMTokenSecretManager(clientToAMTokenSecretManager);
this.SetScheduler(scheduler);
RMStateStore nullStore = new NullRMStateStore();
nullStore.SetRMDispatcher(rmDispatcher);
try
{
nullStore.Init(new YarnConfiguration());
SetStateStore(nullStore);
}
catch (Exception)
{
System.Diagnostics.Debug.Assert(false);
}
}
/// <summary>
/// Get a mock
/// <see cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.RMContext"/>
/// for use in test cases.
/// </summary>
/// <returns>
/// a mock
/// <see cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.RMContext"/>
/// for use in test cases
/// </returns>
public static RMContext GetMockRMContext()
{
// Null dispatcher
Dispatcher nullDispatcher = new _Dispatcher_73();
// No op
ContainerAllocationExpirer cae = new ContainerAllocationExpirer(nullDispatcher);
Configuration conf = new Configuration();
RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
>();
RMContextImpl rmContext = new RMContextImpl(nullDispatcher, cae, null, null, null
, new AMRMTokenSecretManager(conf, null), new RMContainerTokenSecretManager(conf
), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM());
RMNodeLabelsManager nlm = Org.Mockito.Mockito.Mock <RMNodeLabelsManager>();
Org.Mockito.Mockito.When(nlm.GetQueueResource(Matchers.Any <string>(), Matchers.Any
<Set>(), Matchers.Any <Resource>())).ThenAnswer(new _Answer_105());
Org.Mockito.Mockito.When(nlm.GetResourceByLabel(Matchers.Any <string>(), Matchers.Any
<Resource>())).ThenAnswer(new _Answer_114());
rmContext.SetNodeLabelManager(nlm);
rmContext.SetSystemMetricsPublisher(Org.Mockito.Mockito.Mock <SystemMetricsPublisher
>());
rmContext.SetRMApplicationHistoryWriter(Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
>());
return(rmContext);
}
public virtual RMContext MockRMContext(int n, long time)
{
IList <RMApp> apps = NewRMApps(n, time, RMAppState.Finished);
ConcurrentMap <ApplicationId, RMApp> map = Maps.NewConcurrentMap();
foreach (RMApp app in apps)
{
map[app.GetApplicationId()] = app;
}
Dispatcher rmDispatcher = new AsyncDispatcher();
ContainerAllocationExpirer containerAllocationExpirer = new ContainerAllocationExpirer
(rmDispatcher);
AMLivelinessMonitor amLivelinessMonitor = new AMLivelinessMonitor(rmDispatcher);
AMLivelinessMonitor amFinishingMonitor = new AMLivelinessMonitor(rmDispatcher);
RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
>();
RMContext context = new _RMContextImpl_121(map, rmDispatcher, containerAllocationExpirer
, amLivelinessMonitor, amFinishingMonitor, null, null, null, null, null);
((RMContextImpl)context).SetStateStore(Org.Mockito.Mockito.Mock <RMStateStore>());
metricsPublisher = Org.Mockito.Mockito.Mock <SystemMetricsPublisher>();
context.SetSystemMetricsPublisher(metricsPublisher);
context.SetRMApplicationHistoryWriter(writer);
return(context);
}
public _RMContextImpl_159(ConcurrentMap <ApplicationId, RMApp> applicationsMaps, Dispatcher
baseArg1, ContainerAllocationExpirer baseArg2, AMLivelinessMonitor baseArg3, AMLivelinessMonitor
baseArg4, DelegationTokenRenewer baseArg5, AMRMTokenSecretManager baseArg6, RMContainerTokenSecretManager
baseArg7, NMTokenSecretManagerInRM baseArg8, ClientToAMTokenSecretManagerInRM baseArg9
, ResourceScheduler baseArg10)
: base(baseArg1, baseArg2, baseArg3, baseArg4, baseArg5, baseArg6, baseArg7, baseArg8
, baseArg9, baseArg10)
{
this.applicationsMaps = applicationsMaps;
}
public _RMContextImpl_121(ConcurrentMap <ApplicationId, RMApp> map, Dispatcher baseArg1
, ContainerAllocationExpirer baseArg2, AMLivelinessMonitor baseArg3, AMLivelinessMonitor
baseArg4, DelegationTokenRenewer baseArg5, AMRMTokenSecretManager baseArg6, RMContainerTokenSecretManager
baseArg7, NMTokenSecretManagerInRM baseArg8, ClientToAMTokenSecretManagerInRM baseArg9
)
: base(baseArg1, baseArg2, baseArg3, baseArg4, baseArg5, baseArg6, baseArg7, baseArg8
, baseArg9)
{
this.map = map;
}
public RMContextImpl(Dispatcher rmDispatcher, ContainerAllocationExpirer containerAllocationExpirer
, AMLivelinessMonitor amLivelinessMonitor, AMLivelinessMonitor amFinishingMonitor
, DelegationTokenRenewer delegationTokenRenewer, AMRMTokenSecretManager appTokenSecretManager
, RMContainerTokenSecretManager containerTokenSecretManager, NMTokenSecretManagerInRM
nmTokenSecretManager, ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager
)
: this(rmDispatcher, containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor
, delegationTokenRenewer, appTokenSecretManager, containerTokenSecretManager, nmTokenSecretManager
, clientToAMTokenSecretManager, null)
{
}
public _RMContextImpl_183(ConcurrentMap <ApplicationId, RMApp> applicationsMaps, ConcurrentMap
<string, RMNode> deactivatedNodesMap, ConcurrentMap <NodeId, RMNode> nodesMap, Dispatcher
baseArg1, ContainerAllocationExpirer baseArg2, AMLivelinessMonitor baseArg3, AMLivelinessMonitor
baseArg4, DelegationTokenRenewer baseArg5, AMRMTokenSecretManager baseArg6, RMContainerTokenSecretManager
baseArg7, NMTokenSecretManagerInRM baseArg8, ClientToAMTokenSecretManagerInRM baseArg9
, ResourceScheduler baseArg10)
: base(baseArg1, baseArg2, baseArg3, baseArg4, baseArg5, baseArg6, baseArg7, baseArg8
, baseArg9, baseArg10)
{
this.applicationsMaps = applicationsMaps;
this.deactivatedNodesMap = deactivatedNodesMap;
this.nodesMap = nodesMap;
}
public RMContextImpl(Dispatcher rmDispatcher, ContainerAllocationExpirer containerAllocationExpirer
, AMLivelinessMonitor amLivelinessMonitor, AMLivelinessMonitor amFinishingMonitor
, DelegationTokenRenewer delegationTokenRenewer, AMRMTokenSecretManager appTokenSecretManager
, RMContainerTokenSecretManager containerTokenSecretManager, NMTokenSecretManagerInRM
nmTokenSecretManager, ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager
, ResourceScheduler scheduler)
: this()
{
// helper constructor for tests
this.SetDispatcher(rmDispatcher);
SetActiveServiceContext(new RMActiveServiceContext(rmDispatcher, containerAllocationExpirer
, amLivelinessMonitor, amFinishingMonitor, delegationTokenRenewer, appTokenSecretManager
, containerTokenSecretManager, nmTokenSecretManager, clientToAMTokenSecretManager
, scheduler));
ConfigurationProvider provider = new LocalConfigurationProvider();
SetConfigurationProvider(provider);
}
internal virtual void SetContainerAllocationExpirer(ContainerAllocationExpirer containerAllocationExpirer
)
{
activeServiceContext.SetContainerAllocationExpirer(containerAllocationExpirer);
}
internal virtual void SetContainerAllocationExpirer(ContainerAllocationExpirer containerAllocationExpirer
)
{
this.containerAllocationExpirer = containerAllocationExpirer;
}
public virtual void TestSortedQueues()
{
// Setup queue configs
SetupSortedQueues(csConf);
IDictionary <string, CSQueue> queues = new Dictionary <string, CSQueue>();
CSQueue root = CapacityScheduler.ParseQueue(csContext, csConf, null, CapacitySchedulerConfiguration
.Root, queues, queues, TestUtils.spyHook);
// Setup some nodes
int memoryPerNode = 10;
int coresPerNode = 16;
int numNodes = 1;
FiCaSchedulerNode node_0 = TestUtils.GetMockNode("host_0", DefaultRack, 0, memoryPerNode
* Gb);
Org.Mockito.Mockito.DoNothing().When(node_0).ReleaseContainer(Matchers.Any <Container
>());
Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = Resources.CreateResource
(numNodes * (memoryPerNode * Gb), numNodes * coresPerNode);
Org.Mockito.Mockito.When(csContext.GetNumClusterNodes()).ThenReturn(numNodes);
// Start testing
CSQueue a = queues[A];
CSQueue b = queues[B];
CSQueue c = queues[C];
CSQueue d = queues[D];
string user_0 = "user_0";
// Stub an App and its containerCompleted
FiCaSchedulerApp app_0 = GetMockApplication(0, user_0);
Org.Mockito.Mockito.DoReturn(true).When(app_0).ContainerCompleted(Matchers.Any <RMContainer
>(), Matchers.Any <ContainerStatus>(), Matchers.Any <RMContainerEventType>());
Priority priority = TestUtils.CreateMockPriority(1);
ContainerAllocationExpirer expirer = Org.Mockito.Mockito.Mock <ContainerAllocationExpirer
>();
DrainDispatcher drainDispatcher = new DrainDispatcher();
RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
>();
SystemMetricsPublisher publisher = Org.Mockito.Mockito.Mock <SystemMetricsPublisher
>();
RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>();
Org.Mockito.Mockito.When(rmContext.GetContainerAllocationExpirer()).ThenReturn(expirer
);
Org.Mockito.Mockito.When(rmContext.GetDispatcher()).ThenReturn(drainDispatcher);
Org.Mockito.Mockito.When(rmContext.GetRMApplicationHistoryWriter()).ThenReturn(writer
);
Org.Mockito.Mockito.When(rmContext.GetSystemMetricsPublisher()).ThenReturn(publisher
);
Org.Mockito.Mockito.When(rmContext.GetYarnConfiguration()).ThenReturn(new YarnConfiguration
());
ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(app_0.GetApplicationId
(), 1);
ContainerId containerId = BuilderUtils.NewContainerId(appAttemptId, 1);
Container container = TestUtils.GetMockContainer(containerId, node_0.GetNodeID(),
Resources.CreateResource(1 * Gb), priority);
RMContainer rmContainer = new RMContainerImpl(container, appAttemptId, node_0.GetNodeID
(), "user", rmContext);
// Assign {1,2,3,4} 1GB containers respectively to queues
StubQueueAllocation(a, clusterResource, node_0, 1 * Gb);
StubQueueAllocation(b, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(c, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(d, clusterResource, node_0, 0 * Gb);
root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
for (int i = 0; i < 2; i++)
{
StubQueueAllocation(a, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(b, clusterResource, node_0, 1 * Gb);
StubQueueAllocation(c, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(d, clusterResource, node_0, 0 * Gb);
root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
}
for (int i_1 = 0; i_1 < 3; i_1++)
{
StubQueueAllocation(a, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(b, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(c, clusterResource, node_0, 1 * Gb);
StubQueueAllocation(d, clusterResource, node_0, 0 * Gb);
root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
}
for (int i_2 = 0; i_2 < 4; i_2++)
{
StubQueueAllocation(a, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(b, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(c, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(d, clusterResource, node_0, 1 * Gb);
root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
}
VerifyQueueMetrics(a, 1 * Gb, clusterResource);
VerifyQueueMetrics(b, 2 * Gb, clusterResource);
VerifyQueueMetrics(c, 3 * Gb, clusterResource);
VerifyQueueMetrics(d, 4 * Gb, clusterResource);
Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint());
//Release 3 x 1GB containers from D
for (int i_3 = 0; i_3 < 3; i_3++)
{
d.CompletedContainer(clusterResource, app_0, node_0, rmContainer, null, RMContainerEventType
.Kill, null, true);
}
VerifyQueueMetrics(a, 1 * Gb, clusterResource);
VerifyQueueMetrics(b, 2 * Gb, clusterResource);
VerifyQueueMetrics(c, 3 * Gb, clusterResource);
VerifyQueueMetrics(d, 1 * Gb, clusterResource);
//reset manually resources on node
node_0 = TestUtils.GetMockNode("host_0", DefaultRack, 0, (memoryPerNode - 1 - 2 -
3 - 1) * Gb);
Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint());
// Assign 2 x 1GB Containers to A
for (int i_4 = 0; i_4 < 2; i_4++)
{
StubQueueAllocation(a, clusterResource, node_0, 1 * Gb);
StubQueueAllocation(b, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(c, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(d, clusterResource, node_0, 0 * Gb);
root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
}
VerifyQueueMetrics(a, 3 * Gb, clusterResource);
VerifyQueueMetrics(b, 2 * Gb, clusterResource);
VerifyQueueMetrics(c, 3 * Gb, clusterResource);
VerifyQueueMetrics(d, 1 * Gb, clusterResource);
Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint());
//Release 1GB Container from A
a.CompletedContainer(clusterResource, app_0, node_0, rmContainer, null, RMContainerEventType
.Kill, null, true);
VerifyQueueMetrics(a, 2 * Gb, clusterResource);
VerifyQueueMetrics(b, 2 * Gb, clusterResource);
VerifyQueueMetrics(c, 3 * Gb, clusterResource);
VerifyQueueMetrics(d, 1 * Gb, clusterResource);
//reset manually resources on node
node_0 = TestUtils.GetMockNode("host_0", DefaultRack, 0, (memoryPerNode - 2 - 2 -
3 - 1) * Gb);
Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint());
// Assign 1GB container to B
StubQueueAllocation(a, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(b, clusterResource, node_0, 1 * Gb);
StubQueueAllocation(c, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(d, clusterResource, node_0, 0 * Gb);
root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
VerifyQueueMetrics(a, 2 * Gb, clusterResource);
VerifyQueueMetrics(b, 3 * Gb, clusterResource);
VerifyQueueMetrics(c, 3 * Gb, clusterResource);
VerifyQueueMetrics(d, 1 * Gb, clusterResource);
Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint());
//Release 1GB container resources from B
b.CompletedContainer(clusterResource, app_0, node_0, rmContainer, null, RMContainerEventType
.Kill, null, true);
VerifyQueueMetrics(a, 2 * Gb, clusterResource);
VerifyQueueMetrics(b, 2 * Gb, clusterResource);
VerifyQueueMetrics(c, 3 * Gb, clusterResource);
VerifyQueueMetrics(d, 1 * Gb, clusterResource);
//reset manually resources on node
node_0 = TestUtils.GetMockNode("host_0", DefaultRack, 0, (memoryPerNode - 2 - 2 -
3 - 1) * Gb);
Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint());
// Assign 1GB container to A
StubQueueAllocation(a, clusterResource, node_0, 1 * Gb);
StubQueueAllocation(b, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(c, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(d, clusterResource, node_0, 0 * Gb);
root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
VerifyQueueMetrics(a, 3 * Gb, clusterResource);
VerifyQueueMetrics(b, 2 * Gb, clusterResource);
VerifyQueueMetrics(c, 3 * Gb, clusterResource);
VerifyQueueMetrics(d, 1 * Gb, clusterResource);
Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint());
// Now do the real test, where B and D request a 1GB container
// D should should get the next container if the order is correct
StubQueueAllocation(a, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(b, clusterResource, node_0, 1 * Gb);
StubQueueAllocation(c, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(d, clusterResource, node_0, 1 * Gb);
root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
InOrder allocationOrder = Org.Mockito.Mockito.InOrder(d, b);
allocationOrder.Verify(d).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any
<FiCaSchedulerNode>(), Matchers.Any <ResourceLimits>());
allocationOrder.Verify(b).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any
<FiCaSchedulerNode>(), Matchers.Any <ResourceLimits>());
VerifyQueueMetrics(a, 3 * Gb, clusterResource);
VerifyQueueMetrics(b, 2 * Gb, clusterResource);
VerifyQueueMetrics(c, 3 * Gb, clusterResource);
VerifyQueueMetrics(d, 2 * Gb, clusterResource);
//D got the container
Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint());
}
