public virtual void TestRMNMInfoMissmatch()
{
RMContext rmc = Org.Mockito.Mockito.Mock <RMContext>();
ResourceScheduler rms = Org.Mockito.Mockito.Mock <ResourceScheduler>();
ConcurrentMap <NodeId, RMNode> map = new ConcurrentHashMap <NodeId, RMNode>();
RMNode node = MockNodes.NewNodeInfo(1, MockNodes.NewResource(4 * 1024));
map[node.GetNodeID()] = node;
Org.Mockito.Mockito.When(rmc.GetRMNodes()).ThenReturn(map);
RMNMInfo rmInfo = new RMNMInfo(rmc, rms);
string liveNMs = rmInfo.GetLiveNodeManagers();
ObjectMapper mapper = new ObjectMapper();
JsonNode jn = mapper.ReadTree(liveNMs);
NUnit.Framework.Assert.AreEqual("Unexpected number of live nodes:", 1, jn.Size());
IEnumerator <JsonNode> it = jn.GetEnumerator();
while (it.HasNext())
{
JsonNode n = it.Next();
NUnit.Framework.Assert.IsNotNull(n.Get("HostName"));
NUnit.Framework.Assert.IsNotNull(n.Get("Rack"));
NUnit.Framework.Assert.IsTrue("Node " + n.Get("NodeId") + " should be RUNNING", n
.Get("State").AsText().Contains("RUNNING"));
NUnit.Framework.Assert.IsNotNull(n.Get("NodeHTTPAddress"));
NUnit.Framework.Assert.IsNotNull(n.Get("LastHealthUpdate"));
NUnit.Framework.Assert.IsNotNull(n.Get("HealthReport"));
NUnit.Framework.Assert.IsNotNull(n.Get("NodeManagerVersion"));
NUnit.Framework.Assert.IsNull(n.Get("NumContainers"));
NUnit.Framework.Assert.IsNull(n.Get("UsedMemoryMB"));
NUnit.Framework.Assert.IsNull(n.Get("AvailableMemoryMB"));
}
}
public virtual void Init(Clock clock, ResourceScheduler sched, ICollection <Plan>
plans)
{
this.clock = clock;
this.scheduler = sched;
Sharpen.Collections.AddAll(this.plans, plans);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public virtual void Submit()
{
lock (this)
{
ApplicationSubmissionContext context = recordFactory.NewRecordInstance <ApplicationSubmissionContext
>();
context.SetApplicationId(this.applicationId);
context.SetQueue(this.queue);
// Set up the container launch context for the application master
ContainerLaunchContext amContainer = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<ContainerLaunchContext>();
context.SetAMContainerSpec(amContainer);
context.SetResource(Resources.CreateResource(YarnConfiguration.DefaultRmSchedulerMinimumAllocationMb
));
SubmitApplicationRequest request = recordFactory.NewRecordInstance <SubmitApplicationRequest
>();
request.SetApplicationSubmissionContext(context);
ResourceScheduler scheduler = resourceManager.GetResourceScheduler();
resourceManager.GetClientRMService().SubmitApplication(request);
// Notify scheduler
AppAddedSchedulerEvent addAppEvent = new AppAddedSchedulerEvent(this.applicationId
, this.queue, "user");
scheduler.Handle(addAppEvent);
AppAttemptAddedSchedulerEvent addAttemptEvent = new AppAttemptAddedSchedulerEvent
(this.applicationAttemptId, false);
scheduler.Handle(addAttemptEvent);
}
}
public RMActiveServiceContext(Dispatcher rmDispatcher, ContainerAllocationExpirer
containerAllocationExpirer, AMLivelinessMonitor amLivelinessMonitor, AMLivelinessMonitor
amFinishingMonitor, DelegationTokenRenewer delegationTokenRenewer, AMRMTokenSecretManager
appTokenSecretManager, RMContainerTokenSecretManager containerTokenSecretManager
, NMTokenSecretManagerInRM nmTokenSecretManager, ClientToAMTokenSecretManagerInRM
clientToAMTokenSecretManager, ResourceScheduler scheduler)
: this()
{
this.SetContainerAllocationExpirer(containerAllocationExpirer);
this.SetAMLivelinessMonitor(amLivelinessMonitor);
this.SetAMFinishingMonitor(amFinishingMonitor);
this.SetDelegationTokenRenewer(delegationTokenRenewer);
this.SetAMRMTokenSecretManager(appTokenSecretManager);
this.SetContainerTokenSecretManager(containerTokenSecretManager);
this.SetNMTokenSecretManager(nmTokenSecretManager);
this.SetClientToAMTokenSecretManager(clientToAMTokenSecretManager);
this.SetScheduler(scheduler);
RMStateStore nullStore = new NullRMStateStore();
nullStore.SetRMDispatcher(rmDispatcher);
try
{
nullStore.Init(new YarnConfiguration());
SetStateStore(nullStore);
}
catch (Exception)
{
System.Diagnostics.Debug.Assert(false);
}
}
public SchedulerInfo(ResourceManager rm)
{
// JAXB needs this
ResourceScheduler rs = rm.GetResourceScheduler();
if (rs is CapacityScheduler)
{
this.schedulerName = "Capacity Scheduler";
}
else
{
if (rs is FairScheduler)
{
this.schedulerName = "Fair Scheduler";
}
else
{
if (rs is FifoScheduler)
{
this.schedulerName = "Fifo Scheduler";
}
}
}
this.minAllocResource = new ResourceInfo(rs.GetMinimumResourceCapability());
this.maxAllocResource = new ResourceInfo(rs.GetMaximumResourceCapability());
this.schedulingResourceTypes = rs.GetSchedulingResourceTypes();
}
public NodeInfo(RMNode ni, ResourceScheduler sched)
{
// JAXB needs this
NodeId id = ni.GetNodeID();
SchedulerNodeReport report = sched.GetNodeReport(id);
this.numContainers = 0;
this.usedMemoryMB = 0;
this.availMemoryMB = 0;
if (report != null)
{
this.numContainers = report.GetNumContainers();
this.usedMemoryMB = report.GetUsedResource().GetMemory();
this.availMemoryMB = report.GetAvailableResource().GetMemory();
this.usedVirtualCores = report.GetUsedResource().GetVirtualCores();
this.availableVirtualCores = report.GetAvailableResource().GetVirtualCores();
}
this.id = id.ToString();
this.rack = ni.GetRackName();
this.nodeHostName = ni.GetHostName();
this.state = ni.GetState();
this.nodeHTTPAddress = ni.GetHttpAddress();
this.lastHealthUpdate = ni.GetLastHealthReportTime();
this.healthReport = ni.GetHealthReport().ToString();
this.version = ni.GetNodeManagerVersion();
// add labels
ICollection <string> labelSet = ni.GetNodeLabels();
if (labelSet != null)
{
Sharpen.Collections.AddAll(nodeLabels, labelSet);
nodeLabels.Sort();
}
}
public UserMetricsInfo(ResourceManager rm, string user)
{
// JAXB needs this
ResourceScheduler rs = rm.GetResourceScheduler();
QueueMetrics metrics = rs.GetRootQueueMetrics();
QueueMetrics userMetrics = metrics.GetUserMetrics(user);
this.userMetricsAvailable = false;
if (userMetrics != null)
{
this.userMetricsAvailable = true;
this.appsSubmitted = userMetrics.GetAppsSubmitted();
this.appsCompleted = userMetrics.GetAppsCompleted();
this.appsPending = userMetrics.GetAppsPending();
this.appsRunning = userMetrics.GetAppsRunning();
this.appsFailed = userMetrics.GetAppsFailed();
this.appsKilled = userMetrics.GetAppsKilled();
this.runningContainers = userMetrics.GetAllocatedContainers();
this.pendingContainers = userMetrics.GetPendingContainers();
this.reservedContainers = userMetrics.GetReservedContainers();
this.reservedMB = userMetrics.GetReservedMB();
this.pendingMB = userMetrics.GetPendingMB();
this.allocatedMB = userMetrics.GetAllocatedMB();
this.reservedVirtualCores = userMetrics.GetReservedVirtualCores();
this.pendingVirtualCores = userMetrics.GetPendingVirtualCores();
this.allocatedVirtualCores = userMetrics.GetAllocatedVirtualCores();
}
}
public ClusterMetricsInfo(ResourceManager rm)
{
// JAXB needs this
ResourceScheduler rs = rm.GetResourceScheduler();
QueueMetrics metrics = rs.GetRootQueueMetrics();
ClusterMetrics clusterMetrics = ClusterMetrics.GetMetrics();
this.appsSubmitted = metrics.GetAppsSubmitted();
this.appsCompleted = metrics.GetAppsCompleted();
this.appsPending = metrics.GetAppsPending();
this.appsRunning = metrics.GetAppsRunning();
this.appsFailed = metrics.GetAppsFailed();
this.appsKilled = metrics.GetAppsKilled();
this.reservedMB = metrics.GetReservedMB();
this.availableMB = metrics.GetAvailableMB();
this.allocatedMB = metrics.GetAllocatedMB();
this.reservedVirtualCores = metrics.GetReservedVirtualCores();
this.availableVirtualCores = metrics.GetAvailableVirtualCores();
this.allocatedVirtualCores = metrics.GetAllocatedVirtualCores();
this.containersAllocated = metrics.GetAllocatedContainers();
this.containersPending = metrics.GetPendingContainers();
this.containersReserved = metrics.GetReservedContainers();
this.totalMB = availableMB + allocatedMB;
this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores;
this.activeNodes = clusterMetrics.GetNumActiveNMs();
this.lostNodes = clusterMetrics.GetNumLostNMs();
this.unhealthyNodes = clusterMetrics.GetUnhealthyNMs();
this.decommissionedNodes = clusterMetrics.GetNumDecommisionedNMs();
this.rebootedNodes = clusterMetrics.GetNumRebootedNMs();
this.totalNodes = activeNodes + lostNodes + decommissionedNodes + rebootedNodes +
unhealthyNodes;
}
public virtual void TestExistenceOfResourceRequestInRMContainer()
{
Configuration conf = new Configuration();
MockRM rm1 = new MockRM(conf);
rm1.Start();
MockNM nm1 = rm1.RegisterNode("unknownhost:1234", 8000);
RMApp app1 = rm1.SubmitApp(1024);
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
ResourceScheduler scheduler = rm1.GetResourceScheduler();
// request a container.
am1.Allocate("127.0.0.1", 1024, 1, new AList <ContainerId>());
ContainerId containerId2 = ContainerId.NewContainerId(am1.GetApplicationAttemptId
(), 2);
rm1.WaitForState(nm1, containerId2, RMContainerState.Allocated);
// Verify whether list of ResourceRequest is present in RMContainer
// while moving to ALLOCATED state
NUnit.Framework.Assert.IsNotNull(scheduler.GetRMContainer(containerId2).GetResourceRequests
());
// Allocate container
am1.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>()).GetAllocatedContainers
();
rm1.WaitForState(nm1, containerId2, RMContainerState.Acquired);
// After RMContainer moving to ACQUIRED state, list of ResourceRequest will
// be empty
NUnit.Framework.Assert.IsNull(scheduler.GetRMContainer(containerId2).GetResourceRequests
());
}
public override void Init(Clock clock, ResourceScheduler sched, ICollection <Plan>
plans)
{
base.Init(clock, sched, plans);
fs = (FairScheduler)sched;
Log.Info("Initializing Plan Follower Policy:" + this.GetType().GetCanonicalName()
);
}
public ClientRMServiceForTest(TestClientRMTokens _enclosing, Configuration conf,
ResourceScheduler scheduler, RMDelegationTokenSecretManager rmDTSecretManager)
: base(Org.Mockito.Mockito.Mock <RMContext>(), scheduler, Org.Mockito.Mockito.Mock
<RMAppManager>(), new ApplicationACLsManager(conf), new QueueACLsManager(scheduler
, conf), rmDTSecretManager)
{
this._enclosing = _enclosing;
}
private static ResourceScheduler CreateMockScheduler(Configuration conf)
{
ResourceScheduler mockSched = Org.Mockito.Mockito.Mock <ResourceScheduler>();
Org.Mockito.Mockito.DoReturn(BuilderUtils.NewResource(512, 0)).When(mockSched).GetMinimumResourceCapability
();
Org.Mockito.Mockito.DoReturn(BuilderUtils.NewResource(5120, 0)).When(mockSched).GetMaximumResourceCapability
();
return(mockSched);
}
public _RMContextImpl_159(ConcurrentMap <ApplicationId, RMApp> applicationsMaps, Dispatcher
baseArg1, ContainerAllocationExpirer baseArg2, AMLivelinessMonitor baseArg3, AMLivelinessMonitor
baseArg4, DelegationTokenRenewer baseArg5, AMRMTokenSecretManager baseArg6, RMContainerTokenSecretManager
baseArg7, NMTokenSecretManagerInRM baseArg8, ClientToAMTokenSecretManagerInRM baseArg9
, ResourceScheduler baseArg10)
: base(baseArg1, baseArg2, baseArg3, baseArg4, baseArg5, baseArg6, baseArg7, baseArg8
, baseArg9, baseArg10)
{
this.applicationsMaps = applicationsMaps;
}
/// <exception cref="System.Exception"/>
public static ResourceManager MockFifoRm(int apps, int racks, int nodes, int mbsPerNode
)
{
ResourceManager rm = Org.Mockito.Mockito.Mock <ResourceManager>();
RMContext rmContext = MockRMContext(apps, racks, nodes, mbsPerNode);
ResourceScheduler rs = MockFifoScheduler(rmContext);
Org.Mockito.Mockito.When(rm.GetResourceScheduler()).ThenReturn(rs);
Org.Mockito.Mockito.When(rm.GetRMContext()).ThenReturn(rmContext);
return(rm);
}
protected internal override QueueACLsManager CreateQueueACLsManager(ResourceScheduler
scheduler, Configuration conf)
{
QueueACLsManager mockQueueACLsManager = Org.Mockito.Mockito.Mock <QueueACLsManager
>();
Org.Mockito.Mockito.When(mockQueueACLsManager.CheckAccess(Matchers.Any <UserGroupInformation
>(), Matchers.Any <QueueACL>(), Matchers.AnyString())).ThenAnswer(new _Answer_113
());
return(mockQueueACLsManager);
}
/// <exception cref="System.IO.IOException"/>
private static ResourceManager MockRmWithApps(RMContext rmContext)
{
ResourceManager rm = Org.Mockito.Mockito.Mock <ResourceManager>();
ResourceScheduler rs = MockFairSchedulerWithoutApps(rmContext);
ClientRMService clientRMService = MockClientRMService(rmContext);
Org.Mockito.Mockito.When(rm.GetResourceScheduler()).ThenReturn(rs);
Org.Mockito.Mockito.When(rm.GetRMContext()).ThenReturn(rmContext);
Org.Mockito.Mockito.When(rm.GetClientRMService()).ThenReturn(clientRMService);
return(rm);
}
public _RMContextImpl_183(ConcurrentMap <ApplicationId, RMApp> applicationsMaps, ConcurrentMap
<string, RMNode> deactivatedNodesMap, ConcurrentMap <NodeId, RMNode> nodesMap, Dispatcher
baseArg1, ContainerAllocationExpirer baseArg2, AMLivelinessMonitor baseArg3, AMLivelinessMonitor
baseArg4, DelegationTokenRenewer baseArg5, AMRMTokenSecretManager baseArg6, RMContainerTokenSecretManager
baseArg7, NMTokenSecretManagerInRM baseArg8, ClientToAMTokenSecretManagerInRM baseArg9
, ResourceScheduler baseArg10)
: base(baseArg1, baseArg2, baseArg3, baseArg4, baseArg5, baseArg6, baseArg7, baseArg8
, baseArg9, baseArg10)
{
this.applicationsMaps = applicationsMaps;
this.deactivatedNodesMap = deactivatedNodesMap;
this.nodesMap = nodesMap;
}
public override void Init(Clock clock, ResourceScheduler sched, ICollection <Plan>
plans)
{
base.Init(clock, sched, plans);
Log.Info("Initializing Plan Follower Policy:" + this.GetType().GetCanonicalName()
);
if (!(sched is CapacityScheduler))
{
throw new YarnRuntimeException("CapacitySchedulerPlanFollower can only work with CapacityScheduler"
);
}
this.cs = (CapacityScheduler)sched;
}
private static ResourceScheduler MockResourceScheduler()
{
ResourceScheduler scheduler = Org.Mockito.Mockito.Mock <ResourceScheduler>();
Org.Mockito.Mockito.When(scheduler.GetMinimumResourceCapability()).ThenReturn(Resources
.CreateResource(YarnConfiguration.DefaultRmSchedulerMinimumAllocationMb));
Org.Mockito.Mockito.When(scheduler.GetMaximumResourceCapability()).ThenReturn(Resources
.CreateResource(YarnConfiguration.DefaultRmSchedulerMaximumAllocationMb));
ResourceCalculator rc = new DefaultResourceCalculator();
Org.Mockito.Mockito.When(scheduler.GetResourceCalculator()).ThenReturn(rc);
return(scheduler);
}
/// <exception cref="System.IO.IOException"/>
public static ResourceManager MockRm(RMContext rmContext)
{
ResourceManager rm = Org.Mockito.Mockito.Mock <ResourceManager>();
ResourceScheduler rs = MockCapacityScheduler();
ApplicationACLsManager aclMgr = MockAppACLsManager();
ClientRMService clientRMService = MockClientRMService(rmContext);
Org.Mockito.Mockito.When(rm.GetResourceScheduler()).ThenReturn(rs);
Org.Mockito.Mockito.When(rm.GetRMContext()).ThenReturn(rmContext);
Org.Mockito.Mockito.When(rm.GetApplicationACLsManager()).ThenReturn(aclMgr);
Org.Mockito.Mockito.When(rm.GetClientRMService()).ThenReturn(clientRMService);
return(rm);
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public override void Reinitialize(Configuration conf, RMContext rmContext)
{
// Validate if the scheduler is fair scheduler
ResourceScheduler scheduler = rmContext.GetScheduler();
if (!(scheduler is FairScheduler))
{
throw new YarnRuntimeException("Class " + scheduler.GetType().GetCanonicalName()
+ " not instance of " + typeof(FairScheduler).GetCanonicalName());
}
fairScheduler = (FairScheduler)scheduler;
this.conf = conf;
base.Reinitialize(conf, rmContext);
}
/// <summary>Get the default reservation system corresponding to the scheduler</summary>
/// <param name="scheduler">the scheduler for which the reservation system is required
/// </param>
public static string GetDefaultReservationSystem(ResourceScheduler scheduler)
{
if (scheduler is CapacityScheduler)
{
return(typeof(CapacityReservationSystem).FullName);
}
else
{
if (scheduler is FairScheduler)
{
return(typeof(FairReservationSystem).FullName);
}
}
return(null);
}
/// <exception cref="System.Exception"/>
private void WaitForClusterMemory(MockNM nm1, ResourceScheduler rs, int clusterMemory
)
{
int counter = 0;
while (rs.GetRootQueueMetrics().GetAllocatedMB() != clusterMemory)
{
nm1.NodeHeartbeat(true);
Sharpen.Thread.Sleep(100);
if (counter++ == 50)
{
NUnit.Framework.Assert.Fail("Wait for cluster memory is timed out.Expected=" + clusterMemory
+ " Actual=" + rs.GetRootQueueMetrics().GetAllocatedMB());
}
}
}
/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/>
/// <exception cref="System.Exception"/>
public virtual void VerifyClusterMetrics(int submittedApps, int completedApps, int
reservedMB, int availableMB, int allocMB, int reservedVirtualCores, int availableVirtualCores
, int allocVirtualCores, int totalVirtualCores, int containersAlloc, int totalMB
, int totalNodes, int lostNodes, int unhealthyNodes, int decommissionedNodes, int
rebootedNodes, int activeNodes)
{
ResourceScheduler rs = rm.GetResourceScheduler();
QueueMetrics metrics = rs.GetRootQueueMetrics();
ClusterMetrics clusterMetrics = ClusterMetrics.GetMetrics();
long totalMBExpect = metrics.GetAvailableMB() + metrics.GetAllocatedMB();
long totalVirtualCoresExpect = metrics.GetAvailableVirtualCores() + metrics.GetAllocatedVirtualCores
();
NUnit.Framework.Assert.AreEqual("appsSubmitted doesn't match", metrics.GetAppsSubmitted
(), submittedApps);
NUnit.Framework.Assert.AreEqual("appsCompleted doesn't match", metrics.GetAppsCompleted
(), completedApps);
NUnit.Framework.Assert.AreEqual("reservedMB doesn't match", metrics.GetReservedMB
(), reservedMB);
NUnit.Framework.Assert.AreEqual("availableMB doesn't match", metrics.GetAvailableMB
(), availableMB);
NUnit.Framework.Assert.AreEqual("allocatedMB doesn't match", metrics.GetAllocatedMB
(), allocMB);
NUnit.Framework.Assert.AreEqual("reservedVirtualCores doesn't match", metrics.GetReservedVirtualCores
(), reservedVirtualCores);
NUnit.Framework.Assert.AreEqual("availableVirtualCores doesn't match", metrics.GetAvailableVirtualCores
(), availableVirtualCores);
NUnit.Framework.Assert.AreEqual("allocatedVirtualCores doesn't match", totalVirtualCoresExpect
, allocVirtualCores);
NUnit.Framework.Assert.AreEqual("containersAllocated doesn't match", 0, containersAlloc
);
NUnit.Framework.Assert.AreEqual("totalMB doesn't match", totalMBExpect, totalMB);
NUnit.Framework.Assert.AreEqual("totalNodes doesn't match", clusterMetrics.GetNumActiveNMs
() + clusterMetrics.GetNumLostNMs() + clusterMetrics.GetNumDecommisionedNMs() +
clusterMetrics.GetNumRebootedNMs() + clusterMetrics.GetUnhealthyNMs(), totalNodes
);
NUnit.Framework.Assert.AreEqual("lostNodes doesn't match", clusterMetrics.GetNumLostNMs
(), lostNodes);
NUnit.Framework.Assert.AreEqual("unhealthyNodes doesn't match", clusterMetrics.GetUnhealthyNMs
(), unhealthyNodes);
NUnit.Framework.Assert.AreEqual("decommissionedNodes doesn't match", clusterMetrics
.GetNumDecommisionedNMs(), decommissionedNodes);
NUnit.Framework.Assert.AreEqual("rebootedNodes doesn't match", clusterMetrics.GetNumRebootedNMs
(), rebootedNodes);
NUnit.Framework.Assert.AreEqual("activeNodes doesn't match", clusterMetrics.GetNumActiveNMs
(), activeNodes);
}
/// <summary>Constructor for RMNMInfo registers the bean with JMX.</summary>
/// <param name="rmc">resource manager's context object</param>
/// <param name="sched">resource manager's scheduler object</param>
public RMNMInfo(RMContext rmc, ResourceScheduler sched)
{
this.rmContext = rmc;
this.scheduler = sched;
StandardMBean bean;
try
{
bean = new StandardMBean(this, typeof(RMNMInfoBeans));
MBeans.Register("ResourceManager", "RMNMInfo", bean);
}
catch (NotCompliantMBeanException e)
{
Log.Warn("Error registering RMNMInfo MBean", e);
}
Log.Info("Registered RMNMInfo MBean");
}
public virtual void TestGetAppsInQueue()
{
Application application_0 = new Application("user_0", resourceManager);
application_0.Submit();
Application application_1 = new Application("user_0", resourceManager);
application_1.Submit();
ResourceScheduler scheduler = resourceManager.GetResourceScheduler();
IList <ApplicationAttemptId> appsInDefault = scheduler.GetAppsInQueue("default");
NUnit.Framework.Assert.IsTrue(appsInDefault.Contains(application_0.GetApplicationAttemptId
()));
NUnit.Framework.Assert.IsTrue(appsInDefault.Contains(application_1.GetApplicationAttemptId
()));
NUnit.Framework.Assert.AreEqual(2, appsInDefault.Count);
NUnit.Framework.Assert.IsNull(scheduler.GetAppsInQueue("someotherqueue"));
}
public RMContextImpl(Dispatcher rmDispatcher, ContainerAllocationExpirer containerAllocationExpirer
, AMLivelinessMonitor amLivelinessMonitor, AMLivelinessMonitor amFinishingMonitor
, DelegationTokenRenewer delegationTokenRenewer, AMRMTokenSecretManager appTokenSecretManager
, RMContainerTokenSecretManager containerTokenSecretManager, NMTokenSecretManagerInRM
nmTokenSecretManager, ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager
, ResourceScheduler scheduler)
: this()
{
// helper constructor for tests
this.SetDispatcher(rmDispatcher);
SetActiveServiceContext(new RMActiveServiceContext(rmDispatcher, containerAllocationExpirer
, amLivelinessMonitor, amFinishingMonitor, delegationTokenRenewer, appTokenSecretManager
, containerTokenSecretManager, nmTokenSecretManager, clientToAMTokenSecretManager
, scheduler));
ConfigurationProvider provider = new LocalConfigurationProvider();
SetConfigurationProvider(provider);
}
public virtual void SetUp()
{
long now = Runtime.CurrentTimeMillis();
rmContext = MockRMContext(1, now - 10);
ResourceScheduler scheduler = MockResourceScheduler();
Configuration conf = new Configuration();
ApplicationMasterService masterService = new ApplicationMasterService(rmContext,
scheduler);
appMonitor = new TestAppManager.TestRMAppManager(this, rmContext, new ClientToAMTokenSecretManagerInRM
(), scheduler, masterService, new ApplicationACLsManager(conf), conf);
appId = MockApps.NewAppID(1);
RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(null);
asContext = recordFactory.NewRecordInstance <ApplicationSubmissionContext>();
asContext.SetApplicationId(appId);
asContext.SetAMContainerSpec(MockContainerLaunchContext(recordFactory));
asContext.SetResource(MockResource());
SetupDispatcher(rmContext, conf);
}
/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/>
/// <exception cref="System.Exception"/>
public virtual void VerifyNodeInfoGeneric(MockNM nm, string state, string rack, string
id, string nodeHostName, string nodeHTTPAddress, long lastHealthUpdate, string
healthReport, int numContainers, long usedMemoryMB, long availMemoryMB, long usedVirtualCores
, long availVirtualCores, string version)
{
RMNode node = rm.GetRMContext().GetRMNodes()[nm.GetNodeId()];
ResourceScheduler sched = rm.GetResourceScheduler();
SchedulerNodeReport report = sched.GetNodeReport(nm.GetNodeId());
WebServicesTestUtils.CheckStringMatch("state", node.GetState().ToString(), state);
WebServicesTestUtils.CheckStringMatch("rack", node.GetRackName(), rack);
WebServicesTestUtils.CheckStringMatch("id", nm.GetNodeId().ToString(), id);
WebServicesTestUtils.CheckStringMatch("nodeHostName", nm.GetNodeId().GetHost(), nodeHostName
);
WebServicesTestUtils.CheckStringMatch("healthReport", node.GetHealthReport().ToString
(), healthReport);
string expectedHttpAddress = nm.GetNodeId().GetHost() + ":" + nm.GetHttpPort();
WebServicesTestUtils.CheckStringMatch("nodeHTTPAddress", expectedHttpAddress, nodeHTTPAddress
);
WebServicesTestUtils.CheckStringMatch("version", node.GetNodeManagerVersion(), version
);
long expectedHealthUpdate = node.GetLastHealthReportTime();
NUnit.Framework.Assert.AreEqual("lastHealthUpdate doesn't match, got: " + lastHealthUpdate
+ " expected: " + expectedHealthUpdate, expectedHealthUpdate, lastHealthUpdate);
if (report != null)
{
NUnit.Framework.Assert.AreEqual("numContainers doesn't match: " + numContainers,
report.GetNumContainers(), numContainers);
NUnit.Framework.Assert.AreEqual("usedMemoryMB doesn't match: " + usedMemoryMB, report
.GetUsedResource().GetMemory(), usedMemoryMB);
NUnit.Framework.Assert.AreEqual("availMemoryMB doesn't match: " + availMemoryMB,
report.GetAvailableResource().GetMemory(), availMemoryMB);
NUnit.Framework.Assert.AreEqual("usedVirtualCores doesn't match: " + usedVirtualCores
, report.GetUsedResource().GetVirtualCores(), usedVirtualCores);
NUnit.Framework.Assert.AreEqual("availVirtualCores doesn't match: " + availVirtualCores
, report.GetAvailableResource().GetVirtualCores(), availVirtualCores);
}
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
private void Initialize(Configuration conf)
{
Log.Info("Initializing Reservation system");
this.conf = conf;
scheduler = rmContext.GetScheduler();
// Get the plan step size
planStepSize = conf.GetTimeDuration(YarnConfiguration.RmReservationSystemPlanFollowerTimeStep
, YarnConfiguration.DefaultRmReservationSystemPlanFollowerTimeStep, TimeUnit.Milliseconds
);
if (planStepSize < 0)
{
planStepSize = YarnConfiguration.DefaultRmReservationSystemPlanFollowerTimeStep;
}
// Create a plan corresponding to every reservable queue
ICollection <string> planQueueNames = scheduler.GetPlanQueues();
foreach (string planQueueName in planQueueNames)
{
Plan plan = InitializePlan(planQueueName);
plans[planQueueName] = plan;
}
}
