public UserMetricsInfo(ResourceManager rm, string user)
{
// JAXB needs this
ResourceScheduler rs = rm.GetResourceScheduler();
QueueMetrics metrics = rs.GetRootQueueMetrics();
QueueMetrics userMetrics = metrics.GetUserMetrics(user);
this.userMetricsAvailable = false;
if (userMetrics != null)
{
this.userMetricsAvailable = true;
this.appsSubmitted = userMetrics.GetAppsSubmitted();
this.appsCompleted = userMetrics.GetAppsCompleted();
this.appsPending = userMetrics.GetAppsPending();
this.appsRunning = userMetrics.GetAppsRunning();
this.appsFailed = userMetrics.GetAppsFailed();
this.appsKilled = userMetrics.GetAppsKilled();
this.runningContainers = userMetrics.GetAllocatedContainers();
this.pendingContainers = userMetrics.GetPendingContainers();
this.reservedContainers = userMetrics.GetReservedContainers();
this.reservedMB = userMetrics.GetReservedMB();
this.pendingMB = userMetrics.GetPendingMB();
this.allocatedMB = userMetrics.GetAllocatedMB();
this.reservedVirtualCores = userMetrics.GetReservedVirtualCores();
this.pendingVirtualCores = userMetrics.GetPendingVirtualCores();
this.allocatedVirtualCores = userMetrics.GetAllocatedVirtualCores();
}
}
public virtual void Setup()
{
// 24h window
timeWindow = 86400000L;
// 1 sec step
step = 1000L;
// 25% avg cap on capacity
avgConstraint = 25;
// 70% instantaneous cap on capacity
instConstraint = 70;
initTime = Runtime.CurrentTimeMillis();
minAlloc = Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance(1024, 1);
res = new DefaultResourceCalculator();
maxAlloc = Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance(1024 * 8, 8);
mAgent = Org.Mockito.Mockito.Mock <ReservationAgent>();
ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
QueueMetrics rootQueueMetrics = Org.Mockito.Mockito.Mock <QueueMetrics>();
string reservationQ = testUtil.GetFullReservationQueueName();
Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = ReservationSystemTestUtil
.CalculateClusterResource(totCont);
ReservationSchedulerConfiguration conf = ReservationSystemTestUtil.CreateConf(reservationQ
, timeWindow, instConstraint, avgConstraint);
CapacityOverTimePolicy policy = new CapacityOverTimePolicy();
policy.Init(reservationQ, conf);
plan = new InMemoryPlan(rootQueueMetrics, policy, mAgent, clusterResource, step,
res, minAlloc, maxAlloc, "dedicated", null, true);
}
/// <exception cref="System.Exception"/>
private void VerifyClusterMetrics(int activeNodes, int appsSubmitted, int appsPending
, int containersPending, int availableMB, int activeApplications)
{
int timeoutSecs = 0;
QueueMetrics metrics = rm.GetResourceScheduler().GetRootQueueMetrics();
ClusterMetrics clusterMetrics = ClusterMetrics.GetMetrics();
bool isAllMetricAssertionDone = false;
string message = null;
while (timeoutSecs++ < 5)
{
try
{
// verify queue metrics
AssertMetric("appsSubmitted", appsSubmitted, metrics.GetAppsSubmitted());
AssertMetric("appsPending", appsPending, metrics.GetAppsPending());
AssertMetric("containersPending", containersPending, metrics.GetPendingContainers
());
AssertMetric("availableMB", availableMB, metrics.GetAvailableMB());
AssertMetric("activeApplications", activeApplications, metrics.GetActiveApps());
// verify node metric
AssertMetric("activeNodes", activeNodes, clusterMetrics.GetNumActiveNMs());
isAllMetricAssertionDone = true;
break;
}
catch (Exception e)
{
message = e.Message;
System.Console.Out.WriteLine("Waiting for metrics assertion to complete");
Sharpen.Thread.Sleep(1000);
}
}
NUnit.Framework.Assert.IsTrue(message, isAllMetricAssertionDone);
}
public ClusterMetricsInfo(ResourceManager rm)
{
// JAXB needs this
ResourceScheduler rs = rm.GetResourceScheduler();
QueueMetrics metrics = rs.GetRootQueueMetrics();
ClusterMetrics clusterMetrics = ClusterMetrics.GetMetrics();
this.appsSubmitted = metrics.GetAppsSubmitted();
this.appsCompleted = metrics.GetAppsCompleted();
this.appsPending = metrics.GetAppsPending();
this.appsRunning = metrics.GetAppsRunning();
this.appsFailed = metrics.GetAppsFailed();
this.appsKilled = metrics.GetAppsKilled();
this.reservedMB = metrics.GetReservedMB();
this.availableMB = metrics.GetAvailableMB();
this.allocatedMB = metrics.GetAllocatedMB();
this.reservedVirtualCores = metrics.GetReservedVirtualCores();
this.availableVirtualCores = metrics.GetAvailableVirtualCores();
this.allocatedVirtualCores = metrics.GetAllocatedVirtualCores();
this.containersAllocated = metrics.GetAllocatedContainers();
this.containersPending = metrics.GetPendingContainers();
this.containersReserved = metrics.GetReservedContainers();
this.totalMB = availableMB + allocatedMB;
this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores;
this.activeNodes = clusterMetrics.GetNumActiveNMs();
this.lostNodes = clusterMetrics.GetNumLostNMs();
this.unhealthyNodes = clusterMetrics.GetUnhealthyNMs();
this.decommissionedNodes = clusterMetrics.GetNumDecommisionedNMs();
this.rebootedNodes = clusterMetrics.GetNumRebootedNMs();
this.totalNodes = activeNodes + lostNodes + decommissionedNodes + rebootedNodes +
unhealthyNodes;
}
public virtual void Setup()
{
long seed = rand.NextLong();
rand.SetSeed(seed);
Org.Mortbay.Log.Log.Info("Running with seed: " + seed);
// setting completely loose quotas
long timeWindow = 1000000L;
Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterCapacity = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(100 * 1024, 100);
step = 1000L;
ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
string reservationQ = testUtil.GetFullReservationQueueName();
float instConstraint = 100;
float avgConstraint = 100;
ReservationSchedulerConfiguration conf = ReservationSystemTestUtil.CreateConf(reservationQ
, timeWindow, instConstraint, avgConstraint);
CapacityOverTimePolicy policy = new CapacityOverTimePolicy();
policy.Init(reservationQ, conf);
agent = new GreedyReservationAgent();
QueueMetrics queueMetrics = Org.Mockito.Mockito.Mock <QueueMetrics>();
plan = new InMemoryPlan(queueMetrics, policy, agent, clusterCapacity, step, res,
minAlloc, maxAlloc, "dedicated", null, true);
}
public virtual void TestHeadroom()
{
FairScheduler mockScheduler = Org.Mockito.Mockito.Mock <FairScheduler>();
Org.Mockito.Mockito.When(mockScheduler.GetClock()).ThenReturn(scheduler.GetClock(
));
FSLeafQueue mockQueue = Org.Mockito.Mockito.Mock <FSLeafQueue>();
Resource queueMaxResources = Resource.NewInstance(5 * 1024, 3);
Org.Apache.Hadoop.Yarn.Api.Records.Resource queueFairShare = Resources.CreateResource
(4096, 2);
Org.Apache.Hadoop.Yarn.Api.Records.Resource queueUsage = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(2048, 2);
Org.Apache.Hadoop.Yarn.Api.Records.Resource queueStarvation = Resources.Subtract(
queueFairShare, queueUsage);
Org.Apache.Hadoop.Yarn.Api.Records.Resource queueMaxResourcesAvailable = Resources
.Subtract(queueMaxResources, queueUsage);
Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = Resources.CreateResource
(8192, 8);
Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterUsage = Resources.CreateResource
(2048, 2);
Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterAvailable = Resources.Subtract
(clusterResource, clusterUsage);
QueueMetrics fakeRootQueueMetrics = Org.Mockito.Mockito.Mock <QueueMetrics>();
Org.Mockito.Mockito.When(mockQueue.GetMaxShare()).ThenReturn(queueMaxResources);
Org.Mockito.Mockito.When(mockQueue.GetFairShare()).ThenReturn(queueFairShare);
Org.Mockito.Mockito.When(mockQueue.GetResourceUsage()).ThenReturn(queueUsage);
Org.Mockito.Mockito.When(mockScheduler.GetClusterResource()).ThenReturn(clusterResource
);
Org.Mockito.Mockito.When(fakeRootQueueMetrics.GetAllocatedResources()).ThenReturn
(clusterUsage);
Org.Mockito.Mockito.When(mockScheduler.GetRootQueueMetrics()).ThenReturn(fakeRootQueueMetrics
);
ApplicationAttemptId applicationAttemptId = CreateAppAttemptId(1, 1);
RMContext rmContext = resourceManager.GetRMContext();
FSAppAttempt schedulerApp = new FSAppAttempt(mockScheduler, applicationAttemptId,
"user1", mockQueue, null, rmContext);
// Min of Memory and CPU across cluster and queue is used in
// DominantResourceFairnessPolicy
Org.Mockito.Mockito.When(mockQueue.GetPolicy()).ThenReturn(SchedulingPolicy.GetInstance
(typeof(DominantResourceFairnessPolicy)));
VerifyHeadroom(schedulerApp, Min(queueStarvation.GetMemory(), clusterAvailable.GetMemory
(), queueMaxResourcesAvailable.GetMemory()), Min(queueStarvation.GetVirtualCores
(), clusterAvailable.GetVirtualCores(), queueMaxResourcesAvailable.GetVirtualCores
()));
// Fair and Fifo ignore CPU of queue, so use cluster available CPU
Org.Mockito.Mockito.When(mockQueue.GetPolicy()).ThenReturn(SchedulingPolicy.GetInstance
(typeof(FairSharePolicy)));
VerifyHeadroom(schedulerApp, Min(queueStarvation.GetMemory(), clusterAvailable.GetMemory
(), queueMaxResourcesAvailable.GetMemory()), Math.Min(clusterAvailable.GetVirtualCores
(), queueMaxResourcesAvailable.GetVirtualCores()));
Org.Mockito.Mockito.When(mockQueue.GetPolicy()).ThenReturn(SchedulingPolicy.GetInstance
(typeof(FifoPolicy)));
VerifyHeadroom(schedulerApp, Min(queueStarvation.GetMemory(), clusterAvailable.GetMemory
(), queueMaxResourcesAvailable.GetMemory()), Math.Min(clusterAvailable.GetVirtualCores
(), queueMaxResourcesAvailable.GetVirtualCores()));
}
internal InMemoryPlan(QueueMetrics queueMetrics, SharingPolicy policy, ReservationAgent
agent, Org.Apache.Hadoop.Yarn.Api.Records.Resource totalCapacity, long step, ResourceCalculator
resCalc, Org.Apache.Hadoop.Yarn.Api.Records.Resource minAlloc, Org.Apache.Hadoop.Yarn.Api.Records.Resource
maxAlloc, string queueName, Planner replanner, bool getMoveOnExpiry)
: this(queueMetrics, policy, agent, totalCapacity, step, resCalc, minAlloc, maxAlloc
, queueName, replanner, getMoveOnExpiry, new UTCClock())
{
}
public virtual void TearDown()
{
resCalc = null;
minAlloc = null;
maxAlloc = null;
totalCapacity = null;
clock = null;
queueMetrics = null;
policy = null;
replanner = null;
}
public virtual void SetUp()
{
resCalc = new DefaultResourceCalculator();
minAlloc = Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance(1024, 1);
maxAlloc = Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance(64 * 1024, 20);
totalCapacity = Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance(100 * 1024
, 100);
clock = Org.Mockito.Mockito.Mock <Clock>();
queueMetrics = Org.Mockito.Mockito.Mock <QueueMetrics>();
policy = Org.Mockito.Mockito.Mock <SharingPolicy>();
replanner = Org.Mockito.Mockito.Mock <Planner>();
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(1L);
}
/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/>
/// <exception cref="System.Exception"/>
public virtual void VerifyClusterMetrics(int submittedApps, int completedApps, int
reservedMB, int availableMB, int allocMB, int reservedVirtualCores, int availableVirtualCores
, int allocVirtualCores, int totalVirtualCores, int containersAlloc, int totalMB
, int totalNodes, int lostNodes, int unhealthyNodes, int decommissionedNodes, int
rebootedNodes, int activeNodes)
{
ResourceScheduler rs = rm.GetResourceScheduler();
QueueMetrics metrics = rs.GetRootQueueMetrics();
ClusterMetrics clusterMetrics = ClusterMetrics.GetMetrics();
long totalMBExpect = metrics.GetAvailableMB() + metrics.GetAllocatedMB();
long totalVirtualCoresExpect = metrics.GetAvailableVirtualCores() + metrics.GetAllocatedVirtualCores
();
NUnit.Framework.Assert.AreEqual("appsSubmitted doesn't match", metrics.GetAppsSubmitted
(), submittedApps);
NUnit.Framework.Assert.AreEqual("appsCompleted doesn't match", metrics.GetAppsCompleted
(), completedApps);
NUnit.Framework.Assert.AreEqual("reservedMB doesn't match", metrics.GetReservedMB
(), reservedMB);
NUnit.Framework.Assert.AreEqual("availableMB doesn't match", metrics.GetAvailableMB
(), availableMB);
NUnit.Framework.Assert.AreEqual("allocatedMB doesn't match", metrics.GetAllocatedMB
(), allocMB);
NUnit.Framework.Assert.AreEqual("reservedVirtualCores doesn't match", metrics.GetReservedVirtualCores
(), reservedVirtualCores);
NUnit.Framework.Assert.AreEqual("availableVirtualCores doesn't match", metrics.GetAvailableVirtualCores
(), availableVirtualCores);
NUnit.Framework.Assert.AreEqual("allocatedVirtualCores doesn't match", totalVirtualCoresExpect
, allocVirtualCores);
NUnit.Framework.Assert.AreEqual("containersAllocated doesn't match", 0, containersAlloc
);
NUnit.Framework.Assert.AreEqual("totalMB doesn't match", totalMBExpect, totalMB);
NUnit.Framework.Assert.AreEqual("totalNodes doesn't match", clusterMetrics.GetNumActiveNMs
() + clusterMetrics.GetNumLostNMs() + clusterMetrics.GetNumDecommisionedNMs() +
clusterMetrics.GetNumRebootedNMs() + clusterMetrics.GetUnhealthyNMs(), totalNodes
);
NUnit.Framework.Assert.AreEqual("lostNodes doesn't match", clusterMetrics.GetNumLostNMs
(), lostNodes);
NUnit.Framework.Assert.AreEqual("unhealthyNodes doesn't match", clusterMetrics.GetUnhealthyNMs
(), unhealthyNodes);
NUnit.Framework.Assert.AreEqual("decommissionedNodes doesn't match", clusterMetrics
.GetNumDecommisionedNMs(), decommissionedNodes);
NUnit.Framework.Assert.AreEqual("rebootedNodes doesn't match", clusterMetrics.GetNumRebootedNMs
(), rebootedNodes);
NUnit.Framework.Assert.AreEqual("activeNodes doesn't match", clusterMetrics.GetNumActiveNMs
(), activeNodes);
}
/// <summary>Validate killing an application when it is at accepted state.</summary>
/// <exception cref="System.Exception">exception</exception>
public virtual void TestApplicationKillAtAcceptedState()
{
Dispatcher dispatcher = new _AsyncDispatcher_573();
MockRM rm = new _MockRM_596(dispatcher, conf);
// test metrics
QueueMetrics metrics = rm.GetResourceScheduler().GetRootQueueMetrics();
int appsKilled = metrics.GetAppsKilled();
int appsSubmitted = metrics.GetAppsSubmitted();
rm.Start();
MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm.GetResourceTrackerService());
nm1.RegisterNode();
// a failed app
RMApp application = rm.SubmitApp(200);
MockAM am = MockRM.LaunchAM(application, rm, nm1);
am.WaitForState(RMAppAttemptState.Launched);
nm1.NodeHeartbeat(am.GetApplicationAttemptId(), 1, ContainerState.Running);
rm.WaitForState(application.GetApplicationId(), RMAppState.Accepted);
// Now kill the application before new attempt is launched, the app report
// returns the invalid AM host and port.
KillApplicationRequest request = KillApplicationRequest.NewInstance(application.GetApplicationId
());
rm.GetClientRMService().ForceKillApplication(request);
// Specific test for YARN-1689 follows
// Now let's say a race causes AM to register now. This should not crash RM.
am.RegisterAppAttempt(false);
// We explicitly intercepted the kill-event to RMAppAttempt, so app should
// still be in KILLING state.
rm.WaitForState(application.GetApplicationId(), RMAppState.Killing);
// AM should now be in running
rm.WaitForState(am.GetApplicationAttemptId(), RMAppAttemptState.Running);
// Simulate that appAttempt is killed.
rm.GetRMContext().GetDispatcher().GetEventHandler().Handle(new RMAppEvent(application
.GetApplicationId(), RMAppEventType.AttemptKilled));
rm.WaitForState(application.GetApplicationId(), RMAppState.Killed);
// test metrics
metrics = rm.GetResourceScheduler().GetRootQueueMetrics();
NUnit.Framework.Assert.AreEqual(appsKilled + 1, metrics.GetAppsKilled());
NUnit.Framework.Assert.AreEqual(appsSubmitted + 1, metrics.GetAppsSubmitted());
}
internal InMemoryPlan(QueueMetrics queueMetrics, SharingPolicy policy, ReservationAgent
agent, Org.Apache.Hadoop.Yarn.Api.Records.Resource totalCapacity, long step, ResourceCalculator
resCalc, Org.Apache.Hadoop.Yarn.Api.Records.Resource minAlloc, Org.Apache.Hadoop.Yarn.Api.Records.Resource
maxAlloc, string queueName, Planner replanner, bool getMoveOnExpiry, Clock clock
)
{
this.queueMetrics = queueMetrics;
this.policy = policy;
this.agent = agent;
this.step = step;
this.totalCapacity = totalCapacity;
this.resCalc = resCalc;
this.minAlloc = minAlloc;
this.maxAlloc = maxAlloc;
this.rleSparseVector = new RLESparseResourceAllocation(resCalc, minAlloc);
this.queueName = queueName;
this.replanner = replanner;
this.getMoveOnExpiry = getMoveOnExpiry;
this.clock = clock;
}
private void InitScheduler(Configuration conf)
{
lock (this)
{
ValidateConf(conf);
//Use ConcurrentSkipListMap because applications need to be ordered
this.applications = new ConcurrentSkipListMap <ApplicationId, SchedulerApplication
<FiCaSchedulerApp> >();
this.minimumAllocation = Resources.CreateResource(conf.GetInt(YarnConfiguration.RmSchedulerMinimumAllocationMb
, YarnConfiguration.DefaultRmSchedulerMinimumAllocationMb));
InitMaximumResourceCapability(Resources.CreateResource(conf.GetInt(YarnConfiguration
.RmSchedulerMaximumAllocationMb, YarnConfiguration.DefaultRmSchedulerMaximumAllocationMb
), conf.GetInt(YarnConfiguration.RmSchedulerMaximumAllocationVcores, YarnConfiguration
.DefaultRmSchedulerMaximumAllocationVcores)));
this.usePortForNodeName = conf.GetBoolean(YarnConfiguration.RmSchedulerIncludePortInNodeName
, YarnConfiguration.DefaultRmSchedulerUsePortForNodeName);
this.metrics = QueueMetrics.ForQueue(DefaultQueueName, null, false, conf);
this.activeUsersManager = new ActiveUsersManager(metrics);
}
}
public virtual void SetUp()
{
configuration = new Configuration();
UserGroupInformation.SetConfiguration(configuration);
configuration.SetBoolean(YarnConfiguration.RmHaEnabled, true);
configuration.Set(YarnConfiguration.RmHaIds, Rm1NodeId + "," + Rm2NodeId);
foreach (string confKey in YarnConfiguration.GetServiceAddressConfKeys(configuration
))
{
configuration.Set(HAUtil.AddSuffix(confKey, Rm1NodeId), Rm1Address);
configuration.Set(HAUtil.AddSuffix(confKey, Rm2NodeId), Rm2Address);
configuration.Set(HAUtil.AddSuffix(confKey, Rm3NodeId), Rm3Address);
}
// Enable webapp to test web-services also
configuration.SetBoolean(MockRM.EnableWebapp, true);
configuration.SetBoolean(YarnConfiguration.YarnAclEnable, true);
ClusterMetrics.Destroy();
QueueMetrics.ClearQueueMetrics();
DefaultMetricsSystem.Shutdown();
}
/// <exception cref="System.Exception"/>
public virtual void TestAppAttemptMetrics()
{
AsyncDispatcher dispatcher = new InlineDispatcher();
FifoScheduler scheduler = new FifoScheduler();
RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
>();
RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null, null,
null, null, null, scheduler);
((RMContextImpl)rmContext).SetSystemMetricsPublisher(Org.Mockito.Mockito.Mock <SystemMetricsPublisher
>());
Configuration conf = new Configuration();
scheduler.SetRMContext(rmContext);
scheduler.Init(conf);
scheduler.Start();
scheduler.Reinitialize(conf, rmContext);
QueueMetrics metrics = scheduler.GetRootQueueMetrics();
int beforeAppsSubmitted = metrics.GetAppsSubmitted();
ApplicationId appId = BuilderUtils.NewApplicationId(200, 1);
ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
);
SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId, "queue", "user");
scheduler.Handle(appEvent);
SchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId, false
);
scheduler.Handle(attemptEvent);
appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 2);
SchedulerEvent attemptEvent2 = new AppAttemptAddedSchedulerEvent(appAttemptId, false
);
scheduler.Handle(attemptEvent2);
int afterAppsSubmitted = metrics.GetAppsSubmitted();
NUnit.Framework.Assert.AreEqual(1, afterAppsSubmitted - beforeAppsSubmitted);
scheduler.Stop();
}
/// <exception cref="System.IO.IOException"/>
public AbstractCSQueue(CapacitySchedulerContext cs, string queueName, CSQueue parent
, CSQueue old)
{
// Track resource usage-by-label like used-resource/pending-resource, etc.
// Track capacities like used-capcity/abs-used-capacity/capacity/abs-capacity,
// etc.
this.labelManager = cs.GetRMContext().GetNodeLabelManager();
this.parent = parent;
this.queueName = queueName;
this.resourceCalculator = cs.GetResourceCalculator();
// must be called after parent and queueName is set
this.metrics = old != null?old.GetMetrics() : QueueMetrics.ForQueue(GetQueuePath
(), parent, cs.GetConfiguration().GetEnableUserMetrics(), cs.GetConf());
this.csContext = cs;
this.minimumAllocation = csContext.GetMinimumResourceCapability();
// initialize ResourceUsage
queueUsage = new ResourceUsage();
queueEntity = new PrivilegedEntity(PrivilegedEntity.EntityType.Queue, GetQueuePath
());
// initialize QueueCapacities
queueCapacities = new QueueCapacities(parent == null);
}
public static QueueMetrics ForQueue(string queueName, Queue parent, bool enableUserMetrics
, Configuration conf)
{
lock (typeof(FSQueueMetrics))
{
MetricsSystem ms = DefaultMetricsSystem.Instance();
QueueMetrics metrics = queueMetrics[queueName];
if (metrics == null)
{
metrics = new Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.Fair.FSQueueMetrics
(ms, queueName, parent, enableUserMetrics, conf).Tag(QueueInfo, queueName);
// Register with the MetricsSystems
if (ms != null)
{
metrics = ms.Register(SourceName(queueName).ToString(), "Metrics for queue: " + queueName
, metrics);
}
queueMetrics[queueName] = metrics;
}
return((Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.Fair.FSQueueMetrics
)metrics);
}
}
public virtual void Setup()
{
// 1 sec step
step = 1000L;
initTime = Runtime.CurrentTimeMillis();
minAlloc = Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance(1024, 1);
res = new DefaultResourceCalculator();
maxAlloc = Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance(1024 * 8, 8);
mAgent = Org.Mockito.Mockito.Mock <ReservationAgent>();
ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
string reservationQ = testUtil.GetFullReservationQueueName();
QueueMetrics rootQueueMetrics = Org.Mockito.Mockito.Mock <QueueMetrics>();
Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = ReservationSystemTestUtil
.CalculateClusterResource(totCont);
ReservationSchedulerConfiguration conf = Org.Mockito.Mockito.Mock <ReservationSchedulerConfiguration
>();
NoOverCommitPolicy policy = new NoOverCommitPolicy();
policy.Init(reservationQ, conf);
plan = new InMemoryPlan(rootQueueMetrics, policy, mAgent, clusterResource, step,
res, minAlloc, maxAlloc, "dedicated", null, true);
}
public virtual void TearDown()
{
ClusterMetrics.Destroy();
QueueMetrics.ClearQueueMetrics();
DefaultMetricsSystem.Shutdown();
}
public virtual void TestReconnectNode()
{
DrainDispatcher dispatcher = new DrainDispatcher();
rm = new _MockRM_567(this, dispatcher);
rm.Start();
MockNM nm1 = rm.RegisterNode("host1:1234", 5120);
MockNM nm2 = rm.RegisterNode("host2:5678", 5120);
nm1.NodeHeartbeat(true);
nm2.NodeHeartbeat(false);
dispatcher.Await();
CheckUnealthyNMCount(rm, nm2, true, 1);
int expectedNMs = ClusterMetrics.GetMetrics().GetNumActiveNMs();
QueueMetrics metrics = rm.GetResourceScheduler().GetRootQueueMetrics();
// TODO Metrics incorrect in case of the FifoScheduler
NUnit.Framework.Assert.AreEqual(5120, metrics.GetAvailableMB());
// reconnect of healthy node
nm1 = rm.RegisterNode("host1:1234", 5120);
NodeHeartbeatResponse response = nm1.NodeHeartbeat(true);
NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(response.GetNodeAction()));
dispatcher.Await();
NUnit.Framework.Assert.AreEqual(expectedNMs, ClusterMetrics.GetMetrics().GetNumActiveNMs
());
CheckUnealthyNMCount(rm, nm2, true, 1);
// reconnect of unhealthy node
nm2 = rm.RegisterNode("host2:5678", 5120);
response = nm2.NodeHeartbeat(false);
NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(response.GetNodeAction()));
dispatcher.Await();
NUnit.Framework.Assert.AreEqual(expectedNMs, ClusterMetrics.GetMetrics().GetNumActiveNMs
());
CheckUnealthyNMCount(rm, nm2, true, 1);
// unhealthy node changed back to healthy
nm2 = rm.RegisterNode("host2:5678", 5120);
dispatcher.Await();
response = nm2.NodeHeartbeat(true);
response = nm2.NodeHeartbeat(true);
dispatcher.Await();
NUnit.Framework.Assert.AreEqual(5120 + 5120, metrics.GetAvailableMB());
// reconnect of node with changed capability
nm1 = rm.RegisterNode("host2:5678", 10240);
dispatcher.Await();
response = nm1.NodeHeartbeat(true);
dispatcher.Await();
NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(response.GetNodeAction()));
NUnit.Framework.Assert.AreEqual(5120 + 10240, metrics.GetAvailableMB());
// reconnect of node with changed capability and running applications
IList <ApplicationId> runningApps = new AList <ApplicationId>();
runningApps.AddItem(ApplicationId.NewInstance(1, 0));
nm1 = rm.RegisterNode("host2:5678", 15360, 2, runningApps);
dispatcher.Await();
response = nm1.NodeHeartbeat(true);
dispatcher.Await();
NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(response.GetNodeAction()));
NUnit.Framework.Assert.AreEqual(5120 + 15360, metrics.GetAvailableMB());
// reconnect healthy node changing http port
nm1 = new MockNM("host1:1234", 5120, rm.GetResourceTrackerService());
nm1.SetHttpPort(3);
nm1.RegisterNode();
dispatcher.Await();
response = nm1.NodeHeartbeat(true);
response = nm1.NodeHeartbeat(true);
dispatcher.Await();
RMNode rmNode = rm.GetRMContext().GetRMNodes()[nm1.GetNodeId()];
NUnit.Framework.Assert.AreEqual(3, rmNode.GetHttpPort());
NUnit.Framework.Assert.AreEqual(5120, rmNode.GetTotalCapability().GetMemory());
NUnit.Framework.Assert.AreEqual(5120 + 15360, metrics.GetAvailableMB());
}
public virtual void TestReplanningPlanCapacityLoss()
{
Resource clusterCapacity = Resource.NewInstance(100 * 1024, 10);
Resource minAlloc = Resource.NewInstance(1024, 1);
Resource maxAlloc = Resource.NewInstance(1024 * 8, 8);
ResourceCalculator res = new DefaultResourceCalculator();
long step = 1L;
Clock clock = Org.Mockito.Mockito.Mock <Clock>();
ReservationAgent agent = Org.Mockito.Mockito.Mock <ReservationAgent>();
SharingPolicy policy = new NoOverCommitPolicy();
policy.Init("root.dedicated", null);
QueueMetrics queueMetrics = Org.Mockito.Mockito.Mock <QueueMetrics>();
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(0L);
SimpleCapacityReplanner enf = new SimpleCapacityReplanner(clock);
ReservationSchedulerConfiguration conf = Org.Mockito.Mockito.Mock <ReservationSchedulerConfiguration
>();
Org.Mockito.Mockito.When(conf.GetEnforcementWindow(Matchers.Any <string>())).ThenReturn
(6L);
enf.Init("blah", conf);
// Initialize the plan with more resources
InMemoryPlan plan = new InMemoryPlan(queueMetrics, policy, agent, clusterCapacity
, step, res, minAlloc, maxAlloc, "dedicated", enf, true, clock);
// add reservation filling the plan (separating them 1ms, so we are sure
// s2 follows s1 on acceptance
long ts = Runtime.CurrentTimeMillis();
ReservationId r1 = ReservationId.NewInstance(ts, 1);
int[] f5 = new int[] { 20, 20, 20, 20, 20 };
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r1, null, "u3", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res,
minAlloc)));
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(1L);
ReservationId r2 = ReservationId.NewInstance(ts, 2);
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r2, null, "u4", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res,
minAlloc)));
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(2L);
ReservationId r3 = ReservationId.NewInstance(ts, 3);
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r3, null, "u5", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res,
minAlloc)));
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(3L);
ReservationId r4 = ReservationId.NewInstance(ts, 4);
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r4, null, "u6", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res,
minAlloc)));
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(4L);
ReservationId r5 = ReservationId.NewInstance(ts, 5);
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r5, null, "u7", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res,
minAlloc)));
int[] f6 = new int[] { 50, 50, 50, 50, 50 };
ReservationId r6 = ReservationId.NewInstance(ts, 6);
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r6, null, "u3", "dedicated", 10, 10 + f6.Length, GenerateAllocation(10, f6), res
, minAlloc)));
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(6L);
ReservationId r7 = ReservationId.NewInstance(ts, 7);
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r7, null, "u4", "dedicated", 10, 10 + f6.Length, GenerateAllocation(10, f6), res
, minAlloc)));
// remove some of the resources (requires replanning)
plan.SetTotalCapacity(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance(70
* 1024, 70));
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(0L);
// run the replanner
enf.Plan(plan, null);
// check which reservation are still present
NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r1));
NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r2));
NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r3));
NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r6));
NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r7));
// and which ones are removed
NUnit.Framework.Assert.IsNull(plan.GetReservationById(r4));
NUnit.Framework.Assert.IsNull(plan.GetReservationById(r5));
// check resources at each moment in time no more exceed capacity
for (int i = 0; i < 20; i++)
{
int tot = 0;
foreach (ReservationAllocation r in plan.GetReservationsAtTime(i))
{
tot = r.GetResourcesAtTime(i).GetMemory();
}
NUnit.Framework.Assert.IsTrue(tot <= 70 * 1024);
}
}
// Explicitly reset queue metrics for testing.
public virtual void ClearQueueMetrics(RMApp app)
{
QueueMetrics.ClearQueueMetrics();
}
