public virtual void TestDefaultSingleQueueMetrics()
{
string queueName = "single";
string user = "******";
QueueMetrics metrics = QueueMetrics.ForQueue(ms, queueName, null, false, conf);
MetricsSource queueSource = QueueSource(ms, queueName);
AppSchedulingInfo app = MockApp(user);
metrics.SubmitApp(user);
MetricsSource userSource = UserSource(ms, queueName, user);
CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true);
metrics.SubmitAppAttempt(user);
CheckApps(queueSource, 1, 1, 0, 0, 0, 0, true);
metrics.SetAvailableResourcesToQueue(Resources.CreateResource(100 * Gb, 100));
metrics.IncrPendingResources(user, 5, Resources.CreateResource(3 * Gb, 3));
// Available resources is set externally, as it depends on dynamic
// configurable cluster/queue resources
CheckResources(queueSource, 0, 0, 0, 0, 0, 100 * Gb, 100, 15 * Gb, 15, 5, 0, 0, 0
);
metrics.RunAppAttempt(app.GetApplicationId(), user);
CheckApps(queueSource, 1, 0, 1, 0, 0, 0, true);
metrics.AllocateResources(user, 3, Resources.CreateResource(2 * Gb, 2), true);
CheckResources(queueSource, 6 * Gb, 6, 3, 3, 0, 100 * Gb, 100, 9 * Gb, 9, 2, 0, 0
, 0);
metrics.ReleaseResources(user, 1, Resources.CreateResource(2 * Gb, 2));
CheckResources(queueSource, 4 * Gb, 4, 2, 3, 1, 100 * Gb, 100, 9 * Gb, 9, 2, 0, 0
, 0);
metrics.FinishAppAttempt(app.GetApplicationId(), app.IsPending(), app.GetUser());
CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true);
metrics.FinishApp(user, RMAppState.Finished);
CheckApps(queueSource, 1, 0, 0, 1, 0, 0, true);
NUnit.Framework.Assert.IsNull(userSource);
}
public virtual void TestMove()
{
string user = "******";
Queue parentQueue = CreateQueue("parent", null);
Queue oldQueue = CreateQueue("old", parentQueue);
Queue newQueue = CreateQueue("new", parentQueue);
QueueMetrics parentMetrics = parentQueue.GetMetrics();
QueueMetrics oldMetrics = oldQueue.GetMetrics();
QueueMetrics newMetrics = newQueue.GetMetrics();
ApplicationAttemptId appAttId = CreateAppAttemptId(0, 0);
RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>();
Org.Mockito.Mockito.When(rmContext.GetEpoch()).ThenReturn(3L);
SchedulerApplicationAttempt app = new SchedulerApplicationAttempt(appAttId, user,
oldQueue, oldQueue.GetActiveUsersManager(), rmContext);
oldMetrics.SubmitApp(user);
// confirm that containerId is calculated based on epoch.
NUnit.Framework.Assert.AreEqual(unchecked ((long)(0x30000000001L)), app.GetNewContainerId
());
// Resource request
Resource requestedResource = Resource.NewInstance(1536, 2);
Priority requestedPriority = Priority.NewInstance(2);
ResourceRequest request = ResourceRequest.NewInstance(requestedPriority, ResourceRequest
.Any, requestedResource, 3);
app.UpdateResourceRequests(Arrays.AsList(request));
// Allocated container
RMContainer container1 = CreateRMContainer(appAttId, 1, requestedResource);
app.liveContainers[container1.GetContainerId()] = container1;
SchedulerNode node = CreateNode();
app.appSchedulingInfo.Allocate(NodeType.OffSwitch, node, requestedPriority, request
, container1.GetContainer());
// Reserved container
Priority prio1 = Priority.NewInstance(1);
Resource reservedResource = Resource.NewInstance(2048, 3);
RMContainer container2 = CreateReservedRMContainer(appAttId, 1, reservedResource,
node.GetNodeID(), prio1);
IDictionary <NodeId, RMContainer> reservations = new Dictionary <NodeId, RMContainer
>();
reservations[node.GetNodeID()] = container2;
app.reservedContainers[prio1] = reservations;
oldMetrics.ReserveResource(user, reservedResource);
CheckQueueMetrics(oldMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4);
CheckQueueMetrics(newMetrics, 0, 0, 0, 0, 0, 0, 0, 0);
CheckQueueMetrics(parentMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4);
app.Move(newQueue);
CheckQueueMetrics(oldMetrics, 0, 0, 0, 0, 0, 0, 0, 0);
CheckQueueMetrics(newMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4);
CheckQueueMetrics(parentMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4);
}
public virtual void TestQueueAppMetricsForMultipleFailures()
{
string queueName = "single";
string user = "******";
QueueMetrics metrics = QueueMetrics.ForQueue(ms, queueName, null, false, new Configuration
());
MetricsSource queueSource = QueueSource(ms, queueName);
AppSchedulingInfo app = MockApp(user);
metrics.SubmitApp(user);
MetricsSource userSource = UserSource(ms, queueName, user);
CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true);
metrics.SubmitAppAttempt(user);
CheckApps(queueSource, 1, 1, 0, 0, 0, 0, true);
metrics.RunAppAttempt(app.GetApplicationId(), user);
CheckApps(queueSource, 1, 0, 1, 0, 0, 0, true);
metrics.FinishAppAttempt(app.GetApplicationId(), app.IsPending(), app.GetUser());
CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true);
// As the application has failed, framework retries the same application
// based on configuration
metrics.SubmitAppAttempt(user);
CheckApps(queueSource, 1, 1, 0, 0, 0, 0, true);
metrics.RunAppAttempt(app.GetApplicationId(), user);
CheckApps(queueSource, 1, 0, 1, 0, 0, 0, true);
// Suppose say application has failed this time as well.
metrics.FinishAppAttempt(app.GetApplicationId(), app.IsPending(), app.GetUser());
CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true);
// As the application has failed, framework retries the same application
// based on configuration
metrics.SubmitAppAttempt(user);
CheckApps(queueSource, 1, 1, 0, 0, 0, 0, true);
metrics.RunAppAttempt(app.GetApplicationId(), user);
CheckApps(queueSource, 1, 0, 1, 0, 0, 0, true);
// Suppose say application has failed, and there's no more retries.
metrics.FinishAppAttempt(app.GetApplicationId(), app.IsPending(), app.GetUser());
CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true);
metrics.FinishApp(user, RMAppState.Failed);
CheckApps(queueSource, 1, 0, 0, 0, 1, 0, true);
NUnit.Framework.Assert.IsNull(userSource);
}
public virtual void TestTwoLevelWithUserMetrics()
{
string parentQueueName = "root";
string leafQueueName = "root.leaf";
string user = "******";
QueueMetrics parentMetrics = QueueMetrics.ForQueue(ms, parentQueueName, null, true
, conf);
Queue parentQueue = MockitoMaker.Make(MockitoMaker.Stub <Queue>().Returning(parentMetrics
).from.GetMetrics());
QueueMetrics metrics = QueueMetrics.ForQueue(ms, leafQueueName, parentQueue, true
, conf);
MetricsSource parentQueueSource = QueueSource(ms, parentQueueName);
MetricsSource queueSource = QueueSource(ms, leafQueueName);
AppSchedulingInfo app = MockApp(user);
metrics.SubmitApp(user);
MetricsSource userSource = UserSource(ms, leafQueueName, user);
MetricsSource parentUserSource = UserSource(ms, parentQueueName, user);
CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true);
CheckApps(parentQueueSource, 1, 0, 0, 0, 0, 0, true);
CheckApps(userSource, 1, 0, 0, 0, 0, 0, true);
CheckApps(parentUserSource, 1, 0, 0, 0, 0, 0, true);
metrics.SubmitAppAttempt(user);
CheckApps(queueSource, 1, 1, 0, 0, 0, 0, true);
CheckApps(parentQueueSource, 1, 1, 0, 0, 0, 0, true);
CheckApps(userSource, 1, 1, 0, 0, 0, 0, true);
CheckApps(parentUserSource, 1, 1, 0, 0, 0, 0, true);
parentMetrics.SetAvailableResourcesToQueue(Resources.CreateResource(100 * Gb, 100
));
metrics.SetAvailableResourcesToQueue(Resources.CreateResource(100 * Gb, 100));
parentMetrics.SetAvailableResourcesToUser(user, Resources.CreateResource(10 * Gb,
10));
metrics.SetAvailableResourcesToUser(user, Resources.CreateResource(10 * Gb, 10));
metrics.IncrPendingResources(user, 5, Resources.CreateResource(3 * Gb, 3));
CheckResources(queueSource, 0, 0, 0, 0, 0, 100 * Gb, 100, 15 * Gb, 15, 5, 0, 0, 0
);
CheckResources(parentQueueSource, 0, 0, 0, 0, 0, 100 * Gb, 100, 15 * Gb, 15, 5, 0
, 0, 0);
CheckResources(userSource, 0, 0, 0, 0, 0, 10 * Gb, 10, 15 * Gb, 15, 5, 0, 0, 0);
CheckResources(parentUserSource, 0, 0, 0, 0, 0, 10 * Gb, 10, 15 * Gb, 15, 5, 0, 0
, 0);
metrics.RunAppAttempt(app.GetApplicationId(), user);
CheckApps(queueSource, 1, 0, 1, 0, 0, 0, true);
CheckApps(userSource, 1, 0, 1, 0, 0, 0, true);
metrics.AllocateResources(user, 3, Resources.CreateResource(2 * Gb, 2), true);
metrics.ReserveResource(user, Resources.CreateResource(3 * Gb, 3));
// Available resources is set externally, as it depends on dynamic
// configurable cluster/queue resources
CheckResources(queueSource, 6 * Gb, 6, 3, 3, 0, 100 * Gb, 100, 9 * Gb, 9, 2, 3 *
Gb, 3, 1);
CheckResources(parentQueueSource, 6 * Gb, 6, 3, 3, 0, 100 * Gb, 100, 9 * Gb, 9, 2
, 3 * Gb, 3, 1);
CheckResources(userSource, 6 * Gb, 6, 3, 3, 0, 10 * Gb, 10, 9 * Gb, 9, 2, 3 * Gb,
3, 1);
CheckResources(parentUserSource, 6 * Gb, 6, 3, 3, 0, 10 * Gb, 10, 9 * Gb, 9, 2, 3
* Gb, 3, 1);
metrics.ReleaseResources(user, 1, Resources.CreateResource(2 * Gb, 2));
metrics.UnreserveResource(user, Resources.CreateResource(3 * Gb, 3));
CheckResources(queueSource, 4 * Gb, 4, 2, 3, 1, 100 * Gb, 100, 9 * Gb, 9, 2, 0, 0
, 0);
CheckResources(parentQueueSource, 4 * Gb, 4, 2, 3, 1, 100 * Gb, 100, 9 * Gb, 9, 2
, 0, 0, 0);
CheckResources(userSource, 4 * Gb, 4, 2, 3, 1, 10 * Gb, 10, 9 * Gb, 9, 2, 0, 0, 0
);
CheckResources(parentUserSource, 4 * Gb, 4, 2, 3, 1, 10 * Gb, 10, 9 * Gb, 9, 2, 0
, 0, 0);
metrics.FinishAppAttempt(app.GetApplicationId(), app.IsPending(), app.GetUser());
CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true);
CheckApps(parentQueueSource, 1, 0, 0, 0, 0, 0, true);
CheckApps(userSource, 1, 0, 0, 0, 0, 0, true);
CheckApps(parentUserSource, 1, 0, 0, 0, 0, 0, true);
metrics.FinishApp(user, RMAppState.Finished);
CheckApps(queueSource, 1, 0, 0, 1, 0, 0, true);
CheckApps(parentQueueSource, 1, 0, 0, 1, 0, 0, true);
CheckApps(userSource, 1, 0, 0, 1, 0, 0, true);
CheckApps(parentUserSource, 1, 0, 0, 1, 0, 0, true);
}