public virtual void TestDefaultSingleQueueMetrics()
{
string queueName = "single";
string user = "******";
QueueMetrics metrics = QueueMetrics.ForQueue(ms, queueName, null, false, conf);
MetricsSource queueSource = QueueSource(ms, queueName);
AppSchedulingInfo app = MockApp(user);
metrics.SubmitApp(user);
MetricsSource userSource = UserSource(ms, queueName, user);
CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true);
metrics.SubmitAppAttempt(user);
CheckApps(queueSource, 1, 1, 0, 0, 0, 0, true);
metrics.SetAvailableResourcesToQueue(Resources.CreateResource(100 * Gb, 100));
metrics.IncrPendingResources(user, 5, Resources.CreateResource(3 * Gb, 3));
// Available resources is set externally, as it depends on dynamic
// configurable cluster/queue resources
CheckResources(queueSource, 0, 0, 0, 0, 0, 100 * Gb, 100, 15 * Gb, 15, 5, 0, 0, 0
);
metrics.RunAppAttempt(app.GetApplicationId(), user);
CheckApps(queueSource, 1, 0, 1, 0, 0, 0, true);
metrics.AllocateResources(user, 3, Resources.CreateResource(2 * Gb, 2), true);
CheckResources(queueSource, 6 * Gb, 6, 3, 3, 0, 100 * Gb, 100, 9 * Gb, 9, 2, 0, 0
, 0);
metrics.ReleaseResources(user, 1, Resources.CreateResource(2 * Gb, 2));
CheckResources(queueSource, 4 * Gb, 4, 2, 3, 1, 100 * Gb, 100, 9 * Gb, 9, 2, 0, 0
, 0);
metrics.FinishAppAttempt(app.GetApplicationId(), app.IsPending(), app.GetUser());
CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true);
metrics.FinishApp(user, RMAppState.Finished);
CheckApps(queueSource, 1, 0, 0, 1, 0, 0, true);
NUnit.Framework.Assert.IsNull(userSource);
}
public SchedulerApplicationAttempt(ApplicationAttemptId applicationAttemptId, string
user, Queue queue, ActiveUsersManager activeUsersManager, RMContext rmContext)
{
// This pendingRelease is used in work-preserving recovery scenario to keep
// track of the AM's outstanding release requests. RM on recovery could
// receive the release request form AM before it receives the container status
// from NM for recovery. In this case, the to-be-recovered containers reported
// by NM should not be recovered.
// Time of the last container scheduled at the current allowed level
Preconditions.CheckNotNull(rmContext, "RMContext should not be null");
this.rmContext = rmContext;
this.appSchedulingInfo = new AppSchedulingInfo(applicationAttemptId, user, queue,
activeUsersManager, rmContext.GetEpoch());
this.queue = queue;
this.pendingRelease = new HashSet <ContainerId>();
this.attemptId = applicationAttemptId;
if (rmContext.GetRMApps() != null && rmContext.GetRMApps().Contains(applicationAttemptId
.GetApplicationId()))
{
ApplicationSubmissionContext appSubmissionContext = rmContext.GetRMApps()[applicationAttemptId
.GetApplicationId()].GetApplicationSubmissionContext();
if (appSubmissionContext != null)
{
unmanagedAM = appSubmissionContext.GetUnmanagedAM();
this.logAggregationContext = appSubmissionContext.GetLogAggregationContext();
}
}
}
private static AppSchedulingInfo MockApp(string user)
{
AppSchedulingInfo app = Org.Mockito.Mockito.Mock <AppSchedulingInfo>();
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user);
ApplicationId appId = BuilderUtils.NewApplicationId(1, 1);
ApplicationAttemptId id = BuilderUtils.NewApplicationAttemptId(appId, 1);
Org.Mockito.Mockito.When(app.GetApplicationAttemptId()).ThenReturn(id);
return(app);
}
public virtual void TestQueueAppMetricsForMultipleFailures()
{
string queueName = "single";
string user = "******";
QueueMetrics metrics = QueueMetrics.ForQueue(ms, queueName, null, false, new Configuration
());
MetricsSource queueSource = QueueSource(ms, queueName);
AppSchedulingInfo app = MockApp(user);
metrics.SubmitApp(user);
MetricsSource userSource = UserSource(ms, queueName, user);
CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true);
metrics.SubmitAppAttempt(user);
CheckApps(queueSource, 1, 1, 0, 0, 0, 0, true);
metrics.RunAppAttempt(app.GetApplicationId(), user);
CheckApps(queueSource, 1, 0, 1, 0, 0, 0, true);
metrics.FinishAppAttempt(app.GetApplicationId(), app.IsPending(), app.GetUser());
CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true);
// As the application has failed, framework retries the same application
// based on configuration
metrics.SubmitAppAttempt(user);
CheckApps(queueSource, 1, 1, 0, 0, 0, 0, true);
metrics.RunAppAttempt(app.GetApplicationId(), user);
CheckApps(queueSource, 1, 0, 1, 0, 0, 0, true);
// Suppose say application has failed this time as well.
metrics.FinishAppAttempt(app.GetApplicationId(), app.IsPending(), app.GetUser());
CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true);
// As the application has failed, framework retries the same application
// based on configuration
metrics.SubmitAppAttempt(user);
CheckApps(queueSource, 1, 1, 0, 0, 0, 0, true);
metrics.RunAppAttempt(app.GetApplicationId(), user);
CheckApps(queueSource, 1, 0, 1, 0, 0, 0, true);
// Suppose say application has failed, and there's no more retries.
metrics.FinishAppAttempt(app.GetApplicationId(), app.IsPending(), app.GetUser());
CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true);
metrics.FinishApp(user, RMAppState.Failed);
CheckApps(queueSource, 1, 0, 0, 0, 1, 0, true);
NUnit.Framework.Assert.IsNull(userSource);
}
public virtual void MoveAppTo(AppSchedulingInfo app)
{
if (app.IsPending())
{
appsPending.Incr();
}
else
{
appsRunning.Incr();
}
Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.QueueMetrics userMetrics =
GetUserMetrics(app.GetUser());
if (userMetrics != null)
{
userMetrics.MoveAppTo(app);
}
if (parent != null)
{
parent.MoveAppTo(app);
}
}
public virtual void TestTwoLevelWithUserMetrics()
{
string parentQueueName = "root";
string leafQueueName = "root.leaf";
string user = "******";
QueueMetrics parentMetrics = QueueMetrics.ForQueue(ms, parentQueueName, null, true
, conf);
Queue parentQueue = MockitoMaker.Make(MockitoMaker.Stub <Queue>().Returning(parentMetrics
).from.GetMetrics());
QueueMetrics metrics = QueueMetrics.ForQueue(ms, leafQueueName, parentQueue, true
, conf);
MetricsSource parentQueueSource = QueueSource(ms, parentQueueName);
MetricsSource queueSource = QueueSource(ms, leafQueueName);
AppSchedulingInfo app = MockApp(user);
metrics.SubmitApp(user);
MetricsSource userSource = UserSource(ms, leafQueueName, user);
MetricsSource parentUserSource = UserSource(ms, parentQueueName, user);
CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true);
CheckApps(parentQueueSource, 1, 0, 0, 0, 0, 0, true);
CheckApps(userSource, 1, 0, 0, 0, 0, 0, true);
CheckApps(parentUserSource, 1, 0, 0, 0, 0, 0, true);
metrics.SubmitAppAttempt(user);
CheckApps(queueSource, 1, 1, 0, 0, 0, 0, true);
CheckApps(parentQueueSource, 1, 1, 0, 0, 0, 0, true);
CheckApps(userSource, 1, 1, 0, 0, 0, 0, true);
CheckApps(parentUserSource, 1, 1, 0, 0, 0, 0, true);
parentMetrics.SetAvailableResourcesToQueue(Resources.CreateResource(100 * Gb, 100
));
metrics.SetAvailableResourcesToQueue(Resources.CreateResource(100 * Gb, 100));
parentMetrics.SetAvailableResourcesToUser(user, Resources.CreateResource(10 * Gb,
10));
metrics.SetAvailableResourcesToUser(user, Resources.CreateResource(10 * Gb, 10));
metrics.IncrPendingResources(user, 5, Resources.CreateResource(3 * Gb, 3));
CheckResources(queueSource, 0, 0, 0, 0, 0, 100 * Gb, 100, 15 * Gb, 15, 5, 0, 0, 0
);
CheckResources(parentQueueSource, 0, 0, 0, 0, 0, 100 * Gb, 100, 15 * Gb, 15, 5, 0
, 0, 0);
CheckResources(userSource, 0, 0, 0, 0, 0, 10 * Gb, 10, 15 * Gb, 15, 5, 0, 0, 0);
CheckResources(parentUserSource, 0, 0, 0, 0, 0, 10 * Gb, 10, 15 * Gb, 15, 5, 0, 0
, 0);
metrics.RunAppAttempt(app.GetApplicationId(), user);
CheckApps(queueSource, 1, 0, 1, 0, 0, 0, true);
CheckApps(userSource, 1, 0, 1, 0, 0, 0, true);
metrics.AllocateResources(user, 3, Resources.CreateResource(2 * Gb, 2), true);
metrics.ReserveResource(user, Resources.CreateResource(3 * Gb, 3));
// Available resources is set externally, as it depends on dynamic
// configurable cluster/queue resources
CheckResources(queueSource, 6 * Gb, 6, 3, 3, 0, 100 * Gb, 100, 9 * Gb, 9, 2, 3 *
Gb, 3, 1);
CheckResources(parentQueueSource, 6 * Gb, 6, 3, 3, 0, 100 * Gb, 100, 9 * Gb, 9, 2
, 3 * Gb, 3, 1);
CheckResources(userSource, 6 * Gb, 6, 3, 3, 0, 10 * Gb, 10, 9 * Gb, 9, 2, 3 * Gb,
3, 1);
CheckResources(parentUserSource, 6 * Gb, 6, 3, 3, 0, 10 * Gb, 10, 9 * Gb, 9, 2, 3
* Gb, 3, 1);
metrics.ReleaseResources(user, 1, Resources.CreateResource(2 * Gb, 2));
metrics.UnreserveResource(user, Resources.CreateResource(3 * Gb, 3));
CheckResources(queueSource, 4 * Gb, 4, 2, 3, 1, 100 * Gb, 100, 9 * Gb, 9, 2, 0, 0
, 0);
CheckResources(parentQueueSource, 4 * Gb, 4, 2, 3, 1, 100 * Gb, 100, 9 * Gb, 9, 2
, 0, 0, 0);
CheckResources(userSource, 4 * Gb, 4, 2, 3, 1, 10 * Gb, 10, 9 * Gb, 9, 2, 0, 0, 0
);
CheckResources(parentUserSource, 4 * Gb, 4, 2, 3, 1, 10 * Gb, 10, 9 * Gb, 9, 2, 0
, 0, 0);
metrics.FinishAppAttempt(app.GetApplicationId(), app.IsPending(), app.GetUser());
CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true);
CheckApps(parentQueueSource, 1, 0, 0, 0, 0, 0, true);
CheckApps(userSource, 1, 0, 0, 0, 0, 0, true);
CheckApps(parentUserSource, 1, 0, 0, 0, 0, 0, true);
metrics.FinishApp(user, RMAppState.Finished);
CheckApps(queueSource, 1, 0, 0, 1, 0, 0, true);
CheckApps(parentQueueSource, 1, 0, 0, 1, 0, 0, true);
CheckApps(userSource, 1, 0, 0, 1, 0, 0, true);
CheckApps(parentUserSource, 1, 0, 0, 1, 0, 0, true);
}
