public virtual void TestMetricsCache()
{
MetricsSystem ms = new MetricsSystemImpl("cache");
ms.Start();
try
{
string p1 = "root1";
string leafQueueName = "root1.leaf";
QueueMetrics p1Metrics = QueueMetrics.ForQueue(ms, p1, null, true, conf);
Queue parentQueue1 = MockitoMaker.Make(MockitoMaker.Stub <Queue>().Returning(p1Metrics
).from.GetMetrics());
QueueMetrics metrics = QueueMetrics.ForQueue(ms, leafQueueName, parentQueue1, true
, conf);
NUnit.Framework.Assert.IsNotNull("QueueMetrics for A shoudn't be null", metrics);
// Re-register to check for cache hit, shouldn't blow up metrics-system...
// also, verify parent-metrics
QueueMetrics alterMetrics = QueueMetrics.ForQueue(ms, leafQueueName, parentQueue1
, true, conf);
NUnit.Framework.Assert.IsNotNull("QueueMetrics for alterMetrics shoudn't be null"
, alterMetrics);
}
finally
{
ms.Shutdown();
}
}
/// <exception cref="System.Exception"/>
private void _mkdirs(bool exists, FsPermission before, FsPermission after)
{
FilePath localDir = MockitoMaker.Make(MockitoMaker.Stub <FilePath>().Returning(exists
).from.Exists());
Org.Mockito.Mockito.When(localDir.Mkdir()).ThenReturn(true);
Path dir = Org.Mockito.Mockito.Mock <Path>();
// use default stubs
LocalFileSystem fs = MockitoMaker.Make(MockitoMaker.Stub <LocalFileSystem>().Returning
(localDir).from.PathToFile(dir));
FileStatus stat = MockitoMaker.Make(MockitoMaker.Stub <FileStatus>().Returning(after
).from.GetPermission());
Org.Mockito.Mockito.When(fs.GetFileStatus(dir)).ThenReturn(stat);
try
{
DiskChecker.MkdirsWithExistsAndPermissionCheck(fs, dir, before);
if (!exists)
{
Org.Mockito.Mockito.Verify(fs).SetPermission(dir, before);
}
else
{
Org.Mockito.Mockito.Verify(fs).GetFileStatus(dir);
Org.Mockito.Mockito.Verify(stat).GetPermission();
}
}
catch (DiskChecker.DiskErrorException e)
{
if (before != after)
{
Assert.True(e.Message.StartsWith("Incorrect permission"));
}
}
}
public virtual void TestNames()
{
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
>();
Task mapTask = MockitoMaker.Make(MockitoMaker.Stub <Task>().Returning(TaskType.Map
).from.GetType());
Task reduceTask = MockitoMaker.Make(MockitoMaker.Stub <Task>().Returning(TaskType.
Reduce).from.GetType());
MRAppMetrics metrics = MRAppMetrics.Create();
metrics.SubmittedJob(job);
metrics.WaitingTask(mapTask);
metrics.WaitingTask(reduceTask);
metrics.PreparingJob(job);
metrics.SubmittedJob(job);
metrics.WaitingTask(mapTask);
metrics.WaitingTask(reduceTask);
metrics.PreparingJob(job);
metrics.SubmittedJob(job);
metrics.WaitingTask(mapTask);
metrics.WaitingTask(reduceTask);
metrics.PreparingJob(job);
metrics.EndPreparingJob(job);
metrics.EndPreparingJob(job);
metrics.EndPreparingJob(job);
metrics.RunningJob(job);
metrics.LaunchedTask(mapTask);
metrics.RunningTask(mapTask);
metrics.FailedTask(mapTask);
metrics.EndWaitingTask(reduceTask);
metrics.EndRunningTask(mapTask);
metrics.EndRunningJob(job);
metrics.FailedJob(job);
metrics.RunningJob(job);
metrics.LaunchedTask(mapTask);
metrics.RunningTask(mapTask);
metrics.KilledTask(mapTask);
metrics.EndWaitingTask(reduceTask);
metrics.EndRunningTask(mapTask);
metrics.EndRunningJob(job);
metrics.KilledJob(job);
metrics.RunningJob(job);
metrics.LaunchedTask(mapTask);
metrics.RunningTask(mapTask);
metrics.CompletedTask(mapTask);
metrics.EndRunningTask(mapTask);
metrics.LaunchedTask(reduceTask);
metrics.RunningTask(reduceTask);
metrics.CompletedTask(reduceTask);
metrics.EndRunningTask(reduceTask);
metrics.EndRunningJob(job);
metrics.CompletedJob(job);
CheckMetrics(3, 1, 1, 1, 0, 0, 3, 1, 1, 1, 0, 0, 1, 1, 0, 0, 0, 0);
}
public virtual void TestNumInputFiles()
{
Configuration conf = Org.Mockito.Mockito.Spy(new Configuration());
Job job = MockitoMaker.Make(MockitoMaker.Stub <Job>().Returning(conf).from.GetConfiguration
());
FileStatus stat = MockitoMaker.Make(MockitoMaker.Stub <FileStatus>().Returning(0L)
.from.GetLen());
TextInputFormat ispy = Org.Mockito.Mockito.Spy(new TextInputFormat());
Org.Mockito.Mockito.DoReturn(Arrays.AsList(stat)).When(ispy).ListStatus(job);
ispy.GetSplits(job);
Org.Mockito.Mockito.Verify(conf).SetLong(FileInputFormat.NumInputFiles, 1);
}
public virtual void TestTwoLevelWithUserMetrics()
{
string parentQueueName = "root";
string leafQueueName = "root.leaf";
string user = "******";
QueueMetrics parentMetrics = QueueMetrics.ForQueue(ms, parentQueueName, null, true
, conf);
Queue parentQueue = MockitoMaker.Make(MockitoMaker.Stub <Queue>().Returning(parentMetrics
).from.GetMetrics());
QueueMetrics metrics = QueueMetrics.ForQueue(ms, leafQueueName, parentQueue, true
, conf);
MetricsSource parentQueueSource = QueueSource(ms, parentQueueName);
MetricsSource queueSource = QueueSource(ms, leafQueueName);
AppSchedulingInfo app = MockApp(user);
metrics.SubmitApp(user);
MetricsSource userSource = UserSource(ms, leafQueueName, user);
MetricsSource parentUserSource = UserSource(ms, parentQueueName, user);
CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true);
CheckApps(parentQueueSource, 1, 0, 0, 0, 0, 0, true);
CheckApps(userSource, 1, 0, 0, 0, 0, 0, true);
CheckApps(parentUserSource, 1, 0, 0, 0, 0, 0, true);
metrics.SubmitAppAttempt(user);
CheckApps(queueSource, 1, 1, 0, 0, 0, 0, true);
CheckApps(parentQueueSource, 1, 1, 0, 0, 0, 0, true);
CheckApps(userSource, 1, 1, 0, 0, 0, 0, true);
CheckApps(parentUserSource, 1, 1, 0, 0, 0, 0, true);
parentMetrics.SetAvailableResourcesToQueue(Resources.CreateResource(100 * Gb, 100
));
metrics.SetAvailableResourcesToQueue(Resources.CreateResource(100 * Gb, 100));
parentMetrics.SetAvailableResourcesToUser(user, Resources.CreateResource(10 * Gb,
10));
metrics.SetAvailableResourcesToUser(user, Resources.CreateResource(10 * Gb, 10));
metrics.IncrPendingResources(user, 5, Resources.CreateResource(3 * Gb, 3));
CheckResources(queueSource, 0, 0, 0, 0, 0, 100 * Gb, 100, 15 * Gb, 15, 5, 0, 0, 0
);
CheckResources(parentQueueSource, 0, 0, 0, 0, 0, 100 * Gb, 100, 15 * Gb, 15, 5, 0
, 0, 0);
CheckResources(userSource, 0, 0, 0, 0, 0, 10 * Gb, 10, 15 * Gb, 15, 5, 0, 0, 0);
CheckResources(parentUserSource, 0, 0, 0, 0, 0, 10 * Gb, 10, 15 * Gb, 15, 5, 0, 0
, 0);
metrics.RunAppAttempt(app.GetApplicationId(), user);
CheckApps(queueSource, 1, 0, 1, 0, 0, 0, true);
CheckApps(userSource, 1, 0, 1, 0, 0, 0, true);
metrics.AllocateResources(user, 3, Resources.CreateResource(2 * Gb, 2), true);
metrics.ReserveResource(user, Resources.CreateResource(3 * Gb, 3));
// Available resources is set externally, as it depends on dynamic
// configurable cluster/queue resources
CheckResources(queueSource, 6 * Gb, 6, 3, 3, 0, 100 * Gb, 100, 9 * Gb, 9, 2, 3 *
Gb, 3, 1);
CheckResources(parentQueueSource, 6 * Gb, 6, 3, 3, 0, 100 * Gb, 100, 9 * Gb, 9, 2
, 3 * Gb, 3, 1);
CheckResources(userSource, 6 * Gb, 6, 3, 3, 0, 10 * Gb, 10, 9 * Gb, 9, 2, 3 * Gb,
3, 1);
CheckResources(parentUserSource, 6 * Gb, 6, 3, 3, 0, 10 * Gb, 10, 9 * Gb, 9, 2, 3
* Gb, 3, 1);
metrics.ReleaseResources(user, 1, Resources.CreateResource(2 * Gb, 2));
metrics.UnreserveResource(user, Resources.CreateResource(3 * Gb, 3));
CheckResources(queueSource, 4 * Gb, 4, 2, 3, 1, 100 * Gb, 100, 9 * Gb, 9, 2, 0, 0
, 0);
CheckResources(parentQueueSource, 4 * Gb, 4, 2, 3, 1, 100 * Gb, 100, 9 * Gb, 9, 2
, 0, 0, 0);
CheckResources(userSource, 4 * Gb, 4, 2, 3, 1, 10 * Gb, 10, 9 * Gb, 9, 2, 0, 0, 0
);
CheckResources(parentUserSource, 4 * Gb, 4, 2, 3, 1, 10 * Gb, 10, 9 * Gb, 9, 2, 0
, 0, 0);
metrics.FinishAppAttempt(app.GetApplicationId(), app.IsPending(), app.GetUser());
CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true);
CheckApps(parentQueueSource, 1, 0, 0, 0, 0, 0, true);
CheckApps(userSource, 1, 0, 0, 0, 0, 0, true);
CheckApps(parentUserSource, 1, 0, 0, 0, 0, 0, true);
metrics.FinishApp(user, RMAppState.Finished);
CheckApps(queueSource, 1, 0, 0, 1, 0, 0, true);
CheckApps(parentQueueSource, 1, 0, 0, 1, 0, 0, true);
CheckApps(userSource, 1, 0, 0, 1, 0, 0, true);
CheckApps(parentUserSource, 1, 0, 0, 1, 0, 0, true);
}
