Exemplo n.º 1
0
        public virtual void TestDefaultSingleQueueMetrics()
        {
            string            queueName   = "single";
            string            user        = "******";
            QueueMetrics      metrics     = QueueMetrics.ForQueue(ms, queueName, null, false, conf);
            MetricsSource     queueSource = QueueSource(ms, queueName);
            AppSchedulingInfo app         = MockApp(user);

            metrics.SubmitApp(user);
            MetricsSource userSource = UserSource(ms, queueName, user);

            CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true);
            metrics.SubmitAppAttempt(user);
            CheckApps(queueSource, 1, 1, 0, 0, 0, 0, true);
            metrics.SetAvailableResourcesToQueue(Resources.CreateResource(100 * Gb, 100));
            metrics.IncrPendingResources(user, 5, Resources.CreateResource(3 * Gb, 3));
            // Available resources is set externally, as it depends on dynamic
            // configurable cluster/queue resources
            CheckResources(queueSource, 0, 0, 0, 0, 0, 100 * Gb, 100, 15 * Gb, 15, 5, 0, 0, 0
                           );
            metrics.RunAppAttempt(app.GetApplicationId(), user);
            CheckApps(queueSource, 1, 0, 1, 0, 0, 0, true);
            metrics.AllocateResources(user, 3, Resources.CreateResource(2 * Gb, 2), true);
            CheckResources(queueSource, 6 * Gb, 6, 3, 3, 0, 100 * Gb, 100, 9 * Gb, 9, 2, 0, 0
                           , 0);
            metrics.ReleaseResources(user, 1, Resources.CreateResource(2 * Gb, 2));
            CheckResources(queueSource, 4 * Gb, 4, 2, 3, 1, 100 * Gb, 100, 9 * Gb, 9, 2, 0, 0
                           , 0);
            metrics.FinishAppAttempt(app.GetApplicationId(), app.IsPending(), app.GetUser());
            CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true);
            metrics.FinishApp(user, RMAppState.Finished);
            CheckApps(queueSource, 1, 0, 0, 1, 0, 0, true);
            NUnit.Framework.Assert.IsNull(userSource);
        }
Exemplo n.º 2
0
        private static AppSchedulingInfo MockApp(string user)
        {
            AppSchedulingInfo app = Org.Mockito.Mockito.Mock <AppSchedulingInfo>();

            Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user);
            ApplicationId        appId = BuilderUtils.NewApplicationId(1, 1);
            ApplicationAttemptId id    = BuilderUtils.NewApplicationAttemptId(appId, 1);

            Org.Mockito.Mockito.When(app.GetApplicationAttemptId()).ThenReturn(id);
            return(app);
        }
Exemplo n.º 3
0
        public virtual void TestQueueAppMetricsForMultipleFailures()
        {
            string       queueName = "single";
            string       user      = "******";
            QueueMetrics metrics   = QueueMetrics.ForQueue(ms, queueName, null, false, new Configuration
                                                               ());
            MetricsSource     queueSource = QueueSource(ms, queueName);
            AppSchedulingInfo app         = MockApp(user);

            metrics.SubmitApp(user);
            MetricsSource userSource = UserSource(ms, queueName, user);

            CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true);
            metrics.SubmitAppAttempt(user);
            CheckApps(queueSource, 1, 1, 0, 0, 0, 0, true);
            metrics.RunAppAttempt(app.GetApplicationId(), user);
            CheckApps(queueSource, 1, 0, 1, 0, 0, 0, true);
            metrics.FinishAppAttempt(app.GetApplicationId(), app.IsPending(), app.GetUser());
            CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true);
            // As the application has failed, framework retries the same application
            // based on configuration
            metrics.SubmitAppAttempt(user);
            CheckApps(queueSource, 1, 1, 0, 0, 0, 0, true);
            metrics.RunAppAttempt(app.GetApplicationId(), user);
            CheckApps(queueSource, 1, 0, 1, 0, 0, 0, true);
            // Suppose say application has failed this time as well.
            metrics.FinishAppAttempt(app.GetApplicationId(), app.IsPending(), app.GetUser());
            CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true);
            // As the application has failed, framework retries the same application
            // based on configuration
            metrics.SubmitAppAttempt(user);
            CheckApps(queueSource, 1, 1, 0, 0, 0, 0, true);
            metrics.RunAppAttempt(app.GetApplicationId(), user);
            CheckApps(queueSource, 1, 0, 1, 0, 0, 0, true);
            // Suppose say application has failed, and there's no more retries.
            metrics.FinishAppAttempt(app.GetApplicationId(), app.IsPending(), app.GetUser());
            CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true);
            metrics.FinishApp(user, RMAppState.Failed);
            CheckApps(queueSource, 1, 0, 0, 0, 1, 0, true);
            NUnit.Framework.Assert.IsNull(userSource);
        }
Exemplo n.º 4
0
 public virtual void MoveAppTo(AppSchedulingInfo app)
 {
     if (app.IsPending())
     {
         appsPending.Incr();
     }
     else
     {
         appsRunning.Incr();
     }
     Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.QueueMetrics userMetrics =
         GetUserMetrics(app.GetUser());
     if (userMetrics != null)
     {
         userMetrics.MoveAppTo(app);
     }
     if (parent != null)
     {
         parent.MoveAppTo(app);
     }
 }
Exemplo n.º 5
0
        public virtual void TestTwoLevelWithUserMetrics()
        {
            string       parentQueueName = "root";
            string       leafQueueName   = "root.leaf";
            string       user            = "******";
            QueueMetrics parentMetrics   = QueueMetrics.ForQueue(ms, parentQueueName, null, true
                                                                 , conf);
            Queue parentQueue = MockitoMaker.Make(MockitoMaker.Stub <Queue>().Returning(parentMetrics
                                                                                        ).from.GetMetrics());
            QueueMetrics metrics = QueueMetrics.ForQueue(ms, leafQueueName, parentQueue, true
                                                         , conf);
            MetricsSource     parentQueueSource = QueueSource(ms, parentQueueName);
            MetricsSource     queueSource       = QueueSource(ms, leafQueueName);
            AppSchedulingInfo app = MockApp(user);

            metrics.SubmitApp(user);
            MetricsSource userSource       = UserSource(ms, leafQueueName, user);
            MetricsSource parentUserSource = UserSource(ms, parentQueueName, user);

            CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true);
            CheckApps(parentQueueSource, 1, 0, 0, 0, 0, 0, true);
            CheckApps(userSource, 1, 0, 0, 0, 0, 0, true);
            CheckApps(parentUserSource, 1, 0, 0, 0, 0, 0, true);
            metrics.SubmitAppAttempt(user);
            CheckApps(queueSource, 1, 1, 0, 0, 0, 0, true);
            CheckApps(parentQueueSource, 1, 1, 0, 0, 0, 0, true);
            CheckApps(userSource, 1, 1, 0, 0, 0, 0, true);
            CheckApps(parentUserSource, 1, 1, 0, 0, 0, 0, true);
            parentMetrics.SetAvailableResourcesToQueue(Resources.CreateResource(100 * Gb, 100
                                                                                ));
            metrics.SetAvailableResourcesToQueue(Resources.CreateResource(100 * Gb, 100));
            parentMetrics.SetAvailableResourcesToUser(user, Resources.CreateResource(10 * Gb,
                                                                                     10));
            metrics.SetAvailableResourcesToUser(user, Resources.CreateResource(10 * Gb, 10));
            metrics.IncrPendingResources(user, 5, Resources.CreateResource(3 * Gb, 3));
            CheckResources(queueSource, 0, 0, 0, 0, 0, 100 * Gb, 100, 15 * Gb, 15, 5, 0, 0, 0
                           );
            CheckResources(parentQueueSource, 0, 0, 0, 0, 0, 100 * Gb, 100, 15 * Gb, 15, 5, 0
                           , 0, 0);
            CheckResources(userSource, 0, 0, 0, 0, 0, 10 * Gb, 10, 15 * Gb, 15, 5, 0, 0, 0);
            CheckResources(parentUserSource, 0, 0, 0, 0, 0, 10 * Gb, 10, 15 * Gb, 15, 5, 0, 0
                           , 0);
            metrics.RunAppAttempt(app.GetApplicationId(), user);
            CheckApps(queueSource, 1, 0, 1, 0, 0, 0, true);
            CheckApps(userSource, 1, 0, 1, 0, 0, 0, true);
            metrics.AllocateResources(user, 3, Resources.CreateResource(2 * Gb, 2), true);
            metrics.ReserveResource(user, Resources.CreateResource(3 * Gb, 3));
            // Available resources is set externally, as it depends on dynamic
            // configurable cluster/queue resources
            CheckResources(queueSource, 6 * Gb, 6, 3, 3, 0, 100 * Gb, 100, 9 * Gb, 9, 2, 3 *
                           Gb, 3, 1);
            CheckResources(parentQueueSource, 6 * Gb, 6, 3, 3, 0, 100 * Gb, 100, 9 * Gb, 9, 2
                           , 3 * Gb, 3, 1);
            CheckResources(userSource, 6 * Gb, 6, 3, 3, 0, 10 * Gb, 10, 9 * Gb, 9, 2, 3 * Gb,
                           3, 1);
            CheckResources(parentUserSource, 6 * Gb, 6, 3, 3, 0, 10 * Gb, 10, 9 * Gb, 9, 2, 3
                           * Gb, 3, 1);
            metrics.ReleaseResources(user, 1, Resources.CreateResource(2 * Gb, 2));
            metrics.UnreserveResource(user, Resources.CreateResource(3 * Gb, 3));
            CheckResources(queueSource, 4 * Gb, 4, 2, 3, 1, 100 * Gb, 100, 9 * Gb, 9, 2, 0, 0
                           , 0);
            CheckResources(parentQueueSource, 4 * Gb, 4, 2, 3, 1, 100 * Gb, 100, 9 * Gb, 9, 2
                           , 0, 0, 0);
            CheckResources(userSource, 4 * Gb, 4, 2, 3, 1, 10 * Gb, 10, 9 * Gb, 9, 2, 0, 0, 0
                           );
            CheckResources(parentUserSource, 4 * Gb, 4, 2, 3, 1, 10 * Gb, 10, 9 * Gb, 9, 2, 0
                           , 0, 0);
            metrics.FinishAppAttempt(app.GetApplicationId(), app.IsPending(), app.GetUser());
            CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true);
            CheckApps(parentQueueSource, 1, 0, 0, 0, 0, 0, true);
            CheckApps(userSource, 1, 0, 0, 0, 0, 0, true);
            CheckApps(parentUserSource, 1, 0, 0, 0, 0, 0, true);
            metrics.FinishApp(user, RMAppState.Finished);
            CheckApps(queueSource, 1, 0, 0, 1, 0, 0, true);
            CheckApps(parentQueueSource, 1, 0, 0, 1, 0, 0, true);
            CheckApps(userSource, 1, 0, 0, 1, 0, 0, true);
            CheckApps(parentUserSource, 1, 0, 0, 1, 0, 0, true);
        }
Exemplo n.º 6
0
 public virtual string GetUser()
 {
     return(appSchedulingInfo.GetUser());
 }