示例#1
0
        private static RMApp CreateRMApp(ApplicationId appId)
        {
            RMApp app = Org.Mockito.Mockito.Mock <RMApp>();

            Org.Mockito.Mockito.When(app.GetApplicationId()).ThenReturn(appId);
            Org.Mockito.Mockito.When(app.GetName()).ThenReturn("test app");
            Org.Mockito.Mockito.When(app.GetApplicationType()).ThenReturn("test app type");
            Org.Mockito.Mockito.When(app.GetUser()).ThenReturn("test user");
            Org.Mockito.Mockito.When(app.GetQueue()).ThenReturn("test queue");
            Org.Mockito.Mockito.When(app.GetSubmitTime()).ThenReturn(int.MaxValue + 1L);
            Org.Mockito.Mockito.When(app.GetStartTime()).ThenReturn(int.MaxValue + 2L);
            Org.Mockito.Mockito.When(app.GetFinishTime()).ThenReturn(int.MaxValue + 3L);
            Org.Mockito.Mockito.When(app.GetDiagnostics()).ThenReturn(new StringBuilder("test diagnostics info"
                                                                                        ));
            RMAppAttempt appAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>();

            Org.Mockito.Mockito.When(appAttempt.GetAppAttemptId()).ThenReturn(ApplicationAttemptId
                                                                              .NewInstance(appId, 1));
            Org.Mockito.Mockito.When(app.GetCurrentAppAttempt()).ThenReturn(appAttempt);
            Org.Mockito.Mockito.When(app.GetFinalApplicationStatus()).ThenReturn(FinalApplicationStatus
                                                                                 .Undefined);
            Org.Mockito.Mockito.When(app.GetRMAppMetrics()).ThenReturn(new RMAppMetrics(null,
                                                                                        0, 0, int.MaxValue, long.MaxValue));
            return(app);
        }
示例#2
0
        public virtual void TestEscapeApplicationSummary()
        {
            RMApp app = Org.Mockito.Mockito.Mock <RMAppImpl>();

            Org.Mockito.Mockito.When(app.GetApplicationId()).ThenReturn(ApplicationId.NewInstance
                                                                            (100L, 1));
            Org.Mockito.Mockito.When(app.GetName()).ThenReturn("Multiline\n\n\r\rAppName");
            Org.Mockito.Mockito.When(app.GetUser()).ThenReturn("Multiline\n\n\r\rUserName");
            Org.Mockito.Mockito.When(app.GetQueue()).ThenReturn("Multiline\n\n\r\rQueueName");
            Org.Mockito.Mockito.When(app.GetState()).ThenReturn(RMAppState.Running);
            Org.Mockito.Mockito.When(app.GetApplicationType()).ThenReturn("MAPREDUCE");
            RMAppMetrics metrics = new RMAppMetrics(Org.Apache.Hadoop.Yarn.Api.Records.Resource
                                                    .NewInstance(1234, 56), 10, 1, 16384, 64);

            Org.Mockito.Mockito.When(app.GetRMAppMetrics()).ThenReturn(metrics);
            RMAppManager.ApplicationSummary.SummaryBuilder summary = RMAppManager.ApplicationSummary
                                                                     .CreateAppSummary(app);
            string msg = summary.ToString();

            Log.Info("summary: " + msg);
            NUnit.Framework.Assert.IsFalse(msg.Contains("\n"));
            NUnit.Framework.Assert.IsFalse(msg.Contains("\r"));
            string escaped = "\\n\\n\\r\\r";

            NUnit.Framework.Assert.IsTrue(msg.Contains("Multiline" + escaped + "AppName"));
            NUnit.Framework.Assert.IsTrue(msg.Contains("Multiline" + escaped + "UserName"));
            NUnit.Framework.Assert.IsTrue(msg.Contains("Multiline" + escaped + "QueueName"));
            NUnit.Framework.Assert.IsTrue(msg.Contains("memorySeconds=16384"));
            NUnit.Framework.Assert.IsTrue(msg.Contains("vcoreSeconds=64"));
            NUnit.Framework.Assert.IsTrue(msg.Contains("preemptedAMContainers=1"));
            NUnit.Framework.Assert.IsTrue(msg.Contains("preemptedNonAMContainers=10"));
            NUnit.Framework.Assert.IsTrue(msg.Contains("preemptedResources=<memory:1234\\, vCores:56>"
                                                       ));
            NUnit.Framework.Assert.IsTrue(msg.Contains("applicationType=MAPREDUCE"));
        }
        private static RMApp CreateRMApp(ApplicationId appId)
        {
            RMApp app = Org.Mockito.Mockito.Mock <RMApp>();

            Org.Mockito.Mockito.When(app.GetApplicationId()).ThenReturn(appId);
            Org.Mockito.Mockito.When(app.GetName()).ThenReturn("test app");
            Org.Mockito.Mockito.When(app.GetApplicationType()).ThenReturn("test app type");
            Org.Mockito.Mockito.When(app.GetUser()).ThenReturn("test user");
            Org.Mockito.Mockito.When(app.GetQueue()).ThenReturn("test queue");
            Org.Mockito.Mockito.When(app.GetSubmitTime()).ThenReturn(0L);
            Org.Mockito.Mockito.When(app.GetStartTime()).ThenReturn(1L);
            Org.Mockito.Mockito.When(app.GetFinishTime()).ThenReturn(2L);
            Org.Mockito.Mockito.When(app.GetDiagnostics()).ThenReturn(new StringBuilder("test diagnostics info"
                                                                                        ));
            Org.Mockito.Mockito.When(app.GetFinalApplicationStatus()).ThenReturn(FinalApplicationStatus
                                                                                 .Undefined);
            return(app);
        }
示例#4
0
        public virtual void TestAppBlockRenderWithNullCurrentAppAttempt()
        {
            ApplicationId AppId = ApplicationId.NewInstance(1234L, 0);
            Injector      injector;
            // init app
            RMApp app = Org.Mockito.Mockito.Mock <RMApp>();

            Org.Mockito.Mockito.When(app.GetTrackingUrl()).ThenReturn("http://host:123");
            Org.Mockito.Mockito.When(app.GetState()).ThenReturn(RMAppState.Failed);
            Org.Mockito.Mockito.When(app.GetApplicationId()).ThenReturn(AppId);
            Org.Mockito.Mockito.When(app.GetApplicationType()).ThenReturn("Type");
            Org.Mockito.Mockito.When(app.GetUser()).ThenReturn("user");
            Org.Mockito.Mockito.When(app.GetName()).ThenReturn("Name");
            Org.Mockito.Mockito.When(app.GetQueue()).ThenReturn("queue");
            Org.Mockito.Mockito.When(app.GetDiagnostics()).ThenReturn(new StringBuilder());
            Org.Mockito.Mockito.When(app.GetFinalApplicationStatus()).ThenReturn(FinalApplicationStatus
                                                                                 .Failed);
            Org.Mockito.Mockito.When(app.GetFinalApplicationStatus()).ThenReturn(FinalApplicationStatus
                                                                                 .Failed);
            Org.Mockito.Mockito.When(app.GetStartTime()).ThenReturn(0L);
            Org.Mockito.Mockito.When(app.GetFinishTime()).ThenReturn(0L);
            Org.Mockito.Mockito.When(app.CreateApplicationState()).ThenReturn(YarnApplicationState
                                                                              .Failed);
            RMAppMetrics appMetrics = new RMAppMetrics(Resource.NewInstance(0, 0), 0, 0, 0, 0
                                                       );

            Org.Mockito.Mockito.When(app.GetRMAppMetrics()).ThenReturn(appMetrics);
            // initialize RM Context, and create RMApp, without creating RMAppAttempt
            RMContext rmContext = TestRMWebApp.MockRMContext(15, 1, 2, 8);

            rmContext.GetRMApps()[AppId] = app;
            injector = WebAppTests.CreateMockInjector <RMContext>(rmContext, new _Module_76(rmContext
                                                                                            ));
            AppBlock instance = injector.GetInstance <AppBlock>();

            instance.Set(YarnWebParams.ApplicationId, AppId.ToString());
            instance.Render();
        }
示例#5
0
 public AppInfo(ResourceManager rm, RMApp app, bool hasAccess, string schemePrefix
                )
 {
     // these are ok for any user to see
     // these are only allowed if acls allow
     // preemption info fields
     // JAXB needs this
     this.schemePrefix = schemePrefix;
     if (app != null)
     {
         string trackingUrl = app.GetTrackingUrl();
         this.state = app.CreateApplicationState();
         this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.IsEmpty() || YarnApplicationState
                                      .New == this.state || YarnApplicationState.NewSaving == this.state || YarnApplicationState
                                      .Submitted == this.state || YarnApplicationState.Accepted == this.state;
         this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app.GetFinishTime(
                                                                            ) == 0 ? "ApplicationMaster" : "History");
         if (!trackingUrlIsNotReady)
         {
             this.trackingUrl       = WebAppUtils.GetURLWithScheme(schemePrefix, trackingUrl);
             this.trackingUrlPretty = this.trackingUrl;
         }
         else
         {
             this.trackingUrlPretty = "UNASSIGNED";
         }
         this.applicationId   = app.GetApplicationId();
         this.applicationType = app.GetApplicationType();
         this.appIdNum        = app.GetApplicationId().GetId().ToString();
         this.id          = app.GetApplicationId().ToString();
         this.user        = app.GetUser().ToString();
         this.name        = app.GetName().ToString();
         this.queue       = app.GetQueue().ToString();
         this.progress    = app.GetProgress() * 100;
         this.diagnostics = app.GetDiagnostics().ToString();
         if (diagnostics == null || diagnostics.IsEmpty())
         {
             this.diagnostics = string.Empty;
         }
         if (app.GetApplicationTags() != null && !app.GetApplicationTags().IsEmpty())
         {
             this.applicationTags = Joiner.On(',').Join(app.GetApplicationTags());
         }
         this.finalStatus = app.GetFinalApplicationStatus();
         this.clusterId   = ResourceManager.GetClusterTimeStamp();
         if (hasAccess)
         {
             this.startedTime  = app.GetStartTime();
             this.finishedTime = app.GetFinishTime();
             this.elapsedTime  = Times.Elapsed(app.GetStartTime(), app.GetFinishTime());
             RMAppAttempt attempt = app.GetCurrentAppAttempt();
             if (attempt != null)
             {
                 Container masterContainer = attempt.GetMasterContainer();
                 if (masterContainer != null)
                 {
                     this.amContainerLogsExist = true;
                     this.amContainerLogs      = WebAppUtils.GetRunningLogURL(schemePrefix + masterContainer
                                                                              .GetNodeHttpAddress(), ConverterUtils.ToString(masterContainer.GetId()), app.GetUser
                                                                                  ());
                     this.amHostHttpAddress = masterContainer.GetNodeHttpAddress();
                 }
                 ApplicationResourceUsageReport resourceReport = attempt.GetApplicationResourceUsageReport
                                                                     ();
                 if (resourceReport != null)
                 {
                     Resource usedResources = resourceReport.GetUsedResources();
                     allocatedMB       = usedResources.GetMemory();
                     allocatedVCores   = usedResources.GetVirtualCores();
                     runningContainers = resourceReport.GetNumUsedContainers();
                 }
                 resourceRequests = ((AbstractYarnScheduler)rm.GetRMContext().GetScheduler()).GetPendingResourceRequestsForAttempt
                                        (attempt.GetAppAttemptId());
             }
         }
         // copy preemption info fields
         RMAppMetrics appMetrics = app.GetRMAppMetrics();
         numAMContainerPreempted    = appMetrics.GetNumAMContainersPreempted();
         preemptedResourceMB        = appMetrics.GetResourcePreempted().GetMemory();
         numNonAMContainerPreempted = appMetrics.GetNumNonAMContainersPreempted();
         preemptedResourceVCores    = appMetrics.GetResourcePreempted().GetVirtualCores();
         memorySeconds = appMetrics.GetMemorySeconds();
         vcoreSeconds  = appMetrics.GetVcoreSeconds();
     }
 }
示例#6
0
 public virtual void AppCreated(RMApp app, long createdTime)
 {
     if (publishSystemMetrics)
     {
         dispatcher.GetEventHandler().Handle(new ApplicationCreatedEvent(app.GetApplicationId
                                                                             (), app.GetName(), app.GetApplicationType(), app.GetUser(), app.GetQueue(), app.
                                                                         GetSubmitTime(), createdTime));
     }
 }
 public virtual void ApplicationStarted(RMApp app)
 {
     if (historyServiceEnabled)
     {
         dispatcher.GetEventHandler().Handle(new WritingApplicationStartEvent(app.GetApplicationId
                                                                                  (), ApplicationStartData.NewInstance(app.GetApplicationId(), app.GetName(), app.
                                                                                                                       GetApplicationType(), app.GetQueue(), app.GetUser(), app.GetSubmitTime(), app.GetStartTime
                                                                                                                           ())));
     }
 }
示例#8
0
            /// <summary>create a summary of the application's runtime.</summary>
            /// <param name="app">
            ///
            /// <see cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Rmapp.RMApp"/>
            /// whose summary is to be created, cannot
            /// be <code>null</code>.
            /// </param>
            public static RMAppManager.ApplicationSummary.SummaryBuilder CreateAppSummary(RMApp
                                                                                          app)
            {
                string       trackingUrl = "N/A";
                string       host        = "N/A";
                RMAppAttempt attempt     = app.GetCurrentAppAttempt();

                if (attempt != null)
                {
                    trackingUrl = attempt.GetTrackingUrl();
                    host        = attempt.GetHost();
                }
                RMAppMetrics metrics = app.GetRMAppMetrics();

                RMAppManager.ApplicationSummary.SummaryBuilder summary = new RMAppManager.ApplicationSummary.SummaryBuilder
                                                                             ().Add("appId", app.GetApplicationId()).Add("name", app.GetName()).Add("user", app
                                                                                                                                                    .GetUser()).Add("queue", app.GetQueue()).Add("state", app.GetState()).Add("trackingUrl"
                                                                                                                                                                                                                              , trackingUrl).Add("appMasterHost", host).Add("startTime", app.GetStartTime()).Add
                                                                             ("finishTime", app.GetFinishTime()).Add("finalStatus", app.GetFinalApplicationStatus
                                                                                                                         ()).Add("memorySeconds", metrics.GetMemorySeconds()).Add("vcoreSeconds", metrics
                                                                                                                                                                                  .GetVcoreSeconds()).Add("preemptedAMContainers", metrics.GetNumAMContainersPreempted
                                                                                                                                                                                                              ()).Add("preemptedNonAMContainers", metrics.GetNumNonAMContainersPreempted()).Add
                                                                             ("preemptedResources", metrics.GetResourcePreempted()).Add("applicationType", app
                                                                                                                                        .GetApplicationType());
                return(summary);
            }
示例#9
0
 /// <exception cref="System.Exception"/>
 public virtual void TestPublishApplicationMetrics()
 {
     for (int i = 1; i <= 2; ++i)
     {
         ApplicationId appId = ApplicationId.NewInstance(0, i);
         RMApp         app   = CreateRMApp(appId);
         metricsPublisher.AppCreated(app, app.GetStartTime());
         metricsPublisher.AppFinished(app, RMAppState.Finished, app.GetFinishTime());
         if (i == 1)
         {
             metricsPublisher.AppACLsUpdated(app, "uers1,user2", 4L);
         }
         else
         {
             // in case user doesn't specify the ACLs
             metricsPublisher.AppACLsUpdated(app, null, 4L);
         }
         TimelineEntity entity = null;
         do
         {
             entity = store.GetEntity(appId.ToString(), ApplicationMetricsConstants.EntityType
                                      , EnumSet.AllOf <TimelineReader.Field>());
         }while (entity == null || entity.GetEvents().Count < 3);
         // ensure three events are both published before leaving the loop
         // verify all the fields
         NUnit.Framework.Assert.AreEqual(ApplicationMetricsConstants.EntityType, entity.GetEntityType
                                             ());
         NUnit.Framework.Assert.AreEqual(app.GetApplicationId().ToString(), entity.GetEntityId
                                             ());
         NUnit.Framework.Assert.AreEqual(app.GetName(), entity.GetOtherInfo()[ApplicationMetricsConstants
                                                                              .NameEntityInfo]);
         NUnit.Framework.Assert.AreEqual(app.GetQueue(), entity.GetOtherInfo()[ApplicationMetricsConstants
                                                                               .QueueEntityInfo]);
         NUnit.Framework.Assert.AreEqual(app.GetUser(), entity.GetOtherInfo()[ApplicationMetricsConstants
                                                                              .UserEntityInfo]);
         NUnit.Framework.Assert.AreEqual(app.GetApplicationType(), entity.GetOtherInfo()[ApplicationMetricsConstants
                                                                                         .TypeEntityInfo]);
         NUnit.Framework.Assert.AreEqual(app.GetSubmitTime(), entity.GetOtherInfo()[ApplicationMetricsConstants
                                                                                    .SubmittedTimeEntityInfo]);
         if (i == 1)
         {
             NUnit.Framework.Assert.AreEqual("uers1,user2", entity.GetOtherInfo()[ApplicationMetricsConstants
                                                                                  .AppViewAclsEntityInfo]);
         }
         else
         {
             NUnit.Framework.Assert.AreEqual(string.Empty, entity.GetOtherInfo()[ApplicationMetricsConstants
                                                                                 .AppViewAclsEntityInfo]);
             NUnit.Framework.Assert.AreEqual(app.GetRMAppMetrics().GetMemorySeconds(), long.Parse
                                                 (entity.GetOtherInfo()[ApplicationMetricsConstants.AppMemMetrics].ToString()));
             NUnit.Framework.Assert.AreEqual(app.GetRMAppMetrics().GetVcoreSeconds(), long.Parse
                                                 (entity.GetOtherInfo()[ApplicationMetricsConstants.AppCpuMetrics].ToString()));
         }
         bool hasCreatedEvent     = false;
         bool hasFinishedEvent    = false;
         bool hasACLsUpdatedEvent = false;
         foreach (TimelineEvent @event in entity.GetEvents())
         {
             if (@event.GetEventType().Equals(ApplicationMetricsConstants.CreatedEventType))
             {
                 hasCreatedEvent = true;
                 NUnit.Framework.Assert.AreEqual(app.GetStartTime(), @event.GetTimestamp());
             }
             else
             {
                 if (@event.GetEventType().Equals(ApplicationMetricsConstants.FinishedEventType))
                 {
                     hasFinishedEvent = true;
                     NUnit.Framework.Assert.AreEqual(app.GetFinishTime(), @event.GetTimestamp());
                     NUnit.Framework.Assert.AreEqual(app.GetDiagnostics().ToString(), @event.GetEventInfo
                                                         ()[ApplicationMetricsConstants.DiagnosticsInfoEventInfo]);
                     NUnit.Framework.Assert.AreEqual(app.GetFinalApplicationStatus().ToString(), @event
                                                     .GetEventInfo()[ApplicationMetricsConstants.FinalStatusEventInfo]);
                     NUnit.Framework.Assert.AreEqual(YarnApplicationState.Finished.ToString(), @event.
                                                     GetEventInfo()[ApplicationMetricsConstants.StateEventInfo]);
                 }
                 else
                 {
                     if (@event.GetEventType().Equals(ApplicationMetricsConstants.AclsUpdatedEventType
                                                      ))
                     {
                         hasACLsUpdatedEvent = true;
                         NUnit.Framework.Assert.AreEqual(4L, @event.GetTimestamp());
                     }
                 }
             }
         }
         NUnit.Framework.Assert.IsTrue(hasCreatedEvent && hasFinishedEvent && hasACLsUpdatedEvent
                                       );
     }
 }
示例#10
0
        /// <exception cref="System.Exception"/>
        public virtual void TestRMAppSubmitDuplicateApplicationId()
        {
            ApplicationId appId = MockApps.NewAppID(0);

            asContext.SetApplicationId(appId);
            RMApp appOrig = rmContext.GetRMApps()[appId];

            NUnit.Framework.Assert.IsTrue("app name matches but shouldn't", "testApp1" != appOrig
                                          .GetName());
            // our testApp1 should be rejected and original app with same id should be left in place
            try
            {
                appMonitor.SubmitApplication(asContext, "test");
                NUnit.Framework.Assert.Fail("Exception is expected when applicationId is duplicate."
                                            );
            }
            catch (YarnException e)
            {
                NUnit.Framework.Assert.IsTrue("The thrown exception is not the expectd one.", e.Message
                                              .Contains("Cannot add a duplicate!"));
            }
            // make sure original app didn't get removed
            RMApp app = rmContext.GetRMApps()[appId];

            NUnit.Framework.Assert.IsNotNull("app is null", app);
            NUnit.Framework.Assert.AreEqual("app id doesn't match", appId, app.GetApplicationId
                                                ());
            NUnit.Framework.Assert.AreEqual("app state doesn't match", RMAppState.Finished, app
                                            .GetState());
        }