Exemplo n.º 1
0
        private static RMApp CreateRMApp(ApplicationId appId)
        {
            RMApp app = Org.Mockito.Mockito.Mock <RMApp>();

            Org.Mockito.Mockito.When(app.GetApplicationId()).ThenReturn(appId);
            Org.Mockito.Mockito.When(app.GetName()).ThenReturn("test app");
            Org.Mockito.Mockito.When(app.GetApplicationType()).ThenReturn("test app type");
            Org.Mockito.Mockito.When(app.GetUser()).ThenReturn("test user");
            Org.Mockito.Mockito.When(app.GetQueue()).ThenReturn("test queue");
            Org.Mockito.Mockito.When(app.GetSubmitTime()).ThenReturn(int.MaxValue + 1L);
            Org.Mockito.Mockito.When(app.GetStartTime()).ThenReturn(int.MaxValue + 2L);
            Org.Mockito.Mockito.When(app.GetFinishTime()).ThenReturn(int.MaxValue + 3L);
            Org.Mockito.Mockito.When(app.GetDiagnostics()).ThenReturn(new StringBuilder("test diagnostics info"
                                                                                        ));
            RMAppAttempt appAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>();

            Org.Mockito.Mockito.When(appAttempt.GetAppAttemptId()).ThenReturn(ApplicationAttemptId
                                                                              .NewInstance(appId, 1));
            Org.Mockito.Mockito.When(app.GetCurrentAppAttempt()).ThenReturn(appAttempt);
            Org.Mockito.Mockito.When(app.GetFinalApplicationStatus()).ThenReturn(FinalApplicationStatus
                                                                                 .Undefined);
            Org.Mockito.Mockito.When(app.GetRMAppMetrics()).ThenReturn(new RMAppMetrics(null,
                                                                                        0, 0, int.MaxValue, long.MaxValue));
            return(app);
        }
Exemplo n.º 2
0
        public virtual void TestEscapeApplicationSummary()
        {
            RMApp app = Org.Mockito.Mockito.Mock <RMAppImpl>();

            Org.Mockito.Mockito.When(app.GetApplicationId()).ThenReturn(ApplicationId.NewInstance
                                                                            (100L, 1));
            Org.Mockito.Mockito.When(app.GetName()).ThenReturn("Multiline\n\n\r\rAppName");
            Org.Mockito.Mockito.When(app.GetUser()).ThenReturn("Multiline\n\n\r\rUserName");
            Org.Mockito.Mockito.When(app.GetQueue()).ThenReturn("Multiline\n\n\r\rQueueName");
            Org.Mockito.Mockito.When(app.GetState()).ThenReturn(RMAppState.Running);
            Org.Mockito.Mockito.When(app.GetApplicationType()).ThenReturn("MAPREDUCE");
            RMAppMetrics metrics = new RMAppMetrics(Org.Apache.Hadoop.Yarn.Api.Records.Resource
                                                    .NewInstance(1234, 56), 10, 1, 16384, 64);

            Org.Mockito.Mockito.When(app.GetRMAppMetrics()).ThenReturn(metrics);
            RMAppManager.ApplicationSummary.SummaryBuilder summary = RMAppManager.ApplicationSummary
                                                                     .CreateAppSummary(app);
            string msg = summary.ToString();

            Log.Info("summary: " + msg);
            NUnit.Framework.Assert.IsFalse(msg.Contains("\n"));
            NUnit.Framework.Assert.IsFalse(msg.Contains("\r"));
            string escaped = "\\n\\n\\r\\r";

            NUnit.Framework.Assert.IsTrue(msg.Contains("Multiline" + escaped + "AppName"));
            NUnit.Framework.Assert.IsTrue(msg.Contains("Multiline" + escaped + "UserName"));
            NUnit.Framework.Assert.IsTrue(msg.Contains("Multiline" + escaped + "QueueName"));
            NUnit.Framework.Assert.IsTrue(msg.Contains("memorySeconds=16384"));
            NUnit.Framework.Assert.IsTrue(msg.Contains("vcoreSeconds=64"));
            NUnit.Framework.Assert.IsTrue(msg.Contains("preemptedAMContainers=1"));
            NUnit.Framework.Assert.IsTrue(msg.Contains("preemptedNonAMContainers=10"));
            NUnit.Framework.Assert.IsTrue(msg.Contains("preemptedResources=<memory:1234\\, vCores:56>"
                                                       ));
            NUnit.Framework.Assert.IsTrue(msg.Contains("applicationType=MAPREDUCE"));
        }
Exemplo n.º 3
0
        /// <exception cref="System.Exception"/>
        public virtual void TestMoveSuccessful()
        {
            MockRM rm1 = new MockRM(conf);

            rm1.Start();
            RMApp           app             = rm1.SubmitApp(1024);
            ClientRMService clientRMService = rm1.GetClientRMService();

            // FIFO scheduler does not support moves
            clientRMService.MoveApplicationAcrossQueues(MoveApplicationAcrossQueuesRequest.NewInstance
                                                            (app.GetApplicationId(), "newqueue"));
            RMApp rmApp = rm1.GetRMContext().GetRMApps()[app.GetApplicationId()];

            NUnit.Framework.Assert.AreEqual("newqueue", rmApp.GetQueue());
            rm1.Stop();
        }
        private static RMApp CreateRMApp(ApplicationId appId)
        {
            RMApp app = Org.Mockito.Mockito.Mock <RMApp>();

            Org.Mockito.Mockito.When(app.GetApplicationId()).ThenReturn(appId);
            Org.Mockito.Mockito.When(app.GetName()).ThenReturn("test app");
            Org.Mockito.Mockito.When(app.GetApplicationType()).ThenReturn("test app type");
            Org.Mockito.Mockito.When(app.GetUser()).ThenReturn("test user");
            Org.Mockito.Mockito.When(app.GetQueue()).ThenReturn("test queue");
            Org.Mockito.Mockito.When(app.GetSubmitTime()).ThenReturn(0L);
            Org.Mockito.Mockito.When(app.GetStartTime()).ThenReturn(1L);
            Org.Mockito.Mockito.When(app.GetFinishTime()).ThenReturn(2L);
            Org.Mockito.Mockito.When(app.GetDiagnostics()).ThenReturn(new StringBuilder("test diagnostics info"
                                                                                        ));
            Org.Mockito.Mockito.When(app.GetFinalApplicationStatus()).ThenReturn(FinalApplicationStatus
                                                                                 .Undefined);
            return(app);
        }
Exemplo n.º 5
0
        /// <exception cref="System.Exception"/>
        private void CheckAppQueue(MockRM resourceManager, string user, string submissionQueue
                                   , string expected)
        {
            RMApp app = resourceManager.SubmitApp(200, "name", user, new Dictionary <ApplicationAccessType
                                                                                     , string>(), false, submissionQueue, -1, null, "MAPREDUCE", false);
            RMAppState expectedState = expected.IsEmpty() ? RMAppState.Failed : RMAppState.Accepted;

            resourceManager.WaitForState(app.GetApplicationId(), expectedState);
            // get scheduler app
            CapacityScheduler    cs           = (CapacityScheduler)resourceManager.GetResourceScheduler();
            SchedulerApplication schedulerApp = cs.GetSchedulerApplications()[app.GetApplicationId
                                                                                  ()];
            string queue = string.Empty;

            if (schedulerApp != null)
            {
                queue = schedulerApp.GetQueue().GetQueueName();
            }
            NUnit.Framework.Assert.IsTrue("expected " + expected + " actual " + queue, expected
                                          .Equals(queue));
            NUnit.Framework.Assert.AreEqual(expected, app.GetQueue());
        }
Exemplo n.º 6
0
        public virtual void TestAppBlockRenderWithNullCurrentAppAttempt()
        {
            ApplicationId AppId = ApplicationId.NewInstance(1234L, 0);
            Injector      injector;
            // init app
            RMApp app = Org.Mockito.Mockito.Mock <RMApp>();

            Org.Mockito.Mockito.When(app.GetTrackingUrl()).ThenReturn("http://host:123");
            Org.Mockito.Mockito.When(app.GetState()).ThenReturn(RMAppState.Failed);
            Org.Mockito.Mockito.When(app.GetApplicationId()).ThenReturn(AppId);
            Org.Mockito.Mockito.When(app.GetApplicationType()).ThenReturn("Type");
            Org.Mockito.Mockito.When(app.GetUser()).ThenReturn("user");
            Org.Mockito.Mockito.When(app.GetName()).ThenReturn("Name");
            Org.Mockito.Mockito.When(app.GetQueue()).ThenReturn("queue");
            Org.Mockito.Mockito.When(app.GetDiagnostics()).ThenReturn(new StringBuilder());
            Org.Mockito.Mockito.When(app.GetFinalApplicationStatus()).ThenReturn(FinalApplicationStatus
                                                                                 .Failed);
            Org.Mockito.Mockito.When(app.GetFinalApplicationStatus()).ThenReturn(FinalApplicationStatus
                                                                                 .Failed);
            Org.Mockito.Mockito.When(app.GetStartTime()).ThenReturn(0L);
            Org.Mockito.Mockito.When(app.GetFinishTime()).ThenReturn(0L);
            Org.Mockito.Mockito.When(app.CreateApplicationState()).ThenReturn(YarnApplicationState
                                                                              .Failed);
            RMAppMetrics appMetrics = new RMAppMetrics(Resource.NewInstance(0, 0), 0, 0, 0, 0
                                                       );

            Org.Mockito.Mockito.When(app.GetRMAppMetrics()).ThenReturn(appMetrics);
            // initialize RM Context, and create RMApp, without creating RMAppAttempt
            RMContext rmContext = TestRMWebApp.MockRMContext(15, 1, 2, 8);

            rmContext.GetRMApps()[AppId] = app;
            injector = WebAppTests.CreateMockInjector <RMContext>(rmContext, new _Module_76(rmContext
                                                                                            ));
            AppBlock instance = injector.GetInstance <AppBlock>();

            instance.Set(YarnWebParams.ApplicationId, AppId.ToString());
            instance.Render();
        }
Exemplo n.º 7
0
 public AppInfo(ResourceManager rm, RMApp app, bool hasAccess, string schemePrefix
                )
 {
     // these are ok for any user to see
     // these are only allowed if acls allow
     // preemption info fields
     // JAXB needs this
     this.schemePrefix = schemePrefix;
     if (app != null)
     {
         string trackingUrl = app.GetTrackingUrl();
         this.state = app.CreateApplicationState();
         this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.IsEmpty() || YarnApplicationState
                                      .New == this.state || YarnApplicationState.NewSaving == this.state || YarnApplicationState
                                      .Submitted == this.state || YarnApplicationState.Accepted == this.state;
         this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app.GetFinishTime(
                                                                            ) == 0 ? "ApplicationMaster" : "History");
         if (!trackingUrlIsNotReady)
         {
             this.trackingUrl       = WebAppUtils.GetURLWithScheme(schemePrefix, trackingUrl);
             this.trackingUrlPretty = this.trackingUrl;
         }
         else
         {
             this.trackingUrlPretty = "UNASSIGNED";
         }
         this.applicationId   = app.GetApplicationId();
         this.applicationType = app.GetApplicationType();
         this.appIdNum        = app.GetApplicationId().GetId().ToString();
         this.id          = app.GetApplicationId().ToString();
         this.user        = app.GetUser().ToString();
         this.name        = app.GetName().ToString();
         this.queue       = app.GetQueue().ToString();
         this.progress    = app.GetProgress() * 100;
         this.diagnostics = app.GetDiagnostics().ToString();
         if (diagnostics == null || diagnostics.IsEmpty())
         {
             this.diagnostics = string.Empty;
         }
         if (app.GetApplicationTags() != null && !app.GetApplicationTags().IsEmpty())
         {
             this.applicationTags = Joiner.On(',').Join(app.GetApplicationTags());
         }
         this.finalStatus = app.GetFinalApplicationStatus();
         this.clusterId   = ResourceManager.GetClusterTimeStamp();
         if (hasAccess)
         {
             this.startedTime  = app.GetStartTime();
             this.finishedTime = app.GetFinishTime();
             this.elapsedTime  = Times.Elapsed(app.GetStartTime(), app.GetFinishTime());
             RMAppAttempt attempt = app.GetCurrentAppAttempt();
             if (attempt != null)
             {
                 Container masterContainer = attempt.GetMasterContainer();
                 if (masterContainer != null)
                 {
                     this.amContainerLogsExist = true;
                     this.amContainerLogs      = WebAppUtils.GetRunningLogURL(schemePrefix + masterContainer
                                                                              .GetNodeHttpAddress(), ConverterUtils.ToString(masterContainer.GetId()), app.GetUser
                                                                                  ());
                     this.amHostHttpAddress = masterContainer.GetNodeHttpAddress();
                 }
                 ApplicationResourceUsageReport resourceReport = attempt.GetApplicationResourceUsageReport
                                                                     ();
                 if (resourceReport != null)
                 {
                     Resource usedResources = resourceReport.GetUsedResources();
                     allocatedMB       = usedResources.GetMemory();
                     allocatedVCores   = usedResources.GetVirtualCores();
                     runningContainers = resourceReport.GetNumUsedContainers();
                 }
                 resourceRequests = ((AbstractYarnScheduler)rm.GetRMContext().GetScheduler()).GetPendingResourceRequestsForAttempt
                                        (attempt.GetAppAttemptId());
             }
         }
         // copy preemption info fields
         RMAppMetrics appMetrics = app.GetRMAppMetrics();
         numAMContainerPreempted    = appMetrics.GetNumAMContainersPreempted();
         preemptedResourceMB        = appMetrics.GetResourcePreempted().GetMemory();
         numNonAMContainerPreempted = appMetrics.GetNumNonAMContainersPreempted();
         preemptedResourceVCores    = appMetrics.GetResourcePreempted().GetVirtualCores();
         memorySeconds = appMetrics.GetMemorySeconds();
         vcoreSeconds  = appMetrics.GetVcoreSeconds();
     }
 }
Exemplo n.º 8
0
 public virtual void AppCreated(RMApp app, long createdTime)
 {
     if (publishSystemMetrics)
     {
         dispatcher.GetEventHandler().Handle(new ApplicationCreatedEvent(app.GetApplicationId
                                                                             (), app.GetName(), app.GetApplicationType(), app.GetUser(), app.GetQueue(), app.
                                                                         GetSubmitTime(), createdTime));
     }
 }
 public virtual void ApplicationStarted(RMApp app)
 {
     if (historyServiceEnabled)
     {
         dispatcher.GetEventHandler().Handle(new WritingApplicationStartEvent(app.GetApplicationId
                                                                                  (), ApplicationStartData.NewInstance(app.GetApplicationId(), app.GetName(), app.
                                                                                                                       GetApplicationType(), app.GetQueue(), app.GetUser(), app.GetSubmitTime(), app.GetStartTime
                                                                                                                           ())));
     }
 }
Exemplo n.º 10
0
        /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
        /// <exception cref="System.IO.IOException"/>
        public virtual AllocateResponse Allocate(AllocateRequest request)
        {
            AMRMTokenIdentifier  amrmTokenIdentifier = AuthorizeRequest();
            ApplicationAttemptId appAttemptId        = amrmTokenIdentifier.GetApplicationAttemptId();
            ApplicationId        applicationId       = appAttemptId.GetApplicationId();

            this.amLivelinessMonitor.ReceivedPing(appAttemptId);
            /* check if its in cache */
            ApplicationMasterService.AllocateResponseLock Lock = responseMap[appAttemptId];
            if (Lock == null)
            {
                string message = "Application attempt " + appAttemptId + " doesn't exist in ApplicationMasterService cache.";
                Log.Error(message);
                throw new ApplicationAttemptNotFoundException(message);
            }
            lock (Lock)
            {
                AllocateResponse lastResponse = Lock.GetAllocateResponse();
                if (!HasApplicationMasterRegistered(appAttemptId))
                {
                    string message = "AM is not registered for known application attempt: " + appAttemptId
                                     + " or RM had restarted after AM registered . AM should re-register.";
                    Log.Info(message);
                    RMAuditLogger.LogFailure(this.rmContext.GetRMApps()[appAttemptId.GetApplicationId
                                                                            ()].GetUser(), RMAuditLogger.AuditConstants.AmAllocate, string.Empty, "ApplicationMasterService"
                                             , message, applicationId, appAttemptId);
                    throw new ApplicationMasterNotRegisteredException(message);
                }
                if ((request.GetResponseId() + 1) == lastResponse.GetResponseId())
                {
                    /* old heartbeat */
                    return(lastResponse);
                }
                else
                {
                    if (request.GetResponseId() + 1 < lastResponse.GetResponseId())
                    {
                        string message = "Invalid responseId in AllocateRequest from application attempt: "
                                         + appAttemptId + ", expect responseId to be " + (lastResponse.GetResponseId() +
                                                                                          1);
                        throw new InvalidApplicationMasterRequestException(message);
                    }
                }
                //filter illegal progress values
                float filteredProgress = request.GetProgress();
                if (float.IsNaN(filteredProgress) || filteredProgress == float.NegativeInfinity ||
                    filteredProgress < 0)
                {
                    request.SetProgress(0);
                }
                else
                {
                    if (filteredProgress > 1 || filteredProgress == float.PositiveInfinity)
                    {
                        request.SetProgress(1);
                    }
                }
                // Send the status update to the appAttempt.
                this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMAppAttemptStatusupdateEvent
                                                                            (appAttemptId, request.GetProgress()));
                IList <ResourceRequest>  ask                = request.GetAskList();
                IList <ContainerId>      release            = request.GetReleaseList();
                ResourceBlacklistRequest blacklistRequest   = request.GetResourceBlacklistRequest();
                IList <string>           blacklistAdditions = (blacklistRequest != null) ? blacklistRequest.
                                                              GetBlacklistAdditions() : Sharpen.Collections.EmptyList;
                IList <string> blacklistRemovals = (blacklistRequest != null) ? blacklistRequest.GetBlacklistRemovals
                                                       () : Sharpen.Collections.EmptyList;
                RMApp app = this.rmContext.GetRMApps()[applicationId];
                // set label expression for Resource Requests if resourceName=ANY
                ApplicationSubmissionContext asc = app.GetApplicationSubmissionContext();
                foreach (ResourceRequest req in ask)
                {
                    if (null == req.GetNodeLabelExpression() && ResourceRequest.Any.Equals(req.GetResourceName
                                                                                               ()))
                    {
                        req.SetNodeLabelExpression(asc.GetNodeLabelExpression());
                    }
                }
                // sanity check
                try
                {
                    RMServerUtils.NormalizeAndValidateRequests(ask, rScheduler.GetMaximumResourceCapability
                                                                   (), app.GetQueue(), rScheduler, rmContext);
                }
                catch (InvalidResourceRequestException e)
                {
                    Log.Warn("Invalid resource ask by application " + appAttemptId, e);
                    throw;
                }
                try
                {
                    RMServerUtils.ValidateBlacklistRequest(blacklistRequest);
                }
                catch (InvalidResourceBlacklistRequestException e)
                {
                    Log.Warn("Invalid blacklist request by application " + appAttemptId, e);
                    throw;
                }
                // In the case of work-preserving AM restart, it's possible for the
                // AM to release containers from the earlier attempt.
                if (!app.GetApplicationSubmissionContext().GetKeepContainersAcrossApplicationAttempts
                        ())
                {
                    try
                    {
                        RMServerUtils.ValidateContainerReleaseRequest(release, appAttemptId);
                    }
                    catch (InvalidContainerReleaseException e)
                    {
                        Log.Warn("Invalid container release by application " + appAttemptId, e);
                        throw;
                    }
                }
                // Send new requests to appAttempt.
                Allocation allocation = this.rScheduler.Allocate(appAttemptId, ask, release, blacklistAdditions
                                                                 , blacklistRemovals);
                if (!blacklistAdditions.IsEmpty() || !blacklistRemovals.IsEmpty())
                {
                    Log.Info("blacklist are updated in Scheduler." + "blacklistAdditions: " + blacklistAdditions
                             + ", " + "blacklistRemovals: " + blacklistRemovals);
                }
                RMAppAttempt     appAttempt       = app.GetRMAppAttempt(appAttemptId);
                AllocateResponse allocateResponse = recordFactory.NewRecordInstance <AllocateResponse
                                                                                     >();
                if (!allocation.GetContainers().IsEmpty())
                {
                    allocateResponse.SetNMTokens(allocation.GetNMTokens());
                }
                // update the response with the deltas of node status changes
                IList <RMNode> updatedNodes = new AList <RMNode>();
                if (app.PullRMNodeUpdates(updatedNodes) > 0)
                {
                    IList <NodeReport> updatedNodeReports = new AList <NodeReport>();
                    foreach (RMNode rmNode in updatedNodes)
                    {
                        SchedulerNodeReport schedulerNodeReport = rScheduler.GetNodeReport(rmNode.GetNodeID
                                                                                               ());
                        Resource used          = BuilderUtils.NewResource(0, 0);
                        int      numContainers = 0;
                        if (schedulerNodeReport != null)
                        {
                            used          = schedulerNodeReport.GetUsedResource();
                            numContainers = schedulerNodeReport.GetNumContainers();
                        }
                        NodeId     nodeId = rmNode.GetNodeID();
                        NodeReport report = BuilderUtils.NewNodeReport(nodeId, rmNode.GetState(), rmNode.
                                                                       GetHttpAddress(), rmNode.GetRackName(), used, rmNode.GetTotalCapability(), numContainers
                                                                       , rmNode.GetHealthReport(), rmNode.GetLastHealthReportTime(), rmNode.GetNodeLabels
                                                                           ());
                        updatedNodeReports.AddItem(report);
                    }
                    allocateResponse.SetUpdatedNodes(updatedNodeReports);
                }
                allocateResponse.SetAllocatedContainers(allocation.GetContainers());
                allocateResponse.SetCompletedContainersStatuses(appAttempt.PullJustFinishedContainers
                                                                    ());
                allocateResponse.SetResponseId(lastResponse.GetResponseId() + 1);
                allocateResponse.SetAvailableResources(allocation.GetResourceLimit());
                allocateResponse.SetNumClusterNodes(this.rScheduler.GetNumClusterNodes());
                // add preemption to the allocateResponse message (if any)
                allocateResponse.SetPreemptionMessage(GeneratePreemptionMessage(allocation));
                // update AMRMToken if the token is rolled-up
                MasterKeyData nextMasterKey = this.rmContext.GetAMRMTokenSecretManager().GetNextMasterKeyData
                                                  ();
                if (nextMasterKey != null && nextMasterKey.GetMasterKey().GetKeyId() != amrmTokenIdentifier
                    .GetKeyId())
                {
                    RMAppAttemptImpl appAttemptImpl = (RMAppAttemptImpl)appAttempt;
                    Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> amrmToken = appAttempt
                                                                                             .GetAMRMToken();
                    if (nextMasterKey.GetMasterKey().GetKeyId() != appAttemptImpl.GetAMRMTokenKeyId())
                    {
                        Log.Info("The AMRMToken has been rolled-over. Send new AMRMToken back" + " to application: "
                                 + applicationId);
                        amrmToken = rmContext.GetAMRMTokenSecretManager().CreateAndGetAMRMToken(appAttemptId
                                                                                                );
                        appAttemptImpl.SetAMRMToken(amrmToken);
                    }
                    allocateResponse.SetAMRMToken(Org.Apache.Hadoop.Yarn.Api.Records.Token.NewInstance
                                                      (amrmToken.GetIdentifier(), amrmToken.GetKind().ToString(), amrmToken.GetPassword
                                                          (), amrmToken.GetService().ToString()));
                }

                /*
                 * As we are updating the response inside the lock object so we don't
                 * need to worry about unregister call occurring in between (which
                 * removes the lock object).
                 */
                Lock.SetAllocateResponse(allocateResponse);
                return(allocateResponse);
            }
        }
Exemplo n.º 11
0
        /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
        /// <exception cref="System.IO.IOException"/>
        public virtual RegisterApplicationMasterResponse RegisterApplicationMaster(RegisterApplicationMasterRequest
                                                                                   request)
        {
            AMRMTokenIdentifier  amrmTokenIdentifier  = AuthorizeRequest();
            ApplicationAttemptId applicationAttemptId = amrmTokenIdentifier.GetApplicationAttemptId
                                                            ();
            ApplicationId appID = applicationAttemptId.GetApplicationId();

            ApplicationMasterService.AllocateResponseLock Lock = responseMap[applicationAttemptId
                                                                 ];
            if (Lock == null)
            {
                RMAuditLogger.LogFailure(this.rmContext.GetRMApps()[appID].GetUser(), RMAuditLogger.AuditConstants
                                         .RegisterAm, "Application doesn't exist in cache " + applicationAttemptId, "ApplicationMasterService"
                                         , "Error in registering application master", appID, applicationAttemptId);
                ThrowApplicationDoesNotExistInCacheException(applicationAttemptId);
            }
            // Allow only one thread in AM to do registerApp at a time.
            lock (Lock)
            {
                AllocateResponse lastResponse = Lock.GetAllocateResponse();
                if (HasApplicationMasterRegistered(applicationAttemptId))
                {
                    string message = "Application Master is already registered : " + appID;
                    Log.Warn(message);
                    RMAuditLogger.LogFailure(this.rmContext.GetRMApps()[appID].GetUser(), RMAuditLogger.AuditConstants
                                             .RegisterAm, string.Empty, "ApplicationMasterService", message, appID, applicationAttemptId
                                             );
                    throw new InvalidApplicationMasterRequestException(message);
                }
                this.amLivelinessMonitor.ReceivedPing(applicationAttemptId);
                RMApp app = this.rmContext.GetRMApps()[appID];
                // Setting the response id to 0 to identify if the
                // application master is register for the respective attemptid
                lastResponse.SetResponseId(0);
                Lock.SetAllocateResponse(lastResponse);
                Log.Info("AM registration " + applicationAttemptId);
                this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMAppAttemptRegistrationEvent
                                                                            (applicationAttemptId, request.GetHost(), request.GetRpcPort(), request.GetTrackingUrl
                                                                                ()));
                RMAuditLogger.LogSuccess(app.GetUser(), RMAuditLogger.AuditConstants.RegisterAm,
                                         "ApplicationMasterService", appID, applicationAttemptId);
                // Pick up min/max resource from scheduler...
                RegisterApplicationMasterResponse response = recordFactory.NewRecordInstance <RegisterApplicationMasterResponse
                                                                                              >();
                response.SetMaximumResourceCapability(rScheduler.GetMaximumResourceCapability(app
                                                                                              .GetQueue()));
                response.SetApplicationACLs(app.GetRMAppAttempt(applicationAttemptId).GetSubmissionContext
                                                ().GetAMContainerSpec().GetApplicationACLs());
                response.SetQueue(app.GetQueue());
                if (UserGroupInformation.IsSecurityEnabled())
                {
                    Log.Info("Setting client token master key");
                    response.SetClientToAMTokenMasterKey(ByteBuffer.Wrap(rmContext.GetClientToAMTokenSecretManager
                                                                             ().GetMasterKey(applicationAttemptId).GetEncoded()));
                }
                // For work-preserving AM restart, retrieve previous attempts' containers
                // and corresponding NM tokens.
                if (app.GetApplicationSubmissionContext().GetKeepContainersAcrossApplicationAttempts
                        ())
                {
                    IList <Container> transferredContainers = ((AbstractYarnScheduler)rScheduler).GetTransferredContainers
                                                                  (applicationAttemptId);
                    if (!transferredContainers.IsEmpty())
                    {
                        response.SetContainersFromPreviousAttempts(transferredContainers);
                        IList <NMToken> nmTokens = new AList <NMToken>();
                        foreach (Container container in transferredContainers)
                        {
                            try
                            {
                                NMToken token = rmContext.GetNMTokenSecretManager().CreateAndGetNMToken(app.GetUser
                                                                                                            (), applicationAttemptId, container);
                                if (null != token)
                                {
                                    nmTokens.AddItem(token);
                                }
                            }
                            catch (ArgumentException e)
                            {
                                // if it's a DNS issue, throw UnknowHostException directly and
                                // that
                                // will be automatically retried by RMProxy in RPC layer.
                                if (e.InnerException is UnknownHostException)
                                {
                                    throw (UnknownHostException)e.InnerException;
                                }
                            }
                        }
                        response.SetNMTokensFromPreviousAttempts(nmTokens);
                        Log.Info("Application " + appID + " retrieved " + transferredContainers.Count + " containers from previous"
                                 + " attempts and " + nmTokens.Count + " NM tokens.");
                    }
                }
                response.SetSchedulerResourceTypes(rScheduler.GetSchedulingResourceTypes());
                return(response);
            }
        }
Exemplo n.º 12
0
            /// <summary>create a summary of the application's runtime.</summary>
            /// <param name="app">
            ///
            /// <see cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Rmapp.RMApp"/>
            /// whose summary is to be created, cannot
            /// be <code>null</code>.
            /// </param>
            public static RMAppManager.ApplicationSummary.SummaryBuilder CreateAppSummary(RMApp
                                                                                          app)
            {
                string       trackingUrl = "N/A";
                string       host        = "N/A";
                RMAppAttempt attempt     = app.GetCurrentAppAttempt();

                if (attempt != null)
                {
                    trackingUrl = attempt.GetTrackingUrl();
                    host        = attempt.GetHost();
                }
                RMAppMetrics metrics = app.GetRMAppMetrics();

                RMAppManager.ApplicationSummary.SummaryBuilder summary = new RMAppManager.ApplicationSummary.SummaryBuilder
                                                                             ().Add("appId", app.GetApplicationId()).Add("name", app.GetName()).Add("user", app
                                                                                                                                                    .GetUser()).Add("queue", app.GetQueue()).Add("state", app.GetState()).Add("trackingUrl"
                                                                                                                                                                                                                              , trackingUrl).Add("appMasterHost", host).Add("startTime", app.GetStartTime()).Add
                                                                             ("finishTime", app.GetFinishTime()).Add("finalStatus", app.GetFinalApplicationStatus
                                                                                                                         ()).Add("memorySeconds", metrics.GetMemorySeconds()).Add("vcoreSeconds", metrics
                                                                                                                                                                                  .GetVcoreSeconds()).Add("preemptedAMContainers", metrics.GetNumAMContainersPreempted
                                                                                                                                                                                                              ()).Add("preemptedNonAMContainers", metrics.GetNumNonAMContainersPreempted()).Add
                                                                             ("preemptedResources", metrics.GetResourcePreempted()).Add("applicationType", app
                                                                                                                                        .GetApplicationType());
                return(summary);
            }
Exemplo n.º 13
0
 /// <exception cref="System.Exception"/>
 public virtual void TestPublishApplicationMetrics()
 {
     for (int i = 1; i <= 2; ++i)
     {
         ApplicationId appId = ApplicationId.NewInstance(0, i);
         RMApp         app   = CreateRMApp(appId);
         metricsPublisher.AppCreated(app, app.GetStartTime());
         metricsPublisher.AppFinished(app, RMAppState.Finished, app.GetFinishTime());
         if (i == 1)
         {
             metricsPublisher.AppACLsUpdated(app, "uers1,user2", 4L);
         }
         else
         {
             // in case user doesn't specify the ACLs
             metricsPublisher.AppACLsUpdated(app, null, 4L);
         }
         TimelineEntity entity = null;
         do
         {
             entity = store.GetEntity(appId.ToString(), ApplicationMetricsConstants.EntityType
                                      , EnumSet.AllOf <TimelineReader.Field>());
         }while (entity == null || entity.GetEvents().Count < 3);
         // ensure three events are both published before leaving the loop
         // verify all the fields
         NUnit.Framework.Assert.AreEqual(ApplicationMetricsConstants.EntityType, entity.GetEntityType
                                             ());
         NUnit.Framework.Assert.AreEqual(app.GetApplicationId().ToString(), entity.GetEntityId
                                             ());
         NUnit.Framework.Assert.AreEqual(app.GetName(), entity.GetOtherInfo()[ApplicationMetricsConstants
                                                                              .NameEntityInfo]);
         NUnit.Framework.Assert.AreEqual(app.GetQueue(), entity.GetOtherInfo()[ApplicationMetricsConstants
                                                                               .QueueEntityInfo]);
         NUnit.Framework.Assert.AreEqual(app.GetUser(), entity.GetOtherInfo()[ApplicationMetricsConstants
                                                                              .UserEntityInfo]);
         NUnit.Framework.Assert.AreEqual(app.GetApplicationType(), entity.GetOtherInfo()[ApplicationMetricsConstants
                                                                                         .TypeEntityInfo]);
         NUnit.Framework.Assert.AreEqual(app.GetSubmitTime(), entity.GetOtherInfo()[ApplicationMetricsConstants
                                                                                    .SubmittedTimeEntityInfo]);
         if (i == 1)
         {
             NUnit.Framework.Assert.AreEqual("uers1,user2", entity.GetOtherInfo()[ApplicationMetricsConstants
                                                                                  .AppViewAclsEntityInfo]);
         }
         else
         {
             NUnit.Framework.Assert.AreEqual(string.Empty, entity.GetOtherInfo()[ApplicationMetricsConstants
                                                                                 .AppViewAclsEntityInfo]);
             NUnit.Framework.Assert.AreEqual(app.GetRMAppMetrics().GetMemorySeconds(), long.Parse
                                                 (entity.GetOtherInfo()[ApplicationMetricsConstants.AppMemMetrics].ToString()));
             NUnit.Framework.Assert.AreEqual(app.GetRMAppMetrics().GetVcoreSeconds(), long.Parse
                                                 (entity.GetOtherInfo()[ApplicationMetricsConstants.AppCpuMetrics].ToString()));
         }
         bool hasCreatedEvent     = false;
         bool hasFinishedEvent    = false;
         bool hasACLsUpdatedEvent = false;
         foreach (TimelineEvent @event in entity.GetEvents())
         {
             if (@event.GetEventType().Equals(ApplicationMetricsConstants.CreatedEventType))
             {
                 hasCreatedEvent = true;
                 NUnit.Framework.Assert.AreEqual(app.GetStartTime(), @event.GetTimestamp());
             }
             else
             {
                 if (@event.GetEventType().Equals(ApplicationMetricsConstants.FinishedEventType))
                 {
                     hasFinishedEvent = true;
                     NUnit.Framework.Assert.AreEqual(app.GetFinishTime(), @event.GetTimestamp());
                     NUnit.Framework.Assert.AreEqual(app.GetDiagnostics().ToString(), @event.GetEventInfo
                                                         ()[ApplicationMetricsConstants.DiagnosticsInfoEventInfo]);
                     NUnit.Framework.Assert.AreEqual(app.GetFinalApplicationStatus().ToString(), @event
                                                     .GetEventInfo()[ApplicationMetricsConstants.FinalStatusEventInfo]);
                     NUnit.Framework.Assert.AreEqual(YarnApplicationState.Finished.ToString(), @event.
                                                     GetEventInfo()[ApplicationMetricsConstants.StateEventInfo]);
                 }
                 else
                 {
                     if (@event.GetEventType().Equals(ApplicationMetricsConstants.AclsUpdatedEventType
                                                      ))
                     {
                         hasACLsUpdatedEvent = true;
                         NUnit.Framework.Assert.AreEqual(4L, @event.GetTimestamp());
                     }
                 }
             }
         }
         NUnit.Framework.Assert.IsTrue(hasCreatedEvent && hasFinishedEvent && hasACLsUpdatedEvent
                                       );
     }
 }