예제 #1
0
        /// <exception cref="System.Exception"/>
        protected internal virtual ContainerId StoreAttempt(RMStateStore store, ApplicationAttemptId
                                                            attemptId, string containerIdStr, Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier
                                                                                                                                      > appToken, SecretKey clientTokenMasterKey, RMStateStoreTestBase.TestDispatcher
                                                            dispatcher)
        {
            RMAppAttemptMetrics mockRmAppAttemptMetrics = Org.Mockito.Mockito.Mock <RMAppAttemptMetrics
                                                                                    >();
            Container container = new ContainerPBImpl();

            container.SetId(ConverterUtils.ToContainerId(containerIdStr));
            RMAppAttempt mockAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>();

            Org.Mockito.Mockito.When(mockAttempt.GetAppAttemptId()).ThenReturn(attemptId);
            Org.Mockito.Mockito.When(mockAttempt.GetMasterContainer()).ThenReturn(container);
            Org.Mockito.Mockito.When(mockAttempt.GetAMRMToken()).ThenReturn(appToken);
            Org.Mockito.Mockito.When(mockAttempt.GetClientTokenMasterKey()).ThenReturn(clientTokenMasterKey
                                                                                       );
            Org.Mockito.Mockito.When(mockAttempt.GetRMAppAttemptMetrics()).ThenReturn(mockRmAppAttemptMetrics
                                                                                      );
            Org.Mockito.Mockito.When(mockRmAppAttemptMetrics.GetAggregateAppResourceUsage()).
            ThenReturn(new AggregateAppResourceUsage(0, 0));
            dispatcher.attemptId = attemptId;
            store.StoreNewApplicationAttempt(mockAttempt);
            WaitNotify(dispatcher);
            return(container.GetId());
        }
예제 #2
0
        public virtual void Setup()
        {
            // start minicluster
            conf        = new YarnConfiguration();
            yarnCluster = new MiniYARNCluster(typeof(TestAMRMClient).FullName, nodeCount, 1,
                                              1);
            yarnCluster.Init(conf);
            yarnCluster.Start();
            NUnit.Framework.Assert.IsNotNull(yarnCluster);
            NUnit.Framework.Assert.AreEqual(Service.STATE.Started, yarnCluster.GetServiceState
                                                ());
            // start rm client
            yarnClient = (YarnClientImpl)YarnClient.CreateYarnClient();
            yarnClient.Init(conf);
            yarnClient.Start();
            NUnit.Framework.Assert.IsNotNull(yarnClient);
            NUnit.Framework.Assert.AreEqual(Service.STATE.Started, yarnClient.GetServiceState
                                                ());
            // get node info
            nodeReports = yarnClient.GetNodeReports(NodeState.Running);
            // submit new app
            ApplicationSubmissionContext appContext = yarnClient.CreateApplication().GetApplicationSubmissionContext
                                                          ();
            ApplicationId appId = appContext.GetApplicationId();

            // set the application name
            appContext.SetApplicationName("Test");
            // Set the priority for the application master
            Priority pri = Priority.NewInstance(0);

            appContext.SetPriority(pri);
            // Set the queue to which this application is to be submitted in the RM
            appContext.SetQueue("default");
            // Set up the container launch context for the application master
            ContainerLaunchContext amContainer = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
                                                 <ContainerLaunchContext>();

            appContext.SetAMContainerSpec(amContainer);
            // unmanaged AM
            appContext.SetUnmanagedAM(true);
            // Create the request to send to the applications manager
            SubmitApplicationRequest appRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
                                                  <SubmitApplicationRequest>();

            appRequest.SetApplicationSubmissionContext(appContext);
            // Submit the application to the applications manager
            yarnClient.SubmitApplication(appContext);
            // wait for app to start
            int          iterationsLeft = 30;
            RMAppAttempt appAttempt     = null;

            while (iterationsLeft > 0)
            {
                ApplicationReport appReport = yarnClient.GetApplicationReport(appId);
                if (appReport.GetYarnApplicationState() == YarnApplicationState.Accepted)
                {
                    attemptId  = appReport.GetCurrentApplicationAttemptId();
                    appAttempt = yarnCluster.GetResourceManager().GetRMContext().GetRMApps()[attemptId
                                                                                             .GetApplicationId()].GetCurrentAppAttempt();
                    while (true)
                    {
                        if (appAttempt.GetAppAttemptState() == RMAppAttemptState.Launched)
                        {
                            break;
                        }
                    }
                    break;
                }
                Sleep(1000);
                --iterationsLeft;
            }
            if (iterationsLeft == 0)
            {
                NUnit.Framework.Assert.Fail("Application hasn't bee started");
            }
            // Just dig into the ResourceManager and get the AMRMToken just for the sake
            // of testing.
            UserGroupInformation.SetLoginUser(UserGroupInformation.CreateRemoteUser(UserGroupInformation
                                                                                    .GetCurrentUser().GetUserName()));
            UserGroupInformation.GetCurrentUser().AddToken(appAttempt.GetAMRMToken());
            //creating an instance NMTokenCase
            nmTokenCache = new NMTokenCache();
            // start am rm client
            rmClient = (AMRMClientImpl <AMRMClient.ContainerRequest>)AMRMClient.CreateAMRMClient
                       <AMRMClient.ContainerRequest>();
            //setting an instance NMTokenCase
            rmClient.SetNMTokenCache(nmTokenCache);
            rmClient.Init(conf);
            rmClient.Start();
            NUnit.Framework.Assert.IsNotNull(rmClient);
            NUnit.Framework.Assert.AreEqual(Service.STATE.Started, rmClient.GetServiceState()
                                            );
            // start am nm client
            nmClient = (NMClientImpl)NMClient.CreateNMClient();
            //propagating the AMRMClient NMTokenCache instance
            nmClient.SetNMTokenCache(rmClient.GetNMTokenCache());
            nmClient.Init(conf);
            nmClient.Start();
            NUnit.Framework.Assert.IsNotNull(nmClient);
            NUnit.Framework.Assert.AreEqual(Service.STATE.Started, nmClient.GetServiceState()
                                            );
        }
        /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
        /// <exception cref="System.IO.IOException"/>
        public virtual AllocateResponse Allocate(AllocateRequest request)
        {
            AMRMTokenIdentifier  amrmTokenIdentifier = AuthorizeRequest();
            ApplicationAttemptId appAttemptId        = amrmTokenIdentifier.GetApplicationAttemptId();
            ApplicationId        applicationId       = appAttemptId.GetApplicationId();

            this.amLivelinessMonitor.ReceivedPing(appAttemptId);
            /* check if its in cache */
            ApplicationMasterService.AllocateResponseLock Lock = responseMap[appAttemptId];
            if (Lock == null)
            {
                string message = "Application attempt " + appAttemptId + " doesn't exist in ApplicationMasterService cache.";
                Log.Error(message);
                throw new ApplicationAttemptNotFoundException(message);
            }
            lock (Lock)
            {
                AllocateResponse lastResponse = Lock.GetAllocateResponse();
                if (!HasApplicationMasterRegistered(appAttemptId))
                {
                    string message = "AM is not registered for known application attempt: " + appAttemptId
                                     + " or RM had restarted after AM registered . AM should re-register.";
                    Log.Info(message);
                    RMAuditLogger.LogFailure(this.rmContext.GetRMApps()[appAttemptId.GetApplicationId
                                                                            ()].GetUser(), RMAuditLogger.AuditConstants.AmAllocate, string.Empty, "ApplicationMasterService"
                                             , message, applicationId, appAttemptId);
                    throw new ApplicationMasterNotRegisteredException(message);
                }
                if ((request.GetResponseId() + 1) == lastResponse.GetResponseId())
                {
                    /* old heartbeat */
                    return(lastResponse);
                }
                else
                {
                    if (request.GetResponseId() + 1 < lastResponse.GetResponseId())
                    {
                        string message = "Invalid responseId in AllocateRequest from application attempt: "
                                         + appAttemptId + ", expect responseId to be " + (lastResponse.GetResponseId() +
                                                                                          1);
                        throw new InvalidApplicationMasterRequestException(message);
                    }
                }
                //filter illegal progress values
                float filteredProgress = request.GetProgress();
                if (float.IsNaN(filteredProgress) || filteredProgress == float.NegativeInfinity ||
                    filteredProgress < 0)
                {
                    request.SetProgress(0);
                }
                else
                {
                    if (filteredProgress > 1 || filteredProgress == float.PositiveInfinity)
                    {
                        request.SetProgress(1);
                    }
                }
                // Send the status update to the appAttempt.
                this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMAppAttemptStatusupdateEvent
                                                                            (appAttemptId, request.GetProgress()));
                IList <ResourceRequest>  ask                = request.GetAskList();
                IList <ContainerId>      release            = request.GetReleaseList();
                ResourceBlacklistRequest blacklistRequest   = request.GetResourceBlacklistRequest();
                IList <string>           blacklistAdditions = (blacklistRequest != null) ? blacklistRequest.
                                                              GetBlacklistAdditions() : Sharpen.Collections.EmptyList;
                IList <string> blacklistRemovals = (blacklistRequest != null) ? blacklistRequest.GetBlacklistRemovals
                                                       () : Sharpen.Collections.EmptyList;
                RMApp app = this.rmContext.GetRMApps()[applicationId];
                // set label expression for Resource Requests if resourceName=ANY
                ApplicationSubmissionContext asc = app.GetApplicationSubmissionContext();
                foreach (ResourceRequest req in ask)
                {
                    if (null == req.GetNodeLabelExpression() && ResourceRequest.Any.Equals(req.GetResourceName
                                                                                               ()))
                    {
                        req.SetNodeLabelExpression(asc.GetNodeLabelExpression());
                    }
                }
                // sanity check
                try
                {
                    RMServerUtils.NormalizeAndValidateRequests(ask, rScheduler.GetMaximumResourceCapability
                                                                   (), app.GetQueue(), rScheduler, rmContext);
                }
                catch (InvalidResourceRequestException e)
                {
                    Log.Warn("Invalid resource ask by application " + appAttemptId, e);
                    throw;
                }
                try
                {
                    RMServerUtils.ValidateBlacklistRequest(blacklistRequest);
                }
                catch (InvalidResourceBlacklistRequestException e)
                {
                    Log.Warn("Invalid blacklist request by application " + appAttemptId, e);
                    throw;
                }
                // In the case of work-preserving AM restart, it's possible for the
                // AM to release containers from the earlier attempt.
                if (!app.GetApplicationSubmissionContext().GetKeepContainersAcrossApplicationAttempts
                        ())
                {
                    try
                    {
                        RMServerUtils.ValidateContainerReleaseRequest(release, appAttemptId);
                    }
                    catch (InvalidContainerReleaseException e)
                    {
                        Log.Warn("Invalid container release by application " + appAttemptId, e);
                        throw;
                    }
                }
                // Send new requests to appAttempt.
                Allocation allocation = this.rScheduler.Allocate(appAttemptId, ask, release, blacklistAdditions
                                                                 , blacklistRemovals);
                if (!blacklistAdditions.IsEmpty() || !blacklistRemovals.IsEmpty())
                {
                    Log.Info("blacklist are updated in Scheduler." + "blacklistAdditions: " + blacklistAdditions
                             + ", " + "blacklistRemovals: " + blacklistRemovals);
                }
                RMAppAttempt     appAttempt       = app.GetRMAppAttempt(appAttemptId);
                AllocateResponse allocateResponse = recordFactory.NewRecordInstance <AllocateResponse
                                                                                     >();
                if (!allocation.GetContainers().IsEmpty())
                {
                    allocateResponse.SetNMTokens(allocation.GetNMTokens());
                }
                // update the response with the deltas of node status changes
                IList <RMNode> updatedNodes = new AList <RMNode>();
                if (app.PullRMNodeUpdates(updatedNodes) > 0)
                {
                    IList <NodeReport> updatedNodeReports = new AList <NodeReport>();
                    foreach (RMNode rmNode in updatedNodes)
                    {
                        SchedulerNodeReport schedulerNodeReport = rScheduler.GetNodeReport(rmNode.GetNodeID
                                                                                               ());
                        Resource used          = BuilderUtils.NewResource(0, 0);
                        int      numContainers = 0;
                        if (schedulerNodeReport != null)
                        {
                            used          = schedulerNodeReport.GetUsedResource();
                            numContainers = schedulerNodeReport.GetNumContainers();
                        }
                        NodeId     nodeId = rmNode.GetNodeID();
                        NodeReport report = BuilderUtils.NewNodeReport(nodeId, rmNode.GetState(), rmNode.
                                                                       GetHttpAddress(), rmNode.GetRackName(), used, rmNode.GetTotalCapability(), numContainers
                                                                       , rmNode.GetHealthReport(), rmNode.GetLastHealthReportTime(), rmNode.GetNodeLabels
                                                                           ());
                        updatedNodeReports.AddItem(report);
                    }
                    allocateResponse.SetUpdatedNodes(updatedNodeReports);
                }
                allocateResponse.SetAllocatedContainers(allocation.GetContainers());
                allocateResponse.SetCompletedContainersStatuses(appAttempt.PullJustFinishedContainers
                                                                    ());
                allocateResponse.SetResponseId(lastResponse.GetResponseId() + 1);
                allocateResponse.SetAvailableResources(allocation.GetResourceLimit());
                allocateResponse.SetNumClusterNodes(this.rScheduler.GetNumClusterNodes());
                // add preemption to the allocateResponse message (if any)
                allocateResponse.SetPreemptionMessage(GeneratePreemptionMessage(allocation));
                // update AMRMToken if the token is rolled-up
                MasterKeyData nextMasterKey = this.rmContext.GetAMRMTokenSecretManager().GetNextMasterKeyData
                                                  ();
                if (nextMasterKey != null && nextMasterKey.GetMasterKey().GetKeyId() != amrmTokenIdentifier
                    .GetKeyId())
                {
                    RMAppAttemptImpl appAttemptImpl = (RMAppAttemptImpl)appAttempt;
                    Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> amrmToken = appAttempt
                                                                                             .GetAMRMToken();
                    if (nextMasterKey.GetMasterKey().GetKeyId() != appAttemptImpl.GetAMRMTokenKeyId())
                    {
                        Log.Info("The AMRMToken has been rolled-over. Send new AMRMToken back" + " to application: "
                                 + applicationId);
                        amrmToken = rmContext.GetAMRMTokenSecretManager().CreateAndGetAMRMToken(appAttemptId
                                                                                                );
                        appAttemptImpl.SetAMRMToken(amrmToken);
                    }
                    allocateResponse.SetAMRMToken(Org.Apache.Hadoop.Yarn.Api.Records.Token.NewInstance
                                                      (amrmToken.GetIdentifier(), amrmToken.GetKind().ToString(), amrmToken.GetPassword
                                                          (), amrmToken.GetService().ToString()));
                }

                /*
                 * As we are updating the response inside the lock object so we don't
                 * need to worry about unregister call occurring in between (which
                 * removes the lock object).
                 */
                Lock.SetAllocateResponse(allocateResponse);
                return(allocateResponse);
            }
        }