예제 #1
0
        public virtual void TestValidateResourceBlacklistRequest()
        {
            TestAMAuthorization.MyContainerManager containerManager = new TestAMAuthorization.MyContainerManager
                                                                          ();
            TestAMAuthorization.MockRMWithAMS rm = new TestAMAuthorization.MockRMWithAMS(new
                                                                                         YarnConfiguration(), containerManager);
            rm.Start();
            MockNM nm1 = rm.RegisterNode("localhost:1234", 5120);
            IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
                                                                               , string>(2);

            acls[ApplicationAccessType.ViewApp] = "*";
            RMApp app = rm.SubmitApp(1024, "appname", "appuser", acls);

            nm1.NodeHeartbeat(true);
            RMAppAttempt         attempt = app.GetCurrentAppAttempt();
            ApplicationAttemptId applicationAttemptId = attempt.GetAppAttemptId();

            WaitForLaunchedState(attempt);
            // Create a client to the RM.
            Configuration        conf        = rm.GetConfig();
            YarnRPC              rpc         = YarnRPC.Create(conf);
            UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(applicationAttemptId
                                                                                     .ToString());
            Credentials credentials   = containerManager.GetContainerCredentials();
            IPEndPoint  rmBindAddress = rm.GetApplicationMasterService().GetBindAddress();

            Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> amRMToken = TestAMAuthorization.MockRMWithAMS
                                                                                 .SetupAndReturnAMRMToken(rmBindAddress, credentials.GetAllTokens());
            currentUser.AddToken(amRMToken);
            ApplicationMasterProtocol client = currentUser.DoAs(new _PrivilegedAction_626(rpc
                                                                                          , rmBindAddress, conf));
            RegisterApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
                                                       <RegisterApplicationMasterRequest>();

            client.RegisterApplicationMaster(request);
            ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.NewInstance(
                Sharpen.Collections.SingletonList(ResourceRequest.Any), null);
            AllocateRequest allocateRequest = AllocateRequest.NewInstance(0, 0.0f, null, null
                                                                          , blacklistRequest);
            bool error = false;

            try
            {
                client.Allocate(allocateRequest);
            }
            catch (InvalidResourceBlacklistRequestException)
            {
                error = true;
            }
            rm.Stop();
            NUnit.Framework.Assert.IsTrue("Didn't not catch InvalidResourceBlacklistRequestException"
                                          , error);
        }
예제 #2
0
        /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
        /// <exception cref="System.IO.IOException"/>
        protected internal virtual AllocateResponse MakeRemoteRequest()
        {
            ApplyRequestLimits();
            ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.NewInstance(
                new AList <string>(blacklistAdditions), new AList <string>(blacklistRemovals));
            AllocateRequest allocateRequest = AllocateRequest.NewInstance(lastResponseID, base
                                                                          .GetApplicationProgress(), new AList <ResourceRequest>(ask), new AList <ContainerId
                                                                                                                                                  >(release), blacklistRequest);
            AllocateResponse allocateResponse = scheduler.Allocate(allocateRequest);

            lastResponseID     = allocateResponse.GetResponseId();
            availableResources = allocateResponse.GetAvailableResources();
            lastClusterNmCount = clusterNmCount;
            clusterNmCount     = allocateResponse.GetNumClusterNodes();
            int numCompletedContainers = allocateResponse.GetCompletedContainersStatuses().Count;

            if (ask.Count > 0 || release.Count > 0)
            {
                Log.Info("getResources() for " + applicationId + ":" + " ask=" + ask.Count + " release= "
                         + release.Count + " newContainers=" + allocateResponse.GetAllocatedContainers()
                         .Count + " finishedContainers=" + numCompletedContainers + " resourcelimit=" + availableResources
                         + " knownNMs=" + clusterNmCount);
            }
            ask.Clear();
            release.Clear();
            if (numCompletedContainers > 0)
            {
                // re-send limited requests when a container completes to trigger asking
                // for more containers
                Sharpen.Collections.AddAll(requestLimitsToUpdate, requestLimits.Keys);
            }
            if (blacklistAdditions.Count > 0 || blacklistRemovals.Count > 0)
            {
                Log.Info("Update the blacklist for " + applicationId + ": blacklistAdditions=" +
                         blacklistAdditions.Count + " blacklistRemovals=" + blacklistRemovals.Count);
            }
            blacklistAdditions.Clear();
            blacklistRemovals.Clear();
            return(allocateResponse);
        }
        /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
        /// <exception cref="System.IO.IOException"/>
        public virtual void TestAllocateOnHA()
        {
            AllocateRequest request = AllocateRequest.NewInstance(0, 50f, new AList <ResourceRequest
                                                                                     >(), new AList <ContainerId>(), ResourceBlacklistRequest.NewInstance(new AList <string
                                                                                                                                                                     >(), new AList <string>()));
            AllocateResponse response = amClient.Allocate(request);

            NUnit.Framework.Assert.AreEqual(response, this.cluster.CreateFakeAllocateResponse
                                                ());
        }
예제 #4
0
        /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
        /// <exception cref="System.IO.IOException"/>
        public override AllocateResponse Allocate(float progressIndicator)
        {
            Preconditions.CheckArgument(progressIndicator >= 0, "Progress indicator should not be negative"
                                        );
            AllocateResponse        allocateResponse  = null;
            IList <ResourceRequest> askList           = null;
            IList <ContainerId>     releaseList       = null;
            AllocateRequest         allocateRequest   = null;
            IList <string>          blacklistToAdd    = new AList <string>();
            IList <string>          blacklistToRemove = new AList <string>();

            try
            {
                lock (this)
                {
                    askList = new AList <ResourceRequest>(ask.Count);
                    foreach (ResourceRequest r in ask)
                    {
                        // create a copy of ResourceRequest as we might change it while the
                        // RPC layer is using it to send info across
                        askList.AddItem(ResourceRequest.NewInstance(r.GetPriority(), r.GetResourceName(),
                                                                    r.GetCapability(), r.GetNumContainers(), r.GetRelaxLocality(), r.GetNodeLabelExpression
                                                                        ()));
                    }
                    releaseList = new AList <ContainerId>(release);
                    // optimistically clear this collection assuming no RPC failure
                    ask.Clear();
                    release.Clear();
                    Sharpen.Collections.AddAll(blacklistToAdd, blacklistAdditions);
                    Sharpen.Collections.AddAll(blacklistToRemove, blacklistRemovals);
                    ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.NewInstance(
                        blacklistToAdd, blacklistToRemove);
                    allocateRequest = AllocateRequest.NewInstance(lastResponseId, progressIndicator,
                                                                  askList, releaseList, blacklistRequest);
                    // clear blacklistAdditions and blacklistRemovals before
                    // unsynchronized part
                    blacklistAdditions.Clear();
                    blacklistRemovals.Clear();
                }
                try
                {
                    allocateResponse = rmClient.Allocate(allocateRequest);
                }
                catch (ApplicationMasterNotRegisteredException)
                {
                    Log.Warn("ApplicationMaster is out of sync with ResourceManager," + " hence resyncing."
                             );
                    lock (this)
                    {
                        Sharpen.Collections.AddAll(release, this.pendingRelease);
                        Sharpen.Collections.AddAll(blacklistAdditions, this.blacklistedNodes);
                        foreach (IDictionary <string, SortedDictionary <Resource, AMRMClientImpl.ResourceRequestInfo
                                                                        > > rr in remoteRequestsTable.Values)
                        {
                            foreach (IDictionary <Resource, AMRMClientImpl.ResourceRequestInfo> capabalities in
                                     rr.Values)
                            {
                                foreach (AMRMClientImpl.ResourceRequestInfo request in capabalities.Values)
                                {
                                    AddResourceRequestToAsk(request.remoteRequest);
                                }
                            }
                        }
                    }
                    // re register with RM
                    RegisterApplicationMaster();
                    allocateResponse = Allocate(progressIndicator);
                    return(allocateResponse);
                }
                lock (this)
                {
                    // update these on successful RPC
                    clusterNodeCount          = allocateResponse.GetNumClusterNodes();
                    lastResponseId            = allocateResponse.GetResponseId();
                    clusterAvailableResources = allocateResponse.GetAvailableResources();
                    if (!allocateResponse.GetNMTokens().IsEmpty())
                    {
                        PopulateNMTokens(allocateResponse.GetNMTokens());
                    }
                    if (allocateResponse.GetAMRMToken() != null)
                    {
                        UpdateAMRMToken(allocateResponse.GetAMRMToken());
                    }
                    if (!pendingRelease.IsEmpty() && !allocateResponse.GetCompletedContainersStatuses
                            ().IsEmpty())
                    {
                        RemovePendingReleaseRequests(allocateResponse.GetCompletedContainersStatuses());
                    }
                }
            }
            finally
            {
                // TODO how to differentiate remote yarn exception vs error in rpc
                if (allocateResponse == null)
                {
                    // we hit an exception in allocate()
                    // preserve ask and release for next call to allocate()
                    lock (this)
                    {
                        Sharpen.Collections.AddAll(release, releaseList);
                        // requests could have been added or deleted during call to allocate
                        // If requests were added/removed then there is nothing to do since
                        // the ResourceRequest object in ask would have the actual new value.
                        // If ask does not have this ResourceRequest then it was unchanged and
                        // so we can add the value back safely.
                        // This assumes that there will no concurrent calls to allocate() and
                        // so we dont have to worry about ask being changed in the
                        // synchronized block at the beginning of this method.
                        foreach (ResourceRequest oldAsk in askList)
                        {
                            if (!ask.Contains(oldAsk))
                            {
                                ask.AddItem(oldAsk);
                            }
                        }
                        Sharpen.Collections.AddAll(blacklistAdditions, blacklistToAdd);
                        Sharpen.Collections.AddAll(blacklistRemovals, blacklistToRemove);
                    }
                }
            }
            return(allocateResponse);
        }