예제 #1
0
 /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
 /// <exception cref="System.IO.IOException"/>
 public override AllocateResponse Allocate(AllocateRequest request)
 {
     this._enclosing.ResetStartFailoverFlag(true);
     // make sure failover has been triggered
     NUnit.Framework.Assert.IsTrue(this._enclosing.WaittingForFailOver());
     return(this._enclosing.CreateFakeAllocateResponse());
 }
예제 #2
0
        public virtual void TestAllcoateRequestWithIncrease()
        {
            IList <ContainerResourceIncreaseRequest> incRequests = new AList <ContainerResourceIncreaseRequest
                                                                              >();

            for (int i = 0; i < 3; i++)
            {
                incRequests.AddItem(ContainerResourceIncreaseRequest.NewInstance(null, Resource.NewInstance
                                                                                     (0, i)));
            }
            AllocateRequest r = AllocateRequest.NewInstance(123, 0f, null, null, null, incRequests
                                                            );

            // serde
            YarnServiceProtos.AllocateRequestProto p = ((AllocateRequestPBImpl)r).GetProto();
            r = new AllocateRequestPBImpl(p);
            // check value
            NUnit.Framework.Assert.AreEqual(123, r.GetResponseId());
            NUnit.Framework.Assert.AreEqual(incRequests.Count, r.GetIncreaseRequests().Count);
            for (int i_1 = 0; i_1 < incRequests.Count; i_1++)
            {
                NUnit.Framework.Assert.AreEqual(r.GetIncreaseRequests()[i_1].GetCapability().GetVirtualCores
                                                    (), incRequests[i_1].GetCapability().GetVirtualCores());
            }
        }
예제 #3
0
        public virtual void TestPbRecordFactory()
        {
            RecordFactory pbRecordFactory = RecordFactoryPBImpl.Get();

            try
            {
                AllocateResponse response = pbRecordFactory.NewRecordInstance <AllocateResponse>();
                NUnit.Framework.Assert.AreEqual(typeof(AllocateResponsePBImpl), response.GetType(
                                                    ));
            }
            catch (YarnRuntimeException e)
            {
                Sharpen.Runtime.PrintStackTrace(e);
                NUnit.Framework.Assert.Fail("Failed to crete record");
            }
            try
            {
                AllocateRequest response = pbRecordFactory.NewRecordInstance <AllocateRequest>();
                NUnit.Framework.Assert.AreEqual(typeof(AllocateRequestPBImpl), response.GetType()
                                                );
            }
            catch (YarnRuntimeException e)
            {
                Sharpen.Runtime.PrintStackTrace(e);
                NUnit.Framework.Assert.Fail("Failed to crete record");
            }
        }
예제 #4
0
        public IActionResult Allocate([FromBody] AllocateRequest spec)
        {
            if (spec == null)
            {
                return(BadRequest($"{nameof(spec)} can not be null"));
            }

            if (spec.Offers == null || spec.Offers.Length == 0)
            {
                return(BadRequest($"{nameof(spec.Offers)} can not be empty"));
            }

            if (spec.Threshold <= 0)
            {
                return(BadRequest($"{nameof(spec.Threshold)} must be greater than 0"));
            }

            if (spec.BusinessSeats < 0)
            {
                return(BadRequest($"{nameof(spec.BusinessSeats)} must be greater than or equal to 0"));
            }

            if (spec.EconomySeats < 0)
            {
                return(BadRequest($"{nameof(spec.EconomySeats)} must be greater than or equal to 0"));
            }

            var allocateResult = _allocationService.Allocate(spec.ToSpec());

            return(Ok(AllocateResponse.From(allocateResult)));
        }
예제 #5
0
                    /// <exception cref="System.IO.IOException"/>
                    public AllocateResponse Allocate(AllocateRequest request)
                    {
                        AllocateResponse response = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateResponse
                                                                                                   >();
                        IList <ResourceRequest> askList    = request.GetAskList();
                        IList <Container>       containers = new AList <Container>();

                        foreach (ResourceRequest req in askList)
                        {
                            if (!ResourceRequest.IsAnyLocation(req.GetResourceName()))
                            {
                                continue;
                            }
                            int numContainers = req.GetNumContainers();
                            for (int i = 0; i < numContainers; i++)
                            {
                                ContainerId containerId = ContainerId.NewContainerId(this._enclosing.GetContext()
                                                                                     .GetApplicationAttemptId(), request.GetResponseId() + i);
                                containers.AddItem(Container.NewInstance(containerId, NodeId.NewInstance("host" +
                                                                                                         containerId.GetContainerId(), 2345), "host" + containerId.GetContainerId() + ":5678"
                                                                         , req.GetCapability(), req.GetPriority(), null));
                            }
                        }
                        response.SetAllocatedContainers(containers);
                        response.SetResponseId(request.GetResponseId() + 1);
                        response.SetNumClusterNodes(350);
                        return(response);
                    }
예제 #6
0
        public async Task <IPEndPoint> AllocateRequestAsync(
            TimeSpan lifetime,
            CancellationToken cancellationToken = default(CancellationToken))
        {
            NetworkStream stream = _control.GetStream();
            StunMessage   response;
            int           retry = 0;

            do
            {
                var request = new AllocateRequest((int)lifetime.TotalSeconds);
                await SendMessageAsync(stream, request, cancellationToken);

                response = await ReceiveMessageAsync(request.TransactionId, cancellationToken);

                if (response is AllocateErrorResponse allocError)
                {
                    Realm = allocError.Realm;
                    Nonce = allocError.Nonce;
                }

                retry += 1;
            }while (response is AllocateErrorResponse && retry < AllocateRetry);

            if (response is AllocateSuccessResponse allocOk)
            {
                return(allocOk.RelayedEndPoint);
            }
            else
            {
                throw new TurnClientException("Allocate failed.", response);
            }
        }
예제 #7
0
        /// <exception cref="System.Exception"/>
        public virtual AllocateResponse Allocate(IList <ResourceRequest> resourceRequest,
                                                 IList <ContainerId> releases)
        {
            AllocateRequest req = AllocateRequest.NewInstance(0, 0F, resourceRequest, releases
                                                              , null);

            return(Allocate(req));
        }
예제 #8
0
 /// <exception cref="System.Exception"/>
 protected internal override void Heartbeat()
 {
     lock (this)
     {
         AllocateRequest allocateRequest = AllocateRequest.NewInstance(this.lastResponseID
                                                                       , base.GetApplicationProgress(), new AList <ResourceRequest>(), new AList <ContainerId
                                                                                                                                                  >(), null);
         AllocateResponse allocateResponse = null;
         try
         {
             allocateResponse = scheduler.Allocate(allocateRequest);
             // Reset retry count if no exception occurred.
             retrystartTime = Runtime.CurrentTimeMillis();
         }
         catch (ApplicationAttemptNotFoundException e)
         {
             Log.Info("Event from RM: shutting down Application Master");
             // This can happen if the RM has been restarted. If it is in that state,
             // this application must clean itself up.
             eventHandler.Handle(new JobEvent(this.GetJob().GetID(), JobEventType.JobAmReboot)
                                 );
             throw new YarnRuntimeException("Resource Manager doesn't recognize AttemptId: " +
                                            this.GetContext().GetApplicationID(), e);
         }
         catch (ApplicationMasterNotRegisteredException)
         {
             Log.Info("ApplicationMaster is out of sync with ResourceManager," + " hence resync and send outstanding requests."
                      );
             this.lastResponseID = 0;
             Register();
         }
         catch (Exception e)
         {
             // This can happen when the connection to the RM has gone down. Keep
             // re-trying until the retryInterval has expired.
             if (Runtime.CurrentTimeMillis() - retrystartTime >= retryInterval)
             {
                 Log.Error("Could not contact RM after " + retryInterval + " milliseconds.");
                 eventHandler.Handle(new JobEvent(this.GetJob().GetID(), JobEventType.InternalError
                                                  ));
                 throw new YarnRuntimeException("Could not contact RM after " + retryInterval + " milliseconds."
                                                );
             }
             // Throw this up to the caller, which may decide to ignore it and
             // continue to attempt to contact the RM.
             throw;
         }
         if (allocateResponse != null)
         {
             this.lastResponseID = allocateResponse.GetResponseId();
             Token token = allocateResponse.GetAMRMToken();
             if (token != null)
             {
                 UpdateAMRMToken(token);
             }
         }
     }
 }
예제 #9
0
        // Keyboard input handling -each keypress can be interpreted as
        // a control command.
        void KeyboardControl()
        {
            while (true)
            {
                switch (Console.ReadKey(true).Key)
                {
                case ConsoleKey.T:
                    transportFunction.PrintRouteTable();
                    break;

                case ConsoleKey.A:
                    LRM.PrintAssignments();
                    break;

                case ConsoleKey.P:
                    if (Log.IsPaused)
                    {
                        Log.Unpause();
                    }
                    else
                    {
                        Log.Pause();
                    }
                    break;

#if DEBUG
                case ConsoleKey.Enter:
                    MPLSPacket testPacket = new MPLSPacket(new int[] { 2137 }, "This is a test MPLSMessage.");
                    transportFunction.EnqueuePacketOnFirstQueue(testPacket);
                    break;

                case ConsoleKey.U:
                    NHLFEntry        entry         = new NHLFEntry(10, 1, 17, true, 2, new int[] { 35 });
                    AddUpdateRequest testUpdateReq = new AddUpdateRequest("Helo it me", mgmtLocalPort, 2137, entry);
                    SendManagementMsg(mgmtLocalPort, testUpdateReq);
                    break;

                case ConsoleKey.R:
                    RemoveRequest testRemoveReq = new RemoveRequest("Helo it me", mgmtLocalPort, 2137, 10);
                    SendManagementMsg(mgmtLocalPort, testRemoveReq);
                    break;

                case ConsoleKey.L:
                    AllocateRequest testAllocReq = new AllocateRequest(id, mgmtLocalPort, allocCounter++, 1, 30, 1);
                    SendManagementMsg(mgmtLocalPort, testAllocReq);
                    break;

                case ConsoleKey.D:
                    DeallocateRequest testDeallocReq = new DeallocateRequest(id, mgmtLocalPort, allocCounter--, 1);
                    SendManagementMsg(mgmtLocalPort, testDeallocReq);
                    break;
#endif
                default:
                    break;
                }
            }
        }
        /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
        /// <exception cref="System.IO.IOException"/>
        public virtual void TestAllocateOnHA()
        {
            AllocateRequest request = AllocateRequest.NewInstance(0, 50f, new AList <ResourceRequest
                                                                                     >(), new AList <ContainerId>(), ResourceBlacklistRequest.NewInstance(new AList <string
                                                                                                                                                                     >(), new AList <string>()));
            AllocateResponse response = amClient.Allocate(request);

            NUnit.Framework.Assert.AreEqual(response, this.cluster.CreateFakeAllocateResponse
                                                ());
        }
예제 #11
0
        /// <exception cref="System.Exception"/>
        public virtual AllocateResponse Allocate(AllocateRequest allocateRequest)
        {
            UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser(attemptId.ToString
                                                                                 ());

            Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> token = context.GetRMApps
                                                                                     ()[attemptId.GetApplicationId()].GetRMAppAttempt(attemptId).GetAMRMToken();
            ugi.AddTokenIdentifier(token.DecodeIdentifier());
            lastResponse = DoAllocateAs(ugi, allocateRequest);
            return(lastResponse);
        }
예제 #12
0
        public virtual void TestValidateResourceBlacklistRequest()
        {
            TestAMAuthorization.MyContainerManager containerManager = new TestAMAuthorization.MyContainerManager
                                                                          ();
            TestAMAuthorization.MockRMWithAMS rm = new TestAMAuthorization.MockRMWithAMS(new
                                                                                         YarnConfiguration(), containerManager);
            rm.Start();
            MockNM nm1 = rm.RegisterNode("localhost:1234", 5120);
            IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
                                                                               , string>(2);

            acls[ApplicationAccessType.ViewApp] = "*";
            RMApp app = rm.SubmitApp(1024, "appname", "appuser", acls);

            nm1.NodeHeartbeat(true);
            RMAppAttempt         attempt = app.GetCurrentAppAttempt();
            ApplicationAttemptId applicationAttemptId = attempt.GetAppAttemptId();

            WaitForLaunchedState(attempt);
            // Create a client to the RM.
            Configuration        conf        = rm.GetConfig();
            YarnRPC              rpc         = YarnRPC.Create(conf);
            UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(applicationAttemptId
                                                                                     .ToString());
            Credentials credentials   = containerManager.GetContainerCredentials();
            IPEndPoint  rmBindAddress = rm.GetApplicationMasterService().GetBindAddress();

            Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> amRMToken = TestAMAuthorization.MockRMWithAMS
                                                                                 .SetupAndReturnAMRMToken(rmBindAddress, credentials.GetAllTokens());
            currentUser.AddToken(amRMToken);
            ApplicationMasterProtocol client = currentUser.DoAs(new _PrivilegedAction_626(rpc
                                                                                          , rmBindAddress, conf));
            RegisterApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
                                                       <RegisterApplicationMasterRequest>();

            client.RegisterApplicationMaster(request);
            ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.NewInstance(
                Sharpen.Collections.SingletonList(ResourceRequest.Any), null);
            AllocateRequest allocateRequest = AllocateRequest.NewInstance(0, 0.0f, null, null
                                                                          , blacklistRequest);
            bool error = false;

            try
            {
                client.Allocate(allocateRequest);
            }
            catch (InvalidResourceBlacklistRequestException)
            {
                error = true;
            }
            rm.Stop();
            NUnit.Framework.Assert.IsTrue("Didn't not catch InvalidResourceBlacklistRequestException"
                                          , error);
        }
예제 #13
0
        public virtual void TestAllcoateRequestWithoutIncrease()
        {
            AllocateRequest r = AllocateRequest.NewInstance(123, 0f, null, null, null, null);

            // serde
            YarnServiceProtos.AllocateRequestProto p = ((AllocateRequestPBImpl)r).GetProto();
            r = new AllocateRequestPBImpl(p);
            // check value
            NUnit.Framework.Assert.AreEqual(123, r.GetResponseId());
            NUnit.Framework.Assert.AreEqual(0, r.GetIncreaseRequests().Count);
        }
예제 #14
0
        /// <exception cref="System.Exception"/>
        private AllocateResponse Allocate(ApplicationAttemptId attemptId, AllocateRequest
                                          req)
        {
            UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser(attemptId.ToString
                                                                                 ());

            Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> token = rm.GetRMContext
                                                                                     ().GetRMApps()[attemptId.GetApplicationId()].GetRMAppAttempt(attemptId).GetAMRMToken
                                                                                     ();
            ugi.AddTokenIdentifier(token.DecodeIdentifier());
            return(ugi.DoAs(new _PrivilegedExceptionAction_67(this, req)));
        }
예제 #15
0
 /// <exception cref="System.Exception"/>
 public virtual AllocateResponse DoAllocateAs(UserGroupInformation ugi, AllocateRequest
                                              req)
 {
     req.SetResponseId(++responseId);
     try
     {
         return(ugi.DoAs(new _PrivilegedExceptionAction_234(this, req)));
     }
     catch (UndeclaredThrowableException e)
     {
         throw (Exception)e.InnerException;
     }
 }
예제 #16
0
 /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
 /// <exception cref="System.IO.IOException"/>
 public virtual AllocateResponse Allocate(AllocateRequest request)
 {
     YarnServiceProtos.AllocateRequestProto requestProto = ((AllocateRequestPBImpl)request
                                                            ).GetProto();
     try
     {
         return(new AllocateResponsePBImpl(proxy.Allocate(null, requestProto)));
     }
     catch (ServiceException e)
     {
         RPCUtil.UnwrapAndThrowException(e);
         return(null);
     }
 }
예제 #17
0
        public void EncodeToBytesWithCredential()
        {
            var request = new AllocateRequest(777)
            {
                TransactionId = new byte[]
                {
                    0xf5, 0xdb, 0xe7, 0xc1, 0x2a, 0x74, 0xbe, 0xf9, 0x8b, 0x16,
                    0x56, 0x3e,
                }
            };
            var ctx = new TestStunContext()
            {
                Username = "******",
                Password = "******",
                Realm    = "twilio.com",
                Nonce    = new byte[]
                {
                    0x37, 0x35, 0x64, 0x34, 0x35, 0x34, 0x31, 0x39, 0x63, 0x33,
                    0x39, 0x33, 0x34, 0x33, 0x66, 0x65,
                },
            };

            Assert.Equal(
                new byte[]
            {
                0x00, 0x03, 0x00, 0x98, 0x21, 0x12, 0xa4, 0x42, 0xf5, 0xdb,
                0xe7, 0xc1, 0x2a, 0x74, 0xbe, 0xf9, 0x8b, 0x16, 0x56, 0x3e,
                0x00, 0x19, 0x00, 0x04, 0x06, 0x00, 0x00, 0x00, 0x00, 0x0d,
                0x00, 0x04, 0x00, 0x00, 0x03, 0x09, 0x00, 0x06, 0x00, 0x40,
                0x61, 0x65, 0x30, 0x36, 0x33, 0x33, 0x63, 0x64, 0x35, 0x38,
                0x62, 0x61, 0x30, 0x39, 0x37, 0x61, 0x31, 0x31, 0x36, 0x37,
                0x63, 0x36, 0x64, 0x32, 0x63, 0x63, 0x34, 0x65, 0x32, 0x33,
                0x36, 0x64, 0x62, 0x35, 0x32, 0x32, 0x35, 0x36, 0x61, 0x34,
                0x30, 0x64, 0x35, 0x36, 0x35, 0x66, 0x31, 0x31, 0x65, 0x64,
                0x66, 0x37, 0x36, 0x63, 0x30, 0x32, 0x64, 0x31, 0x33, 0x64,
                0x62, 0x39, 0x33, 0x63, 0x00, 0x15, 0x00, 0x10, 0x37, 0x35,
                0x64, 0x34, 0x35, 0x34, 0x31, 0x39, 0x63, 0x33, 0x39, 0x33,
                0x34, 0x33, 0x66, 0x65, 0x00, 0x14, 0x00, 0x0a, 0x74, 0x77,
                0x69, 0x6c, 0x69, 0x6f, 0x2e, 0x63, 0x6f, 0x6d, 0x00, 0x00,
                0x00, 0x08, 0x00, 0x14, 0x77, 0xe8, 0xcf, 0x30, 0x9e, 0x85,
                0x6c, 0x22, 0x72, 0x53, 0xa3, 0xb7, 0xe0, 0x35, 0x7c, 0xc2,
                0x30, 0xfc, 0xbc, 0xf4, 0x80, 0x28, 0x00, 0x04, 0x98, 0x46,
                0x9c, 0x07,
            },
                request.Encode(ctx));
        }
예제 #18
0
 /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
 /// <exception cref="System.IO.IOException"/>
 public virtual AllocateResponse Allocate(AllocateRequest request)
 {
     NUnit.Framework.Assert.AreEqual("response ID mismatch", responseId, request.GetResponseId
                                         ());
     ++responseId;
     Org.Apache.Hadoop.Yarn.Api.Records.Token yarnToken = null;
     if (amToken != null)
     {
         yarnToken = Org.Apache.Hadoop.Yarn.Api.Records.Token.NewInstance(amToken.GetIdentifier
                                                                              (), amToken.GetKind().ToString(), amToken.GetPassword(), amToken.GetService().ToString
                                                                              ());
     }
     return(AllocateResponse.NewInstance(responseId, Collections.EmptyList <ContainerStatus
                                                                            >(), Collections.EmptyList <Container>(), Collections.EmptyList <NodeReport>(), Resources
                                         .None(), null, 1, null, Collections.EmptyList <NMToken>(), yarnToken, Collections
                                         .EmptyList <ContainerResourceIncrease>(), Collections.EmptyList <ContainerResourceDecrease
                                                                                                          >()));
 }
예제 #19
0
        /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
        /// <exception cref="System.IO.IOException"/>
        protected internal virtual AllocateResponse MakeRemoteRequest()
        {
            ApplyRequestLimits();
            ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.NewInstance(
                new AList <string>(blacklistAdditions), new AList <string>(blacklistRemovals));
            AllocateRequest allocateRequest = AllocateRequest.NewInstance(lastResponseID, base
                                                                          .GetApplicationProgress(), new AList <ResourceRequest>(ask), new AList <ContainerId
                                                                                                                                                  >(release), blacklistRequest);
            AllocateResponse allocateResponse = scheduler.Allocate(allocateRequest);

            lastResponseID     = allocateResponse.GetResponseId();
            availableResources = allocateResponse.GetAvailableResources();
            lastClusterNmCount = clusterNmCount;
            clusterNmCount     = allocateResponse.GetNumClusterNodes();
            int numCompletedContainers = allocateResponse.GetCompletedContainersStatuses().Count;

            if (ask.Count > 0 || release.Count > 0)
            {
                Log.Info("getResources() for " + applicationId + ":" + " ask=" + ask.Count + " release= "
                         + release.Count + " newContainers=" + allocateResponse.GetAllocatedContainers()
                         .Count + " finishedContainers=" + numCompletedContainers + " resourcelimit=" + availableResources
                         + " knownNMs=" + clusterNmCount);
            }
            ask.Clear();
            release.Clear();
            if (numCompletedContainers > 0)
            {
                // re-send limited requests when a container completes to trigger asking
                // for more containers
                Sharpen.Collections.AddAll(requestLimitsToUpdate, requestLimits.Keys);
            }
            if (blacklistAdditions.Count > 0 || blacklistRemovals.Count > 0)
            {
                Log.Info("Update the blacklist for " + applicationId + ": blacklistAdditions=" +
                         blacklistAdditions.Count + " blacklistRemovals=" + blacklistRemovals.Count);
            }
            blacklistAdditions.Clear();
            blacklistRemovals.Clear();
            return(allocateResponse);
        }
예제 #20
0
        public void EncodeToBytes()
        {
            var request = new AllocateRequest(777)
            {
                TransactionId = new byte[]
                {
                    0xee, 0x29, 0xdd, 0x2d, 0x7a, 0xe9, 0x9c, 0xf4, 0x00, 0x82,
                    0xf2, 0x5e,
                }
            };

            Assert.Equal(
                new byte[]
            {
                0x00, 0x03, 0x00, 0x18, 0x21, 0x12, 0xa4, 0x42, 0xee, 0x29,
                0xdd, 0x2d, 0x7a, 0xe9, 0x9c, 0xf4, 0x00, 0x82, 0xf2, 0x5e,
                0x00, 0x19, 0x00, 0x04, 0x06, 0x00, 0x00, 0x00, 0x00, 0x0d,
                0x00, 0x04, 0x00, 0x00, 0x03, 0x09, 0x80, 0x28, 0x00, 0x04,
                0x3d, 0xbe, 0x42, 0x75,
            },
                request.Encode(new TestStunContext())
                );
        }
예제 #21
0
        public virtual void TestARRMResponseId()
        {
            MockNM nm1 = rm.RegisterNode("h1:1234", 5000);
            RMApp  app = rm.SubmitApp(2000);

            // Trigger the scheduling so the AM gets 'launched'
            nm1.NodeHeartbeat(true);
            RMAppAttempt attempt = app.GetCurrentAppAttempt();
            MockAM       am      = rm.SendAMLaunched(attempt.GetAppAttemptId());

            am.RegisterAppAttempt();
            AllocateRequest allocateRequest = AllocateRequest.NewInstance(0, 0F, null, null,
                                                                          null);
            AllocateResponse response = Allocate(attempt.GetAppAttemptId(), allocateRequest);

            NUnit.Framework.Assert.AreEqual(1, response.GetResponseId());
            NUnit.Framework.Assert.IsTrue(response.GetAMCommand() == null);
            allocateRequest = AllocateRequest.NewInstance(response.GetResponseId(), 0F, null,
                                                          null, null);
            response = Allocate(attempt.GetAppAttemptId(), allocateRequest);
            NUnit.Framework.Assert.AreEqual(2, response.GetResponseId());
            /* try resending */
            response = Allocate(attempt.GetAppAttemptId(), allocateRequest);
            NUnit.Framework.Assert.AreEqual(2, response.GetResponseId());
            allocateRequest = AllocateRequest.NewInstance(0, 0F, null, null, null);
            try
            {
                Allocate(attempt.GetAppAttemptId(), allocateRequest);
                NUnit.Framework.Assert.Fail();
            }
            catch (Exception e)
            {
                NUnit.Framework.Assert.IsTrue(e.InnerException is InvalidApplicationMasterRequestException
                                              );
            }
        }
예제 #22
0
        public virtual void TestTokenExpiry()
        {
            TestAMAuthorization.MyContainerManager containerManager = new TestAMAuthorization.MyContainerManager
                                                                          ();
            TestAMAuthorization.MockRMWithAMS rm = new TestAMAuthorization.MockRMWithAMS(conf
                                                                                         , containerManager);
            rm.Start();
            Configuration             conf     = rm.GetConfig();
            YarnRPC                   rpc      = YarnRPC.Create(conf);
            ApplicationMasterProtocol rmClient = null;

            try
            {
                MockNM nm1 = rm.RegisterNode("localhost:1234", 5120);
                RMApp  app = rm.SubmitApp(1024);
                nm1.NodeHeartbeat(true);
                int waitCount = 0;
                while (containerManager.containerTokens == null && waitCount++ < 20)
                {
                    Log.Info("Waiting for AM Launch to happen..");
                    Sharpen.Thread.Sleep(1000);
                }
                NUnit.Framework.Assert.IsNotNull(containerManager.containerTokens);
                RMAppAttempt         attempt = app.GetCurrentAppAttempt();
                ApplicationAttemptId applicationAttemptId = attempt.GetAppAttemptId();
                // Create a client to the RM.
                UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(applicationAttemptId
                                                                                         .ToString());
                Credentials credentials   = containerManager.GetContainerCredentials();
                IPEndPoint  rmBindAddress = rm.GetApplicationMasterService().GetBindAddress();
                Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> amRMToken = TestAMAuthorization.MockRMWithAMS
                                                                                     .SetupAndReturnAMRMToken(rmBindAddress, credentials.GetAllTokens());
                currentUser.AddToken(amRMToken);
                rmClient = CreateRMClient(rm, conf, rpc, currentUser);
                RegisterApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
                                                           <RegisterApplicationMasterRequest>();
                rmClient.RegisterApplicationMaster(request);
                FinishApplicationMasterRequest finishAMRequest = Org.Apache.Hadoop.Yarn.Util.Records
                                                                 .NewRecord <FinishApplicationMasterRequest>();
                finishAMRequest.SetFinalApplicationStatus(FinalApplicationStatus.Succeeded);
                finishAMRequest.SetDiagnostics("diagnostics");
                finishAMRequest.SetTrackingUrl("url");
                rmClient.FinishApplicationMaster(finishAMRequest);
                // Send RMAppAttemptEventType.CONTAINER_FINISHED to transit RMAppAttempt
                // from Finishing state to Finished State. Both AMRMToken and
                // ClientToAMToken will be removed.
                ContainerStatus containerStatus = BuilderUtils.NewContainerStatus(attempt.GetMasterContainer
                                                                                      ().GetId(), ContainerState.Complete, "AM Container Finished", 0);
                rm.GetRMContext().GetDispatcher().GetEventHandler().Handle(new RMAppAttemptContainerFinishedEvent
                                                                               (applicationAttemptId, containerStatus, nm1.GetNodeId()));
                // Make sure the RMAppAttempt is at Finished State.
                // Both AMRMToken and ClientToAMToken have been removed.
                int count = 0;
                while (attempt.GetState() != RMAppAttemptState.Finished && count < maxWaitAttempts
                       )
                {
                    Sharpen.Thread.Sleep(100);
                    count++;
                }
                NUnit.Framework.Assert.IsTrue(attempt.GetState() == RMAppAttemptState.Finished);
                // Now simulate trying to allocate. RPC call itself should throw auth
                // exception.
                rpc.StopProxy(rmClient, conf);
                // To avoid using cached client
                rmClient = CreateRMClient(rm, conf, rpc, currentUser);
                AllocateRequest allocateRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest
                                                                                                 >();
                try
                {
                    rmClient.Allocate(allocateRequest);
                    NUnit.Framework.Assert.Fail("You got to be kidding me! " + "Using App tokens after app-finish should fail!"
                                                );
                }
                catch (Exception t)
                {
                    Log.Info("Exception found is ", t);
                    // The exception will still have the earlier appAttemptId as it picks it
                    // up from the token.
                    NUnit.Framework.Assert.IsTrue(t.InnerException.Message.Contains(applicationAttemptId
                                                                                    .ToString() + " not found in AMRMTokenSecretManager."));
                }
            }
            finally
            {
                rm.Stop();
                if (rmClient != null)
                {
                    rpc.StopProxy(rmClient, conf);
                }
            }
        }
예제 #23
0
        public virtual void TestMasterKeyRollOver()
        {
            conf.SetLong(YarnConfiguration.RmAmrmTokenMasterKeyRollingIntervalSecs, rolling_interval_sec
                         );
            conf.SetLong(YarnConfiguration.RmAmExpiryIntervalMs, am_expire_ms);
            TestAMAuthorization.MyContainerManager containerManager = new TestAMAuthorization.MyContainerManager
                                                                          ();
            TestAMAuthorization.MockRMWithAMS rm = new TestAMAuthorization.MockRMWithAMS(conf
                                                                                         , containerManager);
            rm.Start();
            long                      startTime             = Runtime.CurrentTimeMillis();
            Configuration             conf                  = rm.GetConfig();
            YarnRPC                   rpc                   = YarnRPC.Create(conf);
            ApplicationMasterProtocol rmClient              = null;
            AMRMTokenSecretManager    appTokenSecretManager = rm.GetRMContext().GetAMRMTokenSecretManager
                                                                  ();
            MasterKeyData oldKey = appTokenSecretManager.GetMasterKey();

            NUnit.Framework.Assert.IsNotNull(oldKey);
            try
            {
                MockNM nm1 = rm.RegisterNode("localhost:1234", 5120);
                RMApp  app = rm.SubmitApp(1024);
                nm1.NodeHeartbeat(true);
                int waitCount = 0;
                while (containerManager.containerTokens == null && waitCount++ < maxWaitAttempts)
                {
                    Log.Info("Waiting for AM Launch to happen..");
                    Sharpen.Thread.Sleep(1000);
                }
                NUnit.Framework.Assert.IsNotNull(containerManager.containerTokens);
                RMAppAttempt         attempt = app.GetCurrentAppAttempt();
                ApplicationAttemptId applicationAttemptId = attempt.GetAppAttemptId();
                // Create a client to the RM.
                UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(applicationAttemptId
                                                                                         .ToString());
                Credentials credentials   = containerManager.GetContainerCredentials();
                IPEndPoint  rmBindAddress = rm.GetApplicationMasterService().GetBindAddress();
                Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> amRMToken = TestAMAuthorization.MockRMWithAMS
                                                                                     .SetupAndReturnAMRMToken(rmBindAddress, credentials.GetAllTokens());
                currentUser.AddToken(amRMToken);
                rmClient = CreateRMClient(rm, conf, rpc, currentUser);
                RegisterApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
                                                           <RegisterApplicationMasterRequest>();
                rmClient.RegisterApplicationMaster(request);
                // One allocate call.
                AllocateRequest allocateRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest
                                                                                                 >();
                NUnit.Framework.Assert.IsTrue(rmClient.Allocate(allocateRequest).GetAMCommand() ==
                                              null);
                // Wait for enough time and make sure the roll_over happens
                // At mean time, the old AMRMToken should continue to work
                while (Runtime.CurrentTimeMillis() - startTime < rolling_interval_sec * 1000)
                {
                    rmClient.Allocate(allocateRequest);
                    Sharpen.Thread.Sleep(500);
                }
                MasterKeyData newKey = appTokenSecretManager.GetMasterKey();
                NUnit.Framework.Assert.IsNotNull(newKey);
                NUnit.Framework.Assert.IsFalse("Master key should have changed!", oldKey.Equals(newKey
                                                                                                ));
                // Another allocate call with old AMRMToken. Should continue to work.
                rpc.StopProxy(rmClient, conf);
                // To avoid using cached client
                rmClient = CreateRMClient(rm, conf, rpc, currentUser);
                NUnit.Framework.Assert.IsTrue(rmClient.Allocate(allocateRequest).GetAMCommand() ==
                                              null);
                waitCount = 0;
                while (waitCount++ <= maxWaitAttempts)
                {
                    if (appTokenSecretManager.GetCurrnetMasterKeyData() != oldKey)
                    {
                        break;
                    }
                    try
                    {
                        rmClient.Allocate(allocateRequest);
                    }
                    catch (Exception)
                    {
                        break;
                    }
                    Sharpen.Thread.Sleep(200);
                }
                // active the nextMasterKey, and replace the currentMasterKey
                NUnit.Framework.Assert.IsTrue(appTokenSecretManager.GetCurrnetMasterKeyData().Equals
                                                  (newKey));
                NUnit.Framework.Assert.IsTrue(appTokenSecretManager.GetMasterKey().Equals(newKey)
                                              );
                NUnit.Framework.Assert.IsTrue(appTokenSecretManager.GetNextMasterKeyData() == null
                                              );
                // Create a new Token
                Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> newToken = appTokenSecretManager
                                                                                        .CreateAndGetAMRMToken(applicationAttemptId);
                SecurityUtil.SetTokenService(newToken, rmBindAddress);
                currentUser.AddToken(newToken);
                // Another allocate call. Should continue to work.
                rpc.StopProxy(rmClient, conf);
                // To avoid using cached client
                rmClient        = CreateRMClient(rm, conf, rpc, currentUser);
                allocateRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest>(
                    );
                NUnit.Framework.Assert.IsTrue(rmClient.Allocate(allocateRequest).GetAMCommand() ==
                                              null);
                // Should not work by using the old AMRMToken.
                rpc.StopProxy(rmClient, conf);
                // To avoid using cached client
                try
                {
                    currentUser.AddToken(amRMToken);
                    rmClient        = CreateRMClient(rm, conf, rpc, currentUser);
                    allocateRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest>(
                        );
                    NUnit.Framework.Assert.IsTrue(rmClient.Allocate(allocateRequest).GetAMCommand() ==
                                                  null);
                    NUnit.Framework.Assert.Fail("The old Token should not work");
                }
                catch (Exception)
                {
                }
            }
            finally
            {
                // expect exception
                rm.Stop();
                if (rmClient != null)
                {
                    rpc.StopProxy(rmClient, conf);
                }
            }
        }
예제 #24
0
        // The test verifies processing of NMContainerStatuses which are sent during
        // NM registration.
        // 1. Start the cluster-RM,NM,Submit app with 1024MB,Launch & register AM
        // 2. AM sends ResourceRequest for 1 container with memory 2048MB.
        // 3. Verify for number of container allocated by RM
        // 4. Verify Memory Usage by cluster, it should be 3072. AM memory + requested
        // memory. 1024 + 2048=3072
        // 5. Re-register NM by sending completed container status
        // 6. Verify for Memory Used, it should be 1024
        // 7. Send AM heatbeat to RM. Allocated response should contain completed
        // container.
        /// <exception cref="System.Exception"/>
        public virtual void TestProcessingNMContainerStatusesOnNMRestart()
        {
            conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1);
            MemoryRMStateStore memStore = new MemoryRMStateStore();

            memStore.Init(conf);
            // 1. Start the cluster-RM,NM,Submit app with 1024MB,Launch & register AM
            MockRM rm1 = new MockRM(conf, memStore);

            rm1.Start();
            int    nmMemory        = 8192;
            int    amMemory        = 1024;
            int    containerMemory = 2048;
            MockNM nm1             = new MockNM("127.0.0.1:1234", nmMemory, rm1.GetResourceTrackerService
                                                    ());

            nm1.RegisterNode();
            RMApp  app0 = rm1.SubmitApp(amMemory);
            MockAM am0  = MockRM.LaunchAndRegisterAM(app0, rm1, nm1);
            // 2. AM sends ResourceRequest for 1 container with memory 2048MB.
            int noOfContainers = 1;
            IList <Container> allocateContainers = am0.AllocateAndWaitForContainers(noOfContainers
                                                                                    , containerMemory, nm1);

            // 3. Verify for number of container allocated by RM
            NUnit.Framework.Assert.AreEqual(noOfContainers, allocateContainers.Count);
            Container container = allocateContainers[0];

            nm1.NodeHeartbeat(am0.GetApplicationAttemptId(), 1, ContainerState.Running);
            nm1.NodeHeartbeat(am0.GetApplicationAttemptId(), container.GetId().GetContainerId
                                  (), ContainerState.Running);
            rm1.WaitForState(app0.GetApplicationId(), RMAppState.Running);
            // 4. Verify Memory Usage by cluster, it should be 3072. AM memory +
            // requested memory. 1024 + 2048=3072
            ResourceScheduler rs = rm1.GetRMContext().GetScheduler();
            int allocatedMB      = rs.GetRootQueueMetrics().GetAllocatedMB();

            NUnit.Framework.Assert.AreEqual(amMemory + containerMemory, allocatedMB);
            // 5. Re-register NM by sending completed container status
            IList <NMContainerStatus> nMContainerStatusForApp = CreateNMContainerStatusForApp(
                am0);

            nm1.RegisterNode(nMContainerStatusForApp, Arrays.AsList(app0.GetApplicationId()));
            WaitForClusterMemory(nm1, rs, amMemory);
            // 6. Verify for Memory Used, it should be 1024
            NUnit.Framework.Assert.AreEqual(amMemory, rs.GetRootQueueMetrics().GetAllocatedMB
                                                ());
            // 7. Send AM heatbeat to RM. Allocated response should contain completed
            // container
            AllocateRequest req = AllocateRequest.NewInstance(0, 0F, new AList <ResourceRequest
                                                                                >(), new AList <ContainerId>(), null);
            AllocateResponse        allocate = am0.Allocate(req);
            IList <ContainerStatus> completedContainersStatuses = allocate.GetCompletedContainersStatuses
                                                                      ();

            NUnit.Framework.Assert.AreEqual(noOfContainers, completedContainersStatuses.Count
                                            );
            // Application clean up should happen Cluster memory used is 0
            nm1.NodeHeartbeat(am0.GetApplicationAttemptId(), 1, ContainerState.Complete);
            WaitForClusterMemory(nm1, rs, 0);
            rm1.Stop();
        }
        /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
        /// <exception cref="System.IO.IOException"/>
        public virtual AllocateResponse Allocate(AllocateRequest request)
        {
            AMRMTokenIdentifier  amrmTokenIdentifier = AuthorizeRequest();
            ApplicationAttemptId appAttemptId        = amrmTokenIdentifier.GetApplicationAttemptId();
            ApplicationId        applicationId       = appAttemptId.GetApplicationId();

            this.amLivelinessMonitor.ReceivedPing(appAttemptId);
            /* check if its in cache */
            ApplicationMasterService.AllocateResponseLock Lock = responseMap[appAttemptId];
            if (Lock == null)
            {
                string message = "Application attempt " + appAttemptId + " doesn't exist in ApplicationMasterService cache.";
                Log.Error(message);
                throw new ApplicationAttemptNotFoundException(message);
            }
            lock (Lock)
            {
                AllocateResponse lastResponse = Lock.GetAllocateResponse();
                if (!HasApplicationMasterRegistered(appAttemptId))
                {
                    string message = "AM is not registered for known application attempt: " + appAttemptId
                                     + " or RM had restarted after AM registered . AM should re-register.";
                    Log.Info(message);
                    RMAuditLogger.LogFailure(this.rmContext.GetRMApps()[appAttemptId.GetApplicationId
                                                                            ()].GetUser(), RMAuditLogger.AuditConstants.AmAllocate, string.Empty, "ApplicationMasterService"
                                             , message, applicationId, appAttemptId);
                    throw new ApplicationMasterNotRegisteredException(message);
                }
                if ((request.GetResponseId() + 1) == lastResponse.GetResponseId())
                {
                    /* old heartbeat */
                    return(lastResponse);
                }
                else
                {
                    if (request.GetResponseId() + 1 < lastResponse.GetResponseId())
                    {
                        string message = "Invalid responseId in AllocateRequest from application attempt: "
                                         + appAttemptId + ", expect responseId to be " + (lastResponse.GetResponseId() +
                                                                                          1);
                        throw new InvalidApplicationMasterRequestException(message);
                    }
                }
                //filter illegal progress values
                float filteredProgress = request.GetProgress();
                if (float.IsNaN(filteredProgress) || filteredProgress == float.NegativeInfinity ||
                    filteredProgress < 0)
                {
                    request.SetProgress(0);
                }
                else
                {
                    if (filteredProgress > 1 || filteredProgress == float.PositiveInfinity)
                    {
                        request.SetProgress(1);
                    }
                }
                // Send the status update to the appAttempt.
                this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMAppAttemptStatusupdateEvent
                                                                            (appAttemptId, request.GetProgress()));
                IList <ResourceRequest>  ask                = request.GetAskList();
                IList <ContainerId>      release            = request.GetReleaseList();
                ResourceBlacklistRequest blacklistRequest   = request.GetResourceBlacklistRequest();
                IList <string>           blacklistAdditions = (blacklistRequest != null) ? blacklistRequest.
                                                              GetBlacklistAdditions() : Sharpen.Collections.EmptyList;
                IList <string> blacklistRemovals = (blacklistRequest != null) ? blacklistRequest.GetBlacklistRemovals
                                                       () : Sharpen.Collections.EmptyList;
                RMApp app = this.rmContext.GetRMApps()[applicationId];
                // set label expression for Resource Requests if resourceName=ANY
                ApplicationSubmissionContext asc = app.GetApplicationSubmissionContext();
                foreach (ResourceRequest req in ask)
                {
                    if (null == req.GetNodeLabelExpression() && ResourceRequest.Any.Equals(req.GetResourceName
                                                                                               ()))
                    {
                        req.SetNodeLabelExpression(asc.GetNodeLabelExpression());
                    }
                }
                // sanity check
                try
                {
                    RMServerUtils.NormalizeAndValidateRequests(ask, rScheduler.GetMaximumResourceCapability
                                                                   (), app.GetQueue(), rScheduler, rmContext);
                }
                catch (InvalidResourceRequestException e)
                {
                    Log.Warn("Invalid resource ask by application " + appAttemptId, e);
                    throw;
                }
                try
                {
                    RMServerUtils.ValidateBlacklistRequest(blacklistRequest);
                }
                catch (InvalidResourceBlacklistRequestException e)
                {
                    Log.Warn("Invalid blacklist request by application " + appAttemptId, e);
                    throw;
                }
                // In the case of work-preserving AM restart, it's possible for the
                // AM to release containers from the earlier attempt.
                if (!app.GetApplicationSubmissionContext().GetKeepContainersAcrossApplicationAttempts
                        ())
                {
                    try
                    {
                        RMServerUtils.ValidateContainerReleaseRequest(release, appAttemptId);
                    }
                    catch (InvalidContainerReleaseException e)
                    {
                        Log.Warn("Invalid container release by application " + appAttemptId, e);
                        throw;
                    }
                }
                // Send new requests to appAttempt.
                Allocation allocation = this.rScheduler.Allocate(appAttemptId, ask, release, blacklistAdditions
                                                                 , blacklistRemovals);
                if (!blacklistAdditions.IsEmpty() || !blacklistRemovals.IsEmpty())
                {
                    Log.Info("blacklist are updated in Scheduler." + "blacklistAdditions: " + blacklistAdditions
                             + ", " + "blacklistRemovals: " + blacklistRemovals);
                }
                RMAppAttempt     appAttempt       = app.GetRMAppAttempt(appAttemptId);
                AllocateResponse allocateResponse = recordFactory.NewRecordInstance <AllocateResponse
                                                                                     >();
                if (!allocation.GetContainers().IsEmpty())
                {
                    allocateResponse.SetNMTokens(allocation.GetNMTokens());
                }
                // update the response with the deltas of node status changes
                IList <RMNode> updatedNodes = new AList <RMNode>();
                if (app.PullRMNodeUpdates(updatedNodes) > 0)
                {
                    IList <NodeReport> updatedNodeReports = new AList <NodeReport>();
                    foreach (RMNode rmNode in updatedNodes)
                    {
                        SchedulerNodeReport schedulerNodeReport = rScheduler.GetNodeReport(rmNode.GetNodeID
                                                                                               ());
                        Resource used          = BuilderUtils.NewResource(0, 0);
                        int      numContainers = 0;
                        if (schedulerNodeReport != null)
                        {
                            used          = schedulerNodeReport.GetUsedResource();
                            numContainers = schedulerNodeReport.GetNumContainers();
                        }
                        NodeId     nodeId = rmNode.GetNodeID();
                        NodeReport report = BuilderUtils.NewNodeReport(nodeId, rmNode.GetState(), rmNode.
                                                                       GetHttpAddress(), rmNode.GetRackName(), used, rmNode.GetTotalCapability(), numContainers
                                                                       , rmNode.GetHealthReport(), rmNode.GetLastHealthReportTime(), rmNode.GetNodeLabels
                                                                           ());
                        updatedNodeReports.AddItem(report);
                    }
                    allocateResponse.SetUpdatedNodes(updatedNodeReports);
                }
                allocateResponse.SetAllocatedContainers(allocation.GetContainers());
                allocateResponse.SetCompletedContainersStatuses(appAttempt.PullJustFinishedContainers
                                                                    ());
                allocateResponse.SetResponseId(lastResponse.GetResponseId() + 1);
                allocateResponse.SetAvailableResources(allocation.GetResourceLimit());
                allocateResponse.SetNumClusterNodes(this.rScheduler.GetNumClusterNodes());
                // add preemption to the allocateResponse message (if any)
                allocateResponse.SetPreemptionMessage(GeneratePreemptionMessage(allocation));
                // update AMRMToken if the token is rolled-up
                MasterKeyData nextMasterKey = this.rmContext.GetAMRMTokenSecretManager().GetNextMasterKeyData
                                                  ();
                if (nextMasterKey != null && nextMasterKey.GetMasterKey().GetKeyId() != amrmTokenIdentifier
                    .GetKeyId())
                {
                    RMAppAttemptImpl appAttemptImpl = (RMAppAttemptImpl)appAttempt;
                    Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> amrmToken = appAttempt
                                                                                             .GetAMRMToken();
                    if (nextMasterKey.GetMasterKey().GetKeyId() != appAttemptImpl.GetAMRMTokenKeyId())
                    {
                        Log.Info("The AMRMToken has been rolled-over. Send new AMRMToken back" + " to application: "
                                 + applicationId);
                        amrmToken = rmContext.GetAMRMTokenSecretManager().CreateAndGetAMRMToken(appAttemptId
                                                                                                );
                        appAttemptImpl.SetAMRMToken(amrmToken);
                    }
                    allocateResponse.SetAMRMToken(Org.Apache.Hadoop.Yarn.Api.Records.Token.NewInstance
                                                      (amrmToken.GetIdentifier(), amrmToken.GetKind().ToString(), amrmToken.GetPassword
                                                          (), amrmToken.GetService().ToString()));
                }

                /*
                 * As we are updating the response inside the lock object so we don't
                 * need to worry about unregister call occurring in between (which
                 * removes the lock object).
                 */
                Lock.SetAllocateResponse(allocateResponse);
                return(allocateResponse);
            }
        }
예제 #26
0
        // Method invoked in a separate thread, sets up a UDP listener
        // that listens for anything on a management port, then detemines
        // what kind of message it is (with what information) and handles
        // the response.
        void ReceiveManagementMsg()
        {
            UdpClient  listener = new UdpClient(mgmtLocalPort);
            IPEndPoint groupEP  = new IPEndPoint(IPAddress.Parse("127.0.0.1"), mgmtLocalPort);

            try
            {
                while (true)
                {
                    byte[] bytes = listener.Receive(ref groupEP);

                    Communications.Message msg = Communications.Serialization.Deserialize(bytes);

                    switch (msg.messageType)
                    {
                    case "NHLF.AddUpdateRequest":
                        Log.WriteLine("[MGMT CCI] {1}, port {2}: {0})", msg.messageType, msg.senderID, msg.senderPort);
                        AddUpdateRequest addUpdateReq = (AddUpdateRequest)msg;
                        transportFunction.UpdateRoutingTable(addUpdateReq.entry, true);
                        // !!! IT CAN GO TERRIBLY WRONG HERE !!!
                        SendManagementMsg(
                            (ushort)addUpdateReq.senderPort,
                            new AddUpdateResponse(id, mgmtLocalPort, addUpdateReq.seq, true)
                            );
                        Log.WriteLine("[MGMT CCI] Add/update forward: conn {2}, iface {0}, label {1}", addUpdateReq.entry.interface_in, addUpdateReq.entry.label_in, addUpdateReq.entry.connectionID);
                        break;

                    case "NHLF.RemoveRequest":
                        Log.WriteLine("[MGMT CCI] {1}, port {2}: {0})", msg.messageType, msg.senderID, msg.senderPort);
                        RemoveRequest removeReq = (RemoveRequest)msg;
                        bool          status    = transportFunction.RemoveFromRoutingTable(removeReq.connectionID);
                        SendManagementMsg(
                            (ushort)removeReq.senderPort,
                            new RemoveResponse(id, mgmtLocalPort, removeReq.seq, status)
                            );
                        Log.WriteLine("[MGMT CCI] {0} entries for connection {1}", status ? "Removed" : "Could not remove", removeReq.connectionID);
                        break;

                    case "AllocateRequest":
                        Log.WriteLine("[MGMT LRM] {1}, port {2}: {0})", msg.messageType, msg.senderID, msg.senderPort);
                        AllocateRequest allocateReq = (AllocateRequest)msg;
                        uint            label       = LRM.AssignBandwidthOnInterface(allocateReq.interfaceID, (uint)allocateReq.bitrate, allocateReq.connectionID);
                        SendManagementMsg(
                            (ushort)allocateReq.senderPort,
                            new AllocateResponse(
                                id,
                                mgmtLocalPort,
                                (int)label,
                                allocateReq.seq
                                )
                            );
                        if (label != 0)
                        {
                            Log.WriteLine("[MGMT LRM] Allocate {0} Mb/s on iface {1} for connection {2})", allocateReq.bitrate, allocateReq.interfaceID, allocateReq.connectionID);
                        }
                        else
                        {
                            Log.WriteLine("[MGMT LRM] Could not allocate {0} Mb/s on iface {1} for connection {2})", allocateReq.bitrate, allocateReq.interfaceID, allocateReq.connectionID);
                        }
                        break;

                    case "DeallocateRequest":
                        Log.WriteLine("[MGMT LRM] {1}, port {2}: {0})", msg.messageType, msg.senderID, msg.senderPort);
                        DeallocateRequest deallocateReq = (DeallocateRequest)msg;
                        LRM.ReleaseAsignedBandwidth(deallocateReq.connectionID);
                        SendManagementMsg(
                            (ushort)deallocateReq.senderPort,
                            new DeallocateResponse(
                                id,
                                mgmtLocalPort,
                                deallocateReq.seq
                                )
                            );
                        Log.WriteLine("[MGMT LRM] Deallocate resources for connection {0})", deallocateReq.connectionID);
                        break;

#if DEBUG
                    case "NHLF.AddUpdateResponse":
                        AddUpdateResponse addUpdateResponse = (AddUpdateResponse)msg;
                        Log.WriteLine("[CCI] AddUpdateRequest for {0} {1}.", addUpdateResponse.senderID, addUpdateResponse.status ? "successful" : "failed");
                        break;

                    case "NHLF.RemoveResponse":
                        RemoveResponse removeResp = (RemoveResponse)msg;
                        Log.WriteLine("[CCI] Remove {0}", removeResp.status ? "some" : "none");
                        break;

                    case "LRMRC.LinkStateUpdate":
                        LinkStateUpdate linkStateUpdate = (LinkStateUpdate)msg;
                        Log.WriteLine("[RC] {0} --{2}-> {1}", linkStateUpdate.beginNode.id, linkStateUpdate.endNode.id, linkStateUpdate.capacity);
                        break;
#endif
                    default:
                        break;
                    }

                    // NHLFMgmtMessage mgmtMessage = NHLFSerialization.Deserialize(bytes);

                    //NHLFEntry newEntry = mgmtMessage.entry;

                    //string result = transportFunction.UpdateRoutingTable(newEntry, mgmtMessage.addOrSwap);
                }
            }
            catch (Exception e)
            {
                Log.WriteLine(e.ToString());
            }
            finally
            {
                listener.Close();
            }
        }
예제 #27
0
        /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
        /// <exception cref="System.IO.IOException"/>
        public override AllocateResponse Allocate(float progressIndicator)
        {
            Preconditions.CheckArgument(progressIndicator >= 0, "Progress indicator should not be negative"
                                        );
            AllocateResponse        allocateResponse  = null;
            IList <ResourceRequest> askList           = null;
            IList <ContainerId>     releaseList       = null;
            AllocateRequest         allocateRequest   = null;
            IList <string>          blacklistToAdd    = new AList <string>();
            IList <string>          blacklistToRemove = new AList <string>();

            try
            {
                lock (this)
                {
                    askList = new AList <ResourceRequest>(ask.Count);
                    foreach (ResourceRequest r in ask)
                    {
                        // create a copy of ResourceRequest as we might change it while the
                        // RPC layer is using it to send info across
                        askList.AddItem(ResourceRequest.NewInstance(r.GetPriority(), r.GetResourceName(),
                                                                    r.GetCapability(), r.GetNumContainers(), r.GetRelaxLocality(), r.GetNodeLabelExpression
                                                                        ()));
                    }
                    releaseList = new AList <ContainerId>(release);
                    // optimistically clear this collection assuming no RPC failure
                    ask.Clear();
                    release.Clear();
                    Sharpen.Collections.AddAll(blacklistToAdd, blacklistAdditions);
                    Sharpen.Collections.AddAll(blacklistToRemove, blacklistRemovals);
                    ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.NewInstance(
                        blacklistToAdd, blacklistToRemove);
                    allocateRequest = AllocateRequest.NewInstance(lastResponseId, progressIndicator,
                                                                  askList, releaseList, blacklistRequest);
                    // clear blacklistAdditions and blacklistRemovals before
                    // unsynchronized part
                    blacklistAdditions.Clear();
                    blacklistRemovals.Clear();
                }
                try
                {
                    allocateResponse = rmClient.Allocate(allocateRequest);
                }
                catch (ApplicationMasterNotRegisteredException)
                {
                    Log.Warn("ApplicationMaster is out of sync with ResourceManager," + " hence resyncing."
                             );
                    lock (this)
                    {
                        Sharpen.Collections.AddAll(release, this.pendingRelease);
                        Sharpen.Collections.AddAll(blacklistAdditions, this.blacklistedNodes);
                        foreach (IDictionary <string, SortedDictionary <Resource, AMRMClientImpl.ResourceRequestInfo
                                                                        > > rr in remoteRequestsTable.Values)
                        {
                            foreach (IDictionary <Resource, AMRMClientImpl.ResourceRequestInfo> capabalities in
                                     rr.Values)
                            {
                                foreach (AMRMClientImpl.ResourceRequestInfo request in capabalities.Values)
                                {
                                    AddResourceRequestToAsk(request.remoteRequest);
                                }
                            }
                        }
                    }
                    // re register with RM
                    RegisterApplicationMaster();
                    allocateResponse = Allocate(progressIndicator);
                    return(allocateResponse);
                }
                lock (this)
                {
                    // update these on successful RPC
                    clusterNodeCount          = allocateResponse.GetNumClusterNodes();
                    lastResponseId            = allocateResponse.GetResponseId();
                    clusterAvailableResources = allocateResponse.GetAvailableResources();
                    if (!allocateResponse.GetNMTokens().IsEmpty())
                    {
                        PopulateNMTokens(allocateResponse.GetNMTokens());
                    }
                    if (allocateResponse.GetAMRMToken() != null)
                    {
                        UpdateAMRMToken(allocateResponse.GetAMRMToken());
                    }
                    if (!pendingRelease.IsEmpty() && !allocateResponse.GetCompletedContainersStatuses
                            ().IsEmpty())
                    {
                        RemovePendingReleaseRequests(allocateResponse.GetCompletedContainersStatuses());
                    }
                }
            }
            finally
            {
                // TODO how to differentiate remote yarn exception vs error in rpc
                if (allocateResponse == null)
                {
                    // we hit an exception in allocate()
                    // preserve ask and release for next call to allocate()
                    lock (this)
                    {
                        Sharpen.Collections.AddAll(release, releaseList);
                        // requests could have been added or deleted during call to allocate
                        // If requests were added/removed then there is nothing to do since
                        // the ResourceRequest object in ask would have the actual new value.
                        // If ask does not have this ResourceRequest then it was unchanged and
                        // so we can add the value back safely.
                        // This assumes that there will no concurrent calls to allocate() and
                        // so we dont have to worry about ask being changed in the
                        // synchronized block at the beginning of this method.
                        foreach (ResourceRequest oldAsk in askList)
                        {
                            if (!ask.Contains(oldAsk))
                            {
                                ask.AddItem(oldAsk);
                            }
                        }
                        Sharpen.Collections.AddAll(blacklistAdditions, blacklistToAdd);
                        Sharpen.Collections.AddAll(blacklistRemovals, blacklistToRemove);
                    }
                }
            }
            return(allocateResponse);
        }
예제 #28
0
 public _PrivilegedExceptionAction_234(MockAM _enclosing, AllocateRequest req)
 {
     this._enclosing = _enclosing;
     this.req        = req;
 }
예제 #29
0
 public _PrivilegedExceptionAction_67(TestAMRMRPCResponseId _enclosing, AllocateRequest
                                      req)
 {
     this._enclosing = _enclosing;
     this.req        = req;
 }
예제 #30
0
        public virtual void TestAMRMUnusableNodes()
        {
            MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 10000);
            MockNM nm2 = rm.RegisterNode("127.0.0.2:1234", 10000);
            MockNM nm3 = rm.RegisterNode("127.0.0.3:1234", 10000);
            MockNM nm4 = rm.RegisterNode("127.0.0.4:1234", 10000);

            dispatcher.Await();
            RMApp app1 = rm.SubmitApp(2000);

            // Trigger the scheduling so the AM gets 'launched' on nm1
            nm1.NodeHeartbeat(true);
            RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
            MockAM       am1      = rm.SendAMLaunched(attempt1.GetAppAttemptId());

            // register AM returns no unusable node
            am1.RegisterAppAttempt();
            // allocate request returns no updated node
            AllocateRequest allocateRequest1 = AllocateRequest.NewInstance(0, 0F, null, null,
                                                                           null);
            AllocateResponse response1 = Allocate(attempt1.GetAppAttemptId(), allocateRequest1
                                                  );
            IList <NodeReport> updatedNodes = response1.GetUpdatedNodes();

            NUnit.Framework.Assert.AreEqual(0, updatedNodes.Count);
            SyncNodeHeartbeat(nm4, false);
            // allocate request returns updated node
            allocateRequest1 = AllocateRequest.NewInstance(response1.GetResponseId(), 0F, null
                                                           , null, null);
            response1    = Allocate(attempt1.GetAppAttemptId(), allocateRequest1);
            updatedNodes = response1.GetUpdatedNodes();
            NUnit.Framework.Assert.AreEqual(1, updatedNodes.Count);
            NodeReport nr = updatedNodes.GetEnumerator().Next();

            NUnit.Framework.Assert.AreEqual(nm4.GetNodeId(), nr.GetNodeId());
            NUnit.Framework.Assert.AreEqual(NodeState.Unhealthy, nr.GetNodeState());
            // resending the allocate request returns the same result
            response1    = Allocate(attempt1.GetAppAttemptId(), allocateRequest1);
            updatedNodes = response1.GetUpdatedNodes();
            NUnit.Framework.Assert.AreEqual(1, updatedNodes.Count);
            nr = updatedNodes.GetEnumerator().Next();
            NUnit.Framework.Assert.AreEqual(nm4.GetNodeId(), nr.GetNodeId());
            NUnit.Framework.Assert.AreEqual(NodeState.Unhealthy, nr.GetNodeState());
            SyncNodeLost(nm3);
            // subsequent allocate request returns delta
            allocateRequest1 = AllocateRequest.NewInstance(response1.GetResponseId(), 0F, null
                                                           , null, null);
            response1    = Allocate(attempt1.GetAppAttemptId(), allocateRequest1);
            updatedNodes = response1.GetUpdatedNodes();
            NUnit.Framework.Assert.AreEqual(1, updatedNodes.Count);
            nr = updatedNodes.GetEnumerator().Next();
            NUnit.Framework.Assert.AreEqual(nm3.GetNodeId(), nr.GetNodeId());
            NUnit.Framework.Assert.AreEqual(NodeState.Lost, nr.GetNodeState());
            // registering another AM gives it the complete failed list
            RMApp app2 = rm.SubmitApp(2000);

            // Trigger nm2 heartbeat so that AM gets launched on it
            nm2.NodeHeartbeat(true);
            RMAppAttempt attempt2 = app2.GetCurrentAppAttempt();
            MockAM       am2      = rm.SendAMLaunched(attempt2.GetAppAttemptId());

            // register AM returns all unusable nodes
            am2.RegisterAppAttempt();
            // allocate request returns no updated node
            AllocateRequest allocateRequest2 = AllocateRequest.NewInstance(0, 0F, null, null,
                                                                           null);
            AllocateResponse response2 = Allocate(attempt2.GetAppAttemptId(), allocateRequest2
                                                  );

            updatedNodes = response2.GetUpdatedNodes();
            NUnit.Framework.Assert.AreEqual(0, updatedNodes.Count);
            SyncNodeHeartbeat(nm4, true);
            // both AM's should get delta updated nodes
            allocateRequest1 = AllocateRequest.NewInstance(response1.GetResponseId(), 0F, null
                                                           , null, null);
            response1    = Allocate(attempt1.GetAppAttemptId(), allocateRequest1);
            updatedNodes = response1.GetUpdatedNodes();
            NUnit.Framework.Assert.AreEqual(1, updatedNodes.Count);
            nr = updatedNodes.GetEnumerator().Next();
            NUnit.Framework.Assert.AreEqual(nm4.GetNodeId(), nr.GetNodeId());
            NUnit.Framework.Assert.AreEqual(NodeState.Running, nr.GetNodeState());
            allocateRequest2 = AllocateRequest.NewInstance(response2.GetResponseId(), 0F, null
                                                           , null, null);
            response2    = Allocate(attempt2.GetAppAttemptId(), allocateRequest2);
            updatedNodes = response2.GetUpdatedNodes();
            NUnit.Framework.Assert.AreEqual(1, updatedNodes.Count);
            nr = updatedNodes.GetEnumerator().Next();
            NUnit.Framework.Assert.AreEqual(nm4.GetNodeId(), nr.GetNodeId());
            NUnit.Framework.Assert.AreEqual(NodeState.Running, nr.GetNodeState());
            // subsequent allocate calls should return no updated nodes
            allocateRequest2 = AllocateRequest.NewInstance(response2.GetResponseId(), 0F, null
                                                           , null, null);
            response2    = Allocate(attempt2.GetAppAttemptId(), allocateRequest2);
            updatedNodes = response2.GetUpdatedNodes();
            NUnit.Framework.Assert.AreEqual(0, updatedNodes.Count);
        }