/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> public override AllocateResponse Allocate(AllocateRequest request) { this._enclosing.ResetStartFailoverFlag(true); // make sure failover has been triggered NUnit.Framework.Assert.IsTrue(this._enclosing.WaittingForFailOver()); return(this._enclosing.CreateFakeAllocateResponse()); }
public virtual void TestAllcoateRequestWithIncrease() { IList <ContainerResourceIncreaseRequest> incRequests = new AList <ContainerResourceIncreaseRequest >(); for (int i = 0; i < 3; i++) { incRequests.AddItem(ContainerResourceIncreaseRequest.NewInstance(null, Resource.NewInstance (0, i))); } AllocateRequest r = AllocateRequest.NewInstance(123, 0f, null, null, null, incRequests ); // serde YarnServiceProtos.AllocateRequestProto p = ((AllocateRequestPBImpl)r).GetProto(); r = new AllocateRequestPBImpl(p); // check value NUnit.Framework.Assert.AreEqual(123, r.GetResponseId()); NUnit.Framework.Assert.AreEqual(incRequests.Count, r.GetIncreaseRequests().Count); for (int i_1 = 0; i_1 < incRequests.Count; i_1++) { NUnit.Framework.Assert.AreEqual(r.GetIncreaseRequests()[i_1].GetCapability().GetVirtualCores (), incRequests[i_1].GetCapability().GetVirtualCores()); } }
public virtual void TestPbRecordFactory() { RecordFactory pbRecordFactory = RecordFactoryPBImpl.Get(); try { AllocateResponse response = pbRecordFactory.NewRecordInstance <AllocateResponse>(); NUnit.Framework.Assert.AreEqual(typeof(AllocateResponsePBImpl), response.GetType( )); } catch (YarnRuntimeException e) { Sharpen.Runtime.PrintStackTrace(e); NUnit.Framework.Assert.Fail("Failed to crete record"); } try { AllocateRequest response = pbRecordFactory.NewRecordInstance <AllocateRequest>(); NUnit.Framework.Assert.AreEqual(typeof(AllocateRequestPBImpl), response.GetType() ); } catch (YarnRuntimeException e) { Sharpen.Runtime.PrintStackTrace(e); NUnit.Framework.Assert.Fail("Failed to crete record"); } }
public IActionResult Allocate([FromBody] AllocateRequest spec) { if (spec == null) { return(BadRequest($"{nameof(spec)} can not be null")); } if (spec.Offers == null || spec.Offers.Length == 0) { return(BadRequest($"{nameof(spec.Offers)} can not be empty")); } if (spec.Threshold <= 0) { return(BadRequest($"{nameof(spec.Threshold)} must be greater than 0")); } if (spec.BusinessSeats < 0) { return(BadRequest($"{nameof(spec.BusinessSeats)} must be greater than or equal to 0")); } if (spec.EconomySeats < 0) { return(BadRequest($"{nameof(spec.EconomySeats)} must be greater than or equal to 0")); } var allocateResult = _allocationService.Allocate(spec.ToSpec()); return(Ok(AllocateResponse.From(allocateResult))); }
/// <exception cref="System.IO.IOException"/> public AllocateResponse Allocate(AllocateRequest request) { AllocateResponse response = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateResponse >(); IList <ResourceRequest> askList = request.GetAskList(); IList <Container> containers = new AList <Container>(); foreach (ResourceRequest req in askList) { if (!ResourceRequest.IsAnyLocation(req.GetResourceName())) { continue; } int numContainers = req.GetNumContainers(); for (int i = 0; i < numContainers; i++) { ContainerId containerId = ContainerId.NewContainerId(this._enclosing.GetContext() .GetApplicationAttemptId(), request.GetResponseId() + i); containers.AddItem(Container.NewInstance(containerId, NodeId.NewInstance("host" + containerId.GetContainerId(), 2345), "host" + containerId.GetContainerId() + ":5678" , req.GetCapability(), req.GetPriority(), null)); } } response.SetAllocatedContainers(containers); response.SetResponseId(request.GetResponseId() + 1); response.SetNumClusterNodes(350); return(response); }
public async Task <IPEndPoint> AllocateRequestAsync( TimeSpan lifetime, CancellationToken cancellationToken = default(CancellationToken)) { NetworkStream stream = _control.GetStream(); StunMessage response; int retry = 0; do { var request = new AllocateRequest((int)lifetime.TotalSeconds); await SendMessageAsync(stream, request, cancellationToken); response = await ReceiveMessageAsync(request.TransactionId, cancellationToken); if (response is AllocateErrorResponse allocError) { Realm = allocError.Realm; Nonce = allocError.Nonce; } retry += 1; }while (response is AllocateErrorResponse && retry < AllocateRetry); if (response is AllocateSuccessResponse allocOk) { return(allocOk.RelayedEndPoint); } else { throw new TurnClientException("Allocate failed.", response); } }
/// <exception cref="System.Exception"/> public virtual AllocateResponse Allocate(IList <ResourceRequest> resourceRequest, IList <ContainerId> releases) { AllocateRequest req = AllocateRequest.NewInstance(0, 0F, resourceRequest, releases , null); return(Allocate(req)); }
/// <exception cref="System.Exception"/> protected internal override void Heartbeat() { lock (this) { AllocateRequest allocateRequest = AllocateRequest.NewInstance(this.lastResponseID , base.GetApplicationProgress(), new AList <ResourceRequest>(), new AList <ContainerId >(), null); AllocateResponse allocateResponse = null; try { allocateResponse = scheduler.Allocate(allocateRequest); // Reset retry count if no exception occurred. retrystartTime = Runtime.CurrentTimeMillis(); } catch (ApplicationAttemptNotFoundException e) { Log.Info("Event from RM: shutting down Application Master"); // This can happen if the RM has been restarted. If it is in that state, // this application must clean itself up. eventHandler.Handle(new JobEvent(this.GetJob().GetID(), JobEventType.JobAmReboot) ); throw new YarnRuntimeException("Resource Manager doesn't recognize AttemptId: " + this.GetContext().GetApplicationID(), e); } catch (ApplicationMasterNotRegisteredException) { Log.Info("ApplicationMaster is out of sync with ResourceManager," + " hence resync and send outstanding requests." ); this.lastResponseID = 0; Register(); } catch (Exception e) { // This can happen when the connection to the RM has gone down. Keep // re-trying until the retryInterval has expired. if (Runtime.CurrentTimeMillis() - retrystartTime >= retryInterval) { Log.Error("Could not contact RM after " + retryInterval + " milliseconds."); eventHandler.Handle(new JobEvent(this.GetJob().GetID(), JobEventType.InternalError )); throw new YarnRuntimeException("Could not contact RM after " + retryInterval + " milliseconds." ); } // Throw this up to the caller, which may decide to ignore it and // continue to attempt to contact the RM. throw; } if (allocateResponse != null) { this.lastResponseID = allocateResponse.GetResponseId(); Token token = allocateResponse.GetAMRMToken(); if (token != null) { UpdateAMRMToken(token); } } } }
// Keyboard input handling -each keypress can be interpreted as // a control command. void KeyboardControl() { while (true) { switch (Console.ReadKey(true).Key) { case ConsoleKey.T: transportFunction.PrintRouteTable(); break; case ConsoleKey.A: LRM.PrintAssignments(); break; case ConsoleKey.P: if (Log.IsPaused) { Log.Unpause(); } else { Log.Pause(); } break; #if DEBUG case ConsoleKey.Enter: MPLSPacket testPacket = new MPLSPacket(new int[] { 2137 }, "This is a test MPLSMessage."); transportFunction.EnqueuePacketOnFirstQueue(testPacket); break; case ConsoleKey.U: NHLFEntry entry = new NHLFEntry(10, 1, 17, true, 2, new int[] { 35 }); AddUpdateRequest testUpdateReq = new AddUpdateRequest("Helo it me", mgmtLocalPort, 2137, entry); SendManagementMsg(mgmtLocalPort, testUpdateReq); break; case ConsoleKey.R: RemoveRequest testRemoveReq = new RemoveRequest("Helo it me", mgmtLocalPort, 2137, 10); SendManagementMsg(mgmtLocalPort, testRemoveReq); break; case ConsoleKey.L: AllocateRequest testAllocReq = new AllocateRequest(id, mgmtLocalPort, allocCounter++, 1, 30, 1); SendManagementMsg(mgmtLocalPort, testAllocReq); break; case ConsoleKey.D: DeallocateRequest testDeallocReq = new DeallocateRequest(id, mgmtLocalPort, allocCounter--, 1); SendManagementMsg(mgmtLocalPort, testDeallocReq); break; #endif default: break; } } }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> public virtual void TestAllocateOnHA() { AllocateRequest request = AllocateRequest.NewInstance(0, 50f, new AList <ResourceRequest >(), new AList <ContainerId>(), ResourceBlacklistRequest.NewInstance(new AList <string >(), new AList <string>())); AllocateResponse response = amClient.Allocate(request); NUnit.Framework.Assert.AreEqual(response, this.cluster.CreateFakeAllocateResponse ()); }
/// <exception cref="System.Exception"/> public virtual AllocateResponse Allocate(AllocateRequest allocateRequest) { UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser(attemptId.ToString ()); Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> token = context.GetRMApps ()[attemptId.GetApplicationId()].GetRMAppAttempt(attemptId).GetAMRMToken(); ugi.AddTokenIdentifier(token.DecodeIdentifier()); lastResponse = DoAllocateAs(ugi, allocateRequest); return(lastResponse); }
public virtual void TestValidateResourceBlacklistRequest() { TestAMAuthorization.MyContainerManager containerManager = new TestAMAuthorization.MyContainerManager (); TestAMAuthorization.MockRMWithAMS rm = new TestAMAuthorization.MockRMWithAMS(new YarnConfiguration(), containerManager); rm.Start(); MockNM nm1 = rm.RegisterNode("localhost:1234", 5120); IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType , string>(2); acls[ApplicationAccessType.ViewApp] = "*"; RMApp app = rm.SubmitApp(1024, "appname", "appuser", acls); nm1.NodeHeartbeat(true); RMAppAttempt attempt = app.GetCurrentAppAttempt(); ApplicationAttemptId applicationAttemptId = attempt.GetAppAttemptId(); WaitForLaunchedState(attempt); // Create a client to the RM. Configuration conf = rm.GetConfig(); YarnRPC rpc = YarnRPC.Create(conf); UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(applicationAttemptId .ToString()); Credentials credentials = containerManager.GetContainerCredentials(); IPEndPoint rmBindAddress = rm.GetApplicationMasterService().GetBindAddress(); Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> amRMToken = TestAMAuthorization.MockRMWithAMS .SetupAndReturnAMRMToken(rmBindAddress, credentials.GetAllTokens()); currentUser.AddToken(amRMToken); ApplicationMasterProtocol client = currentUser.DoAs(new _PrivilegedAction_626(rpc , rmBindAddress, conf)); RegisterApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <RegisterApplicationMasterRequest>(); client.RegisterApplicationMaster(request); ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.NewInstance( Sharpen.Collections.SingletonList(ResourceRequest.Any), null); AllocateRequest allocateRequest = AllocateRequest.NewInstance(0, 0.0f, null, null , blacklistRequest); bool error = false; try { client.Allocate(allocateRequest); } catch (InvalidResourceBlacklistRequestException) { error = true; } rm.Stop(); NUnit.Framework.Assert.IsTrue("Didn't not catch InvalidResourceBlacklistRequestException" , error); }
public virtual void TestAllcoateRequestWithoutIncrease() { AllocateRequest r = AllocateRequest.NewInstance(123, 0f, null, null, null, null); // serde YarnServiceProtos.AllocateRequestProto p = ((AllocateRequestPBImpl)r).GetProto(); r = new AllocateRequestPBImpl(p); // check value NUnit.Framework.Assert.AreEqual(123, r.GetResponseId()); NUnit.Framework.Assert.AreEqual(0, r.GetIncreaseRequests().Count); }
/// <exception cref="System.Exception"/> private AllocateResponse Allocate(ApplicationAttemptId attemptId, AllocateRequest req) { UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser(attemptId.ToString ()); Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> token = rm.GetRMContext ().GetRMApps()[attemptId.GetApplicationId()].GetRMAppAttempt(attemptId).GetAMRMToken (); ugi.AddTokenIdentifier(token.DecodeIdentifier()); return(ugi.DoAs(new _PrivilegedExceptionAction_67(this, req))); }
/// <exception cref="System.Exception"/> public virtual AllocateResponse DoAllocateAs(UserGroupInformation ugi, AllocateRequest req) { req.SetResponseId(++responseId); try { return(ugi.DoAs(new _PrivilegedExceptionAction_234(this, req))); } catch (UndeclaredThrowableException e) { throw (Exception)e.InnerException; } }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> public virtual AllocateResponse Allocate(AllocateRequest request) { YarnServiceProtos.AllocateRequestProto requestProto = ((AllocateRequestPBImpl)request ).GetProto(); try { return(new AllocateResponsePBImpl(proxy.Allocate(null, requestProto))); } catch (ServiceException e) { RPCUtil.UnwrapAndThrowException(e); return(null); } }
public void EncodeToBytesWithCredential() { var request = new AllocateRequest(777) { TransactionId = new byte[] { 0xf5, 0xdb, 0xe7, 0xc1, 0x2a, 0x74, 0xbe, 0xf9, 0x8b, 0x16, 0x56, 0x3e, } }; var ctx = new TestStunContext() { Username = "******", Password = "******", Realm = "twilio.com", Nonce = new byte[] { 0x37, 0x35, 0x64, 0x34, 0x35, 0x34, 0x31, 0x39, 0x63, 0x33, 0x39, 0x33, 0x34, 0x33, 0x66, 0x65, }, }; Assert.Equal( new byte[] { 0x00, 0x03, 0x00, 0x98, 0x21, 0x12, 0xa4, 0x42, 0xf5, 0xdb, 0xe7, 0xc1, 0x2a, 0x74, 0xbe, 0xf9, 0x8b, 0x16, 0x56, 0x3e, 0x00, 0x19, 0x00, 0x04, 0x06, 0x00, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x04, 0x00, 0x00, 0x03, 0x09, 0x00, 0x06, 0x00, 0x40, 0x61, 0x65, 0x30, 0x36, 0x33, 0x33, 0x63, 0x64, 0x35, 0x38, 0x62, 0x61, 0x30, 0x39, 0x37, 0x61, 0x31, 0x31, 0x36, 0x37, 0x63, 0x36, 0x64, 0x32, 0x63, 0x63, 0x34, 0x65, 0x32, 0x33, 0x36, 0x64, 0x62, 0x35, 0x32, 0x32, 0x35, 0x36, 0x61, 0x34, 0x30, 0x64, 0x35, 0x36, 0x35, 0x66, 0x31, 0x31, 0x65, 0x64, 0x66, 0x37, 0x36, 0x63, 0x30, 0x32, 0x64, 0x31, 0x33, 0x64, 0x62, 0x39, 0x33, 0x63, 0x00, 0x15, 0x00, 0x10, 0x37, 0x35, 0x64, 0x34, 0x35, 0x34, 0x31, 0x39, 0x63, 0x33, 0x39, 0x33, 0x34, 0x33, 0x66, 0x65, 0x00, 0x14, 0x00, 0x0a, 0x74, 0x77, 0x69, 0x6c, 0x69, 0x6f, 0x2e, 0x63, 0x6f, 0x6d, 0x00, 0x00, 0x00, 0x08, 0x00, 0x14, 0x77, 0xe8, 0xcf, 0x30, 0x9e, 0x85, 0x6c, 0x22, 0x72, 0x53, 0xa3, 0xb7, 0xe0, 0x35, 0x7c, 0xc2, 0x30, 0xfc, 0xbc, 0xf4, 0x80, 0x28, 0x00, 0x04, 0x98, 0x46, 0x9c, 0x07, }, request.Encode(ctx)); }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> public virtual AllocateResponse Allocate(AllocateRequest request) { NUnit.Framework.Assert.AreEqual("response ID mismatch", responseId, request.GetResponseId ()); ++responseId; Org.Apache.Hadoop.Yarn.Api.Records.Token yarnToken = null; if (amToken != null) { yarnToken = Org.Apache.Hadoop.Yarn.Api.Records.Token.NewInstance(amToken.GetIdentifier (), amToken.GetKind().ToString(), amToken.GetPassword(), amToken.GetService().ToString ()); } return(AllocateResponse.NewInstance(responseId, Collections.EmptyList <ContainerStatus >(), Collections.EmptyList <Container>(), Collections.EmptyList <NodeReport>(), Resources .None(), null, 1, null, Collections.EmptyList <NMToken>(), yarnToken, Collections .EmptyList <ContainerResourceIncrease>(), Collections.EmptyList <ContainerResourceDecrease >())); }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> protected internal virtual AllocateResponse MakeRemoteRequest() { ApplyRequestLimits(); ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.NewInstance( new AList <string>(blacklistAdditions), new AList <string>(blacklistRemovals)); AllocateRequest allocateRequest = AllocateRequest.NewInstance(lastResponseID, base .GetApplicationProgress(), new AList <ResourceRequest>(ask), new AList <ContainerId >(release), blacklistRequest); AllocateResponse allocateResponse = scheduler.Allocate(allocateRequest); lastResponseID = allocateResponse.GetResponseId(); availableResources = allocateResponse.GetAvailableResources(); lastClusterNmCount = clusterNmCount; clusterNmCount = allocateResponse.GetNumClusterNodes(); int numCompletedContainers = allocateResponse.GetCompletedContainersStatuses().Count; if (ask.Count > 0 || release.Count > 0) { Log.Info("getResources() for " + applicationId + ":" + " ask=" + ask.Count + " release= " + release.Count + " newContainers=" + allocateResponse.GetAllocatedContainers() .Count + " finishedContainers=" + numCompletedContainers + " resourcelimit=" + availableResources + " knownNMs=" + clusterNmCount); } ask.Clear(); release.Clear(); if (numCompletedContainers > 0) { // re-send limited requests when a container completes to trigger asking // for more containers Sharpen.Collections.AddAll(requestLimitsToUpdate, requestLimits.Keys); } if (blacklistAdditions.Count > 0 || blacklistRemovals.Count > 0) { Log.Info("Update the blacklist for " + applicationId + ": blacklistAdditions=" + blacklistAdditions.Count + " blacklistRemovals=" + blacklistRemovals.Count); } blacklistAdditions.Clear(); blacklistRemovals.Clear(); return(allocateResponse); }
public void EncodeToBytes() { var request = new AllocateRequest(777) { TransactionId = new byte[] { 0xee, 0x29, 0xdd, 0x2d, 0x7a, 0xe9, 0x9c, 0xf4, 0x00, 0x82, 0xf2, 0x5e, } }; Assert.Equal( new byte[] { 0x00, 0x03, 0x00, 0x18, 0x21, 0x12, 0xa4, 0x42, 0xee, 0x29, 0xdd, 0x2d, 0x7a, 0xe9, 0x9c, 0xf4, 0x00, 0x82, 0xf2, 0x5e, 0x00, 0x19, 0x00, 0x04, 0x06, 0x00, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x04, 0x00, 0x00, 0x03, 0x09, 0x80, 0x28, 0x00, 0x04, 0x3d, 0xbe, 0x42, 0x75, }, request.Encode(new TestStunContext()) ); }
public virtual void TestARRMResponseId() { MockNM nm1 = rm.RegisterNode("h1:1234", 5000); RMApp app = rm.SubmitApp(2000); // Trigger the scheduling so the AM gets 'launched' nm1.NodeHeartbeat(true); RMAppAttempt attempt = app.GetCurrentAppAttempt(); MockAM am = rm.SendAMLaunched(attempt.GetAppAttemptId()); am.RegisterAppAttempt(); AllocateRequest allocateRequest = AllocateRequest.NewInstance(0, 0F, null, null, null); AllocateResponse response = Allocate(attempt.GetAppAttemptId(), allocateRequest); NUnit.Framework.Assert.AreEqual(1, response.GetResponseId()); NUnit.Framework.Assert.IsTrue(response.GetAMCommand() == null); allocateRequest = AllocateRequest.NewInstance(response.GetResponseId(), 0F, null, null, null); response = Allocate(attempt.GetAppAttemptId(), allocateRequest); NUnit.Framework.Assert.AreEqual(2, response.GetResponseId()); /* try resending */ response = Allocate(attempt.GetAppAttemptId(), allocateRequest); NUnit.Framework.Assert.AreEqual(2, response.GetResponseId()); allocateRequest = AllocateRequest.NewInstance(0, 0F, null, null, null); try { Allocate(attempt.GetAppAttemptId(), allocateRequest); NUnit.Framework.Assert.Fail(); } catch (Exception e) { NUnit.Framework.Assert.IsTrue(e.InnerException is InvalidApplicationMasterRequestException ); } }
public virtual void TestTokenExpiry() { TestAMAuthorization.MyContainerManager containerManager = new TestAMAuthorization.MyContainerManager (); TestAMAuthorization.MockRMWithAMS rm = new TestAMAuthorization.MockRMWithAMS(conf , containerManager); rm.Start(); Configuration conf = rm.GetConfig(); YarnRPC rpc = YarnRPC.Create(conf); ApplicationMasterProtocol rmClient = null; try { MockNM nm1 = rm.RegisterNode("localhost:1234", 5120); RMApp app = rm.SubmitApp(1024); nm1.NodeHeartbeat(true); int waitCount = 0; while (containerManager.containerTokens == null && waitCount++ < 20) { Log.Info("Waiting for AM Launch to happen.."); Sharpen.Thread.Sleep(1000); } NUnit.Framework.Assert.IsNotNull(containerManager.containerTokens); RMAppAttempt attempt = app.GetCurrentAppAttempt(); ApplicationAttemptId applicationAttemptId = attempt.GetAppAttemptId(); // Create a client to the RM. UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(applicationAttemptId .ToString()); Credentials credentials = containerManager.GetContainerCredentials(); IPEndPoint rmBindAddress = rm.GetApplicationMasterService().GetBindAddress(); Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> amRMToken = TestAMAuthorization.MockRMWithAMS .SetupAndReturnAMRMToken(rmBindAddress, credentials.GetAllTokens()); currentUser.AddToken(amRMToken); rmClient = CreateRMClient(rm, conf, rpc, currentUser); RegisterApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <RegisterApplicationMasterRequest>(); rmClient.RegisterApplicationMaster(request); FinishApplicationMasterRequest finishAMRequest = Org.Apache.Hadoop.Yarn.Util.Records .NewRecord <FinishApplicationMasterRequest>(); finishAMRequest.SetFinalApplicationStatus(FinalApplicationStatus.Succeeded); finishAMRequest.SetDiagnostics("diagnostics"); finishAMRequest.SetTrackingUrl("url"); rmClient.FinishApplicationMaster(finishAMRequest); // Send RMAppAttemptEventType.CONTAINER_FINISHED to transit RMAppAttempt // from Finishing state to Finished State. Both AMRMToken and // ClientToAMToken will be removed. ContainerStatus containerStatus = BuilderUtils.NewContainerStatus(attempt.GetMasterContainer ().GetId(), ContainerState.Complete, "AM Container Finished", 0); rm.GetRMContext().GetDispatcher().GetEventHandler().Handle(new RMAppAttemptContainerFinishedEvent (applicationAttemptId, containerStatus, nm1.GetNodeId())); // Make sure the RMAppAttempt is at Finished State. // Both AMRMToken and ClientToAMToken have been removed. int count = 0; while (attempt.GetState() != RMAppAttemptState.Finished && count < maxWaitAttempts ) { Sharpen.Thread.Sleep(100); count++; } NUnit.Framework.Assert.IsTrue(attempt.GetState() == RMAppAttemptState.Finished); // Now simulate trying to allocate. RPC call itself should throw auth // exception. rpc.StopProxy(rmClient, conf); // To avoid using cached client rmClient = CreateRMClient(rm, conf, rpc, currentUser); AllocateRequest allocateRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest >(); try { rmClient.Allocate(allocateRequest); NUnit.Framework.Assert.Fail("You got to be kidding me! " + "Using App tokens after app-finish should fail!" ); } catch (Exception t) { Log.Info("Exception found is ", t); // The exception will still have the earlier appAttemptId as it picks it // up from the token. NUnit.Framework.Assert.IsTrue(t.InnerException.Message.Contains(applicationAttemptId .ToString() + " not found in AMRMTokenSecretManager.")); } } finally { rm.Stop(); if (rmClient != null) { rpc.StopProxy(rmClient, conf); } } }
public virtual void TestMasterKeyRollOver() { conf.SetLong(YarnConfiguration.RmAmrmTokenMasterKeyRollingIntervalSecs, rolling_interval_sec ); conf.SetLong(YarnConfiguration.RmAmExpiryIntervalMs, am_expire_ms); TestAMAuthorization.MyContainerManager containerManager = new TestAMAuthorization.MyContainerManager (); TestAMAuthorization.MockRMWithAMS rm = new TestAMAuthorization.MockRMWithAMS(conf , containerManager); rm.Start(); long startTime = Runtime.CurrentTimeMillis(); Configuration conf = rm.GetConfig(); YarnRPC rpc = YarnRPC.Create(conf); ApplicationMasterProtocol rmClient = null; AMRMTokenSecretManager appTokenSecretManager = rm.GetRMContext().GetAMRMTokenSecretManager (); MasterKeyData oldKey = appTokenSecretManager.GetMasterKey(); NUnit.Framework.Assert.IsNotNull(oldKey); try { MockNM nm1 = rm.RegisterNode("localhost:1234", 5120); RMApp app = rm.SubmitApp(1024); nm1.NodeHeartbeat(true); int waitCount = 0; while (containerManager.containerTokens == null && waitCount++ < maxWaitAttempts) { Log.Info("Waiting for AM Launch to happen.."); Sharpen.Thread.Sleep(1000); } NUnit.Framework.Assert.IsNotNull(containerManager.containerTokens); RMAppAttempt attempt = app.GetCurrentAppAttempt(); ApplicationAttemptId applicationAttemptId = attempt.GetAppAttemptId(); // Create a client to the RM. UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(applicationAttemptId .ToString()); Credentials credentials = containerManager.GetContainerCredentials(); IPEndPoint rmBindAddress = rm.GetApplicationMasterService().GetBindAddress(); Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> amRMToken = TestAMAuthorization.MockRMWithAMS .SetupAndReturnAMRMToken(rmBindAddress, credentials.GetAllTokens()); currentUser.AddToken(amRMToken); rmClient = CreateRMClient(rm, conf, rpc, currentUser); RegisterApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <RegisterApplicationMasterRequest>(); rmClient.RegisterApplicationMaster(request); // One allocate call. AllocateRequest allocateRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest >(); NUnit.Framework.Assert.IsTrue(rmClient.Allocate(allocateRequest).GetAMCommand() == null); // Wait for enough time and make sure the roll_over happens // At mean time, the old AMRMToken should continue to work while (Runtime.CurrentTimeMillis() - startTime < rolling_interval_sec * 1000) { rmClient.Allocate(allocateRequest); Sharpen.Thread.Sleep(500); } MasterKeyData newKey = appTokenSecretManager.GetMasterKey(); NUnit.Framework.Assert.IsNotNull(newKey); NUnit.Framework.Assert.IsFalse("Master key should have changed!", oldKey.Equals(newKey )); // Another allocate call with old AMRMToken. Should continue to work. rpc.StopProxy(rmClient, conf); // To avoid using cached client rmClient = CreateRMClient(rm, conf, rpc, currentUser); NUnit.Framework.Assert.IsTrue(rmClient.Allocate(allocateRequest).GetAMCommand() == null); waitCount = 0; while (waitCount++ <= maxWaitAttempts) { if (appTokenSecretManager.GetCurrnetMasterKeyData() != oldKey) { break; } try { rmClient.Allocate(allocateRequest); } catch (Exception) { break; } Sharpen.Thread.Sleep(200); } // active the nextMasterKey, and replace the currentMasterKey NUnit.Framework.Assert.IsTrue(appTokenSecretManager.GetCurrnetMasterKeyData().Equals (newKey)); NUnit.Framework.Assert.IsTrue(appTokenSecretManager.GetMasterKey().Equals(newKey) ); NUnit.Framework.Assert.IsTrue(appTokenSecretManager.GetNextMasterKeyData() == null ); // Create a new Token Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> newToken = appTokenSecretManager .CreateAndGetAMRMToken(applicationAttemptId); SecurityUtil.SetTokenService(newToken, rmBindAddress); currentUser.AddToken(newToken); // Another allocate call. Should continue to work. rpc.StopProxy(rmClient, conf); // To avoid using cached client rmClient = CreateRMClient(rm, conf, rpc, currentUser); allocateRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest>( ); NUnit.Framework.Assert.IsTrue(rmClient.Allocate(allocateRequest).GetAMCommand() == null); // Should not work by using the old AMRMToken. rpc.StopProxy(rmClient, conf); // To avoid using cached client try { currentUser.AddToken(amRMToken); rmClient = CreateRMClient(rm, conf, rpc, currentUser); allocateRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest>( ); NUnit.Framework.Assert.IsTrue(rmClient.Allocate(allocateRequest).GetAMCommand() == null); NUnit.Framework.Assert.Fail("The old Token should not work"); } catch (Exception) { } } finally { // expect exception rm.Stop(); if (rmClient != null) { rpc.StopProxy(rmClient, conf); } } }
// The test verifies processing of NMContainerStatuses which are sent during // NM registration. // 1. Start the cluster-RM,NM,Submit app with 1024MB,Launch & register AM // 2. AM sends ResourceRequest for 1 container with memory 2048MB. // 3. Verify for number of container allocated by RM // 4. Verify Memory Usage by cluster, it should be 3072. AM memory + requested // memory. 1024 + 2048=3072 // 5. Re-register NM by sending completed container status // 6. Verify for Memory Used, it should be 1024 // 7. Send AM heatbeat to RM. Allocated response should contain completed // container. /// <exception cref="System.Exception"/> public virtual void TestProcessingNMContainerStatusesOnNMRestart() { conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1); MemoryRMStateStore memStore = new MemoryRMStateStore(); memStore.Init(conf); // 1. Start the cluster-RM,NM,Submit app with 1024MB,Launch & register AM MockRM rm1 = new MockRM(conf, memStore); rm1.Start(); int nmMemory = 8192; int amMemory = 1024; int containerMemory = 2048; MockNM nm1 = new MockNM("127.0.0.1:1234", nmMemory, rm1.GetResourceTrackerService ()); nm1.RegisterNode(); RMApp app0 = rm1.SubmitApp(amMemory); MockAM am0 = MockRM.LaunchAndRegisterAM(app0, rm1, nm1); // 2. AM sends ResourceRequest for 1 container with memory 2048MB. int noOfContainers = 1; IList <Container> allocateContainers = am0.AllocateAndWaitForContainers(noOfContainers , containerMemory, nm1); // 3. Verify for number of container allocated by RM NUnit.Framework.Assert.AreEqual(noOfContainers, allocateContainers.Count); Container container = allocateContainers[0]; nm1.NodeHeartbeat(am0.GetApplicationAttemptId(), 1, ContainerState.Running); nm1.NodeHeartbeat(am0.GetApplicationAttemptId(), container.GetId().GetContainerId (), ContainerState.Running); rm1.WaitForState(app0.GetApplicationId(), RMAppState.Running); // 4. Verify Memory Usage by cluster, it should be 3072. AM memory + // requested memory. 1024 + 2048=3072 ResourceScheduler rs = rm1.GetRMContext().GetScheduler(); int allocatedMB = rs.GetRootQueueMetrics().GetAllocatedMB(); NUnit.Framework.Assert.AreEqual(amMemory + containerMemory, allocatedMB); // 5. Re-register NM by sending completed container status IList <NMContainerStatus> nMContainerStatusForApp = CreateNMContainerStatusForApp( am0); nm1.RegisterNode(nMContainerStatusForApp, Arrays.AsList(app0.GetApplicationId())); WaitForClusterMemory(nm1, rs, amMemory); // 6. Verify for Memory Used, it should be 1024 NUnit.Framework.Assert.AreEqual(amMemory, rs.GetRootQueueMetrics().GetAllocatedMB ()); // 7. Send AM heatbeat to RM. Allocated response should contain completed // container AllocateRequest req = AllocateRequest.NewInstance(0, 0F, new AList <ResourceRequest >(), new AList <ContainerId>(), null); AllocateResponse allocate = am0.Allocate(req); IList <ContainerStatus> completedContainersStatuses = allocate.GetCompletedContainersStatuses (); NUnit.Framework.Assert.AreEqual(noOfContainers, completedContainersStatuses.Count ); // Application clean up should happen Cluster memory used is 0 nm1.NodeHeartbeat(am0.GetApplicationAttemptId(), 1, ContainerState.Complete); WaitForClusterMemory(nm1, rs, 0); rm1.Stop(); }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> public virtual AllocateResponse Allocate(AllocateRequest request) { AMRMTokenIdentifier amrmTokenIdentifier = AuthorizeRequest(); ApplicationAttemptId appAttemptId = amrmTokenIdentifier.GetApplicationAttemptId(); ApplicationId applicationId = appAttemptId.GetApplicationId(); this.amLivelinessMonitor.ReceivedPing(appAttemptId); /* check if its in cache */ ApplicationMasterService.AllocateResponseLock Lock = responseMap[appAttemptId]; if (Lock == null) { string message = "Application attempt " + appAttemptId + " doesn't exist in ApplicationMasterService cache."; Log.Error(message); throw new ApplicationAttemptNotFoundException(message); } lock (Lock) { AllocateResponse lastResponse = Lock.GetAllocateResponse(); if (!HasApplicationMasterRegistered(appAttemptId)) { string message = "AM is not registered for known application attempt: " + appAttemptId + " or RM had restarted after AM registered . AM should re-register."; Log.Info(message); RMAuditLogger.LogFailure(this.rmContext.GetRMApps()[appAttemptId.GetApplicationId ()].GetUser(), RMAuditLogger.AuditConstants.AmAllocate, string.Empty, "ApplicationMasterService" , message, applicationId, appAttemptId); throw new ApplicationMasterNotRegisteredException(message); } if ((request.GetResponseId() + 1) == lastResponse.GetResponseId()) { /* old heartbeat */ return(lastResponse); } else { if (request.GetResponseId() + 1 < lastResponse.GetResponseId()) { string message = "Invalid responseId in AllocateRequest from application attempt: " + appAttemptId + ", expect responseId to be " + (lastResponse.GetResponseId() + 1); throw new InvalidApplicationMasterRequestException(message); } } //filter illegal progress values float filteredProgress = request.GetProgress(); if (float.IsNaN(filteredProgress) || filteredProgress == float.NegativeInfinity || filteredProgress < 0) { request.SetProgress(0); } else { if (filteredProgress > 1 || filteredProgress == float.PositiveInfinity) { request.SetProgress(1); } } // Send the status update to the appAttempt. this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMAppAttemptStatusupdateEvent (appAttemptId, request.GetProgress())); IList <ResourceRequest> ask = request.GetAskList(); IList <ContainerId> release = request.GetReleaseList(); ResourceBlacklistRequest blacklistRequest = request.GetResourceBlacklistRequest(); IList <string> blacklistAdditions = (blacklistRequest != null) ? blacklistRequest. GetBlacklistAdditions() : Sharpen.Collections.EmptyList; IList <string> blacklistRemovals = (blacklistRequest != null) ? blacklistRequest.GetBlacklistRemovals () : Sharpen.Collections.EmptyList; RMApp app = this.rmContext.GetRMApps()[applicationId]; // set label expression for Resource Requests if resourceName=ANY ApplicationSubmissionContext asc = app.GetApplicationSubmissionContext(); foreach (ResourceRequest req in ask) { if (null == req.GetNodeLabelExpression() && ResourceRequest.Any.Equals(req.GetResourceName ())) { req.SetNodeLabelExpression(asc.GetNodeLabelExpression()); } } // sanity check try { RMServerUtils.NormalizeAndValidateRequests(ask, rScheduler.GetMaximumResourceCapability (), app.GetQueue(), rScheduler, rmContext); } catch (InvalidResourceRequestException e) { Log.Warn("Invalid resource ask by application " + appAttemptId, e); throw; } try { RMServerUtils.ValidateBlacklistRequest(blacklistRequest); } catch (InvalidResourceBlacklistRequestException e) { Log.Warn("Invalid blacklist request by application " + appAttemptId, e); throw; } // In the case of work-preserving AM restart, it's possible for the // AM to release containers from the earlier attempt. if (!app.GetApplicationSubmissionContext().GetKeepContainersAcrossApplicationAttempts ()) { try { RMServerUtils.ValidateContainerReleaseRequest(release, appAttemptId); } catch (InvalidContainerReleaseException e) { Log.Warn("Invalid container release by application " + appAttemptId, e); throw; } } // Send new requests to appAttempt. Allocation allocation = this.rScheduler.Allocate(appAttemptId, ask, release, blacklistAdditions , blacklistRemovals); if (!blacklistAdditions.IsEmpty() || !blacklistRemovals.IsEmpty()) { Log.Info("blacklist are updated in Scheduler." + "blacklistAdditions: " + blacklistAdditions + ", " + "blacklistRemovals: " + blacklistRemovals); } RMAppAttempt appAttempt = app.GetRMAppAttempt(appAttemptId); AllocateResponse allocateResponse = recordFactory.NewRecordInstance <AllocateResponse >(); if (!allocation.GetContainers().IsEmpty()) { allocateResponse.SetNMTokens(allocation.GetNMTokens()); } // update the response with the deltas of node status changes IList <RMNode> updatedNodes = new AList <RMNode>(); if (app.PullRMNodeUpdates(updatedNodes) > 0) { IList <NodeReport> updatedNodeReports = new AList <NodeReport>(); foreach (RMNode rmNode in updatedNodes) { SchedulerNodeReport schedulerNodeReport = rScheduler.GetNodeReport(rmNode.GetNodeID ()); Resource used = BuilderUtils.NewResource(0, 0); int numContainers = 0; if (schedulerNodeReport != null) { used = schedulerNodeReport.GetUsedResource(); numContainers = schedulerNodeReport.GetNumContainers(); } NodeId nodeId = rmNode.GetNodeID(); NodeReport report = BuilderUtils.NewNodeReport(nodeId, rmNode.GetState(), rmNode. GetHttpAddress(), rmNode.GetRackName(), used, rmNode.GetTotalCapability(), numContainers , rmNode.GetHealthReport(), rmNode.GetLastHealthReportTime(), rmNode.GetNodeLabels ()); updatedNodeReports.AddItem(report); } allocateResponse.SetUpdatedNodes(updatedNodeReports); } allocateResponse.SetAllocatedContainers(allocation.GetContainers()); allocateResponse.SetCompletedContainersStatuses(appAttempt.PullJustFinishedContainers ()); allocateResponse.SetResponseId(lastResponse.GetResponseId() + 1); allocateResponse.SetAvailableResources(allocation.GetResourceLimit()); allocateResponse.SetNumClusterNodes(this.rScheduler.GetNumClusterNodes()); // add preemption to the allocateResponse message (if any) allocateResponse.SetPreemptionMessage(GeneratePreemptionMessage(allocation)); // update AMRMToken if the token is rolled-up MasterKeyData nextMasterKey = this.rmContext.GetAMRMTokenSecretManager().GetNextMasterKeyData (); if (nextMasterKey != null && nextMasterKey.GetMasterKey().GetKeyId() != amrmTokenIdentifier .GetKeyId()) { RMAppAttemptImpl appAttemptImpl = (RMAppAttemptImpl)appAttempt; Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> amrmToken = appAttempt .GetAMRMToken(); if (nextMasterKey.GetMasterKey().GetKeyId() != appAttemptImpl.GetAMRMTokenKeyId()) { Log.Info("The AMRMToken has been rolled-over. Send new AMRMToken back" + " to application: " + applicationId); amrmToken = rmContext.GetAMRMTokenSecretManager().CreateAndGetAMRMToken(appAttemptId ); appAttemptImpl.SetAMRMToken(amrmToken); } allocateResponse.SetAMRMToken(Org.Apache.Hadoop.Yarn.Api.Records.Token.NewInstance (amrmToken.GetIdentifier(), amrmToken.GetKind().ToString(), amrmToken.GetPassword (), amrmToken.GetService().ToString())); } /* * As we are updating the response inside the lock object so we don't * need to worry about unregister call occurring in between (which * removes the lock object). */ Lock.SetAllocateResponse(allocateResponse); return(allocateResponse); } }
// Method invoked in a separate thread, sets up a UDP listener // that listens for anything on a management port, then detemines // what kind of message it is (with what information) and handles // the response. void ReceiveManagementMsg() { UdpClient listener = new UdpClient(mgmtLocalPort); IPEndPoint groupEP = new IPEndPoint(IPAddress.Parse("127.0.0.1"), mgmtLocalPort); try { while (true) { byte[] bytes = listener.Receive(ref groupEP); Communications.Message msg = Communications.Serialization.Deserialize(bytes); switch (msg.messageType) { case "NHLF.AddUpdateRequest": Log.WriteLine("[MGMT CCI] {1}, port {2}: {0})", msg.messageType, msg.senderID, msg.senderPort); AddUpdateRequest addUpdateReq = (AddUpdateRequest)msg; transportFunction.UpdateRoutingTable(addUpdateReq.entry, true); // !!! IT CAN GO TERRIBLY WRONG HERE !!! SendManagementMsg( (ushort)addUpdateReq.senderPort, new AddUpdateResponse(id, mgmtLocalPort, addUpdateReq.seq, true) ); Log.WriteLine("[MGMT CCI] Add/update forward: conn {2}, iface {0}, label {1}", addUpdateReq.entry.interface_in, addUpdateReq.entry.label_in, addUpdateReq.entry.connectionID); break; case "NHLF.RemoveRequest": Log.WriteLine("[MGMT CCI] {1}, port {2}: {0})", msg.messageType, msg.senderID, msg.senderPort); RemoveRequest removeReq = (RemoveRequest)msg; bool status = transportFunction.RemoveFromRoutingTable(removeReq.connectionID); SendManagementMsg( (ushort)removeReq.senderPort, new RemoveResponse(id, mgmtLocalPort, removeReq.seq, status) ); Log.WriteLine("[MGMT CCI] {0} entries for connection {1}", status ? "Removed" : "Could not remove", removeReq.connectionID); break; case "AllocateRequest": Log.WriteLine("[MGMT LRM] {1}, port {2}: {0})", msg.messageType, msg.senderID, msg.senderPort); AllocateRequest allocateReq = (AllocateRequest)msg; uint label = LRM.AssignBandwidthOnInterface(allocateReq.interfaceID, (uint)allocateReq.bitrate, allocateReq.connectionID); SendManagementMsg( (ushort)allocateReq.senderPort, new AllocateResponse( id, mgmtLocalPort, (int)label, allocateReq.seq ) ); if (label != 0) { Log.WriteLine("[MGMT LRM] Allocate {0} Mb/s on iface {1} for connection {2})", allocateReq.bitrate, allocateReq.interfaceID, allocateReq.connectionID); } else { Log.WriteLine("[MGMT LRM] Could not allocate {0} Mb/s on iface {1} for connection {2})", allocateReq.bitrate, allocateReq.interfaceID, allocateReq.connectionID); } break; case "DeallocateRequest": Log.WriteLine("[MGMT LRM] {1}, port {2}: {0})", msg.messageType, msg.senderID, msg.senderPort); DeallocateRequest deallocateReq = (DeallocateRequest)msg; LRM.ReleaseAsignedBandwidth(deallocateReq.connectionID); SendManagementMsg( (ushort)deallocateReq.senderPort, new DeallocateResponse( id, mgmtLocalPort, deallocateReq.seq ) ); Log.WriteLine("[MGMT LRM] Deallocate resources for connection {0})", deallocateReq.connectionID); break; #if DEBUG case "NHLF.AddUpdateResponse": AddUpdateResponse addUpdateResponse = (AddUpdateResponse)msg; Log.WriteLine("[CCI] AddUpdateRequest for {0} {1}.", addUpdateResponse.senderID, addUpdateResponse.status ? "successful" : "failed"); break; case "NHLF.RemoveResponse": RemoveResponse removeResp = (RemoveResponse)msg; Log.WriteLine("[CCI] Remove {0}", removeResp.status ? "some" : "none"); break; case "LRMRC.LinkStateUpdate": LinkStateUpdate linkStateUpdate = (LinkStateUpdate)msg; Log.WriteLine("[RC] {0} --{2}-> {1}", linkStateUpdate.beginNode.id, linkStateUpdate.endNode.id, linkStateUpdate.capacity); break; #endif default: break; } // NHLFMgmtMessage mgmtMessage = NHLFSerialization.Deserialize(bytes); //NHLFEntry newEntry = mgmtMessage.entry; //string result = transportFunction.UpdateRoutingTable(newEntry, mgmtMessage.addOrSwap); } } catch (Exception e) { Log.WriteLine(e.ToString()); } finally { listener.Close(); } }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> public override AllocateResponse Allocate(float progressIndicator) { Preconditions.CheckArgument(progressIndicator >= 0, "Progress indicator should not be negative" ); AllocateResponse allocateResponse = null; IList <ResourceRequest> askList = null; IList <ContainerId> releaseList = null; AllocateRequest allocateRequest = null; IList <string> blacklistToAdd = new AList <string>(); IList <string> blacklistToRemove = new AList <string>(); try { lock (this) { askList = new AList <ResourceRequest>(ask.Count); foreach (ResourceRequest r in ask) { // create a copy of ResourceRequest as we might change it while the // RPC layer is using it to send info across askList.AddItem(ResourceRequest.NewInstance(r.GetPriority(), r.GetResourceName(), r.GetCapability(), r.GetNumContainers(), r.GetRelaxLocality(), r.GetNodeLabelExpression ())); } releaseList = new AList <ContainerId>(release); // optimistically clear this collection assuming no RPC failure ask.Clear(); release.Clear(); Sharpen.Collections.AddAll(blacklistToAdd, blacklistAdditions); Sharpen.Collections.AddAll(blacklistToRemove, blacklistRemovals); ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.NewInstance( blacklistToAdd, blacklistToRemove); allocateRequest = AllocateRequest.NewInstance(lastResponseId, progressIndicator, askList, releaseList, blacklistRequest); // clear blacklistAdditions and blacklistRemovals before // unsynchronized part blacklistAdditions.Clear(); blacklistRemovals.Clear(); } try { allocateResponse = rmClient.Allocate(allocateRequest); } catch (ApplicationMasterNotRegisteredException) { Log.Warn("ApplicationMaster is out of sync with ResourceManager," + " hence resyncing." ); lock (this) { Sharpen.Collections.AddAll(release, this.pendingRelease); Sharpen.Collections.AddAll(blacklistAdditions, this.blacklistedNodes); foreach (IDictionary <string, SortedDictionary <Resource, AMRMClientImpl.ResourceRequestInfo > > rr in remoteRequestsTable.Values) { foreach (IDictionary <Resource, AMRMClientImpl.ResourceRequestInfo> capabalities in rr.Values) { foreach (AMRMClientImpl.ResourceRequestInfo request in capabalities.Values) { AddResourceRequestToAsk(request.remoteRequest); } } } } // re register with RM RegisterApplicationMaster(); allocateResponse = Allocate(progressIndicator); return(allocateResponse); } lock (this) { // update these on successful RPC clusterNodeCount = allocateResponse.GetNumClusterNodes(); lastResponseId = allocateResponse.GetResponseId(); clusterAvailableResources = allocateResponse.GetAvailableResources(); if (!allocateResponse.GetNMTokens().IsEmpty()) { PopulateNMTokens(allocateResponse.GetNMTokens()); } if (allocateResponse.GetAMRMToken() != null) { UpdateAMRMToken(allocateResponse.GetAMRMToken()); } if (!pendingRelease.IsEmpty() && !allocateResponse.GetCompletedContainersStatuses ().IsEmpty()) { RemovePendingReleaseRequests(allocateResponse.GetCompletedContainersStatuses()); } } } finally { // TODO how to differentiate remote yarn exception vs error in rpc if (allocateResponse == null) { // we hit an exception in allocate() // preserve ask and release for next call to allocate() lock (this) { Sharpen.Collections.AddAll(release, releaseList); // requests could have been added or deleted during call to allocate // If requests were added/removed then there is nothing to do since // the ResourceRequest object in ask would have the actual new value. // If ask does not have this ResourceRequest then it was unchanged and // so we can add the value back safely. // This assumes that there will no concurrent calls to allocate() and // so we dont have to worry about ask being changed in the // synchronized block at the beginning of this method. foreach (ResourceRequest oldAsk in askList) { if (!ask.Contains(oldAsk)) { ask.AddItem(oldAsk); } } Sharpen.Collections.AddAll(blacklistAdditions, blacklistToAdd); Sharpen.Collections.AddAll(blacklistRemovals, blacklistToRemove); } } } return(allocateResponse); }
public _PrivilegedExceptionAction_234(MockAM _enclosing, AllocateRequest req) { this._enclosing = _enclosing; this.req = req; }
public _PrivilegedExceptionAction_67(TestAMRMRPCResponseId _enclosing, AllocateRequest req) { this._enclosing = _enclosing; this.req = req; }
public virtual void TestAMRMUnusableNodes() { MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 10000); MockNM nm2 = rm.RegisterNode("127.0.0.2:1234", 10000); MockNM nm3 = rm.RegisterNode("127.0.0.3:1234", 10000); MockNM nm4 = rm.RegisterNode("127.0.0.4:1234", 10000); dispatcher.Await(); RMApp app1 = rm.SubmitApp(2000); // Trigger the scheduling so the AM gets 'launched' on nm1 nm1.NodeHeartbeat(true); RMAppAttempt attempt1 = app1.GetCurrentAppAttempt(); MockAM am1 = rm.SendAMLaunched(attempt1.GetAppAttemptId()); // register AM returns no unusable node am1.RegisterAppAttempt(); // allocate request returns no updated node AllocateRequest allocateRequest1 = AllocateRequest.NewInstance(0, 0F, null, null, null); AllocateResponse response1 = Allocate(attempt1.GetAppAttemptId(), allocateRequest1 ); IList <NodeReport> updatedNodes = response1.GetUpdatedNodes(); NUnit.Framework.Assert.AreEqual(0, updatedNodes.Count); SyncNodeHeartbeat(nm4, false); // allocate request returns updated node allocateRequest1 = AllocateRequest.NewInstance(response1.GetResponseId(), 0F, null , null, null); response1 = Allocate(attempt1.GetAppAttemptId(), allocateRequest1); updatedNodes = response1.GetUpdatedNodes(); NUnit.Framework.Assert.AreEqual(1, updatedNodes.Count); NodeReport nr = updatedNodes.GetEnumerator().Next(); NUnit.Framework.Assert.AreEqual(nm4.GetNodeId(), nr.GetNodeId()); NUnit.Framework.Assert.AreEqual(NodeState.Unhealthy, nr.GetNodeState()); // resending the allocate request returns the same result response1 = Allocate(attempt1.GetAppAttemptId(), allocateRequest1); updatedNodes = response1.GetUpdatedNodes(); NUnit.Framework.Assert.AreEqual(1, updatedNodes.Count); nr = updatedNodes.GetEnumerator().Next(); NUnit.Framework.Assert.AreEqual(nm4.GetNodeId(), nr.GetNodeId()); NUnit.Framework.Assert.AreEqual(NodeState.Unhealthy, nr.GetNodeState()); SyncNodeLost(nm3); // subsequent allocate request returns delta allocateRequest1 = AllocateRequest.NewInstance(response1.GetResponseId(), 0F, null , null, null); response1 = Allocate(attempt1.GetAppAttemptId(), allocateRequest1); updatedNodes = response1.GetUpdatedNodes(); NUnit.Framework.Assert.AreEqual(1, updatedNodes.Count); nr = updatedNodes.GetEnumerator().Next(); NUnit.Framework.Assert.AreEqual(nm3.GetNodeId(), nr.GetNodeId()); NUnit.Framework.Assert.AreEqual(NodeState.Lost, nr.GetNodeState()); // registering another AM gives it the complete failed list RMApp app2 = rm.SubmitApp(2000); // Trigger nm2 heartbeat so that AM gets launched on it nm2.NodeHeartbeat(true); RMAppAttempt attempt2 = app2.GetCurrentAppAttempt(); MockAM am2 = rm.SendAMLaunched(attempt2.GetAppAttemptId()); // register AM returns all unusable nodes am2.RegisterAppAttempt(); // allocate request returns no updated node AllocateRequest allocateRequest2 = AllocateRequest.NewInstance(0, 0F, null, null, null); AllocateResponse response2 = Allocate(attempt2.GetAppAttemptId(), allocateRequest2 ); updatedNodes = response2.GetUpdatedNodes(); NUnit.Framework.Assert.AreEqual(0, updatedNodes.Count); SyncNodeHeartbeat(nm4, true); // both AM's should get delta updated nodes allocateRequest1 = AllocateRequest.NewInstance(response1.GetResponseId(), 0F, null , null, null); response1 = Allocate(attempt1.GetAppAttemptId(), allocateRequest1); updatedNodes = response1.GetUpdatedNodes(); NUnit.Framework.Assert.AreEqual(1, updatedNodes.Count); nr = updatedNodes.GetEnumerator().Next(); NUnit.Framework.Assert.AreEqual(nm4.GetNodeId(), nr.GetNodeId()); NUnit.Framework.Assert.AreEqual(NodeState.Running, nr.GetNodeState()); allocateRequest2 = AllocateRequest.NewInstance(response2.GetResponseId(), 0F, null , null, null); response2 = Allocate(attempt2.GetAppAttemptId(), allocateRequest2); updatedNodes = response2.GetUpdatedNodes(); NUnit.Framework.Assert.AreEqual(1, updatedNodes.Count); nr = updatedNodes.GetEnumerator().Next(); NUnit.Framework.Assert.AreEqual(nm4.GetNodeId(), nr.GetNodeId()); NUnit.Framework.Assert.AreEqual(NodeState.Running, nr.GetNodeState()); // subsequent allocate calls should return no updated nodes allocateRequest2 = AllocateRequest.NewInstance(response2.GetResponseId(), 0F, null , null, null); response2 = Allocate(attempt2.GetAppAttemptId(), allocateRequest2); updatedNodes = response2.GetUpdatedNodes(); NUnit.Framework.Assert.AreEqual(0, updatedNodes.Count); }