private static RMApp CreateRMApp(ApplicationId appId) { RMApp app = Org.Mockito.Mockito.Mock <RMApp>(); Org.Mockito.Mockito.When(app.GetApplicationId()).ThenReturn(appId); Org.Mockito.Mockito.When(app.GetName()).ThenReturn("test app"); Org.Mockito.Mockito.When(app.GetApplicationType()).ThenReturn("test app type"); Org.Mockito.Mockito.When(app.GetUser()).ThenReturn("test user"); Org.Mockito.Mockito.When(app.GetQueue()).ThenReturn("test queue"); Org.Mockito.Mockito.When(app.GetSubmitTime()).ThenReturn(int.MaxValue + 1L); Org.Mockito.Mockito.When(app.GetStartTime()).ThenReturn(int.MaxValue + 2L); Org.Mockito.Mockito.When(app.GetFinishTime()).ThenReturn(int.MaxValue + 3L); Org.Mockito.Mockito.When(app.GetDiagnostics()).ThenReturn(new StringBuilder("test diagnostics info" )); RMAppAttempt appAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>(); Org.Mockito.Mockito.When(appAttempt.GetAppAttemptId()).ThenReturn(ApplicationAttemptId .NewInstance(appId, 1)); Org.Mockito.Mockito.When(app.GetCurrentAppAttempt()).ThenReturn(appAttempt); Org.Mockito.Mockito.When(app.GetFinalApplicationStatus()).ThenReturn(FinalApplicationStatus .Undefined); Org.Mockito.Mockito.When(app.GetRMAppMetrics()).ThenReturn(new RMAppMetrics(null, 0, 0, int.MaxValue, long.MaxValue)); return(app); }
public virtual void TestEscapeApplicationSummary() { RMApp app = Org.Mockito.Mockito.Mock <RMAppImpl>(); Org.Mockito.Mockito.When(app.GetApplicationId()).ThenReturn(ApplicationId.NewInstance (100L, 1)); Org.Mockito.Mockito.When(app.GetName()).ThenReturn("Multiline\n\n\r\rAppName"); Org.Mockito.Mockito.When(app.GetUser()).ThenReturn("Multiline\n\n\r\rUserName"); Org.Mockito.Mockito.When(app.GetQueue()).ThenReturn("Multiline\n\n\r\rQueueName"); Org.Mockito.Mockito.When(app.GetState()).ThenReturn(RMAppState.Running); Org.Mockito.Mockito.When(app.GetApplicationType()).ThenReturn("MAPREDUCE"); RMAppMetrics metrics = new RMAppMetrics(Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(1234, 56), 10, 1, 16384, 64); Org.Mockito.Mockito.When(app.GetRMAppMetrics()).ThenReturn(metrics); RMAppManager.ApplicationSummary.SummaryBuilder summary = RMAppManager.ApplicationSummary .CreateAppSummary(app); string msg = summary.ToString(); Log.Info("summary: " + msg); NUnit.Framework.Assert.IsFalse(msg.Contains("\n")); NUnit.Framework.Assert.IsFalse(msg.Contains("\r")); string escaped = "\\n\\n\\r\\r"; NUnit.Framework.Assert.IsTrue(msg.Contains("Multiline" + escaped + "AppName")); NUnit.Framework.Assert.IsTrue(msg.Contains("Multiline" + escaped + "UserName")); NUnit.Framework.Assert.IsTrue(msg.Contains("Multiline" + escaped + "QueueName")); NUnit.Framework.Assert.IsTrue(msg.Contains("memorySeconds=16384")); NUnit.Framework.Assert.IsTrue(msg.Contains("vcoreSeconds=64")); NUnit.Framework.Assert.IsTrue(msg.Contains("preemptedAMContainers=1")); NUnit.Framework.Assert.IsTrue(msg.Contains("preemptedNonAMContainers=10")); NUnit.Framework.Assert.IsTrue(msg.Contains("preemptedResources=<memory:1234\\, vCores:56>" )); NUnit.Framework.Assert.IsTrue(msg.Contains("applicationType=MAPREDUCE")); }
/// <exception cref="System.Exception"/> public virtual void TestMoveSuccessful() { MockRM rm1 = new MockRM(conf); rm1.Start(); RMApp app = rm1.SubmitApp(1024); ClientRMService clientRMService = rm1.GetClientRMService(); // FIFO scheduler does not support moves clientRMService.MoveApplicationAcrossQueues(MoveApplicationAcrossQueuesRequest.NewInstance (app.GetApplicationId(), "newqueue")); RMApp rmApp = rm1.GetRMContext().GetRMApps()[app.GetApplicationId()]; NUnit.Framework.Assert.AreEqual("newqueue", rmApp.GetQueue()); rm1.Stop(); }
private static RMApp CreateRMApp(ApplicationId appId) { RMApp app = Org.Mockito.Mockito.Mock <RMApp>(); Org.Mockito.Mockito.When(app.GetApplicationId()).ThenReturn(appId); Org.Mockito.Mockito.When(app.GetName()).ThenReturn("test app"); Org.Mockito.Mockito.When(app.GetApplicationType()).ThenReturn("test app type"); Org.Mockito.Mockito.When(app.GetUser()).ThenReturn("test user"); Org.Mockito.Mockito.When(app.GetQueue()).ThenReturn("test queue"); Org.Mockito.Mockito.When(app.GetSubmitTime()).ThenReturn(0L); Org.Mockito.Mockito.When(app.GetStartTime()).ThenReturn(1L); Org.Mockito.Mockito.When(app.GetFinishTime()).ThenReturn(2L); Org.Mockito.Mockito.When(app.GetDiagnostics()).ThenReturn(new StringBuilder("test diagnostics info" )); Org.Mockito.Mockito.When(app.GetFinalApplicationStatus()).ThenReturn(FinalApplicationStatus .Undefined); return(app); }
/// <exception cref="System.Exception"/> private void CheckAppQueue(MockRM resourceManager, string user, string submissionQueue , string expected) { RMApp app = resourceManager.SubmitApp(200, "name", user, new Dictionary <ApplicationAccessType , string>(), false, submissionQueue, -1, null, "MAPREDUCE", false); RMAppState expectedState = expected.IsEmpty() ? RMAppState.Failed : RMAppState.Accepted; resourceManager.WaitForState(app.GetApplicationId(), expectedState); // get scheduler app CapacityScheduler cs = (CapacityScheduler)resourceManager.GetResourceScheduler(); SchedulerApplication schedulerApp = cs.GetSchedulerApplications()[app.GetApplicationId ()]; string queue = string.Empty; if (schedulerApp != null) { queue = schedulerApp.GetQueue().GetQueueName(); } NUnit.Framework.Assert.IsTrue("expected " + expected + " actual " + queue, expected .Equals(queue)); NUnit.Framework.Assert.AreEqual(expected, app.GetQueue()); }
public virtual void TestAppBlockRenderWithNullCurrentAppAttempt() { ApplicationId AppId = ApplicationId.NewInstance(1234L, 0); Injector injector; // init app RMApp app = Org.Mockito.Mockito.Mock <RMApp>(); Org.Mockito.Mockito.When(app.GetTrackingUrl()).ThenReturn("http://host:123"); Org.Mockito.Mockito.When(app.GetState()).ThenReturn(RMAppState.Failed); Org.Mockito.Mockito.When(app.GetApplicationId()).ThenReturn(AppId); Org.Mockito.Mockito.When(app.GetApplicationType()).ThenReturn("Type"); Org.Mockito.Mockito.When(app.GetUser()).ThenReturn("user"); Org.Mockito.Mockito.When(app.GetName()).ThenReturn("Name"); Org.Mockito.Mockito.When(app.GetQueue()).ThenReturn("queue"); Org.Mockito.Mockito.When(app.GetDiagnostics()).ThenReturn(new StringBuilder()); Org.Mockito.Mockito.When(app.GetFinalApplicationStatus()).ThenReturn(FinalApplicationStatus .Failed); Org.Mockito.Mockito.When(app.GetFinalApplicationStatus()).ThenReturn(FinalApplicationStatus .Failed); Org.Mockito.Mockito.When(app.GetStartTime()).ThenReturn(0L); Org.Mockito.Mockito.When(app.GetFinishTime()).ThenReturn(0L); Org.Mockito.Mockito.When(app.CreateApplicationState()).ThenReturn(YarnApplicationState .Failed); RMAppMetrics appMetrics = new RMAppMetrics(Resource.NewInstance(0, 0), 0, 0, 0, 0 ); Org.Mockito.Mockito.When(app.GetRMAppMetrics()).ThenReturn(appMetrics); // initialize RM Context, and create RMApp, without creating RMAppAttempt RMContext rmContext = TestRMWebApp.MockRMContext(15, 1, 2, 8); rmContext.GetRMApps()[AppId] = app; injector = WebAppTests.CreateMockInjector <RMContext>(rmContext, new _Module_76(rmContext )); AppBlock instance = injector.GetInstance <AppBlock>(); instance.Set(YarnWebParams.ApplicationId, AppId.ToString()); instance.Render(); }
public AppInfo(ResourceManager rm, RMApp app, bool hasAccess, string schemePrefix ) { // these are ok for any user to see // these are only allowed if acls allow // preemption info fields // JAXB needs this this.schemePrefix = schemePrefix; if (app != null) { string trackingUrl = app.GetTrackingUrl(); this.state = app.CreateApplicationState(); this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.IsEmpty() || YarnApplicationState .New == this.state || YarnApplicationState.NewSaving == this.state || YarnApplicationState .Submitted == this.state || YarnApplicationState.Accepted == this.state; this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app.GetFinishTime( ) == 0 ? "ApplicationMaster" : "History"); if (!trackingUrlIsNotReady) { this.trackingUrl = WebAppUtils.GetURLWithScheme(schemePrefix, trackingUrl); this.trackingUrlPretty = this.trackingUrl; } else { this.trackingUrlPretty = "UNASSIGNED"; } this.applicationId = app.GetApplicationId(); this.applicationType = app.GetApplicationType(); this.appIdNum = app.GetApplicationId().GetId().ToString(); this.id = app.GetApplicationId().ToString(); this.user = app.GetUser().ToString(); this.name = app.GetName().ToString(); this.queue = app.GetQueue().ToString(); this.progress = app.GetProgress() * 100; this.diagnostics = app.GetDiagnostics().ToString(); if (diagnostics == null || diagnostics.IsEmpty()) { this.diagnostics = string.Empty; } if (app.GetApplicationTags() != null && !app.GetApplicationTags().IsEmpty()) { this.applicationTags = Joiner.On(',').Join(app.GetApplicationTags()); } this.finalStatus = app.GetFinalApplicationStatus(); this.clusterId = ResourceManager.GetClusterTimeStamp(); if (hasAccess) { this.startedTime = app.GetStartTime(); this.finishedTime = app.GetFinishTime(); this.elapsedTime = Times.Elapsed(app.GetStartTime(), app.GetFinishTime()); RMAppAttempt attempt = app.GetCurrentAppAttempt(); if (attempt != null) { Container masterContainer = attempt.GetMasterContainer(); if (masterContainer != null) { this.amContainerLogsExist = true; this.amContainerLogs = WebAppUtils.GetRunningLogURL(schemePrefix + masterContainer .GetNodeHttpAddress(), ConverterUtils.ToString(masterContainer.GetId()), app.GetUser ()); this.amHostHttpAddress = masterContainer.GetNodeHttpAddress(); } ApplicationResourceUsageReport resourceReport = attempt.GetApplicationResourceUsageReport (); if (resourceReport != null) { Resource usedResources = resourceReport.GetUsedResources(); allocatedMB = usedResources.GetMemory(); allocatedVCores = usedResources.GetVirtualCores(); runningContainers = resourceReport.GetNumUsedContainers(); } resourceRequests = ((AbstractYarnScheduler)rm.GetRMContext().GetScheduler()).GetPendingResourceRequestsForAttempt (attempt.GetAppAttemptId()); } } // copy preemption info fields RMAppMetrics appMetrics = app.GetRMAppMetrics(); numAMContainerPreempted = appMetrics.GetNumAMContainersPreempted(); preemptedResourceMB = appMetrics.GetResourcePreempted().GetMemory(); numNonAMContainerPreempted = appMetrics.GetNumNonAMContainersPreempted(); preemptedResourceVCores = appMetrics.GetResourcePreempted().GetVirtualCores(); memorySeconds = appMetrics.GetMemorySeconds(); vcoreSeconds = appMetrics.GetVcoreSeconds(); } }
public virtual void AppCreated(RMApp app, long createdTime) { if (publishSystemMetrics) { dispatcher.GetEventHandler().Handle(new ApplicationCreatedEvent(app.GetApplicationId (), app.GetName(), app.GetApplicationType(), app.GetUser(), app.GetQueue(), app. GetSubmitTime(), createdTime)); } }
public virtual void ApplicationStarted(RMApp app) { if (historyServiceEnabled) { dispatcher.GetEventHandler().Handle(new WritingApplicationStartEvent(app.GetApplicationId (), ApplicationStartData.NewInstance(app.GetApplicationId(), app.GetName(), app. GetApplicationType(), app.GetQueue(), app.GetUser(), app.GetSubmitTime(), app.GetStartTime ()))); } }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> public virtual AllocateResponse Allocate(AllocateRequest request) { AMRMTokenIdentifier amrmTokenIdentifier = AuthorizeRequest(); ApplicationAttemptId appAttemptId = amrmTokenIdentifier.GetApplicationAttemptId(); ApplicationId applicationId = appAttemptId.GetApplicationId(); this.amLivelinessMonitor.ReceivedPing(appAttemptId); /* check if its in cache */ ApplicationMasterService.AllocateResponseLock Lock = responseMap[appAttemptId]; if (Lock == null) { string message = "Application attempt " + appAttemptId + " doesn't exist in ApplicationMasterService cache."; Log.Error(message); throw new ApplicationAttemptNotFoundException(message); } lock (Lock) { AllocateResponse lastResponse = Lock.GetAllocateResponse(); if (!HasApplicationMasterRegistered(appAttemptId)) { string message = "AM is not registered for known application attempt: " + appAttemptId + " or RM had restarted after AM registered . AM should re-register."; Log.Info(message); RMAuditLogger.LogFailure(this.rmContext.GetRMApps()[appAttemptId.GetApplicationId ()].GetUser(), RMAuditLogger.AuditConstants.AmAllocate, string.Empty, "ApplicationMasterService" , message, applicationId, appAttemptId); throw new ApplicationMasterNotRegisteredException(message); } if ((request.GetResponseId() + 1) == lastResponse.GetResponseId()) { /* old heartbeat */ return(lastResponse); } else { if (request.GetResponseId() + 1 < lastResponse.GetResponseId()) { string message = "Invalid responseId in AllocateRequest from application attempt: " + appAttemptId + ", expect responseId to be " + (lastResponse.GetResponseId() + 1); throw new InvalidApplicationMasterRequestException(message); } } //filter illegal progress values float filteredProgress = request.GetProgress(); if (float.IsNaN(filteredProgress) || filteredProgress == float.NegativeInfinity || filteredProgress < 0) { request.SetProgress(0); } else { if (filteredProgress > 1 || filteredProgress == float.PositiveInfinity) { request.SetProgress(1); } } // Send the status update to the appAttempt. this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMAppAttemptStatusupdateEvent (appAttemptId, request.GetProgress())); IList <ResourceRequest> ask = request.GetAskList(); IList <ContainerId> release = request.GetReleaseList(); ResourceBlacklistRequest blacklistRequest = request.GetResourceBlacklistRequest(); IList <string> blacklistAdditions = (blacklistRequest != null) ? blacklistRequest. GetBlacklistAdditions() : Sharpen.Collections.EmptyList; IList <string> blacklistRemovals = (blacklistRequest != null) ? blacklistRequest.GetBlacklistRemovals () : Sharpen.Collections.EmptyList; RMApp app = this.rmContext.GetRMApps()[applicationId]; // set label expression for Resource Requests if resourceName=ANY ApplicationSubmissionContext asc = app.GetApplicationSubmissionContext(); foreach (ResourceRequest req in ask) { if (null == req.GetNodeLabelExpression() && ResourceRequest.Any.Equals(req.GetResourceName ())) { req.SetNodeLabelExpression(asc.GetNodeLabelExpression()); } } // sanity check try { RMServerUtils.NormalizeAndValidateRequests(ask, rScheduler.GetMaximumResourceCapability (), app.GetQueue(), rScheduler, rmContext); } catch (InvalidResourceRequestException e) { Log.Warn("Invalid resource ask by application " + appAttemptId, e); throw; } try { RMServerUtils.ValidateBlacklistRequest(blacklistRequest); } catch (InvalidResourceBlacklistRequestException e) { Log.Warn("Invalid blacklist request by application " + appAttemptId, e); throw; } // In the case of work-preserving AM restart, it's possible for the // AM to release containers from the earlier attempt. if (!app.GetApplicationSubmissionContext().GetKeepContainersAcrossApplicationAttempts ()) { try { RMServerUtils.ValidateContainerReleaseRequest(release, appAttemptId); } catch (InvalidContainerReleaseException e) { Log.Warn("Invalid container release by application " + appAttemptId, e); throw; } } // Send new requests to appAttempt. Allocation allocation = this.rScheduler.Allocate(appAttemptId, ask, release, blacklistAdditions , blacklistRemovals); if (!blacklistAdditions.IsEmpty() || !blacklistRemovals.IsEmpty()) { Log.Info("blacklist are updated in Scheduler." + "blacklistAdditions: " + blacklistAdditions + ", " + "blacklistRemovals: " + blacklistRemovals); } RMAppAttempt appAttempt = app.GetRMAppAttempt(appAttemptId); AllocateResponse allocateResponse = recordFactory.NewRecordInstance <AllocateResponse >(); if (!allocation.GetContainers().IsEmpty()) { allocateResponse.SetNMTokens(allocation.GetNMTokens()); } // update the response with the deltas of node status changes IList <RMNode> updatedNodes = new AList <RMNode>(); if (app.PullRMNodeUpdates(updatedNodes) > 0) { IList <NodeReport> updatedNodeReports = new AList <NodeReport>(); foreach (RMNode rmNode in updatedNodes) { SchedulerNodeReport schedulerNodeReport = rScheduler.GetNodeReport(rmNode.GetNodeID ()); Resource used = BuilderUtils.NewResource(0, 0); int numContainers = 0; if (schedulerNodeReport != null) { used = schedulerNodeReport.GetUsedResource(); numContainers = schedulerNodeReport.GetNumContainers(); } NodeId nodeId = rmNode.GetNodeID(); NodeReport report = BuilderUtils.NewNodeReport(nodeId, rmNode.GetState(), rmNode. GetHttpAddress(), rmNode.GetRackName(), used, rmNode.GetTotalCapability(), numContainers , rmNode.GetHealthReport(), rmNode.GetLastHealthReportTime(), rmNode.GetNodeLabels ()); updatedNodeReports.AddItem(report); } allocateResponse.SetUpdatedNodes(updatedNodeReports); } allocateResponse.SetAllocatedContainers(allocation.GetContainers()); allocateResponse.SetCompletedContainersStatuses(appAttempt.PullJustFinishedContainers ()); allocateResponse.SetResponseId(lastResponse.GetResponseId() + 1); allocateResponse.SetAvailableResources(allocation.GetResourceLimit()); allocateResponse.SetNumClusterNodes(this.rScheduler.GetNumClusterNodes()); // add preemption to the allocateResponse message (if any) allocateResponse.SetPreemptionMessage(GeneratePreemptionMessage(allocation)); // update AMRMToken if the token is rolled-up MasterKeyData nextMasterKey = this.rmContext.GetAMRMTokenSecretManager().GetNextMasterKeyData (); if (nextMasterKey != null && nextMasterKey.GetMasterKey().GetKeyId() != amrmTokenIdentifier .GetKeyId()) { RMAppAttemptImpl appAttemptImpl = (RMAppAttemptImpl)appAttempt; Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> amrmToken = appAttempt .GetAMRMToken(); if (nextMasterKey.GetMasterKey().GetKeyId() != appAttemptImpl.GetAMRMTokenKeyId()) { Log.Info("The AMRMToken has been rolled-over. Send new AMRMToken back" + " to application: " + applicationId); amrmToken = rmContext.GetAMRMTokenSecretManager().CreateAndGetAMRMToken(appAttemptId ); appAttemptImpl.SetAMRMToken(amrmToken); } allocateResponse.SetAMRMToken(Org.Apache.Hadoop.Yarn.Api.Records.Token.NewInstance (amrmToken.GetIdentifier(), amrmToken.GetKind().ToString(), amrmToken.GetPassword (), amrmToken.GetService().ToString())); } /* * As we are updating the response inside the lock object so we don't * need to worry about unregister call occurring in between (which * removes the lock object). */ Lock.SetAllocateResponse(allocateResponse); return(allocateResponse); } }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> public virtual RegisterApplicationMasterResponse RegisterApplicationMaster(RegisterApplicationMasterRequest request) { AMRMTokenIdentifier amrmTokenIdentifier = AuthorizeRequest(); ApplicationAttemptId applicationAttemptId = amrmTokenIdentifier.GetApplicationAttemptId (); ApplicationId appID = applicationAttemptId.GetApplicationId(); ApplicationMasterService.AllocateResponseLock Lock = responseMap[applicationAttemptId ]; if (Lock == null) { RMAuditLogger.LogFailure(this.rmContext.GetRMApps()[appID].GetUser(), RMAuditLogger.AuditConstants .RegisterAm, "Application doesn't exist in cache " + applicationAttemptId, "ApplicationMasterService" , "Error in registering application master", appID, applicationAttemptId); ThrowApplicationDoesNotExistInCacheException(applicationAttemptId); } // Allow only one thread in AM to do registerApp at a time. lock (Lock) { AllocateResponse lastResponse = Lock.GetAllocateResponse(); if (HasApplicationMasterRegistered(applicationAttemptId)) { string message = "Application Master is already registered : " + appID; Log.Warn(message); RMAuditLogger.LogFailure(this.rmContext.GetRMApps()[appID].GetUser(), RMAuditLogger.AuditConstants .RegisterAm, string.Empty, "ApplicationMasterService", message, appID, applicationAttemptId ); throw new InvalidApplicationMasterRequestException(message); } this.amLivelinessMonitor.ReceivedPing(applicationAttemptId); RMApp app = this.rmContext.GetRMApps()[appID]; // Setting the response id to 0 to identify if the // application master is register for the respective attemptid lastResponse.SetResponseId(0); Lock.SetAllocateResponse(lastResponse); Log.Info("AM registration " + applicationAttemptId); this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMAppAttemptRegistrationEvent (applicationAttemptId, request.GetHost(), request.GetRpcPort(), request.GetTrackingUrl ())); RMAuditLogger.LogSuccess(app.GetUser(), RMAuditLogger.AuditConstants.RegisterAm, "ApplicationMasterService", appID, applicationAttemptId); // Pick up min/max resource from scheduler... RegisterApplicationMasterResponse response = recordFactory.NewRecordInstance <RegisterApplicationMasterResponse >(); response.SetMaximumResourceCapability(rScheduler.GetMaximumResourceCapability(app .GetQueue())); response.SetApplicationACLs(app.GetRMAppAttempt(applicationAttemptId).GetSubmissionContext ().GetAMContainerSpec().GetApplicationACLs()); response.SetQueue(app.GetQueue()); if (UserGroupInformation.IsSecurityEnabled()) { Log.Info("Setting client token master key"); response.SetClientToAMTokenMasterKey(ByteBuffer.Wrap(rmContext.GetClientToAMTokenSecretManager ().GetMasterKey(applicationAttemptId).GetEncoded())); } // For work-preserving AM restart, retrieve previous attempts' containers // and corresponding NM tokens. if (app.GetApplicationSubmissionContext().GetKeepContainersAcrossApplicationAttempts ()) { IList <Container> transferredContainers = ((AbstractYarnScheduler)rScheduler).GetTransferredContainers (applicationAttemptId); if (!transferredContainers.IsEmpty()) { response.SetContainersFromPreviousAttempts(transferredContainers); IList <NMToken> nmTokens = new AList <NMToken>(); foreach (Container container in transferredContainers) { try { NMToken token = rmContext.GetNMTokenSecretManager().CreateAndGetNMToken(app.GetUser (), applicationAttemptId, container); if (null != token) { nmTokens.AddItem(token); } } catch (ArgumentException e) { // if it's a DNS issue, throw UnknowHostException directly and // that // will be automatically retried by RMProxy in RPC layer. if (e.InnerException is UnknownHostException) { throw (UnknownHostException)e.InnerException; } } } response.SetNMTokensFromPreviousAttempts(nmTokens); Log.Info("Application " + appID + " retrieved " + transferredContainers.Count + " containers from previous" + " attempts and " + nmTokens.Count + " NM tokens."); } } response.SetSchedulerResourceTypes(rScheduler.GetSchedulingResourceTypes()); return(response); } }
/// <summary>create a summary of the application's runtime.</summary> /// <param name="app"> /// /// <see cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Rmapp.RMApp"/> /// whose summary is to be created, cannot /// be <code>null</code>. /// </param> public static RMAppManager.ApplicationSummary.SummaryBuilder CreateAppSummary(RMApp app) { string trackingUrl = "N/A"; string host = "N/A"; RMAppAttempt attempt = app.GetCurrentAppAttempt(); if (attempt != null) { trackingUrl = attempt.GetTrackingUrl(); host = attempt.GetHost(); } RMAppMetrics metrics = app.GetRMAppMetrics(); RMAppManager.ApplicationSummary.SummaryBuilder summary = new RMAppManager.ApplicationSummary.SummaryBuilder ().Add("appId", app.GetApplicationId()).Add("name", app.GetName()).Add("user", app .GetUser()).Add("queue", app.GetQueue()).Add("state", app.GetState()).Add("trackingUrl" , trackingUrl).Add("appMasterHost", host).Add("startTime", app.GetStartTime()).Add ("finishTime", app.GetFinishTime()).Add("finalStatus", app.GetFinalApplicationStatus ()).Add("memorySeconds", metrics.GetMemorySeconds()).Add("vcoreSeconds", metrics .GetVcoreSeconds()).Add("preemptedAMContainers", metrics.GetNumAMContainersPreempted ()).Add("preemptedNonAMContainers", metrics.GetNumNonAMContainersPreempted()).Add ("preemptedResources", metrics.GetResourcePreempted()).Add("applicationType", app .GetApplicationType()); return(summary); }
/// <exception cref="System.Exception"/> public virtual void TestPublishApplicationMetrics() { for (int i = 1; i <= 2; ++i) { ApplicationId appId = ApplicationId.NewInstance(0, i); RMApp app = CreateRMApp(appId); metricsPublisher.AppCreated(app, app.GetStartTime()); metricsPublisher.AppFinished(app, RMAppState.Finished, app.GetFinishTime()); if (i == 1) { metricsPublisher.AppACLsUpdated(app, "uers1,user2", 4L); } else { // in case user doesn't specify the ACLs metricsPublisher.AppACLsUpdated(app, null, 4L); } TimelineEntity entity = null; do { entity = store.GetEntity(appId.ToString(), ApplicationMetricsConstants.EntityType , EnumSet.AllOf <TimelineReader.Field>()); }while (entity == null || entity.GetEvents().Count < 3); // ensure three events are both published before leaving the loop // verify all the fields NUnit.Framework.Assert.AreEqual(ApplicationMetricsConstants.EntityType, entity.GetEntityType ()); NUnit.Framework.Assert.AreEqual(app.GetApplicationId().ToString(), entity.GetEntityId ()); NUnit.Framework.Assert.AreEqual(app.GetName(), entity.GetOtherInfo()[ApplicationMetricsConstants .NameEntityInfo]); NUnit.Framework.Assert.AreEqual(app.GetQueue(), entity.GetOtherInfo()[ApplicationMetricsConstants .QueueEntityInfo]); NUnit.Framework.Assert.AreEqual(app.GetUser(), entity.GetOtherInfo()[ApplicationMetricsConstants .UserEntityInfo]); NUnit.Framework.Assert.AreEqual(app.GetApplicationType(), entity.GetOtherInfo()[ApplicationMetricsConstants .TypeEntityInfo]); NUnit.Framework.Assert.AreEqual(app.GetSubmitTime(), entity.GetOtherInfo()[ApplicationMetricsConstants .SubmittedTimeEntityInfo]); if (i == 1) { NUnit.Framework.Assert.AreEqual("uers1,user2", entity.GetOtherInfo()[ApplicationMetricsConstants .AppViewAclsEntityInfo]); } else { NUnit.Framework.Assert.AreEqual(string.Empty, entity.GetOtherInfo()[ApplicationMetricsConstants .AppViewAclsEntityInfo]); NUnit.Framework.Assert.AreEqual(app.GetRMAppMetrics().GetMemorySeconds(), long.Parse (entity.GetOtherInfo()[ApplicationMetricsConstants.AppMemMetrics].ToString())); NUnit.Framework.Assert.AreEqual(app.GetRMAppMetrics().GetVcoreSeconds(), long.Parse (entity.GetOtherInfo()[ApplicationMetricsConstants.AppCpuMetrics].ToString())); } bool hasCreatedEvent = false; bool hasFinishedEvent = false; bool hasACLsUpdatedEvent = false; foreach (TimelineEvent @event in entity.GetEvents()) { if (@event.GetEventType().Equals(ApplicationMetricsConstants.CreatedEventType)) { hasCreatedEvent = true; NUnit.Framework.Assert.AreEqual(app.GetStartTime(), @event.GetTimestamp()); } else { if (@event.GetEventType().Equals(ApplicationMetricsConstants.FinishedEventType)) { hasFinishedEvent = true; NUnit.Framework.Assert.AreEqual(app.GetFinishTime(), @event.GetTimestamp()); NUnit.Framework.Assert.AreEqual(app.GetDiagnostics().ToString(), @event.GetEventInfo ()[ApplicationMetricsConstants.DiagnosticsInfoEventInfo]); NUnit.Framework.Assert.AreEqual(app.GetFinalApplicationStatus().ToString(), @event .GetEventInfo()[ApplicationMetricsConstants.FinalStatusEventInfo]); NUnit.Framework.Assert.AreEqual(YarnApplicationState.Finished.ToString(), @event. GetEventInfo()[ApplicationMetricsConstants.StateEventInfo]); } else { if (@event.GetEventType().Equals(ApplicationMetricsConstants.AclsUpdatedEventType )) { hasACLsUpdatedEvent = true; NUnit.Framework.Assert.AreEqual(4L, @event.GetTimestamp()); } } } } NUnit.Framework.Assert.IsTrue(hasCreatedEvent && hasFinishedEvent && hasACLsUpdatedEvent ); } }