private static RMApp CreateRMApp(ApplicationId appId)
{
RMApp app = Org.Mockito.Mockito.Mock <RMApp>();
Org.Mockito.Mockito.When(app.GetApplicationId()).ThenReturn(appId);
Org.Mockito.Mockito.When(app.GetName()).ThenReturn("test app");
Org.Mockito.Mockito.When(app.GetApplicationType()).ThenReturn("test app type");
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn("test user");
Org.Mockito.Mockito.When(app.GetQueue()).ThenReturn("test queue");
Org.Mockito.Mockito.When(app.GetSubmitTime()).ThenReturn(int.MaxValue + 1L);
Org.Mockito.Mockito.When(app.GetStartTime()).ThenReturn(int.MaxValue + 2L);
Org.Mockito.Mockito.When(app.GetFinishTime()).ThenReturn(int.MaxValue + 3L);
Org.Mockito.Mockito.When(app.GetDiagnostics()).ThenReturn(new StringBuilder("test diagnostics info"
));
RMAppAttempt appAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>();
Org.Mockito.Mockito.When(appAttempt.GetAppAttemptId()).ThenReturn(ApplicationAttemptId
.NewInstance(appId, 1));
Org.Mockito.Mockito.When(app.GetCurrentAppAttempt()).ThenReturn(appAttempt);
Org.Mockito.Mockito.When(app.GetFinalApplicationStatus()).ThenReturn(FinalApplicationStatus
.Undefined);
Org.Mockito.Mockito.When(app.GetRMAppMetrics()).ThenReturn(new RMAppMetrics(null,
0, 0, int.MaxValue, long.MaxValue));
return(app);
}
public virtual void TestEscapeApplicationSummary()
{
RMApp app = Org.Mockito.Mockito.Mock <RMAppImpl>();
Org.Mockito.Mockito.When(app.GetApplicationId()).ThenReturn(ApplicationId.NewInstance
(100L, 1));
Org.Mockito.Mockito.When(app.GetName()).ThenReturn("Multiline\n\n\r\rAppName");
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn("Multiline\n\n\r\rUserName");
Org.Mockito.Mockito.When(app.GetQueue()).ThenReturn("Multiline\n\n\r\rQueueName");
Org.Mockito.Mockito.When(app.GetState()).ThenReturn(RMAppState.Running);
Org.Mockito.Mockito.When(app.GetApplicationType()).ThenReturn("MAPREDUCE");
RMAppMetrics metrics = new RMAppMetrics(Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(1234, 56), 10, 1, 16384, 64);
Org.Mockito.Mockito.When(app.GetRMAppMetrics()).ThenReturn(metrics);
RMAppManager.ApplicationSummary.SummaryBuilder summary = RMAppManager.ApplicationSummary
.CreateAppSummary(app);
string msg = summary.ToString();
Log.Info("summary: " + msg);
NUnit.Framework.Assert.IsFalse(msg.Contains("\n"));
NUnit.Framework.Assert.IsFalse(msg.Contains("\r"));
string escaped = "\\n\\n\\r\\r";
NUnit.Framework.Assert.IsTrue(msg.Contains("Multiline" + escaped + "AppName"));
NUnit.Framework.Assert.IsTrue(msg.Contains("Multiline" + escaped + "UserName"));
NUnit.Framework.Assert.IsTrue(msg.Contains("Multiline" + escaped + "QueueName"));
NUnit.Framework.Assert.IsTrue(msg.Contains("memorySeconds=16384"));
NUnit.Framework.Assert.IsTrue(msg.Contains("vcoreSeconds=64"));
NUnit.Framework.Assert.IsTrue(msg.Contains("preemptedAMContainers=1"));
NUnit.Framework.Assert.IsTrue(msg.Contains("preemptedNonAMContainers=10"));
NUnit.Framework.Assert.IsTrue(msg.Contains("preemptedResources=<memory:1234\\, vCores:56>"
));
NUnit.Framework.Assert.IsTrue(msg.Contains("applicationType=MAPREDUCE"));
}
/// <exception cref="System.Exception"/>
public virtual void TestMoveSuccessful()
{
MockRM rm1 = new MockRM(conf);
rm1.Start();
RMApp app = rm1.SubmitApp(1024);
ClientRMService clientRMService = rm1.GetClientRMService();
// FIFO scheduler does not support moves
clientRMService.MoveApplicationAcrossQueues(MoveApplicationAcrossQueuesRequest.NewInstance
(app.GetApplicationId(), "newqueue"));
RMApp rmApp = rm1.GetRMContext().GetRMApps()[app.GetApplicationId()];
NUnit.Framework.Assert.AreEqual("newqueue", rmApp.GetQueue());
rm1.Stop();
}
private static RMApp CreateRMApp(ApplicationId appId)
{
RMApp app = Org.Mockito.Mockito.Mock <RMApp>();
Org.Mockito.Mockito.When(app.GetApplicationId()).ThenReturn(appId);
Org.Mockito.Mockito.When(app.GetName()).ThenReturn("test app");
Org.Mockito.Mockito.When(app.GetApplicationType()).ThenReturn("test app type");
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn("test user");
Org.Mockito.Mockito.When(app.GetQueue()).ThenReturn("test queue");
Org.Mockito.Mockito.When(app.GetSubmitTime()).ThenReturn(0L);
Org.Mockito.Mockito.When(app.GetStartTime()).ThenReturn(1L);
Org.Mockito.Mockito.When(app.GetFinishTime()).ThenReturn(2L);
Org.Mockito.Mockito.When(app.GetDiagnostics()).ThenReturn(new StringBuilder("test diagnostics info"
));
Org.Mockito.Mockito.When(app.GetFinalApplicationStatus()).ThenReturn(FinalApplicationStatus
.Undefined);
return(app);
}
/// <exception cref="System.Exception"/>
private void CheckAppQueue(MockRM resourceManager, string user, string submissionQueue
, string expected)
{
RMApp app = resourceManager.SubmitApp(200, "name", user, new Dictionary <ApplicationAccessType
, string>(), false, submissionQueue, -1, null, "MAPREDUCE", false);
RMAppState expectedState = expected.IsEmpty() ? RMAppState.Failed : RMAppState.Accepted;
resourceManager.WaitForState(app.GetApplicationId(), expectedState);
// get scheduler app
CapacityScheduler cs = (CapacityScheduler)resourceManager.GetResourceScheduler();
SchedulerApplication schedulerApp = cs.GetSchedulerApplications()[app.GetApplicationId
()];
string queue = string.Empty;
if (schedulerApp != null)
{
queue = schedulerApp.GetQueue().GetQueueName();
}
NUnit.Framework.Assert.IsTrue("expected " + expected + " actual " + queue, expected
.Equals(queue));
NUnit.Framework.Assert.AreEqual(expected, app.GetQueue());
}
public virtual void TestAppBlockRenderWithNullCurrentAppAttempt()
{
ApplicationId AppId = ApplicationId.NewInstance(1234L, 0);
Injector injector;
// init app
RMApp app = Org.Mockito.Mockito.Mock <RMApp>();
Org.Mockito.Mockito.When(app.GetTrackingUrl()).ThenReturn("http://host:123");
Org.Mockito.Mockito.When(app.GetState()).ThenReturn(RMAppState.Failed);
Org.Mockito.Mockito.When(app.GetApplicationId()).ThenReturn(AppId);
Org.Mockito.Mockito.When(app.GetApplicationType()).ThenReturn("Type");
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn("user");
Org.Mockito.Mockito.When(app.GetName()).ThenReturn("Name");
Org.Mockito.Mockito.When(app.GetQueue()).ThenReturn("queue");
Org.Mockito.Mockito.When(app.GetDiagnostics()).ThenReturn(new StringBuilder());
Org.Mockito.Mockito.When(app.GetFinalApplicationStatus()).ThenReturn(FinalApplicationStatus
.Failed);
Org.Mockito.Mockito.When(app.GetFinalApplicationStatus()).ThenReturn(FinalApplicationStatus
.Failed);
Org.Mockito.Mockito.When(app.GetStartTime()).ThenReturn(0L);
Org.Mockito.Mockito.When(app.GetFinishTime()).ThenReturn(0L);
Org.Mockito.Mockito.When(app.CreateApplicationState()).ThenReturn(YarnApplicationState
.Failed);
RMAppMetrics appMetrics = new RMAppMetrics(Resource.NewInstance(0, 0), 0, 0, 0, 0
);
Org.Mockito.Mockito.When(app.GetRMAppMetrics()).ThenReturn(appMetrics);
// initialize RM Context, and create RMApp, without creating RMAppAttempt
RMContext rmContext = TestRMWebApp.MockRMContext(15, 1, 2, 8);
rmContext.GetRMApps()[AppId] = app;
injector = WebAppTests.CreateMockInjector <RMContext>(rmContext, new _Module_76(rmContext
));
AppBlock instance = injector.GetInstance <AppBlock>();
instance.Set(YarnWebParams.ApplicationId, AppId.ToString());
instance.Render();
}
public AppInfo(ResourceManager rm, RMApp app, bool hasAccess, string schemePrefix
)
{
// these are ok for any user to see
// these are only allowed if acls allow
// preemption info fields
// JAXB needs this
this.schemePrefix = schemePrefix;
if (app != null)
{
string trackingUrl = app.GetTrackingUrl();
this.state = app.CreateApplicationState();
this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.IsEmpty() || YarnApplicationState
.New == this.state || YarnApplicationState.NewSaving == this.state || YarnApplicationState
.Submitted == this.state || YarnApplicationState.Accepted == this.state;
this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app.GetFinishTime(
) == 0 ? "ApplicationMaster" : "History");
if (!trackingUrlIsNotReady)
{
this.trackingUrl = WebAppUtils.GetURLWithScheme(schemePrefix, trackingUrl);
this.trackingUrlPretty = this.trackingUrl;
}
else
{
this.trackingUrlPretty = "UNASSIGNED";
}
this.applicationId = app.GetApplicationId();
this.applicationType = app.GetApplicationType();
this.appIdNum = app.GetApplicationId().GetId().ToString();
this.id = app.GetApplicationId().ToString();
this.user = app.GetUser().ToString();
this.name = app.GetName().ToString();
this.queue = app.GetQueue().ToString();
this.progress = app.GetProgress() * 100;
this.diagnostics = app.GetDiagnostics().ToString();
if (diagnostics == null || diagnostics.IsEmpty())
{
this.diagnostics = string.Empty;
}
if (app.GetApplicationTags() != null && !app.GetApplicationTags().IsEmpty())
{
this.applicationTags = Joiner.On(',').Join(app.GetApplicationTags());
}
this.finalStatus = app.GetFinalApplicationStatus();
this.clusterId = ResourceManager.GetClusterTimeStamp();
if (hasAccess)
{
this.startedTime = app.GetStartTime();
this.finishedTime = app.GetFinishTime();
this.elapsedTime = Times.Elapsed(app.GetStartTime(), app.GetFinishTime());
RMAppAttempt attempt = app.GetCurrentAppAttempt();
if (attempt != null)
{
Container masterContainer = attempt.GetMasterContainer();
if (masterContainer != null)
{
this.amContainerLogsExist = true;
this.amContainerLogs = WebAppUtils.GetRunningLogURL(schemePrefix + masterContainer
.GetNodeHttpAddress(), ConverterUtils.ToString(masterContainer.GetId()), app.GetUser
());
this.amHostHttpAddress = masterContainer.GetNodeHttpAddress();
}
ApplicationResourceUsageReport resourceReport = attempt.GetApplicationResourceUsageReport
();
if (resourceReport != null)
{
Resource usedResources = resourceReport.GetUsedResources();
allocatedMB = usedResources.GetMemory();
allocatedVCores = usedResources.GetVirtualCores();
runningContainers = resourceReport.GetNumUsedContainers();
}
resourceRequests = ((AbstractYarnScheduler)rm.GetRMContext().GetScheduler()).GetPendingResourceRequestsForAttempt
(attempt.GetAppAttemptId());
}
}
// copy preemption info fields
RMAppMetrics appMetrics = app.GetRMAppMetrics();
numAMContainerPreempted = appMetrics.GetNumAMContainersPreempted();
preemptedResourceMB = appMetrics.GetResourcePreempted().GetMemory();
numNonAMContainerPreempted = appMetrics.GetNumNonAMContainersPreempted();
preemptedResourceVCores = appMetrics.GetResourcePreempted().GetVirtualCores();
memorySeconds = appMetrics.GetMemorySeconds();
vcoreSeconds = appMetrics.GetVcoreSeconds();
}
}
public virtual void AppCreated(RMApp app, long createdTime)
{
if (publishSystemMetrics)
{
dispatcher.GetEventHandler().Handle(new ApplicationCreatedEvent(app.GetApplicationId
(), app.GetName(), app.GetApplicationType(), app.GetUser(), app.GetQueue(), app.
GetSubmitTime(), createdTime));
}
}
public virtual void ApplicationStarted(RMApp app)
{
if (historyServiceEnabled)
{
dispatcher.GetEventHandler().Handle(new WritingApplicationStartEvent(app.GetApplicationId
(), ApplicationStartData.NewInstance(app.GetApplicationId(), app.GetName(), app.
GetApplicationType(), app.GetQueue(), app.GetUser(), app.GetSubmitTime(), app.GetStartTime
())));
}
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual AllocateResponse Allocate(AllocateRequest request)
{
AMRMTokenIdentifier amrmTokenIdentifier = AuthorizeRequest();
ApplicationAttemptId appAttemptId = amrmTokenIdentifier.GetApplicationAttemptId();
ApplicationId applicationId = appAttemptId.GetApplicationId();
this.amLivelinessMonitor.ReceivedPing(appAttemptId);
/* check if its in cache */
ApplicationMasterService.AllocateResponseLock Lock = responseMap[appAttemptId];
if (Lock == null)
{
string message = "Application attempt " + appAttemptId + " doesn't exist in ApplicationMasterService cache.";
Log.Error(message);
throw new ApplicationAttemptNotFoundException(message);
}
lock (Lock)
{
AllocateResponse lastResponse = Lock.GetAllocateResponse();
if (!HasApplicationMasterRegistered(appAttemptId))
{
string message = "AM is not registered for known application attempt: " + appAttemptId
+ " or RM had restarted after AM registered . AM should re-register.";
Log.Info(message);
RMAuditLogger.LogFailure(this.rmContext.GetRMApps()[appAttemptId.GetApplicationId
()].GetUser(), RMAuditLogger.AuditConstants.AmAllocate, string.Empty, "ApplicationMasterService"
, message, applicationId, appAttemptId);
throw new ApplicationMasterNotRegisteredException(message);
}
if ((request.GetResponseId() + 1) == lastResponse.GetResponseId())
{
/* old heartbeat */
return(lastResponse);
}
else
{
if (request.GetResponseId() + 1 < lastResponse.GetResponseId())
{
string message = "Invalid responseId in AllocateRequest from application attempt: "
+ appAttemptId + ", expect responseId to be " + (lastResponse.GetResponseId() +
1);
throw new InvalidApplicationMasterRequestException(message);
}
}
//filter illegal progress values
float filteredProgress = request.GetProgress();
if (float.IsNaN(filteredProgress) || filteredProgress == float.NegativeInfinity ||
filteredProgress < 0)
{
request.SetProgress(0);
}
else
{
if (filteredProgress > 1 || filteredProgress == float.PositiveInfinity)
{
request.SetProgress(1);
}
}
// Send the status update to the appAttempt.
this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMAppAttemptStatusupdateEvent
(appAttemptId, request.GetProgress()));
IList <ResourceRequest> ask = request.GetAskList();
IList <ContainerId> release = request.GetReleaseList();
ResourceBlacklistRequest blacklistRequest = request.GetResourceBlacklistRequest();
IList <string> blacklistAdditions = (blacklistRequest != null) ? blacklistRequest.
GetBlacklistAdditions() : Sharpen.Collections.EmptyList;
IList <string> blacklistRemovals = (blacklistRequest != null) ? blacklistRequest.GetBlacklistRemovals
() : Sharpen.Collections.EmptyList;
RMApp app = this.rmContext.GetRMApps()[applicationId];
// set label expression for Resource Requests if resourceName=ANY
ApplicationSubmissionContext asc = app.GetApplicationSubmissionContext();
foreach (ResourceRequest req in ask)
{
if (null == req.GetNodeLabelExpression() && ResourceRequest.Any.Equals(req.GetResourceName
()))
{
req.SetNodeLabelExpression(asc.GetNodeLabelExpression());
}
}
// sanity check
try
{
RMServerUtils.NormalizeAndValidateRequests(ask, rScheduler.GetMaximumResourceCapability
(), app.GetQueue(), rScheduler, rmContext);
}
catch (InvalidResourceRequestException e)
{
Log.Warn("Invalid resource ask by application " + appAttemptId, e);
throw;
}
try
{
RMServerUtils.ValidateBlacklistRequest(blacklistRequest);
}
catch (InvalidResourceBlacklistRequestException e)
{
Log.Warn("Invalid blacklist request by application " + appAttemptId, e);
throw;
}
// In the case of work-preserving AM restart, it's possible for the
// AM to release containers from the earlier attempt.
if (!app.GetApplicationSubmissionContext().GetKeepContainersAcrossApplicationAttempts
())
{
try
{
RMServerUtils.ValidateContainerReleaseRequest(release, appAttemptId);
}
catch (InvalidContainerReleaseException e)
{
Log.Warn("Invalid container release by application " + appAttemptId, e);
throw;
}
}
// Send new requests to appAttempt.
Allocation allocation = this.rScheduler.Allocate(appAttemptId, ask, release, blacklistAdditions
, blacklistRemovals);
if (!blacklistAdditions.IsEmpty() || !blacklistRemovals.IsEmpty())
{
Log.Info("blacklist are updated in Scheduler." + "blacklistAdditions: " + blacklistAdditions
+ ", " + "blacklistRemovals: " + blacklistRemovals);
}
RMAppAttempt appAttempt = app.GetRMAppAttempt(appAttemptId);
AllocateResponse allocateResponse = recordFactory.NewRecordInstance <AllocateResponse
>();
if (!allocation.GetContainers().IsEmpty())
{
allocateResponse.SetNMTokens(allocation.GetNMTokens());
}
// update the response with the deltas of node status changes
IList <RMNode> updatedNodes = new AList <RMNode>();
if (app.PullRMNodeUpdates(updatedNodes) > 0)
{
IList <NodeReport> updatedNodeReports = new AList <NodeReport>();
foreach (RMNode rmNode in updatedNodes)
{
SchedulerNodeReport schedulerNodeReport = rScheduler.GetNodeReport(rmNode.GetNodeID
());
Resource used = BuilderUtils.NewResource(0, 0);
int numContainers = 0;
if (schedulerNodeReport != null)
{
used = schedulerNodeReport.GetUsedResource();
numContainers = schedulerNodeReport.GetNumContainers();
}
NodeId nodeId = rmNode.GetNodeID();
NodeReport report = BuilderUtils.NewNodeReport(nodeId, rmNode.GetState(), rmNode.
GetHttpAddress(), rmNode.GetRackName(), used, rmNode.GetTotalCapability(), numContainers
, rmNode.GetHealthReport(), rmNode.GetLastHealthReportTime(), rmNode.GetNodeLabels
());
updatedNodeReports.AddItem(report);
}
allocateResponse.SetUpdatedNodes(updatedNodeReports);
}
allocateResponse.SetAllocatedContainers(allocation.GetContainers());
allocateResponse.SetCompletedContainersStatuses(appAttempt.PullJustFinishedContainers
());
allocateResponse.SetResponseId(lastResponse.GetResponseId() + 1);
allocateResponse.SetAvailableResources(allocation.GetResourceLimit());
allocateResponse.SetNumClusterNodes(this.rScheduler.GetNumClusterNodes());
// add preemption to the allocateResponse message (if any)
allocateResponse.SetPreemptionMessage(GeneratePreemptionMessage(allocation));
// update AMRMToken if the token is rolled-up
MasterKeyData nextMasterKey = this.rmContext.GetAMRMTokenSecretManager().GetNextMasterKeyData
();
if (nextMasterKey != null && nextMasterKey.GetMasterKey().GetKeyId() != amrmTokenIdentifier
.GetKeyId())
{
RMAppAttemptImpl appAttemptImpl = (RMAppAttemptImpl)appAttempt;
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> amrmToken = appAttempt
.GetAMRMToken();
if (nextMasterKey.GetMasterKey().GetKeyId() != appAttemptImpl.GetAMRMTokenKeyId())
{
Log.Info("The AMRMToken has been rolled-over. Send new AMRMToken back" + " to application: "
+ applicationId);
amrmToken = rmContext.GetAMRMTokenSecretManager().CreateAndGetAMRMToken(appAttemptId
);
appAttemptImpl.SetAMRMToken(amrmToken);
}
allocateResponse.SetAMRMToken(Org.Apache.Hadoop.Yarn.Api.Records.Token.NewInstance
(amrmToken.GetIdentifier(), amrmToken.GetKind().ToString(), amrmToken.GetPassword
(), amrmToken.GetService().ToString()));
}
/*
* As we are updating the response inside the lock object so we don't
* need to worry about unregister call occurring in between (which
* removes the lock object).
*/
Lock.SetAllocateResponse(allocateResponse);
return(allocateResponse);
}
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual RegisterApplicationMasterResponse RegisterApplicationMaster(RegisterApplicationMasterRequest
request)
{
AMRMTokenIdentifier amrmTokenIdentifier = AuthorizeRequest();
ApplicationAttemptId applicationAttemptId = amrmTokenIdentifier.GetApplicationAttemptId
();
ApplicationId appID = applicationAttemptId.GetApplicationId();
ApplicationMasterService.AllocateResponseLock Lock = responseMap[applicationAttemptId
];
if (Lock == null)
{
RMAuditLogger.LogFailure(this.rmContext.GetRMApps()[appID].GetUser(), RMAuditLogger.AuditConstants
.RegisterAm, "Application doesn't exist in cache " + applicationAttemptId, "ApplicationMasterService"
, "Error in registering application master", appID, applicationAttemptId);
ThrowApplicationDoesNotExistInCacheException(applicationAttemptId);
}
// Allow only one thread in AM to do registerApp at a time.
lock (Lock)
{
AllocateResponse lastResponse = Lock.GetAllocateResponse();
if (HasApplicationMasterRegistered(applicationAttemptId))
{
string message = "Application Master is already registered : " + appID;
Log.Warn(message);
RMAuditLogger.LogFailure(this.rmContext.GetRMApps()[appID].GetUser(), RMAuditLogger.AuditConstants
.RegisterAm, string.Empty, "ApplicationMasterService", message, appID, applicationAttemptId
);
throw new InvalidApplicationMasterRequestException(message);
}
this.amLivelinessMonitor.ReceivedPing(applicationAttemptId);
RMApp app = this.rmContext.GetRMApps()[appID];
// Setting the response id to 0 to identify if the
// application master is register for the respective attemptid
lastResponse.SetResponseId(0);
Lock.SetAllocateResponse(lastResponse);
Log.Info("AM registration " + applicationAttemptId);
this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMAppAttemptRegistrationEvent
(applicationAttemptId, request.GetHost(), request.GetRpcPort(), request.GetTrackingUrl
()));
RMAuditLogger.LogSuccess(app.GetUser(), RMAuditLogger.AuditConstants.RegisterAm,
"ApplicationMasterService", appID, applicationAttemptId);
// Pick up min/max resource from scheduler...
RegisterApplicationMasterResponse response = recordFactory.NewRecordInstance <RegisterApplicationMasterResponse
>();
response.SetMaximumResourceCapability(rScheduler.GetMaximumResourceCapability(app
.GetQueue()));
response.SetApplicationACLs(app.GetRMAppAttempt(applicationAttemptId).GetSubmissionContext
().GetAMContainerSpec().GetApplicationACLs());
response.SetQueue(app.GetQueue());
if (UserGroupInformation.IsSecurityEnabled())
{
Log.Info("Setting client token master key");
response.SetClientToAMTokenMasterKey(ByteBuffer.Wrap(rmContext.GetClientToAMTokenSecretManager
().GetMasterKey(applicationAttemptId).GetEncoded()));
}
// For work-preserving AM restart, retrieve previous attempts' containers
// and corresponding NM tokens.
if (app.GetApplicationSubmissionContext().GetKeepContainersAcrossApplicationAttempts
())
{
IList <Container> transferredContainers = ((AbstractYarnScheduler)rScheduler).GetTransferredContainers
(applicationAttemptId);
if (!transferredContainers.IsEmpty())
{
response.SetContainersFromPreviousAttempts(transferredContainers);
IList <NMToken> nmTokens = new AList <NMToken>();
foreach (Container container in transferredContainers)
{
try
{
NMToken token = rmContext.GetNMTokenSecretManager().CreateAndGetNMToken(app.GetUser
(), applicationAttemptId, container);
if (null != token)
{
nmTokens.AddItem(token);
}
}
catch (ArgumentException e)
{
// if it's a DNS issue, throw UnknowHostException directly and
// that
// will be automatically retried by RMProxy in RPC layer.
if (e.InnerException is UnknownHostException)
{
throw (UnknownHostException)e.InnerException;
}
}
}
response.SetNMTokensFromPreviousAttempts(nmTokens);
Log.Info("Application " + appID + " retrieved " + transferredContainers.Count + " containers from previous"
+ " attempts and " + nmTokens.Count + " NM tokens.");
}
}
response.SetSchedulerResourceTypes(rScheduler.GetSchedulingResourceTypes());
return(response);
}
}
/// <summary>create a summary of the application's runtime.</summary>
/// <param name="app">
///
/// <see cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Rmapp.RMApp"/>
/// whose summary is to be created, cannot
/// be <code>null</code>.
/// </param>
public static RMAppManager.ApplicationSummary.SummaryBuilder CreateAppSummary(RMApp
app)
{
string trackingUrl = "N/A";
string host = "N/A";
RMAppAttempt attempt = app.GetCurrentAppAttempt();
if (attempt != null)
{
trackingUrl = attempt.GetTrackingUrl();
host = attempt.GetHost();
}
RMAppMetrics metrics = app.GetRMAppMetrics();
RMAppManager.ApplicationSummary.SummaryBuilder summary = new RMAppManager.ApplicationSummary.SummaryBuilder
().Add("appId", app.GetApplicationId()).Add("name", app.GetName()).Add("user", app
.GetUser()).Add("queue", app.GetQueue()).Add("state", app.GetState()).Add("trackingUrl"
, trackingUrl).Add("appMasterHost", host).Add("startTime", app.GetStartTime()).Add
("finishTime", app.GetFinishTime()).Add("finalStatus", app.GetFinalApplicationStatus
()).Add("memorySeconds", metrics.GetMemorySeconds()).Add("vcoreSeconds", metrics
.GetVcoreSeconds()).Add("preemptedAMContainers", metrics.GetNumAMContainersPreempted
()).Add("preemptedNonAMContainers", metrics.GetNumNonAMContainersPreempted()).Add
("preemptedResources", metrics.GetResourcePreempted()).Add("applicationType", app
.GetApplicationType());
return(summary);
}
/// <exception cref="System.Exception"/>
public virtual void TestPublishApplicationMetrics()
{
for (int i = 1; i <= 2; ++i)
{
ApplicationId appId = ApplicationId.NewInstance(0, i);
RMApp app = CreateRMApp(appId);
metricsPublisher.AppCreated(app, app.GetStartTime());
metricsPublisher.AppFinished(app, RMAppState.Finished, app.GetFinishTime());
if (i == 1)
{
metricsPublisher.AppACLsUpdated(app, "uers1,user2", 4L);
}
else
{
// in case user doesn't specify the ACLs
metricsPublisher.AppACLsUpdated(app, null, 4L);
}
TimelineEntity entity = null;
do
{
entity = store.GetEntity(appId.ToString(), ApplicationMetricsConstants.EntityType
, EnumSet.AllOf <TimelineReader.Field>());
}while (entity == null || entity.GetEvents().Count < 3);
// ensure three events are both published before leaving the loop
// verify all the fields
NUnit.Framework.Assert.AreEqual(ApplicationMetricsConstants.EntityType, entity.GetEntityType
());
NUnit.Framework.Assert.AreEqual(app.GetApplicationId().ToString(), entity.GetEntityId
());
NUnit.Framework.Assert.AreEqual(app.GetName(), entity.GetOtherInfo()[ApplicationMetricsConstants
.NameEntityInfo]);
NUnit.Framework.Assert.AreEqual(app.GetQueue(), entity.GetOtherInfo()[ApplicationMetricsConstants
.QueueEntityInfo]);
NUnit.Framework.Assert.AreEqual(app.GetUser(), entity.GetOtherInfo()[ApplicationMetricsConstants
.UserEntityInfo]);
NUnit.Framework.Assert.AreEqual(app.GetApplicationType(), entity.GetOtherInfo()[ApplicationMetricsConstants
.TypeEntityInfo]);
NUnit.Framework.Assert.AreEqual(app.GetSubmitTime(), entity.GetOtherInfo()[ApplicationMetricsConstants
.SubmittedTimeEntityInfo]);
if (i == 1)
{
NUnit.Framework.Assert.AreEqual("uers1,user2", entity.GetOtherInfo()[ApplicationMetricsConstants
.AppViewAclsEntityInfo]);
}
else
{
NUnit.Framework.Assert.AreEqual(string.Empty, entity.GetOtherInfo()[ApplicationMetricsConstants
.AppViewAclsEntityInfo]);
NUnit.Framework.Assert.AreEqual(app.GetRMAppMetrics().GetMemorySeconds(), long.Parse
(entity.GetOtherInfo()[ApplicationMetricsConstants.AppMemMetrics].ToString()));
NUnit.Framework.Assert.AreEqual(app.GetRMAppMetrics().GetVcoreSeconds(), long.Parse
(entity.GetOtherInfo()[ApplicationMetricsConstants.AppCpuMetrics].ToString()));
}
bool hasCreatedEvent = false;
bool hasFinishedEvent = false;
bool hasACLsUpdatedEvent = false;
foreach (TimelineEvent @event in entity.GetEvents())
{
if (@event.GetEventType().Equals(ApplicationMetricsConstants.CreatedEventType))
{
hasCreatedEvent = true;
NUnit.Framework.Assert.AreEqual(app.GetStartTime(), @event.GetTimestamp());
}
else
{
if (@event.GetEventType().Equals(ApplicationMetricsConstants.FinishedEventType))
{
hasFinishedEvent = true;
NUnit.Framework.Assert.AreEqual(app.GetFinishTime(), @event.GetTimestamp());
NUnit.Framework.Assert.AreEqual(app.GetDiagnostics().ToString(), @event.GetEventInfo
()[ApplicationMetricsConstants.DiagnosticsInfoEventInfo]);
NUnit.Framework.Assert.AreEqual(app.GetFinalApplicationStatus().ToString(), @event
.GetEventInfo()[ApplicationMetricsConstants.FinalStatusEventInfo]);
NUnit.Framework.Assert.AreEqual(YarnApplicationState.Finished.ToString(), @event.
GetEventInfo()[ApplicationMetricsConstants.StateEventInfo]);
}
else
{
if (@event.GetEventType().Equals(ApplicationMetricsConstants.AclsUpdatedEventType
))
{
hasACLsUpdatedEvent = true;
NUnit.Framework.Assert.AreEqual(4L, @event.GetTimestamp());
}
}
}
}
NUnit.Framework.Assert.IsTrue(hasCreatedEvent && hasFinishedEvent && hasACLsUpdatedEvent
);
}
}
