private YarnServiceProtos.NMTokenProto ConvertToProtoFormat(NMToken token)
{
lock (this)
{
return(((NMTokenPBImpl)token).GetProto());
}
}
public virtual NMToken CreateAndGetNMToken(string applicationSubmitter, ApplicationAttemptId
appAttemptId, Container container)
{
try
{
this.readLock.Lock();
HashSet <NodeId> nodeSet = this.appAttemptToNodeKeyMap[appAttemptId];
NMToken nmToken = null;
if (nodeSet != null)
{
if (!nodeSet.Contains(container.GetNodeId()))
{
Log.Info("Sending NMToken for nodeId : " + container.GetNodeId() + " for container : "
+ container.GetId());
Token token = CreateNMToken(container.GetId().GetApplicationAttemptId(), container
.GetNodeId(), applicationSubmitter);
nmToken = NMToken.NewInstance(container.GetNodeId(), token);
nodeSet.AddItem(container.GetNodeId());
}
}
return(nmToken);
}
finally
{
this.readLock.Unlock();
}
}
// Create container token and NMToken altogether, if either of them fails for
// some reason like DNS unavailable, do not return this container and keep it
// in the newlyAllocatedContainers waiting to be refetched.
public virtual SchedulerApplicationAttempt.ContainersAndNMTokensAllocation PullNewlyAllocatedContainersAndNMTokens
()
{
lock (this)
{
IList <Container> returnContainerList = new AList <Container>(newlyAllocatedContainers
.Count);
IList <NMToken> nmTokens = new AList <NMToken>();
for (IEnumerator <RMContainer> i = newlyAllocatedContainers.GetEnumerator(); i.HasNext
();)
{
RMContainer rmContainer = i.Next();
Container container = rmContainer.GetContainer();
try
{
// create container token and NMToken altogether.
container.SetContainerToken(rmContext.GetContainerTokenSecretManager().CreateContainerToken
(container.GetId(), container.GetNodeId(), GetUser(), container.GetResource(), container
.GetPriority(), rmContainer.GetCreationTime(), this.logAggregationContext));
NMToken nmToken = rmContext.GetNMTokenSecretManager().CreateAndGetNMToken(GetUser
(), GetApplicationAttemptId(), container);
if (nmToken != null)
{
nmTokens.AddItem(nmToken);
}
}
catch (ArgumentException e)
{
// DNS might be down, skip returning this container.
Log.Error("Error trying to assign container token and NM token to" + " an allocated container "
+ container.GetId(), e);
continue;
}
returnContainerList.AddItem(container);
i.Remove();
rmContainer.Handle(new RMContainerEvent(rmContainer.GetContainerId(), RMContainerEventType
.Acquired));
}
return(new SchedulerApplicationAttempt.ContainersAndNMTokensAllocation(returnContainerList
, nmTokens));
}
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual RegisterApplicationMasterResponse RegisterApplicationMaster(RegisterApplicationMasterRequest
request)
{
AMRMTokenIdentifier amrmTokenIdentifier = AuthorizeRequest();
ApplicationAttemptId applicationAttemptId = amrmTokenIdentifier.GetApplicationAttemptId
();
ApplicationId appID = applicationAttemptId.GetApplicationId();
ApplicationMasterService.AllocateResponseLock Lock = responseMap[applicationAttemptId
];
if (Lock == null)
{
RMAuditLogger.LogFailure(this.rmContext.GetRMApps()[appID].GetUser(), RMAuditLogger.AuditConstants
.RegisterAm, "Application doesn't exist in cache " + applicationAttemptId, "ApplicationMasterService"
, "Error in registering application master", appID, applicationAttemptId);
ThrowApplicationDoesNotExistInCacheException(applicationAttemptId);
}
// Allow only one thread in AM to do registerApp at a time.
lock (Lock)
{
AllocateResponse lastResponse = Lock.GetAllocateResponse();
if (HasApplicationMasterRegistered(applicationAttemptId))
{
string message = "Application Master is already registered : " + appID;
Log.Warn(message);
RMAuditLogger.LogFailure(this.rmContext.GetRMApps()[appID].GetUser(), RMAuditLogger.AuditConstants
.RegisterAm, string.Empty, "ApplicationMasterService", message, appID, applicationAttemptId
);
throw new InvalidApplicationMasterRequestException(message);
}
this.amLivelinessMonitor.ReceivedPing(applicationAttemptId);
RMApp app = this.rmContext.GetRMApps()[appID];
// Setting the response id to 0 to identify if the
// application master is register for the respective attemptid
lastResponse.SetResponseId(0);
Lock.SetAllocateResponse(lastResponse);
Log.Info("AM registration " + applicationAttemptId);
this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMAppAttemptRegistrationEvent
(applicationAttemptId, request.GetHost(), request.GetRpcPort(), request.GetTrackingUrl
()));
RMAuditLogger.LogSuccess(app.GetUser(), RMAuditLogger.AuditConstants.RegisterAm,
"ApplicationMasterService", appID, applicationAttemptId);
// Pick up min/max resource from scheduler...
RegisterApplicationMasterResponse response = recordFactory.NewRecordInstance <RegisterApplicationMasterResponse
>();
response.SetMaximumResourceCapability(rScheduler.GetMaximumResourceCapability(app
.GetQueue()));
response.SetApplicationACLs(app.GetRMAppAttempt(applicationAttemptId).GetSubmissionContext
().GetAMContainerSpec().GetApplicationACLs());
response.SetQueue(app.GetQueue());
if (UserGroupInformation.IsSecurityEnabled())
{
Log.Info("Setting client token master key");
response.SetClientToAMTokenMasterKey(ByteBuffer.Wrap(rmContext.GetClientToAMTokenSecretManager
().GetMasterKey(applicationAttemptId).GetEncoded()));
}
// For work-preserving AM restart, retrieve previous attempts' containers
// and corresponding NM tokens.
if (app.GetApplicationSubmissionContext().GetKeepContainersAcrossApplicationAttempts
())
{
IList <Container> transferredContainers = ((AbstractYarnScheduler)rScheduler).GetTransferredContainers
(applicationAttemptId);
if (!transferredContainers.IsEmpty())
{
response.SetContainersFromPreviousAttempts(transferredContainers);
IList <NMToken> nmTokens = new AList <NMToken>();
foreach (Container container in transferredContainers)
{
try
{
NMToken token = rmContext.GetNMTokenSecretManager().CreateAndGetNMToken(app.GetUser
(), applicationAttemptId, container);
if (null != token)
{
nmTokens.AddItem(token);
}
}
catch (ArgumentException e)
{
// if it's a DNS issue, throw UnknowHostException directly and
// that
// will be automatically retried by RMProxy in RPC layer.
if (e.InnerException is UnknownHostException)
{
throw (UnknownHostException)e.InnerException;
}
}
}
response.SetNMTokensFromPreviousAttempts(nmTokens);
Log.Info("Application " + appID + " retrieved " + transferredContainers.Count + " containers from previous"
+ " attempts and " + nmTokens.Count + " NM tokens.");
}
}
response.SetSchedulerResourceTypes(rScheduler.GetSchedulingResourceTypes());
return(response);
}
}
