public virtual void RollMasterKey()
{
base.writeLock.Lock();
try
{
Log.Info("Rolling master-key for container-tokens");
if (this.currentMasterKey == null)
{
// Setting up for the first time.
this.currentMasterKey = CreateNewMasterKey();
}
else
{
this.nextMasterKey = CreateNewMasterKey();
Log.Info("Going to activate master-key with key-id " + this.nextMasterKey.GetMasterKey
().GetKeyId() + " in " + this.activationDelay + "ms");
this.timer.Schedule(new RMContainerTokenSecretManager.NextKeyActivator(this), this
.activationDelay);
}
}
finally
{
base.writeLock.Unlock();
}
}
/// <exception cref="System.Exception"/>
public virtual void TestAMRMTokenSecretManagerStateStore(RMStateStoreTestBase.RMStateStoreHelper
stateStoreHelper)
{
System.Console.Out.WriteLine("Start testing");
RMStateStore store = stateStoreHelper.GetRMStateStore();
RMStateStoreTestBase.TestDispatcher dispatcher = new RMStateStoreTestBase.TestDispatcher
();
store.SetRMDispatcher(dispatcher);
RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>();
Org.Mockito.Mockito.When(rmContext.GetStateStore()).ThenReturn(store);
Configuration conf = new YarnConfiguration();
AMRMTokenSecretManager appTokenMgr = new AMRMTokenSecretManager(conf, rmContext);
//create and save the first masterkey
MasterKeyData firstMasterKeyData = appTokenMgr.CreateNewMasterKey();
AMRMTokenSecretManagerState state1 = AMRMTokenSecretManagerState.NewInstance(firstMasterKeyData
.GetMasterKey(), null);
rmContext.GetStateStore().StoreOrUpdateAMRMTokenSecretManager(state1, false);
// load state
store = stateStoreHelper.GetRMStateStore();
Org.Mockito.Mockito.When(rmContext.GetStateStore()).ThenReturn(store);
store.SetRMDispatcher(dispatcher);
RMStateStore.RMState state = store.LoadState();
NUnit.Framework.Assert.IsNotNull(state.GetAMRMTokenSecretManagerState());
NUnit.Framework.Assert.AreEqual(firstMasterKeyData.GetMasterKey(), state.GetAMRMTokenSecretManagerState
().GetCurrentMasterKey());
NUnit.Framework.Assert.IsNull(state.GetAMRMTokenSecretManagerState().GetNextMasterKey
());
//create and save the second masterkey
MasterKeyData secondMasterKeyData = appTokenMgr.CreateNewMasterKey();
AMRMTokenSecretManagerState state2 = AMRMTokenSecretManagerState.NewInstance(firstMasterKeyData
.GetMasterKey(), secondMasterKeyData.GetMasterKey());
rmContext.GetStateStore().StoreOrUpdateAMRMTokenSecretManager(state2, true);
// load state
store = stateStoreHelper.GetRMStateStore();
Org.Mockito.Mockito.When(rmContext.GetStateStore()).ThenReturn(store);
store.SetRMDispatcher(dispatcher);
RMStateStore.RMState state_2 = store.LoadState();
NUnit.Framework.Assert.IsNotNull(state_2.GetAMRMTokenSecretManagerState());
NUnit.Framework.Assert.AreEqual(firstMasterKeyData.GetMasterKey(), state_2.GetAMRMTokenSecretManagerState
().GetCurrentMasterKey());
NUnit.Framework.Assert.AreEqual(secondMasterKeyData.GetMasterKey(), state_2.GetAMRMTokenSecretManagerState
().GetNextMasterKey());
// re-create the masterKeyData based on the recovered masterkey
// should have the same secretKey
appTokenMgr.Recover(state_2);
NUnit.Framework.Assert.AreEqual(appTokenMgr.GetCurrnetMasterKeyData().GetSecretKey
(), firstMasterKeyData.GetSecretKey());
NUnit.Framework.Assert.AreEqual(appTokenMgr.GetNextMasterKeyData().GetSecretKey()
, secondMasterKeyData.GetSecretKey());
store.Close();
}
private void UpdateAppAttemptKey(ApplicationAttemptId attempt, MasterKeyData key)
{
this.oldMasterKeys[attempt] = key;
try
{
stateStore.StoreNMTokenApplicationMasterKey(attempt, key.GetMasterKey());
}
catch (IOException e)
{
Log.Error("Unable to store master key for application " + attempt, e);
}
}
private void UpdateCurrentMasterKey(MasterKeyData key)
{
base.currentMasterKey = key;
try
{
stateStore.StoreContainerTokenCurrentMasterKey(key.GetMasterKey());
}
catch (IOException e)
{
Log.Error("Unable to update current master key in state store", e);
}
}
public virtual void Start()
{
if (this.currentMasterKey == null)
{
this.currentMasterKey = CreateNewMasterKey();
AMRMTokenSecretManagerState state = AMRMTokenSecretManagerState.NewInstance(this.
currentMasterKey.GetMasterKey(), null);
rmContext.GetStateStore().StoreOrUpdateAMRMTokenSecretManager(state, false);
}
this.timer.ScheduleAtFixedRate(new AMRMTokenSecretManager.MasterKeyRoller(this),
rollingInterval, rollingInterval);
}
private void UpdatePreviousMasterKey(MasterKeyData key)
{
previousMasterKey = key;
try
{
stateStore.StoreContainerTokenPreviousMasterKey(key.GetMasterKey());
}
catch (IOException e)
{
Log.Error("Unable to update previous master key in state store", e);
}
}
public virtual void ActivateNextMasterKey()
{
base.writeLock.Lock();
try
{
Log.Info("Activating next master key with id: " + this.nextMasterKey.GetMasterKey
().GetKeyId());
this.currentMasterKey = this.nextMasterKey;
this.nextMasterKey = null;
}
finally
{
base.writeLock.Unlock();
}
}
public virtual void ActivateNextMasterKey()
{
this.writeLock.Lock();
try
{
Log.Info("Activating next master key with id: " + this.nextMasterKey.GetMasterKey
().GetKeyId());
this.currentMasterKey = this.nextMasterKey;
this.nextMasterKey = null;
AMRMTokenSecretManagerState state = AMRMTokenSecretManagerState.NewInstance(this.
currentMasterKey.GetMasterKey(), null);
rmContext.GetStateStore().StoreOrUpdateAMRMTokenSecretManager(state, true);
}
finally
{
this.writeLock.Unlock();
}
}
internal virtual void RollMasterKey()
{
this.writeLock.Lock();
try
{
Log.Info("Rolling master-key for amrm-tokens");
this.nextMasterKey = CreateNewMasterKey();
AMRMTokenSecretManagerState state = AMRMTokenSecretManagerState.NewInstance(this.
currentMasterKey.GetMasterKey(), this.nextMasterKey.GetMasterKey());
rmContext.GetStateStore().StoreOrUpdateAMRMTokenSecretManager(state, true);
this.timer.Schedule(new AMRMTokenSecretManager.NextKeyActivator(this), this.activationDelay
);
}
finally
{
this.writeLock.Unlock();
}
}
/// <summary>
/// This method will be used to verify NMTokens generated by different master
/// keys.
/// </summary>
/// <exception cref="Org.Apache.Hadoop.Security.Token.SecretManager.InvalidToken"/>
public override byte[] RetrievePassword(NMTokenIdentifier identifier)
{
lock (this)
{
int keyId = identifier.GetKeyId();
ApplicationAttemptId appAttemptId = identifier.GetApplicationAttemptId();
/*
* MasterKey used for retrieving password will be as follows. 1) By default
* older saved master key will be used. 2) If identifier's master key id
* matches that of previous master key id then previous key will be used. 3)
* If identifier's master key id matches that of current master key id then
* current key will be used.
*/
MasterKeyData oldMasterKey = oldMasterKeys[appAttemptId];
MasterKeyData masterKeyToUse = oldMasterKey;
if (previousMasterKey != null && keyId == previousMasterKey.GetMasterKey().GetKeyId
())
{
masterKeyToUse = previousMasterKey;
}
else
{
if (keyId == currentMasterKey.GetMasterKey().GetKeyId())
{
masterKeyToUse = currentMasterKey;
}
}
if (nodeId != null && !identifier.GetNodeId().Equals(nodeId))
{
throw new SecretManager.InvalidToken("Given NMToken for application : " + appAttemptId
.ToString() + " is not valid for current node manager." + "expected : " + nodeId
.ToString() + " found : " + identifier.GetNodeId().ToString());
}
if (masterKeyToUse != null)
{
byte[] password = RetrivePasswordInternal(identifier, masterKeyToUse);
Log.Debug("NMToken password retrieved successfully!!");
return(password);
}
throw new SecretManager.InvalidToken("Given NMToken for application : " + appAttemptId
.ToString() + " seems to have been generated illegally.");
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void Recover()
{
lock (this)
{
NMStateStoreService.RecoveredNMTokensState state = stateStore.LoadNMTokensState();
MasterKey key = state.GetCurrentMasterKey();
if (key != null)
{
base.currentMasterKey = new MasterKeyData(key, CreateSecretKey(((byte[])key.GetBytes
().Array())));
}
key = state.GetPreviousMasterKey();
if (key != null)
{
previousMasterKey = new MasterKeyData(key, CreateSecretKey(((byte[])key.GetBytes(
).Array())));
}
// restore the serial number from the current master key
if (base.currentMasterKey != null)
{
base.serialNo = base.currentMasterKey.GetMasterKey().GetKeyId() + 1;
}
foreach (KeyValuePair <ApplicationAttemptId, MasterKey> entry in state.GetApplicationMasterKeys
())
{
key = entry.Value;
oldMasterKeys[entry.Key] = new MasterKeyData(key, CreateSecretKey(((byte[])key.GetBytes
().Array())));
}
// reconstruct app to app attempts map
appToAppAttemptMap.Clear();
foreach (ApplicationAttemptId attempt in oldMasterKeys.Keys)
{
ApplicationId app = attempt.GetApplicationId();
IList <ApplicationAttemptId> attempts = appToAppAttemptMap[app];
if (attempts == null)
{
attempts = new AList <ApplicationAttemptId>();
appToAppAttemptMap[app] = attempts;
}
attempts.AddItem(attempt);
}
}
}
/// <summary>This will be called by startContainer.</summary>
/// <remarks>
/// This will be called by startContainer. It will add the master key into
/// the cache used for starting this container. This should be called before
/// validating the startContainer request.
/// </remarks>
/// <exception cref="Org.Apache.Hadoop.Security.Token.SecretManager.InvalidToken"/>
public virtual void AppAttemptStartContainer(NMTokenIdentifier identifier)
{
lock (this)
{
ApplicationAttemptId appAttemptId = identifier.GetApplicationAttemptId();
if (!appToAppAttemptMap.Contains(appAttemptId.GetApplicationId()))
{
// First application attempt for the given application
appToAppAttemptMap[appAttemptId.GetApplicationId()] = new AList <ApplicationAttemptId
>();
}
MasterKeyData oldKey = oldMasterKeys[appAttemptId];
if (oldKey == null)
{
// This is a new application attempt.
appToAppAttemptMap[appAttemptId.GetApplicationId()].AddItem(appAttemptId);
}
if (oldKey == null || oldKey.GetMasterKey().GetKeyId() != identifier.GetKeyId())
{
// Update key only if it is modified.
Log.Debug("NMToken key updated for application attempt : " + identifier.GetApplicationAttemptId
().ToString());
if (identifier.GetKeyId() == currentMasterKey.GetMasterKey().GetKeyId())
{
UpdateAppAttemptKey(appAttemptId, currentMasterKey);
}
else
{
if (previousMasterKey != null && identifier.GetKeyId() == previousMasterKey.GetMasterKey
().GetKeyId())
{
UpdateAppAttemptKey(appAttemptId, previousMasterKey);
}
else
{
throw new SecretManager.InvalidToken("Older NMToken should not be used while starting the container."
);
}
}
}
}
}
public virtual void Recover(RMStateStore.RMState state)
{
if (state.GetAMRMTokenSecretManagerState() != null)
{
// recover the current master key
MasterKey currentKey = state.GetAMRMTokenSecretManagerState().GetCurrentMasterKey
();
this.currentMasterKey = new MasterKeyData(currentKey, CreateSecretKey(((byte[])currentKey
.GetBytes().Array())));
// recover the next master key if not null
MasterKey nextKey = state.GetAMRMTokenSecretManagerState().GetNextMasterKey();
if (nextKey != null)
{
this.nextMasterKey = new MasterKeyData(nextKey, CreateSecretKey(((byte[])nextKey.
GetBytes().Array())));
this.timer.Schedule(new AMRMTokenSecretManager.NextKeyActivator(this), this.activationDelay
);
}
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void Recover()
{
lock (this)
{
NMStateStoreService.RecoveredContainerTokensState state = stateStore.LoadContainerTokensState
();
MasterKey key = state.GetCurrentMasterKey();
if (key != null)
{
base.currentMasterKey = new MasterKeyData(key, CreateSecretKey(((byte[])key.GetBytes
().Array())));
}
key = state.GetPreviousMasterKey();
if (key != null)
{
previousMasterKey = new MasterKeyData(key, CreateSecretKey(((byte[])key.GetBytes(
).Array())));
}
// restore the serial number from the current master key
if (base.currentMasterKey != null)
{
base.serialNo = base.currentMasterKey.GetMasterKey().GetKeyId() + 1;
}
foreach (KeyValuePair <ContainerId, long> entry in state.GetActiveTokens())
{
ContainerId containerId = entry.Key;
long expTime = entry.Value;
IList <ContainerId> containerList = recentlyStartedContainerTracker[expTime];
if (containerList == null)
{
containerList = new AList <ContainerId>();
recentlyStartedContainerTracker[expTime] = containerList;
}
if (!containerList.Contains(containerId))
{
containerList.AddItem(containerId);
}
}
}
}
/// <summary>
/// Override of this is to validate ContainerTokens generated by using
/// different
/// <see cref="Org.Apache.Hadoop.Yarn.Server.Api.Records.MasterKey"/>
/// s.
/// </summary>
/// <exception cref="Org.Apache.Hadoop.Security.Token.SecretManager.InvalidToken"/>
public override byte[] RetrievePassword(ContainerTokenIdentifier identifier)
{
lock (this)
{
int keyId = identifier.GetMasterKeyId();
MasterKeyData masterKeyToUse = null;
if (this.previousMasterKey != null && keyId == this.previousMasterKey.GetMasterKey
().GetKeyId())
{
// A container-launch has come in with a token generated off the last
// master-key
masterKeyToUse = this.previousMasterKey;
}
else
{
if (keyId == base.currentMasterKey.GetMasterKey().GetKeyId())
{
// A container-launch has come in with a token generated off the current
// master-key
masterKeyToUse = base.currentMasterKey;
}
}
if (nodeHostAddr != null && !identifier.GetNmHostAddress().Equals(nodeHostAddr))
{
// Valid container token used for incorrect node.
throw new SecretManager.InvalidToken("Given Container " + identifier.GetContainerID
().ToString() + " identifier is not valid for current Node manager. Expected : "
+ nodeHostAddr + " Found : " + identifier.GetNmHostAddress());
}
if (masterKeyToUse != null)
{
return(RetrievePasswordInternal(identifier, masterKeyToUse));
}
// Invalid request. Like startContainer() with token generated off
// old-master-keys.
throw new SecretManager.InvalidToken("Given Container " + identifier.GetContainerID
().ToString() + " seems to have an illegally generated token.");
}
}
public virtual void TestMasterKeyRollOver()
{
conf.SetLong(YarnConfiguration.RmAmrmTokenMasterKeyRollingIntervalSecs, rolling_interval_sec
);
conf.SetLong(YarnConfiguration.RmAmExpiryIntervalMs, am_expire_ms);
TestAMAuthorization.MyContainerManager containerManager = new TestAMAuthorization.MyContainerManager
();
TestAMAuthorization.MockRMWithAMS rm = new TestAMAuthorization.MockRMWithAMS(conf
, containerManager);
rm.Start();
long startTime = Runtime.CurrentTimeMillis();
Configuration conf = rm.GetConfig();
YarnRPC rpc = YarnRPC.Create(conf);
ApplicationMasterProtocol rmClient = null;
AMRMTokenSecretManager appTokenSecretManager = rm.GetRMContext().GetAMRMTokenSecretManager
();
MasterKeyData oldKey = appTokenSecretManager.GetMasterKey();
NUnit.Framework.Assert.IsNotNull(oldKey);
try
{
MockNM nm1 = rm.RegisterNode("localhost:1234", 5120);
RMApp app = rm.SubmitApp(1024);
nm1.NodeHeartbeat(true);
int waitCount = 0;
while (containerManager.containerTokens == null && waitCount++ < maxWaitAttempts)
{
Log.Info("Waiting for AM Launch to happen..");
Sharpen.Thread.Sleep(1000);
}
NUnit.Framework.Assert.IsNotNull(containerManager.containerTokens);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
ApplicationAttemptId applicationAttemptId = attempt.GetAppAttemptId();
// Create a client to the RM.
UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(applicationAttemptId
.ToString());
Credentials credentials = containerManager.GetContainerCredentials();
IPEndPoint rmBindAddress = rm.GetApplicationMasterService().GetBindAddress();
Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> amRMToken = TestAMAuthorization.MockRMWithAMS
.SetupAndReturnAMRMToken(rmBindAddress, credentials.GetAllTokens());
currentUser.AddToken(amRMToken);
rmClient = CreateRMClient(rm, conf, rpc, currentUser);
RegisterApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<RegisterApplicationMasterRequest>();
rmClient.RegisterApplicationMaster(request);
// One allocate call.
AllocateRequest allocateRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest
>();
NUnit.Framework.Assert.IsTrue(rmClient.Allocate(allocateRequest).GetAMCommand() ==
null);
// Wait for enough time and make sure the roll_over happens
// At mean time, the old AMRMToken should continue to work
while (Runtime.CurrentTimeMillis() - startTime < rolling_interval_sec * 1000)
{
rmClient.Allocate(allocateRequest);
Sharpen.Thread.Sleep(500);
}
MasterKeyData newKey = appTokenSecretManager.GetMasterKey();
NUnit.Framework.Assert.IsNotNull(newKey);
NUnit.Framework.Assert.IsFalse("Master key should have changed!", oldKey.Equals(newKey
));
// Another allocate call with old AMRMToken. Should continue to work.
rpc.StopProxy(rmClient, conf);
// To avoid using cached client
rmClient = CreateRMClient(rm, conf, rpc, currentUser);
NUnit.Framework.Assert.IsTrue(rmClient.Allocate(allocateRequest).GetAMCommand() ==
null);
waitCount = 0;
while (waitCount++ <= maxWaitAttempts)
{
if (appTokenSecretManager.GetCurrnetMasterKeyData() != oldKey)
{
break;
}
try
{
rmClient.Allocate(allocateRequest);
}
catch (Exception)
{
break;
}
Sharpen.Thread.Sleep(200);
}
// active the nextMasterKey, and replace the currentMasterKey
NUnit.Framework.Assert.IsTrue(appTokenSecretManager.GetCurrnetMasterKeyData().Equals
(newKey));
NUnit.Framework.Assert.IsTrue(appTokenSecretManager.GetMasterKey().Equals(newKey)
);
NUnit.Framework.Assert.IsTrue(appTokenSecretManager.GetNextMasterKeyData() == null
);
// Create a new Token
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> newToken = appTokenSecretManager
.CreateAndGetAMRMToken(applicationAttemptId);
SecurityUtil.SetTokenService(newToken, rmBindAddress);
currentUser.AddToken(newToken);
// Another allocate call. Should continue to work.
rpc.StopProxy(rmClient, conf);
// To avoid using cached client
rmClient = CreateRMClient(rm, conf, rpc, currentUser);
allocateRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest>(
);
NUnit.Framework.Assert.IsTrue(rmClient.Allocate(allocateRequest).GetAMCommand() ==
null);
// Should not work by using the old AMRMToken.
rpc.StopProxy(rmClient, conf);
// To avoid using cached client
try
{
currentUser.AddToken(amRMToken);
rmClient = CreateRMClient(rm, conf, rpc, currentUser);
allocateRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest>(
);
NUnit.Framework.Assert.IsTrue(rmClient.Allocate(allocateRequest).GetAMCommand() ==
null);
NUnit.Framework.Assert.Fail("The old Token should not work");
}
catch (Exception)
{
}
}
finally
{
// expect exception
rm.Stop();
if (rmClient != null)
{
rpc.StopProxy(rmClient, conf);
}
}
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual AllocateResponse Allocate(AllocateRequest request)
{
AMRMTokenIdentifier amrmTokenIdentifier = AuthorizeRequest();
ApplicationAttemptId appAttemptId = amrmTokenIdentifier.GetApplicationAttemptId();
ApplicationId applicationId = appAttemptId.GetApplicationId();
this.amLivelinessMonitor.ReceivedPing(appAttemptId);
/* check if its in cache */
ApplicationMasterService.AllocateResponseLock Lock = responseMap[appAttemptId];
if (Lock == null)
{
string message = "Application attempt " + appAttemptId + " doesn't exist in ApplicationMasterService cache.";
Log.Error(message);
throw new ApplicationAttemptNotFoundException(message);
}
lock (Lock)
{
AllocateResponse lastResponse = Lock.GetAllocateResponse();
if (!HasApplicationMasterRegistered(appAttemptId))
{
string message = "AM is not registered for known application attempt: " + appAttemptId
+ " or RM had restarted after AM registered . AM should re-register.";
Log.Info(message);
RMAuditLogger.LogFailure(this.rmContext.GetRMApps()[appAttemptId.GetApplicationId
()].GetUser(), RMAuditLogger.AuditConstants.AmAllocate, string.Empty, "ApplicationMasterService"
, message, applicationId, appAttemptId);
throw new ApplicationMasterNotRegisteredException(message);
}
if ((request.GetResponseId() + 1) == lastResponse.GetResponseId())
{
/* old heartbeat */
return(lastResponse);
}
else
{
if (request.GetResponseId() + 1 < lastResponse.GetResponseId())
{
string message = "Invalid responseId in AllocateRequest from application attempt: "
+ appAttemptId + ", expect responseId to be " + (lastResponse.GetResponseId() +
1);
throw new InvalidApplicationMasterRequestException(message);
}
}
//filter illegal progress values
float filteredProgress = request.GetProgress();
if (float.IsNaN(filteredProgress) || filteredProgress == float.NegativeInfinity ||
filteredProgress < 0)
{
request.SetProgress(0);
}
else
{
if (filteredProgress > 1 || filteredProgress == float.PositiveInfinity)
{
request.SetProgress(1);
}
}
// Send the status update to the appAttempt.
this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMAppAttemptStatusupdateEvent
(appAttemptId, request.GetProgress()));
IList <ResourceRequest> ask = request.GetAskList();
IList <ContainerId> release = request.GetReleaseList();
ResourceBlacklistRequest blacklistRequest = request.GetResourceBlacklistRequest();
IList <string> blacklistAdditions = (blacklistRequest != null) ? blacklistRequest.
GetBlacklistAdditions() : Sharpen.Collections.EmptyList;
IList <string> blacklistRemovals = (blacklistRequest != null) ? blacklistRequest.GetBlacklistRemovals
() : Sharpen.Collections.EmptyList;
RMApp app = this.rmContext.GetRMApps()[applicationId];
// set label expression for Resource Requests if resourceName=ANY
ApplicationSubmissionContext asc = app.GetApplicationSubmissionContext();
foreach (ResourceRequest req in ask)
{
if (null == req.GetNodeLabelExpression() && ResourceRequest.Any.Equals(req.GetResourceName
()))
{
req.SetNodeLabelExpression(asc.GetNodeLabelExpression());
}
}
// sanity check
try
{
RMServerUtils.NormalizeAndValidateRequests(ask, rScheduler.GetMaximumResourceCapability
(), app.GetQueue(), rScheduler, rmContext);
}
catch (InvalidResourceRequestException e)
{
Log.Warn("Invalid resource ask by application " + appAttemptId, e);
throw;
}
try
{
RMServerUtils.ValidateBlacklistRequest(blacklistRequest);
}
catch (InvalidResourceBlacklistRequestException e)
{
Log.Warn("Invalid blacklist request by application " + appAttemptId, e);
throw;
}
// In the case of work-preserving AM restart, it's possible for the
// AM to release containers from the earlier attempt.
if (!app.GetApplicationSubmissionContext().GetKeepContainersAcrossApplicationAttempts
())
{
try
{
RMServerUtils.ValidateContainerReleaseRequest(release, appAttemptId);
}
catch (InvalidContainerReleaseException e)
{
Log.Warn("Invalid container release by application " + appAttemptId, e);
throw;
}
}
// Send new requests to appAttempt.
Allocation allocation = this.rScheduler.Allocate(appAttemptId, ask, release, blacklistAdditions
, blacklistRemovals);
if (!blacklistAdditions.IsEmpty() || !blacklistRemovals.IsEmpty())
{
Log.Info("blacklist are updated in Scheduler." + "blacklistAdditions: " + blacklistAdditions
+ ", " + "blacklistRemovals: " + blacklistRemovals);
}
RMAppAttempt appAttempt = app.GetRMAppAttempt(appAttemptId);
AllocateResponse allocateResponse = recordFactory.NewRecordInstance <AllocateResponse
>();
if (!allocation.GetContainers().IsEmpty())
{
allocateResponse.SetNMTokens(allocation.GetNMTokens());
}
// update the response with the deltas of node status changes
IList <RMNode> updatedNodes = new AList <RMNode>();
if (app.PullRMNodeUpdates(updatedNodes) > 0)
{
IList <NodeReport> updatedNodeReports = new AList <NodeReport>();
foreach (RMNode rmNode in updatedNodes)
{
SchedulerNodeReport schedulerNodeReport = rScheduler.GetNodeReport(rmNode.GetNodeID
());
Resource used = BuilderUtils.NewResource(0, 0);
int numContainers = 0;
if (schedulerNodeReport != null)
{
used = schedulerNodeReport.GetUsedResource();
numContainers = schedulerNodeReport.GetNumContainers();
}
NodeId nodeId = rmNode.GetNodeID();
NodeReport report = BuilderUtils.NewNodeReport(nodeId, rmNode.GetState(), rmNode.
GetHttpAddress(), rmNode.GetRackName(), used, rmNode.GetTotalCapability(), numContainers
, rmNode.GetHealthReport(), rmNode.GetLastHealthReportTime(), rmNode.GetNodeLabels
());
updatedNodeReports.AddItem(report);
}
allocateResponse.SetUpdatedNodes(updatedNodeReports);
}
allocateResponse.SetAllocatedContainers(allocation.GetContainers());
allocateResponse.SetCompletedContainersStatuses(appAttempt.PullJustFinishedContainers
());
allocateResponse.SetResponseId(lastResponse.GetResponseId() + 1);
allocateResponse.SetAvailableResources(allocation.GetResourceLimit());
allocateResponse.SetNumClusterNodes(this.rScheduler.GetNumClusterNodes());
// add preemption to the allocateResponse message (if any)
allocateResponse.SetPreemptionMessage(GeneratePreemptionMessage(allocation));
// update AMRMToken if the token is rolled-up
MasterKeyData nextMasterKey = this.rmContext.GetAMRMTokenSecretManager().GetNextMasterKeyData
();
if (nextMasterKey != null && nextMasterKey.GetMasterKey().GetKeyId() != amrmTokenIdentifier
.GetKeyId())
{
RMAppAttemptImpl appAttemptImpl = (RMAppAttemptImpl)appAttempt;
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> amrmToken = appAttempt
.GetAMRMToken();
if (nextMasterKey.GetMasterKey().GetKeyId() != appAttemptImpl.GetAMRMTokenKeyId())
{
Log.Info("The AMRMToken has been rolled-over. Send new AMRMToken back" + " to application: "
+ applicationId);
amrmToken = rmContext.GetAMRMTokenSecretManager().CreateAndGetAMRMToken(appAttemptId
);
appAttemptImpl.SetAMRMToken(amrmToken);
}
allocateResponse.SetAMRMToken(Org.Apache.Hadoop.Yarn.Api.Records.Token.NewInstance
(amrmToken.GetIdentifier(), amrmToken.GetKind().ToString(), amrmToken.GetPassword
(), amrmToken.GetService().ToString()));
}
/*
* As we are updating the response inside the lock object so we don't
* need to worry about unregister call occurring in between (which
* removes the lock object).
*/
Lock.SetAllocateResponse(allocateResponse);
return(allocateResponse);
}
}
/// <exception cref="System.Exception"/>
internal virtual void TestRMAppStateStore(RMStateStoreTestBase.RMStateStoreHelper
stateStoreHelper, RMStateStoreTestBase.StoreStateVerifier verifier)
{
long submitTime = Runtime.CurrentTimeMillis();
long startTime = Runtime.CurrentTimeMillis() + 1234;
Configuration conf = new YarnConfiguration();
RMStateStore store = stateStoreHelper.GetRMStateStore();
RMStateStoreTestBase.TestDispatcher dispatcher = new RMStateStoreTestBase.TestDispatcher
();
store.SetRMDispatcher(dispatcher);
RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>();
Org.Mockito.Mockito.When(rmContext.GetStateStore()).ThenReturn(store);
AMRMTokenSecretManager appTokenMgr = Org.Mockito.Mockito.Spy(new AMRMTokenSecretManager
(conf, rmContext));
MasterKeyData masterKeyData = appTokenMgr.CreateNewMasterKey();
Org.Mockito.Mockito.When(appTokenMgr.GetMasterKey()).ThenReturn(masterKeyData);
ClientToAMTokenSecretManagerInRM clientToAMTokenMgr = new ClientToAMTokenSecretManagerInRM
();
ApplicationAttemptId attemptId1 = ConverterUtils.ToApplicationAttemptId("appattempt_1352994193343_0001_000001"
);
ApplicationId appId1 = attemptId1.GetApplicationId();
StoreApp(store, appId1, submitTime, startTime);
verifier.AfterStoreApp(store, appId1);
// create application token and client token key for attempt1
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> appAttemptToken1 = GenerateAMRMToken
(attemptId1, appTokenMgr);
SecretKey clientTokenKey1 = clientToAMTokenMgr.CreateMasterKey(attemptId1);
ContainerId containerId1 = StoreAttempt(store, attemptId1, "container_1352994193343_0001_01_000001"
, appAttemptToken1, clientTokenKey1, dispatcher);
string appAttemptIdStr2 = "appattempt_1352994193343_0001_000002";
ApplicationAttemptId attemptId2 = ConverterUtils.ToApplicationAttemptId(appAttemptIdStr2
);
// create application token and client token key for attempt2
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> appAttemptToken2 = GenerateAMRMToken
(attemptId2, appTokenMgr);
SecretKey clientTokenKey2 = clientToAMTokenMgr.CreateMasterKey(attemptId2);
ContainerId containerId2 = StoreAttempt(store, attemptId2, "container_1352994193343_0001_02_000001"
, appAttemptToken2, clientTokenKey2, dispatcher);
ApplicationAttemptId attemptIdRemoved = ConverterUtils.ToApplicationAttemptId("appattempt_1352994193343_0002_000001"
);
ApplicationId appIdRemoved = attemptIdRemoved.GetApplicationId();
StoreApp(store, appIdRemoved, submitTime, startTime);
StoreAttempt(store, attemptIdRemoved, "container_1352994193343_0002_01_000001", null
, null, dispatcher);
verifier.AfterStoreAppAttempt(store, attemptIdRemoved);
RMApp mockRemovedApp = Org.Mockito.Mockito.Mock <RMApp>();
RMAppAttemptMetrics mockRmAppAttemptMetrics = Org.Mockito.Mockito.Mock <RMAppAttemptMetrics
>();
Dictionary <ApplicationAttemptId, RMAppAttempt> attempts = new Dictionary <ApplicationAttemptId
, RMAppAttempt>();
ApplicationSubmissionContext context = new ApplicationSubmissionContextPBImpl();
context.SetApplicationId(appIdRemoved);
Org.Mockito.Mockito.When(mockRemovedApp.GetSubmitTime()).ThenReturn(submitTime);
Org.Mockito.Mockito.When(mockRemovedApp.GetApplicationSubmissionContext()).ThenReturn
(context);
Org.Mockito.Mockito.When(mockRemovedApp.GetAppAttempts()).ThenReturn(attempts);
Org.Mockito.Mockito.When(mockRemovedApp.GetUser()).ThenReturn("user1");
RMAppAttempt mockRemovedAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>();
Org.Mockito.Mockito.When(mockRemovedAttempt.GetAppAttemptId()).ThenReturn(attemptIdRemoved
);
Org.Mockito.Mockito.When(mockRemovedAttempt.GetRMAppAttemptMetrics()).ThenReturn(
mockRmAppAttemptMetrics);
Org.Mockito.Mockito.When(mockRmAppAttemptMetrics.GetAggregateAppResourceUsage()).
ThenReturn(new AggregateAppResourceUsage(0, 0));
attempts[attemptIdRemoved] = mockRemovedAttempt;
store.RemoveApplication(mockRemovedApp);
// remove application directory recursively.
StoreApp(store, appIdRemoved, submitTime, startTime);
StoreAttempt(store, attemptIdRemoved, "container_1352994193343_0002_01_000001", null
, null, dispatcher);
store.RemoveApplication(mockRemovedApp);
// let things settle down
Sharpen.Thread.Sleep(1000);
store.Close();
// give tester a chance to modify app state in the store
ModifyAppState();
// load state
store = stateStoreHelper.GetRMStateStore();
store.SetRMDispatcher(dispatcher);
RMStateStore.RMState state = store.LoadState();
IDictionary <ApplicationId, ApplicationStateData> rmAppState = state.GetApplicationState
();
ApplicationStateData appState = rmAppState[appId1];
// app is loaded
NUnit.Framework.Assert.IsNotNull(appState);
// app is loaded correctly
NUnit.Framework.Assert.AreEqual(submitTime, appState.GetSubmitTime());
NUnit.Framework.Assert.AreEqual(startTime, appState.GetStartTime());
// submission context is loaded correctly
NUnit.Framework.Assert.AreEqual(appId1, appState.GetApplicationSubmissionContext(
).GetApplicationId());
ApplicationAttemptStateData attemptState = appState.GetAttempt(attemptId1);
// attempt1 is loaded correctly
NUnit.Framework.Assert.IsNotNull(attemptState);
NUnit.Framework.Assert.AreEqual(attemptId1, attemptState.GetAttemptId());
NUnit.Framework.Assert.AreEqual(-1000, attemptState.GetAMContainerExitStatus());
// attempt1 container is loaded correctly
NUnit.Framework.Assert.AreEqual(containerId1, attemptState.GetMasterContainer().GetId
());
// attempt1 client token master key is loaded correctly
Assert.AssertArrayEquals(clientTokenKey1.GetEncoded(), attemptState.GetAppAttemptTokens
().GetSecretKey(RMStateStore.AmClientTokenMasterKeyName));
attemptState = appState.GetAttempt(attemptId2);
// attempt2 is loaded correctly
NUnit.Framework.Assert.IsNotNull(attemptState);
NUnit.Framework.Assert.AreEqual(attemptId2, attemptState.GetAttemptId());
// attempt2 container is loaded correctly
NUnit.Framework.Assert.AreEqual(containerId2, attemptState.GetMasterContainer().GetId
());
// attempt2 client token master key is loaded correctly
Assert.AssertArrayEquals(clientTokenKey2.GetEncoded(), attemptState.GetAppAttemptTokens
().GetSecretKey(RMStateStore.AmClientTokenMasterKeyName));
//******* update application/attempt state *******//
ApplicationStateData appState2 = ApplicationStateData.NewInstance(appState.GetSubmitTime
(), appState.GetStartTime(), appState.GetUser(), appState.GetApplicationSubmissionContext
(), RMAppState.Finished, "appDiagnostics", 1234);
appState2.attempts.PutAll(appState.attempts);
store.UpdateApplicationState(appState2);
ApplicationAttemptStateData oldAttemptState = attemptState;
ApplicationAttemptStateData newAttemptState = ApplicationAttemptStateData.NewInstance
(oldAttemptState.GetAttemptId(), oldAttemptState.GetMasterContainer(), oldAttemptState
.GetAppAttemptTokens(), oldAttemptState.GetStartTime(), RMAppAttemptState.Finished
, "myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.Succeeded, 100,
oldAttemptState.GetFinishTime(), 0, 0);
store.UpdateApplicationAttemptState(newAttemptState);
// test updating the state of an app/attempt whose initial state was not
// saved.
ApplicationId dummyAppId = ApplicationId.NewInstance(1234, 10);
ApplicationSubmissionContext dummyContext = new ApplicationSubmissionContextPBImpl
();
dummyContext.SetApplicationId(dummyAppId);
ApplicationStateData dummyApp = ApplicationStateData.NewInstance(appState.GetSubmitTime
(), appState.GetStartTime(), appState.GetUser(), dummyContext, RMAppState.Finished
, "appDiagnostics", 1234);
store.UpdateApplicationState(dummyApp);
ApplicationAttemptId dummyAttemptId = ApplicationAttemptId.NewInstance(dummyAppId
, 6);
ApplicationAttemptStateData dummyAttempt = ApplicationAttemptStateData.NewInstance
(dummyAttemptId, oldAttemptState.GetMasterContainer(), oldAttemptState.GetAppAttemptTokens
(), oldAttemptState.GetStartTime(), RMAppAttemptState.Finished, "myTrackingUrl",
"attemptDiagnostics", FinalApplicationStatus.Succeeded, 111, oldAttemptState.GetFinishTime
(), 0, 0);
store.UpdateApplicationAttemptState(dummyAttempt);
// let things settle down
Sharpen.Thread.Sleep(1000);
store.Close();
// check updated application state.
store = stateStoreHelper.GetRMStateStore();
store.SetRMDispatcher(dispatcher);
RMStateStore.RMState newRMState = store.LoadState();
IDictionary <ApplicationId, ApplicationStateData> newRMAppState = newRMState.GetApplicationState
();
NUnit.Framework.Assert.IsNotNull(newRMAppState[dummyApp.GetApplicationSubmissionContext
().GetApplicationId()]);
ApplicationStateData updatedAppState = newRMAppState[appId1];
NUnit.Framework.Assert.AreEqual(appState.GetApplicationSubmissionContext().GetApplicationId
(), updatedAppState.GetApplicationSubmissionContext().GetApplicationId());
NUnit.Framework.Assert.AreEqual(appState.GetSubmitTime(), updatedAppState.GetSubmitTime
());
NUnit.Framework.Assert.AreEqual(appState.GetStartTime(), updatedAppState.GetStartTime
());
NUnit.Framework.Assert.AreEqual(appState.GetUser(), updatedAppState.GetUser());
// new app state fields
NUnit.Framework.Assert.AreEqual(RMAppState.Finished, updatedAppState.GetState());
NUnit.Framework.Assert.AreEqual("appDiagnostics", updatedAppState.GetDiagnostics(
));
NUnit.Framework.Assert.AreEqual(1234, updatedAppState.GetFinishTime());
// check updated attempt state
NUnit.Framework.Assert.IsNotNull(newRMAppState[dummyApp.GetApplicationSubmissionContext
().GetApplicationId()].GetAttempt(dummyAttemptId));
ApplicationAttemptStateData updatedAttemptState = updatedAppState.GetAttempt(newAttemptState
.GetAttemptId());
NUnit.Framework.Assert.AreEqual(oldAttemptState.GetAttemptId(), updatedAttemptState
.GetAttemptId());
NUnit.Framework.Assert.AreEqual(containerId2, updatedAttemptState.GetMasterContainer
().GetId());
Assert.AssertArrayEquals(clientTokenKey2.GetEncoded(), attemptState.GetAppAttemptTokens
().GetSecretKey(RMStateStore.AmClientTokenMasterKeyName));
// new attempt state fields
NUnit.Framework.Assert.AreEqual(RMAppAttemptState.Finished, updatedAttemptState.GetState
());
NUnit.Framework.Assert.AreEqual("myTrackingUrl", updatedAttemptState.GetFinalTrackingUrl
());
NUnit.Framework.Assert.AreEqual("attemptDiagnostics", updatedAttemptState.GetDiagnostics
());
NUnit.Framework.Assert.AreEqual(100, updatedAttemptState.GetAMContainerExitStatus
());
NUnit.Framework.Assert.AreEqual(FinalApplicationStatus.Succeeded, updatedAttemptState
.GetFinalApplicationStatus());
// assert store is in expected state after everything is cleaned
NUnit.Framework.Assert.IsTrue(stateStoreHelper.IsFinalStateValid());
store.Close();
}
