// Test the DT mast key in the state-store when the mast key is being rolled.
/// <exception cref="System.Exception"/>
public virtual void TestRMDTMasterKeyStateOnRollingMasterKey()
{
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.Init(conf);
RMStateStore.RMState rmState = memStore.GetState();
IDictionary <RMDelegationTokenIdentifier, long> rmDTState = rmState.GetRMDTSecretManagerState
().GetTokenState();
ICollection <DelegationKey> rmDTMasterKeyState = rmState.GetRMDTSecretManagerState
().GetMasterKeyState();
MockRM rm1 = new TestRMDelegationTokens.MyMockRM(this, conf, memStore);
rm1.Start();
// on rm start, two master keys are created.
// One is created at RMDTSecretMgr.startThreads.updateCurrentKey();
// the other is created on the first run of
// tokenRemoverThread.rollMasterKey()
RMDelegationTokenSecretManager dtSecretManager = rm1.GetRMContext().GetRMDelegationTokenSecretManager
();
// assert all master keys are saved
NUnit.Framework.Assert.AreEqual(dtSecretManager.GetAllMasterKeys(), rmDTMasterKeyState
);
ICollection <DelegationKey> expiringKeys = new HashSet <DelegationKey>();
Sharpen.Collections.AddAll(expiringKeys, dtSecretManager.GetAllMasterKeys());
// request to generate a RMDelegationToken
GetDelegationTokenRequest request = Org.Mockito.Mockito.Mock <GetDelegationTokenRequest
>();
Org.Mockito.Mockito.When(request.GetRenewer()).ThenReturn("renewer1");
GetDelegationTokenResponse response = rm1.GetClientRMService().GetDelegationToken
(request);
Org.Apache.Hadoop.Yarn.Api.Records.Token delegationToken = response.GetRMDelegationToken
();
Org.Apache.Hadoop.Security.Token.Token <RMDelegationTokenIdentifier> token1 = ConverterUtils
.ConvertFromYarn(delegationToken, (Text)null);
RMDelegationTokenIdentifier dtId1 = token1.DecodeIdentifier();
// For all keys that still remain in memory, we should have them stored
// in state-store also.
while (((TestRMDelegationTokens.TestRMDelegationTokenSecretManager)dtSecretManager
).numUpdatedKeys.Get() < 3)
{
((TestRMDelegationTokens.TestRMDelegationTokenSecretManager)dtSecretManager).CheckCurrentKeyInStateStore
(rmDTMasterKeyState);
Sharpen.Thread.Sleep(100);
}
// wait for token to expire and remove from state-store
// rollMasterKey is called every 1 second.
int count = 0;
while (rmDTState.Contains(dtId1) && count < 100)
{
Sharpen.Thread.Sleep(100);
count++;
}
rm1.Stop();
}
/// <exception cref="System.Exception"/>
public override RMStateStore.RMState LoadState()
{
RMStateStore.RMState rmState = new RMStateStore.RMState();
LoadRMDTSecretManagerState(rmState);
LoadRMApps(rmState);
LoadAMRMTokenSecretManagerState(rmState);
return(rmState);
}
/// <exception cref="System.Exception"/>
private void LoadRMDTSecretManagerState(RMStateStore.RMState rmState)
{
CheckAndResumeUpdateOperation(rmDTSecretManagerRoot);
FileStatus[] childNodes = ListStatusWithRetries(rmDTSecretManagerRoot);
foreach (FileStatus childNodeStatus in childNodes)
{
System.Diagnostics.Debug.Assert(childNodeStatus.IsFile());
string childNodeName = childNodeStatus.GetPath().GetName();
if (CheckAndRemovePartialRecordWithRetries(childNodeStatus.GetPath()))
{
continue;
}
if (childNodeName.StartsWith(DelegationTokenSequenceNumberPrefix))
{
rmState.rmSecretManagerState.dtSequenceNumber = System.Convert.ToInt32(childNodeName
.Split("_")[1]);
continue;
}
Path childNodePath = GetNodePath(rmDTSecretManagerRoot, childNodeName);
byte[] childData = ReadFileWithRetries(childNodePath, childNodeStatus.GetLen());
ByteArrayInputStream @is = new ByteArrayInputStream(childData);
using (DataInputStream fsIn = new DataInputStream(@is))
{
if (childNodeName.StartsWith(DelegationKeyPrefix))
{
DelegationKey key = new DelegationKey();
key.ReadFields(fsIn);
rmState.rmSecretManagerState.masterKeyState.AddItem(key);
if (Log.IsDebugEnabled())
{
Log.Debug("Loaded delegation key: keyId=" + key.GetKeyId() + ", expirationDate="
+ key.GetExpiryDate());
}
}
else
{
if (childNodeName.StartsWith(DelegationTokenPrefix))
{
RMDelegationTokenIdentifierData identifierData = new RMDelegationTokenIdentifierData
();
identifierData.ReadFields(fsIn);
RMDelegationTokenIdentifier identifier = identifierData.GetTokenIdentifier();
long renewDate = identifierData.GetRenewDate();
rmState.rmSecretManagerState.delegationTokenState[identifier] = renewDate;
if (Log.IsDebugEnabled())
{
Log.Debug("Loaded RMDelegationTokenIdentifier: " + identifier + " renewDate=" + renewDate
);
}
}
else
{
Log.Warn("Unknown file for recovering RMDelegationTokenSecretManager");
}
}
}
}
}
/// <exception cref="System.IO.IOException"/>
private void LoadRMDTSecretManagerState(RMStateStore.RMState state)
{
int numKeys = LoadRMDTSecretManagerKeys(state);
Log.Info("Recovered " + numKeys + " RM delegation token master keys");
int numTokens = LoadRMDTSecretManagerTokens(state);
Log.Info("Recovered " + numTokens + " RM delegation tokens");
LoadRMDTSecretManagerTokenSequenceNumber(state);
}
/// <exception cref="System.Exception"/>
public virtual void TestAMRMTokenSecretManagerStateStore(RMStateStoreTestBase.RMStateStoreHelper
stateStoreHelper)
{
System.Console.Out.WriteLine("Start testing");
RMStateStore store = stateStoreHelper.GetRMStateStore();
RMStateStoreTestBase.TestDispatcher dispatcher = new RMStateStoreTestBase.TestDispatcher
();
store.SetRMDispatcher(dispatcher);
RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>();
Org.Mockito.Mockito.When(rmContext.GetStateStore()).ThenReturn(store);
Configuration conf = new YarnConfiguration();
AMRMTokenSecretManager appTokenMgr = new AMRMTokenSecretManager(conf, rmContext);
//create and save the first masterkey
MasterKeyData firstMasterKeyData = appTokenMgr.CreateNewMasterKey();
AMRMTokenSecretManagerState state1 = AMRMTokenSecretManagerState.NewInstance(firstMasterKeyData
.GetMasterKey(), null);
rmContext.GetStateStore().StoreOrUpdateAMRMTokenSecretManager(state1, false);
// load state
store = stateStoreHelper.GetRMStateStore();
Org.Mockito.Mockito.When(rmContext.GetStateStore()).ThenReturn(store);
store.SetRMDispatcher(dispatcher);
RMStateStore.RMState state = store.LoadState();
NUnit.Framework.Assert.IsNotNull(state.GetAMRMTokenSecretManagerState());
NUnit.Framework.Assert.AreEqual(firstMasterKeyData.GetMasterKey(), state.GetAMRMTokenSecretManagerState
().GetCurrentMasterKey());
NUnit.Framework.Assert.IsNull(state.GetAMRMTokenSecretManagerState().GetNextMasterKey
());
//create and save the second masterkey
MasterKeyData secondMasterKeyData = appTokenMgr.CreateNewMasterKey();
AMRMTokenSecretManagerState state2 = AMRMTokenSecretManagerState.NewInstance(firstMasterKeyData
.GetMasterKey(), secondMasterKeyData.GetMasterKey());
rmContext.GetStateStore().StoreOrUpdateAMRMTokenSecretManager(state2, true);
// load state
store = stateStoreHelper.GetRMStateStore();
Org.Mockito.Mockito.When(rmContext.GetStateStore()).ThenReturn(store);
store.SetRMDispatcher(dispatcher);
RMStateStore.RMState state_2 = store.LoadState();
NUnit.Framework.Assert.IsNotNull(state_2.GetAMRMTokenSecretManagerState());
NUnit.Framework.Assert.AreEqual(firstMasterKeyData.GetMasterKey(), state_2.GetAMRMTokenSecretManagerState
().GetCurrentMasterKey());
NUnit.Framework.Assert.AreEqual(secondMasterKeyData.GetMasterKey(), state_2.GetAMRMTokenSecretManagerState
().GetNextMasterKey());
// re-create the masterKeyData based on the recovered masterkey
// should have the same secretKey
appTokenMgr.Recover(state_2);
NUnit.Framework.Assert.AreEqual(appTokenMgr.GetCurrnetMasterKeyData().GetSecretKey
(), firstMasterKeyData.GetSecretKey());
NUnit.Framework.Assert.AreEqual(appTokenMgr.GetNextMasterKeyData().GetSecretKey()
, secondMasterKeyData.GetSecretKey());
store.Close();
}
/// <exception cref="System.Exception"/>
protected internal virtual void RecoverApplication(ApplicationStateData appState,
RMStateStore.RMState rmState)
{
ApplicationSubmissionContext appContext = appState.GetApplicationSubmissionContext
();
ApplicationId appId = appContext.GetApplicationId();
// create and recover app.
RMAppImpl application = CreateAndPopulateNewRMApp(appContext, appState.GetSubmitTime
(), appState.GetUser(), true);
application.Handle(new RMAppRecoverEvent(appId, rmState));
}
/// <exception cref="System.Exception"/>
public override RMStateStore.RMState LoadState()
{
lock (this)
{
RMStateStore.RMState rmState = new RMStateStore.RMState();
// recover DelegationTokenSecretManager
LoadRMDTSecretManagerState(rmState);
// recover RM applications
LoadRMAppState(rmState);
// recover AMRMTokenSecretManager
LoadAMRMTokenSecretManagerState(rmState);
return(rmState);
}
}
/// <exception cref="System.Exception"/>
public virtual void Recover(RMStateStore.RMState state)
{
RMStateStore store = rmContext.GetStateStore();
System.Diagnostics.Debug.Assert(store != null);
// recover applications
IDictionary <ApplicationId, ApplicationStateData> appStates = state.GetApplicationState
();
Log.Info("Recovering " + appStates.Count + " applications");
foreach (ApplicationStateData appState in appStates.Values)
{
RecoverApplication(appState, state);
}
}
/// <exception cref="System.Exception"/>
public override RMStateStore.RMState LoadState()
{
lock (this)
{
// return a copy of the state to allow for modification of the real state
RMStateStore.RMState returnState = new RMStateStore.RMState();
returnState.appState.PutAll(state.appState);
Sharpen.Collections.AddAll(returnState.rmSecretManagerState.GetMasterKeyState(),
state.rmSecretManagerState.GetMasterKeyState());
returnState.rmSecretManagerState.GetTokenState().PutAll(state.rmSecretManagerState
.GetTokenState());
returnState.rmSecretManagerState.dtSequenceNumber = state.rmSecretManagerState.dtSequenceNumber;
returnState.amrmTokenSecretManagerState = state.amrmTokenSecretManagerState == null
? null : AMRMTokenSecretManagerState.NewInstance(state.amrmTokenSecretManagerState
);
return(returnState);
}
}
/// <exception cref="System.IO.IOException"/>
private void LoadAMRMTokenSecretManagerState(RMStateStore.RMState rmState)
{
try
{
byte[] data = db.Get(JniDBFactory.Bytes(AmrmtokenSecretManagerRoot));
if (data != null)
{
AMRMTokenSecretManagerStatePBImpl stateData = new AMRMTokenSecretManagerStatePBImpl
(YarnServerResourceManagerRecoveryProtos.AMRMTokenSecretManagerStateProto.ParseFrom
(data));
rmState.amrmTokenSecretManagerState = AMRMTokenSecretManagerState.NewInstance(stateData
.GetCurrentMasterKey(), stateData.GetNextMasterKey());
}
}
catch (DBException e)
{
throw new IOException(e);
}
}
/// <exception cref="System.IO.IOException"/>
private void LoadRMApps(RMStateStore.RMState state)
{
int numApps = 0;
int numAppAttempts = 0;
LeveldbIterator iter = null;
try
{
iter = new LeveldbIterator(db);
iter.Seek(JniDBFactory.Bytes(RmAppKeyPrefix));
while (iter.HasNext())
{
KeyValuePair <byte[], byte[]> entry = iter.Next();
string key = JniDBFactory.AsString(entry.Key);
if (!key.StartsWith(RmAppKeyPrefix))
{
break;
}
string appIdStr = Sharpen.Runtime.Substring(key, RmAppRoot.Length + 1);
if (appIdStr.Contains(Separator))
{
Log.Warn("Skipping extraneous data " + key);
continue;
}
numAppAttempts += LoadRMApp(state, iter, appIdStr, entry.Value);
++numApps;
}
}
catch (DBException e)
{
throw new IOException(e);
}
finally
{
if (iter != null)
{
iter.Close();
}
}
Log.Info("Recovered " + numApps + " applications and " + numAppAttempts + " application attempts"
);
}
/// <exception cref="System.IO.IOException"/>
private int LoadRMDTSecretManagerTokens(RMStateStore.RMState state)
{
int numTokens = 0;
LeveldbIterator iter = null;
try
{
iter = new LeveldbIterator(db);
iter.Seek(JniDBFactory.Bytes(RmDtTokenKeyPrefix));
while (iter.HasNext())
{
KeyValuePair <byte[], byte[]> entry = iter.Next();
string key = JniDBFactory.AsString(entry.Key);
if (!key.StartsWith(RmDtTokenKeyPrefix))
{
break;
}
RMDelegationTokenIdentifierData tokenData = LoadDelegationToken(entry.Value);
RMDelegationTokenIdentifier tokenId = tokenData.GetTokenIdentifier();
long renewDate = tokenData.GetRenewDate();
state.rmSecretManagerState.delegationTokenState[tokenId] = renewDate;
++numTokens;
if (Log.IsDebugEnabled())
{
Log.Debug("Loaded RM delegation token from " + key + ": tokenId=" + tokenId + ", renewDate="
+ renewDate);
}
}
}
catch (DBException e)
{
throw new IOException(e);
}
finally
{
if (iter != null)
{
iter.Close();
}
}
return(numTokens);
}
public virtual void Recover(RMStateStore.RMState state)
{
if (state.GetAMRMTokenSecretManagerState() != null)
{
// recover the current master key
MasterKey currentKey = state.GetAMRMTokenSecretManagerState().GetCurrentMasterKey
();
this.currentMasterKey = new MasterKeyData(currentKey, CreateSecretKey(((byte[])currentKey
.GetBytes().Array())));
// recover the next master key if not null
MasterKey nextKey = state.GetAMRMTokenSecretManagerState().GetNextMasterKey();
if (nextKey != null)
{
this.nextMasterKey = new MasterKeyData(nextKey, CreateSecretKey(((byte[])nextKey.
GetBytes().Array())));
this.timer.Schedule(new AMRMTokenSecretManager.NextKeyActivator(this), this.activationDelay
);
}
}
}
/// <exception cref="System.Exception"/>
public virtual void Recover(RMStateStore.RMState rmState)
{
Log.Info("recovering RMDelegationTokenSecretManager.");
// recover RMDTMasterKeys
foreach (DelegationKey dtKey in rmState.GetRMDTSecretManagerState().GetMasterKeyState
())
{
AddKey(dtKey);
}
// recover RMDelegationTokens
IDictionary <RMDelegationTokenIdentifier, long> rmDelegationTokens = rmState.GetRMDTSecretManagerState
().GetTokenState();
this.delegationTokenSequenceNumber = rmState.GetRMDTSecretManagerState().GetDTSequenceNumber
();
foreach (KeyValuePair <RMDelegationTokenIdentifier, long> entry in rmDelegationTokens)
{
AddPersistedDelegationToken(entry.Key, entry.Value);
}
}
/// <exception cref="System.Exception"/>
private void LoadAMRMTokenSecretManagerState(RMStateStore.RMState rmState)
{
CheckAndResumeUpdateOperation(amrmTokenSecretManagerRoot);
Path amrmTokenSecretManagerStateDataDir = new Path(amrmTokenSecretManagerRoot, AmrmtokenSecretManagerNode
);
FileStatus status = GetFileStatusWithRetries(amrmTokenSecretManagerStateDataDir);
if (status == null)
{
return;
}
System.Diagnostics.Debug.Assert(status.IsFile());
byte[] data = ReadFileWithRetries(amrmTokenSecretManagerStateDataDir, status.GetLen
());
AMRMTokenSecretManagerStatePBImpl stateData = new AMRMTokenSecretManagerStatePBImpl
(YarnServerResourceManagerRecoveryProtos.AMRMTokenSecretManagerStateProto.ParseFrom
(data));
rmState.amrmTokenSecretManagerState = AMRMTokenSecretManagerState.NewInstance(stateData
.GetCurrentMasterKey(), stateData.GetNextMasterKey());
}
/// <exception cref="System.IO.IOException"/>
private int LoadRMDTSecretManagerKeys(RMStateStore.RMState state)
{
int numKeys = 0;
LeveldbIterator iter = null;
try
{
iter = new LeveldbIterator(db);
iter.Seek(JniDBFactory.Bytes(RmDtMasterKeyKeyPrefix));
while (iter.HasNext())
{
KeyValuePair <byte[], byte[]> entry = iter.Next();
string key = JniDBFactory.AsString(entry.Key);
if (!key.StartsWith(RmDtMasterKeyKeyPrefix))
{
break;
}
DelegationKey masterKey = LoadDelegationKey(entry.Value);
state.rmSecretManagerState.masterKeyState.AddItem(masterKey);
++numKeys;
if (Log.IsDebugEnabled())
{
Log.Debug("Loaded RM delegation key from " + key + ": keyId=" + masterKey.GetKeyId
() + ", expirationDate=" + masterKey.GetExpiryDate());
}
}
}
catch (DBException e)
{
throw new IOException(e);
}
finally
{
if (iter != null)
{
iter.Close();
}
}
return(numKeys);
}
/// <exception cref="System.IO.IOException"/>
private int LoadRMApp(RMStateStore.RMState rmState, LeveldbIterator iter, string
appIdStr, byte[] appData)
{
ApplicationStateData appState = CreateApplicationState(appIdStr, appData);
ApplicationId appId = appState.GetApplicationSubmissionContext().GetApplicationId
();
rmState.appState[appId] = appState;
string attemptNodePrefix = GetApplicationNodeKey(appId) + Separator;
while (iter.HasNext())
{
KeyValuePair <byte[], byte[]> entry = iter.PeekNext();
string key = JniDBFactory.AsString(entry.Key);
if (!key.StartsWith(attemptNodePrefix))
{
break;
}
string attemptId = Sharpen.Runtime.Substring(key, attemptNodePrefix.Length);
if (attemptId.StartsWith(ApplicationAttemptId.appAttemptIdStrPrefix))
{
ApplicationAttemptStateData attemptState = CreateAttemptState(attemptId, entry.Value
);
appState.attempts[attemptState.GetAttemptId()] = attemptState;
}
else
{
Log.Warn("Ignoring unknown application key: " + key);
}
iter.Next();
}
int numAttempts = appState.attempts.Count;
if (Log.IsDebugEnabled())
{
Log.Debug("Loaded application " + appId + " with " + numAttempts + " attempts");
}
return(numAttempts);
}
// Test all expired keys are removed from state-store.
/// <exception cref="System.Exception"/>
public virtual void TestRemoveExpiredMasterKeyInRMStateStore()
{
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.Init(conf);
RMStateStore.RMState rmState = memStore.GetState();
ICollection <DelegationKey> rmDTMasterKeyState = rmState.GetRMDTSecretManagerState
().GetMasterKeyState();
MockRM rm1 = new TestRMDelegationTokens.MyMockRM(this, conf, memStore);
rm1.Start();
RMDelegationTokenSecretManager dtSecretManager = rm1.GetRMContext().GetRMDelegationTokenSecretManager
();
// assert all master keys are saved
NUnit.Framework.Assert.AreEqual(dtSecretManager.GetAllMasterKeys(), rmDTMasterKeyState
);
ICollection <DelegationKey> expiringKeys = new HashSet <DelegationKey>();
Sharpen.Collections.AddAll(expiringKeys, dtSecretManager.GetAllMasterKeys());
// wait for expiringKeys to expire
while (true)
{
bool allExpired = true;
foreach (DelegationKey key in expiringKeys)
{
if (rmDTMasterKeyState.Contains(key))
{
allExpired = false;
}
}
if (allExpired)
{
break;
}
Sharpen.Thread.Sleep(500);
}
}
/// <exception cref="System.IO.IOException"/>
private void LoadRMDTSecretManagerTokenSequenceNumber(RMStateStore.RMState state)
{
byte[] data = null;
try
{
data = db.Get(JniDBFactory.Bytes(RmDtSequenceNumberKey));
}
catch (DBException e)
{
throw new IOException(e);
}
if (data != null)
{
DataInputStream @in = new DataInputStream(new ByteArrayInputStream(data));
try
{
state.rmSecretManagerState.dtSequenceNumber = @in.ReadInt();
}
finally
{
IOUtils.Cleanup(Log, @in);
}
}
}
/// <exception cref="System.Exception"/>
private void LoadRMAppState(RMStateStore.RMState rmState)
{
try
{
IList <ApplicationAttemptStateData> attempts = new AList <ApplicationAttemptStateData
>();
foreach (FileStatus appDir in ListStatusWithRetries(rmAppRoot))
{
CheckAndResumeUpdateOperation(appDir.GetPath());
foreach (FileStatus childNodeStatus in ListStatusWithRetries(appDir.GetPath()))
{
System.Diagnostics.Debug.Assert(childNodeStatus.IsFile());
string childNodeName = childNodeStatus.GetPath().GetName();
if (CheckAndRemovePartialRecordWithRetries(childNodeStatus.GetPath()))
{
continue;
}
byte[] childData = ReadFileWithRetries(childNodeStatus.GetPath(), childNodeStatus
.GetLen());
// Set attribute if not already set
SetUnreadableBySuperuserXattrib(childNodeStatus.GetPath());
if (childNodeName.StartsWith(ApplicationId.appIdStrPrefix))
{
// application
if (Log.IsDebugEnabled())
{
Log.Debug("Loading application from node: " + childNodeName);
}
ApplicationStateDataPBImpl appState = new ApplicationStateDataPBImpl(YarnServerResourceManagerRecoveryProtos.ApplicationStateDataProto
.ParseFrom(childData));
ApplicationId appId = appState.GetApplicationSubmissionContext().GetApplicationId
();
rmState.appState[appId] = appState;
}
else
{
if (childNodeName.StartsWith(ApplicationAttemptId.appAttemptIdStrPrefix))
{
// attempt
if (Log.IsDebugEnabled())
{
Log.Debug("Loading application attempt from node: " + childNodeName);
}
ApplicationAttemptStateDataPBImpl attemptState = new ApplicationAttemptStateDataPBImpl
(YarnServerResourceManagerRecoveryProtos.ApplicationAttemptStateDataProto.ParseFrom
(childData));
attempts.AddItem(attemptState);
}
else
{
Log.Info("Unknown child node with name: " + childNodeName);
}
}
}
}
// go through all attempts and add them to their apps, Ideally, each
// attempt node must have a corresponding app node, because remove
// directory operation remove both at the same time
foreach (ApplicationAttemptStateData attemptState_1 in attempts)
{
ApplicationId appId = attemptState_1.GetAttemptId().GetApplicationId();
ApplicationStateData appState = rmState.appState[appId];
System.Diagnostics.Debug.Assert(appState != null);
appState.attempts[attemptState_1.GetAttemptId()] = attemptState_1;
}
Log.Info("Done loading applications from FS state store");
}
catch (Exception e)
{
Log.Error("Failed to load state.", e);
throw;
}
}
public abstract void Recover(RMStateStore.RMState arg1);
public override void Recover(RMStateStore.RMState state)
{
}
public RMAppRecoverEvent(ApplicationId appId, RMStateStore.RMState state)
: base(appId, RMAppEventType.Recover)
{
this.state = state;
}
/// <exception cref="System.Exception"/>
internal virtual void TestRMAppStateStore(RMStateStoreTestBase.RMStateStoreHelper
stateStoreHelper, RMStateStoreTestBase.StoreStateVerifier verifier)
{
long submitTime = Runtime.CurrentTimeMillis();
long startTime = Runtime.CurrentTimeMillis() + 1234;
Configuration conf = new YarnConfiguration();
RMStateStore store = stateStoreHelper.GetRMStateStore();
RMStateStoreTestBase.TestDispatcher dispatcher = new RMStateStoreTestBase.TestDispatcher
();
store.SetRMDispatcher(dispatcher);
RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>();
Org.Mockito.Mockito.When(rmContext.GetStateStore()).ThenReturn(store);
AMRMTokenSecretManager appTokenMgr = Org.Mockito.Mockito.Spy(new AMRMTokenSecretManager
(conf, rmContext));
MasterKeyData masterKeyData = appTokenMgr.CreateNewMasterKey();
Org.Mockito.Mockito.When(appTokenMgr.GetMasterKey()).ThenReturn(masterKeyData);
ClientToAMTokenSecretManagerInRM clientToAMTokenMgr = new ClientToAMTokenSecretManagerInRM
();
ApplicationAttemptId attemptId1 = ConverterUtils.ToApplicationAttemptId("appattempt_1352994193343_0001_000001"
);
ApplicationId appId1 = attemptId1.GetApplicationId();
StoreApp(store, appId1, submitTime, startTime);
verifier.AfterStoreApp(store, appId1);
// create application token and client token key for attempt1
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> appAttemptToken1 = GenerateAMRMToken
(attemptId1, appTokenMgr);
SecretKey clientTokenKey1 = clientToAMTokenMgr.CreateMasterKey(attemptId1);
ContainerId containerId1 = StoreAttempt(store, attemptId1, "container_1352994193343_0001_01_000001"
, appAttemptToken1, clientTokenKey1, dispatcher);
string appAttemptIdStr2 = "appattempt_1352994193343_0001_000002";
ApplicationAttemptId attemptId2 = ConverterUtils.ToApplicationAttemptId(appAttemptIdStr2
);
// create application token and client token key for attempt2
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> appAttemptToken2 = GenerateAMRMToken
(attemptId2, appTokenMgr);
SecretKey clientTokenKey2 = clientToAMTokenMgr.CreateMasterKey(attemptId2);
ContainerId containerId2 = StoreAttempt(store, attemptId2, "container_1352994193343_0001_02_000001"
, appAttemptToken2, clientTokenKey2, dispatcher);
ApplicationAttemptId attemptIdRemoved = ConverterUtils.ToApplicationAttemptId("appattempt_1352994193343_0002_000001"
);
ApplicationId appIdRemoved = attemptIdRemoved.GetApplicationId();
StoreApp(store, appIdRemoved, submitTime, startTime);
StoreAttempt(store, attemptIdRemoved, "container_1352994193343_0002_01_000001", null
, null, dispatcher);
verifier.AfterStoreAppAttempt(store, attemptIdRemoved);
RMApp mockRemovedApp = Org.Mockito.Mockito.Mock <RMApp>();
RMAppAttemptMetrics mockRmAppAttemptMetrics = Org.Mockito.Mockito.Mock <RMAppAttemptMetrics
>();
Dictionary <ApplicationAttemptId, RMAppAttempt> attempts = new Dictionary <ApplicationAttemptId
, RMAppAttempt>();
ApplicationSubmissionContext context = new ApplicationSubmissionContextPBImpl();
context.SetApplicationId(appIdRemoved);
Org.Mockito.Mockito.When(mockRemovedApp.GetSubmitTime()).ThenReturn(submitTime);
Org.Mockito.Mockito.When(mockRemovedApp.GetApplicationSubmissionContext()).ThenReturn
(context);
Org.Mockito.Mockito.When(mockRemovedApp.GetAppAttempts()).ThenReturn(attempts);
Org.Mockito.Mockito.When(mockRemovedApp.GetUser()).ThenReturn("user1");
RMAppAttempt mockRemovedAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>();
Org.Mockito.Mockito.When(mockRemovedAttempt.GetAppAttemptId()).ThenReturn(attemptIdRemoved
);
Org.Mockito.Mockito.When(mockRemovedAttempt.GetRMAppAttemptMetrics()).ThenReturn(
mockRmAppAttemptMetrics);
Org.Mockito.Mockito.When(mockRmAppAttemptMetrics.GetAggregateAppResourceUsage()).
ThenReturn(new AggregateAppResourceUsage(0, 0));
attempts[attemptIdRemoved] = mockRemovedAttempt;
store.RemoveApplication(mockRemovedApp);
// remove application directory recursively.
StoreApp(store, appIdRemoved, submitTime, startTime);
StoreAttempt(store, attemptIdRemoved, "container_1352994193343_0002_01_000001", null
, null, dispatcher);
store.RemoveApplication(mockRemovedApp);
// let things settle down
Sharpen.Thread.Sleep(1000);
store.Close();
// give tester a chance to modify app state in the store
ModifyAppState();
// load state
store = stateStoreHelper.GetRMStateStore();
store.SetRMDispatcher(dispatcher);
RMStateStore.RMState state = store.LoadState();
IDictionary <ApplicationId, ApplicationStateData> rmAppState = state.GetApplicationState
();
ApplicationStateData appState = rmAppState[appId1];
// app is loaded
NUnit.Framework.Assert.IsNotNull(appState);
// app is loaded correctly
NUnit.Framework.Assert.AreEqual(submitTime, appState.GetSubmitTime());
NUnit.Framework.Assert.AreEqual(startTime, appState.GetStartTime());
// submission context is loaded correctly
NUnit.Framework.Assert.AreEqual(appId1, appState.GetApplicationSubmissionContext(
).GetApplicationId());
ApplicationAttemptStateData attemptState = appState.GetAttempt(attemptId1);
// attempt1 is loaded correctly
NUnit.Framework.Assert.IsNotNull(attemptState);
NUnit.Framework.Assert.AreEqual(attemptId1, attemptState.GetAttemptId());
NUnit.Framework.Assert.AreEqual(-1000, attemptState.GetAMContainerExitStatus());
// attempt1 container is loaded correctly
NUnit.Framework.Assert.AreEqual(containerId1, attemptState.GetMasterContainer().GetId
());
// attempt1 client token master key is loaded correctly
Assert.AssertArrayEquals(clientTokenKey1.GetEncoded(), attemptState.GetAppAttemptTokens
().GetSecretKey(RMStateStore.AmClientTokenMasterKeyName));
attemptState = appState.GetAttempt(attemptId2);
// attempt2 is loaded correctly
NUnit.Framework.Assert.IsNotNull(attemptState);
NUnit.Framework.Assert.AreEqual(attemptId2, attemptState.GetAttemptId());
// attempt2 container is loaded correctly
NUnit.Framework.Assert.AreEqual(containerId2, attemptState.GetMasterContainer().GetId
());
// attempt2 client token master key is loaded correctly
Assert.AssertArrayEquals(clientTokenKey2.GetEncoded(), attemptState.GetAppAttemptTokens
().GetSecretKey(RMStateStore.AmClientTokenMasterKeyName));
//******* update application/attempt state *******//
ApplicationStateData appState2 = ApplicationStateData.NewInstance(appState.GetSubmitTime
(), appState.GetStartTime(), appState.GetUser(), appState.GetApplicationSubmissionContext
(), RMAppState.Finished, "appDiagnostics", 1234);
appState2.attempts.PutAll(appState.attempts);
store.UpdateApplicationState(appState2);
ApplicationAttemptStateData oldAttemptState = attemptState;
ApplicationAttemptStateData newAttemptState = ApplicationAttemptStateData.NewInstance
(oldAttemptState.GetAttemptId(), oldAttemptState.GetMasterContainer(), oldAttemptState
.GetAppAttemptTokens(), oldAttemptState.GetStartTime(), RMAppAttemptState.Finished
, "myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.Succeeded, 100,
oldAttemptState.GetFinishTime(), 0, 0);
store.UpdateApplicationAttemptState(newAttemptState);
// test updating the state of an app/attempt whose initial state was not
// saved.
ApplicationId dummyAppId = ApplicationId.NewInstance(1234, 10);
ApplicationSubmissionContext dummyContext = new ApplicationSubmissionContextPBImpl
();
dummyContext.SetApplicationId(dummyAppId);
ApplicationStateData dummyApp = ApplicationStateData.NewInstance(appState.GetSubmitTime
(), appState.GetStartTime(), appState.GetUser(), dummyContext, RMAppState.Finished
, "appDiagnostics", 1234);
store.UpdateApplicationState(dummyApp);
ApplicationAttemptId dummyAttemptId = ApplicationAttemptId.NewInstance(dummyAppId
, 6);
ApplicationAttemptStateData dummyAttempt = ApplicationAttemptStateData.NewInstance
(dummyAttemptId, oldAttemptState.GetMasterContainer(), oldAttemptState.GetAppAttemptTokens
(), oldAttemptState.GetStartTime(), RMAppAttemptState.Finished, "myTrackingUrl",
"attemptDiagnostics", FinalApplicationStatus.Succeeded, 111, oldAttemptState.GetFinishTime
(), 0, 0);
store.UpdateApplicationAttemptState(dummyAttempt);
// let things settle down
Sharpen.Thread.Sleep(1000);
store.Close();
// check updated application state.
store = stateStoreHelper.GetRMStateStore();
store.SetRMDispatcher(dispatcher);
RMStateStore.RMState newRMState = store.LoadState();
IDictionary <ApplicationId, ApplicationStateData> newRMAppState = newRMState.GetApplicationState
();
NUnit.Framework.Assert.IsNotNull(newRMAppState[dummyApp.GetApplicationSubmissionContext
().GetApplicationId()]);
ApplicationStateData updatedAppState = newRMAppState[appId1];
NUnit.Framework.Assert.AreEqual(appState.GetApplicationSubmissionContext().GetApplicationId
(), updatedAppState.GetApplicationSubmissionContext().GetApplicationId());
NUnit.Framework.Assert.AreEqual(appState.GetSubmitTime(), updatedAppState.GetSubmitTime
());
NUnit.Framework.Assert.AreEqual(appState.GetStartTime(), updatedAppState.GetStartTime
());
NUnit.Framework.Assert.AreEqual(appState.GetUser(), updatedAppState.GetUser());
// new app state fields
NUnit.Framework.Assert.AreEqual(RMAppState.Finished, updatedAppState.GetState());
NUnit.Framework.Assert.AreEqual("appDiagnostics", updatedAppState.GetDiagnostics(
));
NUnit.Framework.Assert.AreEqual(1234, updatedAppState.GetFinishTime());
// check updated attempt state
NUnit.Framework.Assert.IsNotNull(newRMAppState[dummyApp.GetApplicationSubmissionContext
().GetApplicationId()].GetAttempt(dummyAttemptId));
ApplicationAttemptStateData updatedAttemptState = updatedAppState.GetAttempt(newAttemptState
.GetAttemptId());
NUnit.Framework.Assert.AreEqual(oldAttemptState.GetAttemptId(), updatedAttemptState
.GetAttemptId());
NUnit.Framework.Assert.AreEqual(containerId2, updatedAttemptState.GetMasterContainer
().GetId());
Assert.AssertArrayEquals(clientTokenKey2.GetEncoded(), attemptState.GetAppAttemptTokens
().GetSecretKey(RMStateStore.AmClientTokenMasterKeyName));
// new attempt state fields
NUnit.Framework.Assert.AreEqual(RMAppAttemptState.Finished, updatedAttemptState.GetState
());
NUnit.Framework.Assert.AreEqual("myTrackingUrl", updatedAttemptState.GetFinalTrackingUrl
());
NUnit.Framework.Assert.AreEqual("attemptDiagnostics", updatedAttemptState.GetDiagnostics
());
NUnit.Framework.Assert.AreEqual(100, updatedAttemptState.GetAMContainerExitStatus
());
NUnit.Framework.Assert.AreEqual(FinalApplicationStatus.Succeeded, updatedAttemptState
.GetFinalApplicationStatus());
// assert store is in expected state after everything is cleaned
NUnit.Framework.Assert.IsTrue(stateStoreHelper.IsFinalStateValid());
store.Close();
}
