/// <exception cref="System.Exception"/>
public virtual void TestAMRMTokenSecretManagerStateStore(RMStateStoreTestBase.RMStateStoreHelper
stateStoreHelper)
{
System.Console.Out.WriteLine("Start testing");
RMStateStore store = stateStoreHelper.GetRMStateStore();
RMStateStoreTestBase.TestDispatcher dispatcher = new RMStateStoreTestBase.TestDispatcher
();
store.SetRMDispatcher(dispatcher);
RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>();
Org.Mockito.Mockito.When(rmContext.GetStateStore()).ThenReturn(store);
Configuration conf = new YarnConfiguration();
AMRMTokenSecretManager appTokenMgr = new AMRMTokenSecretManager(conf, rmContext);
//create and save the first masterkey
MasterKeyData firstMasterKeyData = appTokenMgr.CreateNewMasterKey();
AMRMTokenSecretManagerState state1 = AMRMTokenSecretManagerState.NewInstance(firstMasterKeyData
.GetMasterKey(), null);
rmContext.GetStateStore().StoreOrUpdateAMRMTokenSecretManager(state1, false);
// load state
store = stateStoreHelper.GetRMStateStore();
Org.Mockito.Mockito.When(rmContext.GetStateStore()).ThenReturn(store);
store.SetRMDispatcher(dispatcher);
RMStateStore.RMState state = store.LoadState();
NUnit.Framework.Assert.IsNotNull(state.GetAMRMTokenSecretManagerState());
NUnit.Framework.Assert.AreEqual(firstMasterKeyData.GetMasterKey(), state.GetAMRMTokenSecretManagerState
().GetCurrentMasterKey());
NUnit.Framework.Assert.IsNull(state.GetAMRMTokenSecretManagerState().GetNextMasterKey
());
//create and save the second masterkey
MasterKeyData secondMasterKeyData = appTokenMgr.CreateNewMasterKey();
AMRMTokenSecretManagerState state2 = AMRMTokenSecretManagerState.NewInstance(firstMasterKeyData
.GetMasterKey(), secondMasterKeyData.GetMasterKey());
rmContext.GetStateStore().StoreOrUpdateAMRMTokenSecretManager(state2, true);
// load state
store = stateStoreHelper.GetRMStateStore();
Org.Mockito.Mockito.When(rmContext.GetStateStore()).ThenReturn(store);
store.SetRMDispatcher(dispatcher);
RMStateStore.RMState state_2 = store.LoadState();
NUnit.Framework.Assert.IsNotNull(state_2.GetAMRMTokenSecretManagerState());
NUnit.Framework.Assert.AreEqual(firstMasterKeyData.GetMasterKey(), state_2.GetAMRMTokenSecretManagerState
().GetCurrentMasterKey());
NUnit.Framework.Assert.AreEqual(secondMasterKeyData.GetMasterKey(), state_2.GetAMRMTokenSecretManagerState
().GetNextMasterKey());
// re-create the masterKeyData based on the recovered masterkey
// should have the same secretKey
appTokenMgr.Recover(state_2);
NUnit.Framework.Assert.AreEqual(appTokenMgr.GetCurrnetMasterKeyData().GetSecretKey
(), firstMasterKeyData.GetSecretKey());
NUnit.Framework.Assert.AreEqual(appTokenMgr.GetNextMasterKeyData().GetSecretKey()
, secondMasterKeyData.GetSecretKey());
store.Close();
}
/// <exception cref="System.Exception"/>
public virtual void TestZKSessionTimeout()
{
TestZKRMStateStoreZKClientConnections.TestZKClient zkClientTester = new TestZKRMStateStoreZKClientConnections.TestZKClient
(this);
string path = "/test";
YarnConfiguration conf = new YarnConfiguration();
conf.SetInt(YarnConfiguration.RmZkTimeoutMs, ZkTimeoutMs);
ZKRMStateStore store = (ZKRMStateStore)zkClientTester.GetRMStateStore(conf);
RMStateStoreTestBase.TestDispatcher dispatcher = new RMStateStoreTestBase.TestDispatcher
();
store.SetRMDispatcher(dispatcher);
// a hack to trigger expired event
zkClientTester.forExpire = true;
// trigger watch
store.CreateWithRetries(path, null, ZooDefs.Ids.OpenAclUnsafe, CreateMode.Persistent
);
store.GetDataWithRetries(path, true);
store.SetDataWithRetries(path, Sharpen.Runtime.GetBytesForString("bytes"), 0);
zkClientTester.syncBarrier.Await();
// after this point, expired event has already been processed.
try
{
byte[] ret = store.GetDataWithRetries(path, false);
NUnit.Framework.Assert.AreEqual("bytes", Sharpen.Runtime.GetStringForBytes(ret));
}
catch (Exception e)
{
string error = "New session creation failed";
Log.Error(error, e);
NUnit.Framework.Assert.Fail(error);
}
// send Disconnected event from old client session to ZKRMStateStore
// check the current client session is not affected.
NUnit.Framework.Assert.IsTrue(zkClientTester.oldWatcher != null);
WatchedEvent disconnectedEvent = new WatchedEvent(Watcher.Event.EventType.None, Watcher.Event.KeeperState
.Disconnected, null);
zkClientTester.oldWatcher.Process(disconnectedEvent);
NUnit.Framework.Assert.IsTrue(store.zkClient != null);
zkClientTester.watcher.Process(disconnectedEvent);
NUnit.Framework.Assert.IsTrue(store.zkClient == null);
WatchedEvent connectedEvent = new WatchedEvent(Watcher.Event.EventType.None, Watcher.Event.KeeperState
.SyncConnected, null);
zkClientTester.watcher.Process(connectedEvent);
NUnit.Framework.Assert.IsTrue(store.zkClient != null);
NUnit.Framework.Assert.IsTrue(store.zkClient == store.activeZkClient);
}
/// <exception cref="System.Exception"/>
public virtual void TestZKClientDisconnectAndReconnect()
{
TestZKRMStateStoreZKClientConnections.TestZKClient zkClientTester = new TestZKRMStateStoreZKClientConnections.TestZKClient
(this);
string path = "/test";
YarnConfiguration conf = new YarnConfiguration();
conf.SetInt(YarnConfiguration.RmZkTimeoutMs, ZkTimeoutMs);
ZKRMStateStore store = (ZKRMStateStore)zkClientTester.GetRMStateStore(conf);
RMStateStoreTestBase.TestDispatcher dispatcher = new RMStateStoreTestBase.TestDispatcher
();
store.SetRMDispatcher(dispatcher);
// trigger watch
store.CreateWithRetries(path, null, ZooDefs.Ids.OpenAclUnsafe, CreateMode.Persistent
);
store.GetDataWithRetries(path, true);
store.SetDataWithRetries(path, Sharpen.Runtime.GetBytesForString("newBytes"), 0);
StopServer();
AtomicBoolean isSucceeded = new AtomicBoolean(false);
zkClientTester.watcher.WaitForDisconnected(ZkOpWaitTime);
Sharpen.Thread thread = new _Thread_190(store, path, isSucceeded);
thread.Start();
// ZKRMStateStore Session restored
StartServer();
zkClientTester.watcher.WaitForConnected(ZkOpWaitTime);
byte[] ret = null;
try
{
ret = store.GetDataWithRetries(path, true);
}
catch (Exception e)
{
string error = "ZKRMStateStore Session restore failed";
Log.Error(error, e);
NUnit.Framework.Assert.Fail(error);
}
NUnit.Framework.Assert.AreEqual("newBytes", Sharpen.Runtime.GetStringForBytes(ret
));
thread.Join();
NUnit.Framework.Assert.IsTrue(isSucceeded.Get());
}
/// <exception cref="System.Exception"/>
public virtual void TestZKClientRetry()
{
TestZKRMStateStoreZKClientConnections.TestZKClient zkClientTester = new TestZKRMStateStoreZKClientConnections.TestZKClient
(this);
string path = "/test";
YarnConfiguration conf = new YarnConfiguration();
conf.SetInt(YarnConfiguration.RmZkTimeoutMs, ZkTimeoutMs);
conf.SetLong(YarnConfiguration.RmZkRetryIntervalMs, 100);
ZKRMStateStore store = (ZKRMStateStore)zkClientTester.GetRMStateStore(conf);
RMStateStoreTestBase.TestDispatcher dispatcher = new RMStateStoreTestBase.TestDispatcher
();
store.SetRMDispatcher(dispatcher);
AtomicBoolean assertionFailedInThread = new AtomicBoolean(false);
StopServer();
Sharpen.Thread clientThread = new _Thread_151(store, path, assertionFailedInThread
);
Sharpen.Thread.Sleep(2000);
StartServer();
clientThread.Join();
NUnit.Framework.Assert.IsFalse(assertionFailedInThread.Get());
}
public virtual void TestDuplicateRMAppDeletion()
{
TestZKRMStateStore.TestZKRMStateStoreTester zkTester = new TestZKRMStateStore.TestZKRMStateStoreTester
(this);
long submitTime = Runtime.CurrentTimeMillis();
long startTime = Runtime.CurrentTimeMillis() + 1234;
RMStateStore store = zkTester.GetRMStateStore();
RMStateStoreTestBase.TestDispatcher dispatcher = new RMStateStoreTestBase.TestDispatcher
();
store.SetRMDispatcher(dispatcher);
ApplicationAttemptId attemptIdRemoved = ConverterUtils.ToApplicationAttemptId("appattempt_1352994193343_0002_000001"
);
ApplicationId appIdRemoved = attemptIdRemoved.GetApplicationId();
StoreApp(store, appIdRemoved, submitTime, startTime);
StoreAttempt(store, attemptIdRemoved, "container_1352994193343_0002_01_000001", null
, null, dispatcher);
ApplicationSubmissionContext context = new ApplicationSubmissionContextPBImpl();
context.SetApplicationId(appIdRemoved);
ApplicationStateData appStateRemoved = ApplicationStateData.NewInstance(submitTime
, startTime, context, "user1");
appStateRemoved.attempts[attemptIdRemoved] = null;
store.RemoveApplicationStateInternal(appStateRemoved);
try
{
store.RemoveApplicationStateInternal(appStateRemoved);
}
catch (KeeperException.NoNodeException)
{
NUnit.Framework.Assert.Fail("NoNodeException should not happen.");
}
store.Close();
}
internal virtual void WaitNotify(RMStateStoreTestBase.TestDispatcher dispatcher)
{
long startTime = Runtime.CurrentTimeMillis();
while (!dispatcher.notified)
{
lock (dispatcher)
{
try
{
Sharpen.Runtime.Wait(dispatcher, 1000);
}
catch (Exception e)
{
Sharpen.Runtime.PrintStackTrace(e);
}
}
if (Runtime.CurrentTimeMillis() - startTime > 1000 * 60)
{
NUnit.Framework.Assert.Fail("Timed out attempt store notification");
}
}
dispatcher.notified = false;
}
public virtual int Run(string[] args)
{
Log.Info("Starting ZKRMStateStorePerf ver." + version);
int numApp = ZkPerfNumAppDefault;
int numAppAttemptPerApp = ZkPerfNumAppattemptPerApp;
string hostPort = null;
bool launchLocalZK = true;
if (args.Length == 0)
{
System.Console.Error.WriteLine("Missing arguments.");
return(-1);
}
for (int i = 0; i < args.Length; i++)
{
// parse command line
if (Sharpen.Runtime.EqualsIgnoreCase(args[i], "-appsize"))
{
numApp = System.Convert.ToInt32(args[++i]);
}
else
{
if (Sharpen.Runtime.EqualsIgnoreCase(args[i], "-appattemptsize"))
{
numAppAttemptPerApp = System.Convert.ToInt32(args[++i]);
}
else
{
if (Sharpen.Runtime.EqualsIgnoreCase(args[i], "-hostPort"))
{
hostPort = args[++i];
launchLocalZK = false;
}
else
{
if (Sharpen.Runtime.EqualsIgnoreCase(args[i], "-workingZnode"))
{
workingZnode = args[++i];
}
else
{
System.Console.Error.WriteLine("Illegal argument: " + args[i]);
return(-1);
}
}
}
}
}
if (launchLocalZK)
{
try
{
SetUp();
}
catch (Exception e)
{
System.Console.Error.WriteLine("failed to setup. : " + e.Message);
return(-1);
}
}
InitStore(hostPort);
long submitTime = Runtime.CurrentTimeMillis();
long startTime = Runtime.CurrentTimeMillis() + 1234;
AList <ApplicationId> applicationIds = new AList <ApplicationId>();
AList <RMApp> rmApps = new AList <RMApp>();
AList <ApplicationAttemptId> attemptIds = new AList <ApplicationAttemptId>();
Dictionary <ApplicationId, ICollection <ApplicationAttemptId> > appIdsToAttemptId =
new Dictionary <ApplicationId, ICollection <ApplicationAttemptId> >();
RMStateStoreTestBase.TestDispatcher dispatcher = new RMStateStoreTestBase.TestDispatcher
();
store.SetRMDispatcher(dispatcher);
for (int i_1 = 0; i_1 < numApp; i_1++)
{
ApplicationId appId = ApplicationId.NewInstance(clusterTimeStamp, i_1);
applicationIds.AddItem(appId);
AList <ApplicationAttemptId> attemptIdsForThisApp = new AList <ApplicationAttemptId
>();
for (int j = 0; j < numAppAttemptPerApp; j++)
{
ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, j);
attemptIdsForThisApp.AddItem(attemptId);
}
appIdsToAttemptId[appId] = new LinkedHashSet(attemptIdsForThisApp);
Sharpen.Collections.AddAll(attemptIds, attemptIdsForThisApp);
}
foreach (ApplicationId appId_1 in applicationIds)
{
RMApp app = null;
try
{
app = StoreApp(store, appId_1, submitTime, startTime);
}
catch (Exception e)
{
System.Console.Error.WriteLine("failed to create Application Znode. : " + e.Message
);
return(-1);
}
WaitNotify(dispatcher);
rmApps.AddItem(app);
}
foreach (ApplicationAttemptId attemptId_1 in attemptIds)
{
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> tokenId = GenerateAMRMToken
(attemptId_1, appTokenMgr);
SecretKey clientTokenKey = clientToAMTokenMgr.CreateMasterKey(attemptId_1);
try
{
StoreAttempt(store, attemptId_1, ContainerId.NewContainerId(attemptId_1, 0L).ToString
(), tokenId, clientTokenKey, dispatcher);
}
catch (Exception e)
{
System.Console.Error.WriteLine("failed to create AppAttempt Znode. : " + e.Message
);
return(-1);
}
}
long storeStart = Runtime.CurrentTimeMillis();
try
{
store.LoadState();
}
catch (Exception e)
{
System.Console.Error.WriteLine("failed to locaState from ZKRMStateStore. : " + e.
Message);
return(-1);
}
long storeEnd = Runtime.CurrentTimeMillis();
long loadTime = storeEnd - storeStart;
string resultMsg = "ZKRMStateStore takes " + loadTime + " msec to loadState.";
Log.Info(resultMsg);
System.Console.Out.WriteLine(resultMsg);
// cleanup
try
{
foreach (RMApp app in rmApps)
{
ApplicationStateData appState = ApplicationStateData.NewInstance(app.GetSubmitTime
(), app.GetStartTime(), app.GetApplicationSubmissionContext(), app.GetUser());
ApplicationId appId = app.GetApplicationId();
IDictionary m = Org.Mockito.Mockito.Mock <IDictionary>();
Org.Mockito.Mockito.When(m.Keys).ThenReturn(appIdsToAttemptId[appId_1]);
appState.attempts = m;
store.RemoveApplicationStateInternal(appState);
}
}
catch (Exception e)
{
System.Console.Error.WriteLine("failed to cleanup. : " + e.Message);
return(-1);
}
return(0);
}
/// <exception cref="System.Exception"/>
public virtual void TestRMDTSecretManagerStateStore(RMStateStoreTestBase.RMStateStoreHelper
stateStoreHelper)
{
RMStateStore store = stateStoreHelper.GetRMStateStore();
RMStateStoreTestBase.TestDispatcher dispatcher = new RMStateStoreTestBase.TestDispatcher
();
store.SetRMDispatcher(dispatcher);
// store RM delegation token;
RMDelegationTokenIdentifier dtId1 = new RMDelegationTokenIdentifier(new Text("owner1"
), new Text("renewer1"), new Text("realuser1"));
int sequenceNumber = 1111;
dtId1.SetSequenceNumber(sequenceNumber);
byte[] tokenBeforeStore = dtId1.GetBytes();
long renewDate1 = Runtime.CurrentTimeMillis();
store.StoreRMDelegationToken(dtId1, renewDate1);
ModifyRMDelegationTokenState();
IDictionary <RMDelegationTokenIdentifier, long> token1 = new Dictionary <RMDelegationTokenIdentifier
, long>();
token1[dtId1] = renewDate1;
// store delegation key;
DelegationKey key = new DelegationKey(1234, 4321, Sharpen.Runtime.GetBytesForString
("keyBytes"));
HashSet <DelegationKey> keySet = new HashSet <DelegationKey>();
keySet.AddItem(key);
store.StoreRMDTMasterKey(key);
RMStateStore.RMDTSecretManagerState secretManagerState = store.LoadState().GetRMDTSecretManagerState
();
NUnit.Framework.Assert.AreEqual(token1, secretManagerState.GetTokenState());
NUnit.Framework.Assert.AreEqual(keySet, secretManagerState.GetMasterKeyState());
NUnit.Framework.Assert.AreEqual(sequenceNumber, secretManagerState.GetDTSequenceNumber
());
RMDelegationTokenIdentifier tokenAfterStore = secretManagerState.GetTokenState().
Keys.GetEnumerator().Next();
NUnit.Framework.Assert.IsTrue(Arrays.Equals(tokenBeforeStore, tokenAfterStore.GetBytes
()));
// update RM delegation token;
renewDate1 = Runtime.CurrentTimeMillis();
store.UpdateRMDelegationToken(dtId1, renewDate1);
token1[dtId1] = renewDate1;
RMStateStore.RMDTSecretManagerState updateSecretManagerState = store.LoadState().
GetRMDTSecretManagerState();
NUnit.Framework.Assert.AreEqual(token1, updateSecretManagerState.GetTokenState());
NUnit.Framework.Assert.AreEqual(keySet, updateSecretManagerState.GetMasterKeyState
());
NUnit.Framework.Assert.AreEqual(sequenceNumber, updateSecretManagerState.GetDTSequenceNumber
());
// check to delete delegationKey
store.RemoveRMDTMasterKey(key);
keySet.Clear();
RMStateStore.RMDTSecretManagerState noKeySecretManagerState = store.LoadState().GetRMDTSecretManagerState
();
NUnit.Framework.Assert.AreEqual(token1, noKeySecretManagerState.GetTokenState());
NUnit.Framework.Assert.AreEqual(keySet, noKeySecretManagerState.GetMasterKeyState
());
NUnit.Framework.Assert.AreEqual(sequenceNumber, noKeySecretManagerState.GetDTSequenceNumber
());
// check to delete delegationToken
store.RemoveRMDelegationToken(dtId1);
RMStateStore.RMDTSecretManagerState noKeyAndTokenSecretManagerState = store.LoadState
().GetRMDTSecretManagerState();
token1.Clear();
NUnit.Framework.Assert.AreEqual(token1, noKeyAndTokenSecretManagerState.GetTokenState
());
NUnit.Framework.Assert.AreEqual(keySet, noKeyAndTokenSecretManagerState.GetMasterKeyState
());
NUnit.Framework.Assert.AreEqual(sequenceNumber, noKeySecretManagerState.GetDTSequenceNumber
());
store.Close();
}
/// <exception cref="System.Exception"/>
internal virtual void TestRMAppStateStore(RMStateStoreTestBase.RMStateStoreHelper
stateStoreHelper, RMStateStoreTestBase.StoreStateVerifier verifier)
{
long submitTime = Runtime.CurrentTimeMillis();
long startTime = Runtime.CurrentTimeMillis() + 1234;
Configuration conf = new YarnConfiguration();
RMStateStore store = stateStoreHelper.GetRMStateStore();
RMStateStoreTestBase.TestDispatcher dispatcher = new RMStateStoreTestBase.TestDispatcher
();
store.SetRMDispatcher(dispatcher);
RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>();
Org.Mockito.Mockito.When(rmContext.GetStateStore()).ThenReturn(store);
AMRMTokenSecretManager appTokenMgr = Org.Mockito.Mockito.Spy(new AMRMTokenSecretManager
(conf, rmContext));
MasterKeyData masterKeyData = appTokenMgr.CreateNewMasterKey();
Org.Mockito.Mockito.When(appTokenMgr.GetMasterKey()).ThenReturn(masterKeyData);
ClientToAMTokenSecretManagerInRM clientToAMTokenMgr = new ClientToAMTokenSecretManagerInRM
();
ApplicationAttemptId attemptId1 = ConverterUtils.ToApplicationAttemptId("appattempt_1352994193343_0001_000001"
);
ApplicationId appId1 = attemptId1.GetApplicationId();
StoreApp(store, appId1, submitTime, startTime);
verifier.AfterStoreApp(store, appId1);
// create application token and client token key for attempt1
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> appAttemptToken1 = GenerateAMRMToken
(attemptId1, appTokenMgr);
SecretKey clientTokenKey1 = clientToAMTokenMgr.CreateMasterKey(attemptId1);
ContainerId containerId1 = StoreAttempt(store, attemptId1, "container_1352994193343_0001_01_000001"
, appAttemptToken1, clientTokenKey1, dispatcher);
string appAttemptIdStr2 = "appattempt_1352994193343_0001_000002";
ApplicationAttemptId attemptId2 = ConverterUtils.ToApplicationAttemptId(appAttemptIdStr2
);
// create application token and client token key for attempt2
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> appAttemptToken2 = GenerateAMRMToken
(attemptId2, appTokenMgr);
SecretKey clientTokenKey2 = clientToAMTokenMgr.CreateMasterKey(attemptId2);
ContainerId containerId2 = StoreAttempt(store, attemptId2, "container_1352994193343_0001_02_000001"
, appAttemptToken2, clientTokenKey2, dispatcher);
ApplicationAttemptId attemptIdRemoved = ConverterUtils.ToApplicationAttemptId("appattempt_1352994193343_0002_000001"
);
ApplicationId appIdRemoved = attemptIdRemoved.GetApplicationId();
StoreApp(store, appIdRemoved, submitTime, startTime);
StoreAttempt(store, attemptIdRemoved, "container_1352994193343_0002_01_000001", null
, null, dispatcher);
verifier.AfterStoreAppAttempt(store, attemptIdRemoved);
RMApp mockRemovedApp = Org.Mockito.Mockito.Mock <RMApp>();
RMAppAttemptMetrics mockRmAppAttemptMetrics = Org.Mockito.Mockito.Mock <RMAppAttemptMetrics
>();
Dictionary <ApplicationAttemptId, RMAppAttempt> attempts = new Dictionary <ApplicationAttemptId
, RMAppAttempt>();
ApplicationSubmissionContext context = new ApplicationSubmissionContextPBImpl();
context.SetApplicationId(appIdRemoved);
Org.Mockito.Mockito.When(mockRemovedApp.GetSubmitTime()).ThenReturn(submitTime);
Org.Mockito.Mockito.When(mockRemovedApp.GetApplicationSubmissionContext()).ThenReturn
(context);
Org.Mockito.Mockito.When(mockRemovedApp.GetAppAttempts()).ThenReturn(attempts);
Org.Mockito.Mockito.When(mockRemovedApp.GetUser()).ThenReturn("user1");
RMAppAttempt mockRemovedAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>();
Org.Mockito.Mockito.When(mockRemovedAttempt.GetAppAttemptId()).ThenReturn(attemptIdRemoved
);
Org.Mockito.Mockito.When(mockRemovedAttempt.GetRMAppAttemptMetrics()).ThenReturn(
mockRmAppAttemptMetrics);
Org.Mockito.Mockito.When(mockRmAppAttemptMetrics.GetAggregateAppResourceUsage()).
ThenReturn(new AggregateAppResourceUsage(0, 0));
attempts[attemptIdRemoved] = mockRemovedAttempt;
store.RemoveApplication(mockRemovedApp);
// remove application directory recursively.
StoreApp(store, appIdRemoved, submitTime, startTime);
StoreAttempt(store, attemptIdRemoved, "container_1352994193343_0002_01_000001", null
, null, dispatcher);
store.RemoveApplication(mockRemovedApp);
// let things settle down
Sharpen.Thread.Sleep(1000);
store.Close();
// give tester a chance to modify app state in the store
ModifyAppState();
// load state
store = stateStoreHelper.GetRMStateStore();
store.SetRMDispatcher(dispatcher);
RMStateStore.RMState state = store.LoadState();
IDictionary <ApplicationId, ApplicationStateData> rmAppState = state.GetApplicationState
();
ApplicationStateData appState = rmAppState[appId1];
// app is loaded
NUnit.Framework.Assert.IsNotNull(appState);
// app is loaded correctly
NUnit.Framework.Assert.AreEqual(submitTime, appState.GetSubmitTime());
NUnit.Framework.Assert.AreEqual(startTime, appState.GetStartTime());
// submission context is loaded correctly
NUnit.Framework.Assert.AreEqual(appId1, appState.GetApplicationSubmissionContext(
).GetApplicationId());
ApplicationAttemptStateData attemptState = appState.GetAttempt(attemptId1);
// attempt1 is loaded correctly
NUnit.Framework.Assert.IsNotNull(attemptState);
NUnit.Framework.Assert.AreEqual(attemptId1, attemptState.GetAttemptId());
NUnit.Framework.Assert.AreEqual(-1000, attemptState.GetAMContainerExitStatus());
// attempt1 container is loaded correctly
NUnit.Framework.Assert.AreEqual(containerId1, attemptState.GetMasterContainer().GetId
());
// attempt1 client token master key is loaded correctly
Assert.AssertArrayEquals(clientTokenKey1.GetEncoded(), attemptState.GetAppAttemptTokens
().GetSecretKey(RMStateStore.AmClientTokenMasterKeyName));
attemptState = appState.GetAttempt(attemptId2);
// attempt2 is loaded correctly
NUnit.Framework.Assert.IsNotNull(attemptState);
NUnit.Framework.Assert.AreEqual(attemptId2, attemptState.GetAttemptId());
// attempt2 container is loaded correctly
NUnit.Framework.Assert.AreEqual(containerId2, attemptState.GetMasterContainer().GetId
());
// attempt2 client token master key is loaded correctly
Assert.AssertArrayEquals(clientTokenKey2.GetEncoded(), attemptState.GetAppAttemptTokens
().GetSecretKey(RMStateStore.AmClientTokenMasterKeyName));
//******* update application/attempt state *******//
ApplicationStateData appState2 = ApplicationStateData.NewInstance(appState.GetSubmitTime
(), appState.GetStartTime(), appState.GetUser(), appState.GetApplicationSubmissionContext
(), RMAppState.Finished, "appDiagnostics", 1234);
appState2.attempts.PutAll(appState.attempts);
store.UpdateApplicationState(appState2);
ApplicationAttemptStateData oldAttemptState = attemptState;
ApplicationAttemptStateData newAttemptState = ApplicationAttemptStateData.NewInstance
(oldAttemptState.GetAttemptId(), oldAttemptState.GetMasterContainer(), oldAttemptState
.GetAppAttemptTokens(), oldAttemptState.GetStartTime(), RMAppAttemptState.Finished
, "myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.Succeeded, 100,
oldAttemptState.GetFinishTime(), 0, 0);
store.UpdateApplicationAttemptState(newAttemptState);
// test updating the state of an app/attempt whose initial state was not
// saved.
ApplicationId dummyAppId = ApplicationId.NewInstance(1234, 10);
ApplicationSubmissionContext dummyContext = new ApplicationSubmissionContextPBImpl
();
dummyContext.SetApplicationId(dummyAppId);
ApplicationStateData dummyApp = ApplicationStateData.NewInstance(appState.GetSubmitTime
(), appState.GetStartTime(), appState.GetUser(), dummyContext, RMAppState.Finished
, "appDiagnostics", 1234);
store.UpdateApplicationState(dummyApp);
ApplicationAttemptId dummyAttemptId = ApplicationAttemptId.NewInstance(dummyAppId
, 6);
ApplicationAttemptStateData dummyAttempt = ApplicationAttemptStateData.NewInstance
(dummyAttemptId, oldAttemptState.GetMasterContainer(), oldAttemptState.GetAppAttemptTokens
(), oldAttemptState.GetStartTime(), RMAppAttemptState.Finished, "myTrackingUrl",
"attemptDiagnostics", FinalApplicationStatus.Succeeded, 111, oldAttemptState.GetFinishTime
(), 0, 0);
store.UpdateApplicationAttemptState(dummyAttempt);
// let things settle down
Sharpen.Thread.Sleep(1000);
store.Close();
// check updated application state.
store = stateStoreHelper.GetRMStateStore();
store.SetRMDispatcher(dispatcher);
RMStateStore.RMState newRMState = store.LoadState();
IDictionary <ApplicationId, ApplicationStateData> newRMAppState = newRMState.GetApplicationState
();
NUnit.Framework.Assert.IsNotNull(newRMAppState[dummyApp.GetApplicationSubmissionContext
().GetApplicationId()]);
ApplicationStateData updatedAppState = newRMAppState[appId1];
NUnit.Framework.Assert.AreEqual(appState.GetApplicationSubmissionContext().GetApplicationId
(), updatedAppState.GetApplicationSubmissionContext().GetApplicationId());
NUnit.Framework.Assert.AreEqual(appState.GetSubmitTime(), updatedAppState.GetSubmitTime
());
NUnit.Framework.Assert.AreEqual(appState.GetStartTime(), updatedAppState.GetStartTime
());
NUnit.Framework.Assert.AreEqual(appState.GetUser(), updatedAppState.GetUser());
// new app state fields
NUnit.Framework.Assert.AreEqual(RMAppState.Finished, updatedAppState.GetState());
NUnit.Framework.Assert.AreEqual("appDiagnostics", updatedAppState.GetDiagnostics(
));
NUnit.Framework.Assert.AreEqual(1234, updatedAppState.GetFinishTime());
// check updated attempt state
NUnit.Framework.Assert.IsNotNull(newRMAppState[dummyApp.GetApplicationSubmissionContext
().GetApplicationId()].GetAttempt(dummyAttemptId));
ApplicationAttemptStateData updatedAttemptState = updatedAppState.GetAttempt(newAttemptState
.GetAttemptId());
NUnit.Framework.Assert.AreEqual(oldAttemptState.GetAttemptId(), updatedAttemptState
.GetAttemptId());
NUnit.Framework.Assert.AreEqual(containerId2, updatedAttemptState.GetMasterContainer
().GetId());
Assert.AssertArrayEquals(clientTokenKey2.GetEncoded(), attemptState.GetAppAttemptTokens
().GetSecretKey(RMStateStore.AmClientTokenMasterKeyName));
// new attempt state fields
NUnit.Framework.Assert.AreEqual(RMAppAttemptState.Finished, updatedAttemptState.GetState
());
NUnit.Framework.Assert.AreEqual("myTrackingUrl", updatedAttemptState.GetFinalTrackingUrl
());
NUnit.Framework.Assert.AreEqual("attemptDiagnostics", updatedAttemptState.GetDiagnostics
());
NUnit.Framework.Assert.AreEqual(100, updatedAttemptState.GetAMContainerExitStatus
());
NUnit.Framework.Assert.AreEqual(FinalApplicationStatus.Succeeded, updatedAttemptState
.GetFinalApplicationStatus());
// assert store is in expected state after everything is cleaned
NUnit.Framework.Assert.IsTrue(stateStoreHelper.IsFinalStateValid());
store.Close();
}
/// <exception cref="System.Exception"/>
protected internal virtual ContainerId StoreAttempt(RMStateStore store, ApplicationAttemptId
attemptId, string containerIdStr, Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier
> appToken, SecretKey clientTokenMasterKey, RMStateStoreTestBase.TestDispatcher
dispatcher)
{
RMAppAttemptMetrics mockRmAppAttemptMetrics = Org.Mockito.Mockito.Mock <RMAppAttemptMetrics
>();
Container container = new ContainerPBImpl();
container.SetId(ConverterUtils.ToContainerId(containerIdStr));
RMAppAttempt mockAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>();
Org.Mockito.Mockito.When(mockAttempt.GetAppAttemptId()).ThenReturn(attemptId);
Org.Mockito.Mockito.When(mockAttempt.GetMasterContainer()).ThenReturn(container);
Org.Mockito.Mockito.When(mockAttempt.GetAMRMToken()).ThenReturn(appToken);
Org.Mockito.Mockito.When(mockAttempt.GetClientTokenMasterKey()).ThenReturn(clientTokenMasterKey
);
Org.Mockito.Mockito.When(mockAttempt.GetRMAppAttemptMetrics()).ThenReturn(mockRmAppAttemptMetrics
);
Org.Mockito.Mockito.When(mockRmAppAttemptMetrics.GetAggregateAppResourceUsage()).
ThenReturn(new AggregateAppResourceUsage(0, 0));
dispatcher.attemptId = attemptId;
store.StoreNewApplicationAttempt(mockAttempt);
WaitNotify(dispatcher);
return(container.GetId());
}