public virtual void TestRecovery()
{
YarnConfiguration conf = new YarnConfiguration();
conf.SetBoolean(YarnConfiguration.NmRecoveryEnabled, true);
NodeId nodeId = NodeId.NewInstance("somehost", 1234);
ApplicationAttemptId attempt1 = ApplicationAttemptId.NewInstance(ApplicationId.NewInstance
(1, 1), 1);
ApplicationAttemptId attempt2 = ApplicationAttemptId.NewInstance(ApplicationId.NewInstance
(2, 2), 2);
TestNMTokenSecretManagerInNM.NMTokenKeyGeneratorForTest keygen = new TestNMTokenSecretManagerInNM.NMTokenKeyGeneratorForTest
();
NMMemoryStateStoreService stateStore = new NMMemoryStateStoreService();
stateStore.Init(conf);
stateStore.Start();
NMTokenSecretManagerInNM secretMgr = new NMTokenSecretManagerInNM(stateStore);
secretMgr.SetNodeId(nodeId);
MasterKey currentKey = keygen.GenerateKey();
secretMgr.SetMasterKey(currentKey);
NMTokenIdentifier attemptToken1 = GetNMTokenId(secretMgr.CreateNMToken(attempt1,
nodeId, "user1"));
NMTokenIdentifier attemptToken2 = GetNMTokenId(secretMgr.CreateNMToken(attempt2,
nodeId, "user2"));
secretMgr.AppAttemptStartContainer(attemptToken1);
secretMgr.AppAttemptStartContainer(attemptToken2);
NUnit.Framework.Assert.IsTrue(secretMgr.IsAppAttemptNMTokenKeyPresent(attempt1));
NUnit.Framework.Assert.IsTrue(secretMgr.IsAppAttemptNMTokenKeyPresent(attempt2));
NUnit.Framework.Assert.IsNotNull(secretMgr.RetrievePassword(attemptToken1));
NUnit.Framework.Assert.IsNotNull(secretMgr.RetrievePassword(attemptToken2));
// restart and verify key is still there and token still valid
secretMgr = new NMTokenSecretManagerInNM(stateStore);
secretMgr.Recover();
secretMgr.SetNodeId(nodeId);
NUnit.Framework.Assert.AreEqual(currentKey, secretMgr.GetCurrentKey());
NUnit.Framework.Assert.IsTrue(secretMgr.IsAppAttemptNMTokenKeyPresent(attempt1));
NUnit.Framework.Assert.IsTrue(secretMgr.IsAppAttemptNMTokenKeyPresent(attempt2));
NUnit.Framework.Assert.IsNotNull(secretMgr.RetrievePassword(attemptToken1));
NUnit.Framework.Assert.IsNotNull(secretMgr.RetrievePassword(attemptToken2));
// roll master key and remove an app
currentKey = keygen.GenerateKey();
secretMgr.SetMasterKey(currentKey);
secretMgr.AppFinished(attempt1.GetApplicationId());
// restart and verify attempt1 key is still valid due to prev key persist
secretMgr = new NMTokenSecretManagerInNM(stateStore);
secretMgr.Recover();
secretMgr.SetNodeId(nodeId);
NUnit.Framework.Assert.AreEqual(currentKey, secretMgr.GetCurrentKey());
NUnit.Framework.Assert.IsFalse(secretMgr.IsAppAttemptNMTokenKeyPresent(attempt1));
NUnit.Framework.Assert.IsTrue(secretMgr.IsAppAttemptNMTokenKeyPresent(attempt2));
NUnit.Framework.Assert.IsNotNull(secretMgr.RetrievePassword(attemptToken1));
NUnit.Framework.Assert.IsNotNull(secretMgr.RetrievePassword(attemptToken2));
// roll master key again, restart, and verify attempt1 key is bad but
// attempt2 is still good due to app key persist
currentKey = keygen.GenerateKey();
secretMgr.SetMasterKey(currentKey);
secretMgr = new NMTokenSecretManagerInNM(stateStore);
secretMgr.Recover();
secretMgr.SetNodeId(nodeId);
NUnit.Framework.Assert.AreEqual(currentKey, secretMgr.GetCurrentKey());
NUnit.Framework.Assert.IsFalse(secretMgr.IsAppAttemptNMTokenKeyPresent(attempt1));
NUnit.Framework.Assert.IsTrue(secretMgr.IsAppAttemptNMTokenKeyPresent(attempt2));
try
{
secretMgr.RetrievePassword(attemptToken1);
NUnit.Framework.Assert.Fail("attempt token should not still be valid");
}
catch (SecretManager.InvalidToken)
{
}
// expected
NUnit.Framework.Assert.IsNotNull(secretMgr.RetrievePassword(attemptToken2));
// remove last attempt, restart, verify both tokens are now bad
secretMgr.AppFinished(attempt2.GetApplicationId());
secretMgr = new NMTokenSecretManagerInNM(stateStore);
secretMgr.Recover();
secretMgr.SetNodeId(nodeId);
NUnit.Framework.Assert.AreEqual(currentKey, secretMgr.GetCurrentKey());
NUnit.Framework.Assert.IsFalse(secretMgr.IsAppAttemptNMTokenKeyPresent(attempt1));
NUnit.Framework.Assert.IsFalse(secretMgr.IsAppAttemptNMTokenKeyPresent(attempt2));
try
{
secretMgr.RetrievePassword(attemptToken1);
NUnit.Framework.Assert.Fail("attempt token should not still be valid");
}
catch (SecretManager.InvalidToken)
{
}
// expected
try
{
secretMgr.RetrievePassword(attemptToken2);
NUnit.Framework.Assert.Fail("attempt token should not still be valid");
}
catch (SecretManager.InvalidToken)
{
}
// expected
stateStore.Close();
}
public virtual void TestRecovery()
{
YarnConfiguration conf = new YarnConfiguration();
conf.SetBoolean(YarnConfiguration.NmRecoveryEnabled, true);
NodeId nodeId = NodeId.NewInstance("somehost", 1234);
ContainerId cid1 = BuilderUtils.NewContainerId(1, 1, 1, 1);
ContainerId cid2 = BuilderUtils.NewContainerId(2, 2, 2, 2);
TestNMContainerTokenSecretManager.ContainerTokenKeyGeneratorForTest keygen = new
TestNMContainerTokenSecretManager.ContainerTokenKeyGeneratorForTest(conf);
NMMemoryStateStoreService stateStore = new NMMemoryStateStoreService();
stateStore.Init(conf);
stateStore.Start();
NMContainerTokenSecretManager secretMgr = new NMContainerTokenSecretManager(conf,
stateStore);
secretMgr.SetNodeId(nodeId);
MasterKey currentKey = keygen.GenerateKey();
secretMgr.SetMasterKey(currentKey);
ContainerTokenIdentifier tokenId1 = CreateContainerTokenId(cid1, nodeId, "user1",
secretMgr);
ContainerTokenIdentifier tokenId2 = CreateContainerTokenId(cid2, nodeId, "user2",
secretMgr);
NUnit.Framework.Assert.IsNotNull(secretMgr.RetrievePassword(tokenId1));
NUnit.Framework.Assert.IsNotNull(secretMgr.RetrievePassword(tokenId2));
// restart and verify tokens still valid
secretMgr = new NMContainerTokenSecretManager(conf, stateStore);
secretMgr.SetNodeId(nodeId);
secretMgr.Recover();
NUnit.Framework.Assert.AreEqual(currentKey, secretMgr.GetCurrentKey());
NUnit.Framework.Assert.IsTrue(secretMgr.IsValidStartContainerRequest(tokenId1));
NUnit.Framework.Assert.IsTrue(secretMgr.IsValidStartContainerRequest(tokenId2));
NUnit.Framework.Assert.IsNotNull(secretMgr.RetrievePassword(tokenId1));
NUnit.Framework.Assert.IsNotNull(secretMgr.RetrievePassword(tokenId2));
// roll master key and start a container
secretMgr.StartContainerSuccessful(tokenId2);
currentKey = keygen.GenerateKey();
secretMgr.SetMasterKey(currentKey);
// restart and verify tokens still valid due to prev key persist
secretMgr = new NMContainerTokenSecretManager(conf, stateStore);
secretMgr.SetNodeId(nodeId);
secretMgr.Recover();
NUnit.Framework.Assert.AreEqual(currentKey, secretMgr.GetCurrentKey());
NUnit.Framework.Assert.IsTrue(secretMgr.IsValidStartContainerRequest(tokenId1));
NUnit.Framework.Assert.IsFalse(secretMgr.IsValidStartContainerRequest(tokenId2));
NUnit.Framework.Assert.IsNotNull(secretMgr.RetrievePassword(tokenId1));
NUnit.Framework.Assert.IsNotNull(secretMgr.RetrievePassword(tokenId2));
// roll master key again, restart, and verify keys no longer valid
currentKey = keygen.GenerateKey();
secretMgr.SetMasterKey(currentKey);
secretMgr = new NMContainerTokenSecretManager(conf, stateStore);
secretMgr.SetNodeId(nodeId);
secretMgr.Recover();
NUnit.Framework.Assert.AreEqual(currentKey, secretMgr.GetCurrentKey());
NUnit.Framework.Assert.IsTrue(secretMgr.IsValidStartContainerRequest(tokenId1));
NUnit.Framework.Assert.IsFalse(secretMgr.IsValidStartContainerRequest(tokenId2));
try
{
secretMgr.RetrievePassword(tokenId1);
NUnit.Framework.Assert.Fail("token should not be valid");
}
catch (SecretManager.InvalidToken)
{
}
// expected
try
{
secretMgr.RetrievePassword(tokenId2);
NUnit.Framework.Assert.Fail("token should not be valid");
}
catch (SecretManager.InvalidToken)
{
}
// expected
stateStore.Close();
}
public virtual void TestRecovery()
{
Random r = new Random();
long seed = r.NextLong();
r.SetSeed(seed);
System.Console.Out.WriteLine("SEED: " + seed);
IList <Path> baseDirs = BuildDirs(r, @base, 4);
CreateDirs(new Path("."), baseDirs);
IList <Path> content = BuildDirs(r, new Path("."), 10);
foreach (Path b in baseDirs)
{
CreateDirs(b, content);
}
Configuration conf = new YarnConfiguration();
conf.SetBoolean(YarnConfiguration.NmRecoveryEnabled, true);
conf.SetInt(YarnConfiguration.DebugNmDeleteDelaySec, 1);
NMMemoryStateStoreService stateStore = new NMMemoryStateStoreService();
stateStore.Init(conf);
stateStore.Start();
DeletionService del = new DeletionService(new TestDeletionService.FakeDefaultContainerExecutor
(), stateStore);
try
{
del.Init(conf);
del.Start();
foreach (Path p in content)
{
NUnit.Framework.Assert.IsTrue(lfs.Util().Exists(new Path(baseDirs[0], p)));
del.Delete((long.Parse(p.GetName()) % 2) == 0 ? null : "dingo", p, Sharpen.Collections.ToArray
(baseDirs, new Path[4]));
}
// restart the deletion service
del.Stop();
del = new DeletionService(new TestDeletionService.FakeDefaultContainerExecutor(),
stateStore);
del.Init(conf);
del.Start();
// verify paths are still eventually deleted
int msecToWait = 10 * 1000;
foreach (Path p_1 in baseDirs)
{
foreach (Path q in content)
{
Path fp = new Path(p_1, q);
while (msecToWait > 0 && lfs.Util().Exists(fp))
{
Sharpen.Thread.Sleep(100);
msecToWait -= 100;
}
NUnit.Framework.Assert.IsFalse(lfs.Util().Exists(fp));
}
}
}
finally
{
del.Close();
stateStore.Close();
}
}
