public virtual void TestRecovery() { YarnConfiguration conf = new YarnConfiguration(); conf.SetBoolean(YarnConfiguration.NmRecoveryEnabled, true); NodeId nodeId = NodeId.NewInstance("somehost", 1234); ApplicationAttemptId attempt1 = ApplicationAttemptId.NewInstance(ApplicationId.NewInstance (1, 1), 1); ApplicationAttemptId attempt2 = ApplicationAttemptId.NewInstance(ApplicationId.NewInstance (2, 2), 2); TestNMTokenSecretManagerInNM.NMTokenKeyGeneratorForTest keygen = new TestNMTokenSecretManagerInNM.NMTokenKeyGeneratorForTest (); NMMemoryStateStoreService stateStore = new NMMemoryStateStoreService(); stateStore.Init(conf); stateStore.Start(); NMTokenSecretManagerInNM secretMgr = new NMTokenSecretManagerInNM(stateStore); secretMgr.SetNodeId(nodeId); MasterKey currentKey = keygen.GenerateKey(); secretMgr.SetMasterKey(currentKey); NMTokenIdentifier attemptToken1 = GetNMTokenId(secretMgr.CreateNMToken(attempt1, nodeId, "user1")); NMTokenIdentifier attemptToken2 = GetNMTokenId(secretMgr.CreateNMToken(attempt2, nodeId, "user2")); secretMgr.AppAttemptStartContainer(attemptToken1); secretMgr.AppAttemptStartContainer(attemptToken2); NUnit.Framework.Assert.IsTrue(secretMgr.IsAppAttemptNMTokenKeyPresent(attempt1)); NUnit.Framework.Assert.IsTrue(secretMgr.IsAppAttemptNMTokenKeyPresent(attempt2)); NUnit.Framework.Assert.IsNotNull(secretMgr.RetrievePassword(attemptToken1)); NUnit.Framework.Assert.IsNotNull(secretMgr.RetrievePassword(attemptToken2)); // restart and verify key is still there and token still valid secretMgr = new NMTokenSecretManagerInNM(stateStore); secretMgr.Recover(); secretMgr.SetNodeId(nodeId); NUnit.Framework.Assert.AreEqual(currentKey, secretMgr.GetCurrentKey()); NUnit.Framework.Assert.IsTrue(secretMgr.IsAppAttemptNMTokenKeyPresent(attempt1)); NUnit.Framework.Assert.IsTrue(secretMgr.IsAppAttemptNMTokenKeyPresent(attempt2)); NUnit.Framework.Assert.IsNotNull(secretMgr.RetrievePassword(attemptToken1)); NUnit.Framework.Assert.IsNotNull(secretMgr.RetrievePassword(attemptToken2)); // roll master key and remove an app currentKey = keygen.GenerateKey(); secretMgr.SetMasterKey(currentKey); secretMgr.AppFinished(attempt1.GetApplicationId()); // restart and verify attempt1 key is still valid due to prev key persist secretMgr = new NMTokenSecretManagerInNM(stateStore); secretMgr.Recover(); secretMgr.SetNodeId(nodeId); NUnit.Framework.Assert.AreEqual(currentKey, secretMgr.GetCurrentKey()); NUnit.Framework.Assert.IsFalse(secretMgr.IsAppAttemptNMTokenKeyPresent(attempt1)); NUnit.Framework.Assert.IsTrue(secretMgr.IsAppAttemptNMTokenKeyPresent(attempt2)); NUnit.Framework.Assert.IsNotNull(secretMgr.RetrievePassword(attemptToken1)); NUnit.Framework.Assert.IsNotNull(secretMgr.RetrievePassword(attemptToken2)); // roll master key again, restart, and verify attempt1 key is bad but // attempt2 is still good due to app key persist currentKey = keygen.GenerateKey(); secretMgr.SetMasterKey(currentKey); secretMgr = new NMTokenSecretManagerInNM(stateStore); secretMgr.Recover(); secretMgr.SetNodeId(nodeId); NUnit.Framework.Assert.AreEqual(currentKey, secretMgr.GetCurrentKey()); NUnit.Framework.Assert.IsFalse(secretMgr.IsAppAttemptNMTokenKeyPresent(attempt1)); NUnit.Framework.Assert.IsTrue(secretMgr.IsAppAttemptNMTokenKeyPresent(attempt2)); try { secretMgr.RetrievePassword(attemptToken1); NUnit.Framework.Assert.Fail("attempt token should not still be valid"); } catch (SecretManager.InvalidToken) { } // expected NUnit.Framework.Assert.IsNotNull(secretMgr.RetrievePassword(attemptToken2)); // remove last attempt, restart, verify both tokens are now bad secretMgr.AppFinished(attempt2.GetApplicationId()); secretMgr = new NMTokenSecretManagerInNM(stateStore); secretMgr.Recover(); secretMgr.SetNodeId(nodeId); NUnit.Framework.Assert.AreEqual(currentKey, secretMgr.GetCurrentKey()); NUnit.Framework.Assert.IsFalse(secretMgr.IsAppAttemptNMTokenKeyPresent(attempt1)); NUnit.Framework.Assert.IsFalse(secretMgr.IsAppAttemptNMTokenKeyPresent(attempt2)); try { secretMgr.RetrievePassword(attemptToken1); NUnit.Framework.Assert.Fail("attempt token should not still be valid"); } catch (SecretManager.InvalidToken) { } // expected try { secretMgr.RetrievePassword(attemptToken2); NUnit.Framework.Assert.Fail("attempt token should not still be valid"); } catch (SecretManager.InvalidToken) { } // expected stateStore.Close(); }
public virtual void TestRecovery() { YarnConfiguration conf = new YarnConfiguration(); conf.SetBoolean(YarnConfiguration.NmRecoveryEnabled, true); NodeId nodeId = NodeId.NewInstance("somehost", 1234); ContainerId cid1 = BuilderUtils.NewContainerId(1, 1, 1, 1); ContainerId cid2 = BuilderUtils.NewContainerId(2, 2, 2, 2); TestNMContainerTokenSecretManager.ContainerTokenKeyGeneratorForTest keygen = new TestNMContainerTokenSecretManager.ContainerTokenKeyGeneratorForTest(conf); NMMemoryStateStoreService stateStore = new NMMemoryStateStoreService(); stateStore.Init(conf); stateStore.Start(); NMContainerTokenSecretManager secretMgr = new NMContainerTokenSecretManager(conf, stateStore); secretMgr.SetNodeId(nodeId); MasterKey currentKey = keygen.GenerateKey(); secretMgr.SetMasterKey(currentKey); ContainerTokenIdentifier tokenId1 = CreateContainerTokenId(cid1, nodeId, "user1", secretMgr); ContainerTokenIdentifier tokenId2 = CreateContainerTokenId(cid2, nodeId, "user2", secretMgr); NUnit.Framework.Assert.IsNotNull(secretMgr.RetrievePassword(tokenId1)); NUnit.Framework.Assert.IsNotNull(secretMgr.RetrievePassword(tokenId2)); // restart and verify tokens still valid secretMgr = new NMContainerTokenSecretManager(conf, stateStore); secretMgr.SetNodeId(nodeId); secretMgr.Recover(); NUnit.Framework.Assert.AreEqual(currentKey, secretMgr.GetCurrentKey()); NUnit.Framework.Assert.IsTrue(secretMgr.IsValidStartContainerRequest(tokenId1)); NUnit.Framework.Assert.IsTrue(secretMgr.IsValidStartContainerRequest(tokenId2)); NUnit.Framework.Assert.IsNotNull(secretMgr.RetrievePassword(tokenId1)); NUnit.Framework.Assert.IsNotNull(secretMgr.RetrievePassword(tokenId2)); // roll master key and start a container secretMgr.StartContainerSuccessful(tokenId2); currentKey = keygen.GenerateKey(); secretMgr.SetMasterKey(currentKey); // restart and verify tokens still valid due to prev key persist secretMgr = new NMContainerTokenSecretManager(conf, stateStore); secretMgr.SetNodeId(nodeId); secretMgr.Recover(); NUnit.Framework.Assert.AreEqual(currentKey, secretMgr.GetCurrentKey()); NUnit.Framework.Assert.IsTrue(secretMgr.IsValidStartContainerRequest(tokenId1)); NUnit.Framework.Assert.IsFalse(secretMgr.IsValidStartContainerRequest(tokenId2)); NUnit.Framework.Assert.IsNotNull(secretMgr.RetrievePassword(tokenId1)); NUnit.Framework.Assert.IsNotNull(secretMgr.RetrievePassword(tokenId2)); // roll master key again, restart, and verify keys no longer valid currentKey = keygen.GenerateKey(); secretMgr.SetMasterKey(currentKey); secretMgr = new NMContainerTokenSecretManager(conf, stateStore); secretMgr.SetNodeId(nodeId); secretMgr.Recover(); NUnit.Framework.Assert.AreEqual(currentKey, secretMgr.GetCurrentKey()); NUnit.Framework.Assert.IsTrue(secretMgr.IsValidStartContainerRequest(tokenId1)); NUnit.Framework.Assert.IsFalse(secretMgr.IsValidStartContainerRequest(tokenId2)); try { secretMgr.RetrievePassword(tokenId1); NUnit.Framework.Assert.Fail("token should not be valid"); } catch (SecretManager.InvalidToken) { } // expected try { secretMgr.RetrievePassword(tokenId2); NUnit.Framework.Assert.Fail("token should not be valid"); } catch (SecretManager.InvalidToken) { } // expected stateStore.Close(); }
public virtual void TestRecovery() { Random r = new Random(); long seed = r.NextLong(); r.SetSeed(seed); System.Console.Out.WriteLine("SEED: " + seed); IList <Path> baseDirs = BuildDirs(r, @base, 4); CreateDirs(new Path("."), baseDirs); IList <Path> content = BuildDirs(r, new Path("."), 10); foreach (Path b in baseDirs) { CreateDirs(b, content); } Configuration conf = new YarnConfiguration(); conf.SetBoolean(YarnConfiguration.NmRecoveryEnabled, true); conf.SetInt(YarnConfiguration.DebugNmDeleteDelaySec, 1); NMMemoryStateStoreService stateStore = new NMMemoryStateStoreService(); stateStore.Init(conf); stateStore.Start(); DeletionService del = new DeletionService(new TestDeletionService.FakeDefaultContainerExecutor (), stateStore); try { del.Init(conf); del.Start(); foreach (Path p in content) { NUnit.Framework.Assert.IsTrue(lfs.Util().Exists(new Path(baseDirs[0], p))); del.Delete((long.Parse(p.GetName()) % 2) == 0 ? null : "dingo", p, Sharpen.Collections.ToArray (baseDirs, new Path[4])); } // restart the deletion service del.Stop(); del = new DeletionService(new TestDeletionService.FakeDefaultContainerExecutor(), stateStore); del.Init(conf); del.Start(); // verify paths are still eventually deleted int msecToWait = 10 * 1000; foreach (Path p_1 in baseDirs) { foreach (Path q in content) { Path fp = new Path(p_1, q); while (msecToWait > 0 && lfs.Util().Exists(fp)) { Sharpen.Thread.Sleep(100); msecToWait -= 100; } NUnit.Framework.Assert.IsFalse(lfs.Util().Exists(fp)); } } } finally { del.Close(); stateStore.Close(); } }