/// <exception cref="System.Exception"/> protected override void ServiceInit(Configuration conf) { Configuration config = new YarnConfiguration(conf); config.SetBoolean(Dispatcher.DispatcherExitOnErrorKey, true); // This is required for WebApps to use https if enabled. MRWebAppUtil.Initialize(GetConfig()); try { DoSecureLogin(conf); } catch (IOException ie) { throw new YarnRuntimeException("History Server Failed to login", ie); } jobHistoryService = new JobHistory(); historyContext = (HistoryContext)jobHistoryService; stateStore = CreateStateStore(conf); this.jhsDTSecretManager = CreateJHSSecretManager(conf, stateStore); clientService = CreateHistoryClientService(); aggLogDelService = new AggregatedLogDeletionService(); hsAdminServer = new HSAdminServer(aggLogDelService, jobHistoryService); AddService(stateStore); AddService(new JobHistoryServer.HistoryServerSecretManagerService(this)); AddService(jobHistoryService); AddService(clientService); AddService(aggLogDelService); AddService(hsAdminServer); base.ServiceInit(config); }
/// <summary>Create a secret manager</summary> /// <param name="delegationKeyUpdateInterval"> /// the number of seconds for rolling new /// secret keys. /// </param> /// <param name="delegationTokenMaxLifetime"> /// the maximum lifetime of the delegation /// tokens /// </param> /// <param name="delegationTokenRenewInterval">how often the tokens must be renewed</param> /// <param name="delegationTokenRemoverScanInterval"> /// how often the tokens are scanned /// for expired tokens /// </param> /// <param name="store">history server state store for persisting state</param> public JHSDelegationTokenSecretManager(long delegationKeyUpdateInterval, long delegationTokenMaxLifetime , long delegationTokenRenewInterval, long delegationTokenRemoverScanInterval, HistoryServerStateStoreService store) : base(delegationKeyUpdateInterval, delegationTokenMaxLifetime, delegationTokenRenewInterval , delegationTokenRemoverScanInterval) { this.store = store; }
/// <exception cref="System.IO.IOException"/> private HistoryServerStateStoreService CreateAndStartStore() { HistoryServerStateStoreService store = HistoryServerStateStoreServiceFactory.GetStore (conf); NUnit.Framework.Assert.IsTrue("Factory did not create a leveldb store", store is HistoryServerLeveldbStateStoreService); store.Init(conf); store.Start(); return(store); }
protected internal virtual JHSDelegationTokenSecretManager CreateJHSSecretManager (Configuration conf, HistoryServerStateStoreService store) { long secretKeyInterval = conf.GetLong(MRConfig.DelegationKeyUpdateIntervalKey, MRConfig .DelegationKeyUpdateIntervalDefault); long tokenMaxLifetime = conf.GetLong(MRConfig.DelegationTokenMaxLifetimeKey, MRConfig .DelegationTokenMaxLifetimeDefault); long tokenRenewInterval = conf.GetLong(MRConfig.DelegationTokenRenewIntervalKey, MRConfig.DelegationTokenRenewIntervalDefault); return(new JHSDelegationTokenSecretManager(secretKeyInterval, tokenMaxLifetime, tokenRenewInterval , 3600000, store)); }
public virtual void TestTokenStore() { HistoryServerStateStoreService store = CreateAndStartStore(); // verify initially the store is empty HistoryServerStateStoreService.HistoryServerState state = store.LoadState(); NUnit.Framework.Assert.IsTrue("token state not empty", state.tokenState.IsEmpty() ); NUnit.Framework.Assert.IsTrue("key state not empty", state.tokenMasterKeyState.IsEmpty ()); // store a key and some tokens DelegationKey key1 = new DelegationKey(1, 2, Sharpen.Runtime.GetBytesForString("keyData1" )); MRDelegationTokenIdentifier token1 = new MRDelegationTokenIdentifier(new Text("tokenOwner1" ), new Text("tokenRenewer1"), new Text("tokenUser1")); token1.SetSequenceNumber(1); long tokenDate1 = 1L; MRDelegationTokenIdentifier token2 = new MRDelegationTokenIdentifier(new Text("tokenOwner2" ), new Text("tokenRenewer2"), new Text("tokenUser2")); token2.SetSequenceNumber(12345678); long tokenDate2 = 87654321L; store.StoreTokenMasterKey(key1); store.StoreToken(token1, tokenDate1); store.StoreToken(token2, tokenDate2); store.Close(); // verify the key and tokens can be recovered store = CreateAndStartStore(); state = store.LoadState(); NUnit.Framework.Assert.AreEqual("incorrect loaded token count", 2, state.tokenState .Count); NUnit.Framework.Assert.IsTrue("missing token 1", state.tokenState.Contains(token1 )); NUnit.Framework.Assert.AreEqual("incorrect token 1 date", tokenDate1, state.tokenState [token1]); NUnit.Framework.Assert.IsTrue("missing token 2", state.tokenState.Contains(token2 )); NUnit.Framework.Assert.AreEqual("incorrect token 2 date", tokenDate2, state.tokenState [token2]); NUnit.Framework.Assert.AreEqual("incorrect master key count", 1, state.tokenMasterKeyState .Count); NUnit.Framework.Assert.IsTrue("missing master key 1", state.tokenMasterKeyState.Contains (key1)); // store some more keys and tokens, remove the previous key and one // of the tokens, and renew a previous token DelegationKey key2 = new DelegationKey(3, 4, Sharpen.Runtime.GetBytesForString("keyData2" )); DelegationKey key3 = new DelegationKey(5, 6, Sharpen.Runtime.GetBytesForString("keyData3" )); MRDelegationTokenIdentifier token3 = new MRDelegationTokenIdentifier(new Text("tokenOwner3" ), new Text("tokenRenewer3"), new Text("tokenUser3")); token3.SetSequenceNumber(12345679); long tokenDate3 = 87654321L; store.RemoveToken(token1); store.StoreTokenMasterKey(key2); long newTokenDate2 = 975318642L; store.UpdateToken(token2, newTokenDate2); store.RemoveTokenMasterKey(key1); store.StoreTokenMasterKey(key3); store.StoreToken(token3, tokenDate3); store.Close(); // verify the new keys and tokens are recovered, the removed key and // token are no longer present, and the renewed token has the updated // expiration date store = CreateAndStartStore(); state = store.LoadState(); NUnit.Framework.Assert.AreEqual("incorrect loaded token count", 2, state.tokenState .Count); NUnit.Framework.Assert.IsFalse("token 1 not removed", state.tokenState.Contains(token1 )); NUnit.Framework.Assert.IsTrue("missing token 2", state.tokenState.Contains(token2 )); NUnit.Framework.Assert.AreEqual("incorrect token 2 date", newTokenDate2, state.tokenState [token2]); NUnit.Framework.Assert.IsTrue("missing token 3", state.tokenState.Contains(token3 )); NUnit.Framework.Assert.AreEqual("incorrect token 3 date", tokenDate3, state.tokenState [token3]); NUnit.Framework.Assert.AreEqual("incorrect master key count", 2, state.tokenMasterKeyState .Count); NUnit.Framework.Assert.IsFalse("master key 1 not removed", state.tokenMasterKeyState .Contains(key1)); NUnit.Framework.Assert.IsTrue("missing master key 2", state.tokenMasterKeyState.Contains (key2)); NUnit.Framework.Assert.IsTrue("missing master key 3", state.tokenMasterKeyState.Contains (key3)); store.Close(); }
/// <exception cref="System.IO.IOException"/> private void TestTokenStore(string stateStoreUri) { conf.Set(JHAdminConfig.MrHsFsStateStoreUri, stateStoreUri); HistoryServerStateStoreService store = CreateAndStartStore(); HistoryServerStateStoreService.HistoryServerState state = store.LoadState(); NUnit.Framework.Assert.IsTrue("token state not empty", state.tokenState.IsEmpty() ); NUnit.Framework.Assert.IsTrue("key state not empty", state.tokenMasterKeyState.IsEmpty ()); DelegationKey key1 = new DelegationKey(1, 2, Sharpen.Runtime.GetBytesForString("keyData1" )); MRDelegationTokenIdentifier token1 = new MRDelegationTokenIdentifier(new Text("tokenOwner1" ), new Text("tokenRenewer1"), new Text("tokenUser1")); token1.SetSequenceNumber(1); long tokenDate1 = 1L; MRDelegationTokenIdentifier token2 = new MRDelegationTokenIdentifier(new Text("tokenOwner2" ), new Text("tokenRenewer2"), new Text("tokenUser2")); token2.SetSequenceNumber(12345678); long tokenDate2 = 87654321L; store.StoreTokenMasterKey(key1); try { store.StoreTokenMasterKey(key1); NUnit.Framework.Assert.Fail("redundant store of key undetected"); } catch (IOException) { } // expected store.StoreToken(token1, tokenDate1); store.StoreToken(token2, tokenDate2); try { store.StoreToken(token1, tokenDate1); NUnit.Framework.Assert.Fail("redundant store of token undetected"); } catch (IOException) { } // expected store.Close(); store = CreateAndStartStore(); state = store.LoadState(); NUnit.Framework.Assert.AreEqual("incorrect loaded token count", 2, state.tokenState .Count); NUnit.Framework.Assert.IsTrue("missing token 1", state.tokenState.Contains(token1 )); NUnit.Framework.Assert.AreEqual("incorrect token 1 date", tokenDate1, state.tokenState [token1]); NUnit.Framework.Assert.IsTrue("missing token 2", state.tokenState.Contains(token2 )); NUnit.Framework.Assert.AreEqual("incorrect token 2 date", tokenDate2, state.tokenState [token2]); NUnit.Framework.Assert.AreEqual("incorrect master key count", 1, state.tokenMasterKeyState .Count); NUnit.Framework.Assert.IsTrue("missing master key 1", state.tokenMasterKeyState.Contains (key1)); DelegationKey key2 = new DelegationKey(3, 4, Sharpen.Runtime.GetBytesForString("keyData2" )); DelegationKey key3 = new DelegationKey(5, 6, Sharpen.Runtime.GetBytesForString("keyData3" )); MRDelegationTokenIdentifier token3 = new MRDelegationTokenIdentifier(new Text("tokenOwner3" ), new Text("tokenRenewer3"), new Text("tokenUser3")); token3.SetSequenceNumber(12345679); long tokenDate3 = 87654321L; store.RemoveToken(token1); store.StoreTokenMasterKey(key2); long newTokenDate2 = 975318642L; store.UpdateToken(token2, newTokenDate2); store.RemoveTokenMasterKey(key1); store.StoreTokenMasterKey(key3); store.StoreToken(token3, tokenDate3); store.Close(); store = CreateAndStartStore(); state = store.LoadState(); NUnit.Framework.Assert.AreEqual("incorrect loaded token count", 2, state.tokenState .Count); NUnit.Framework.Assert.IsFalse("token 1 not removed", state.tokenState.Contains(token1 )); NUnit.Framework.Assert.IsTrue("missing token 2", state.tokenState.Contains(token2 )); NUnit.Framework.Assert.AreEqual("incorrect token 2 date", newTokenDate2, state.tokenState [token2]); NUnit.Framework.Assert.IsTrue("missing token 3", state.tokenState.Contains(token3 )); NUnit.Framework.Assert.AreEqual("incorrect token 3 date", tokenDate3, state.tokenState [token3]); NUnit.Framework.Assert.AreEqual("incorrect master key count", 2, state.tokenMasterKeyState .Count); NUnit.Framework.Assert.IsFalse("master key 1 not removed", state.tokenMasterKeyState .Contains(key1)); NUnit.Framework.Assert.IsTrue("missing master key 2", state.tokenMasterKeyState.Contains (key2)); NUnit.Framework.Assert.IsTrue("missing master key 3", state.tokenMasterKeyState.Contains (key3)); }
public JHSDelegationTokenSecretManagerForTest(HistoryServerStateStoreService store ) : base(10000, 10000, 10000, 10000, store) { }