예제 #1
0
        /// <exception cref="System.Exception"/>
        protected override void ServiceInit(Configuration conf)
        {
            Configuration config = new YarnConfiguration(conf);

            config.SetBoolean(Dispatcher.DispatcherExitOnErrorKey, true);
            // This is required for WebApps to use https if enabled.
            MRWebAppUtil.Initialize(GetConfig());
            try
            {
                DoSecureLogin(conf);
            }
            catch (IOException ie)
            {
                throw new YarnRuntimeException("History Server Failed to login", ie);
            }
            jobHistoryService       = new JobHistory();
            historyContext          = (HistoryContext)jobHistoryService;
            stateStore              = CreateStateStore(conf);
            this.jhsDTSecretManager = CreateJHSSecretManager(conf, stateStore);
            clientService           = CreateHistoryClientService();
            aggLogDelService        = new AggregatedLogDeletionService();
            hsAdminServer           = new HSAdminServer(aggLogDelService, jobHistoryService);
            AddService(stateStore);
            AddService(new JobHistoryServer.HistoryServerSecretManagerService(this));
            AddService(jobHistoryService);
            AddService(clientService);
            AddService(aggLogDelService);
            AddService(hsAdminServer);
            base.ServiceInit(config);
        }
 /// <summary>Create a secret manager</summary>
 /// <param name="delegationKeyUpdateInterval">
 /// the number of seconds for rolling new
 /// secret keys.
 /// </param>
 /// <param name="delegationTokenMaxLifetime">
 /// the maximum lifetime of the delegation
 /// tokens
 /// </param>
 /// <param name="delegationTokenRenewInterval">how often the tokens must be renewed</param>
 /// <param name="delegationTokenRemoverScanInterval">
 /// how often the tokens are scanned
 /// for expired tokens
 /// </param>
 /// <param name="store">history server state store for persisting state</param>
 public JHSDelegationTokenSecretManager(long delegationKeyUpdateInterval, long delegationTokenMaxLifetime
                                        , long delegationTokenRenewInterval, long delegationTokenRemoverScanInterval, HistoryServerStateStoreService
                                        store)
     : base(delegationKeyUpdateInterval, delegationTokenMaxLifetime, delegationTokenRenewInterval
            , delegationTokenRemoverScanInterval)
 {
     this.store = store;
 }
        /// <exception cref="System.IO.IOException"/>
        private HistoryServerStateStoreService CreateAndStartStore()
        {
            HistoryServerStateStoreService store = HistoryServerStateStoreServiceFactory.GetStore
                                                       (conf);

            NUnit.Framework.Assert.IsTrue("Factory did not create a leveldb store", store is
                                          HistoryServerLeveldbStateStoreService);
            store.Init(conf);
            store.Start();
            return(store);
        }
예제 #4
0
        protected internal virtual JHSDelegationTokenSecretManager CreateJHSSecretManager
            (Configuration conf, HistoryServerStateStoreService store)
        {
            long secretKeyInterval = conf.GetLong(MRConfig.DelegationKeyUpdateIntervalKey, MRConfig
                                                  .DelegationKeyUpdateIntervalDefault);
            long tokenMaxLifetime = conf.GetLong(MRConfig.DelegationTokenMaxLifetimeKey, MRConfig
                                                 .DelegationTokenMaxLifetimeDefault);
            long tokenRenewInterval = conf.GetLong(MRConfig.DelegationTokenRenewIntervalKey,
                                                   MRConfig.DelegationTokenRenewIntervalDefault);

            return(new JHSDelegationTokenSecretManager(secretKeyInterval, tokenMaxLifetime, tokenRenewInterval
                                                       , 3600000, store));
        }
        public virtual void TestTokenStore()
        {
            HistoryServerStateStoreService store = CreateAndStartStore();

            // verify initially the store is empty
            HistoryServerStateStoreService.HistoryServerState state = store.LoadState();
            NUnit.Framework.Assert.IsTrue("token state not empty", state.tokenState.IsEmpty()
                                          );
            NUnit.Framework.Assert.IsTrue("key state not empty", state.tokenMasterKeyState.IsEmpty
                                              ());
            // store a key and some tokens
            DelegationKey key1 = new DelegationKey(1, 2, Sharpen.Runtime.GetBytesForString("keyData1"
                                                                                           ));
            MRDelegationTokenIdentifier token1 = new MRDelegationTokenIdentifier(new Text("tokenOwner1"
                                                                                          ), new Text("tokenRenewer1"), new Text("tokenUser1"));

            token1.SetSequenceNumber(1);
            long tokenDate1 = 1L;
            MRDelegationTokenIdentifier token2 = new MRDelegationTokenIdentifier(new Text("tokenOwner2"
                                                                                          ), new Text("tokenRenewer2"), new Text("tokenUser2"));

            token2.SetSequenceNumber(12345678);
            long tokenDate2 = 87654321L;

            store.StoreTokenMasterKey(key1);
            store.StoreToken(token1, tokenDate1);
            store.StoreToken(token2, tokenDate2);
            store.Close();
            // verify the key and tokens can be recovered
            store = CreateAndStartStore();
            state = store.LoadState();
            NUnit.Framework.Assert.AreEqual("incorrect loaded token count", 2, state.tokenState
                                            .Count);
            NUnit.Framework.Assert.IsTrue("missing token 1", state.tokenState.Contains(token1
                                                                                       ));
            NUnit.Framework.Assert.AreEqual("incorrect token 1 date", tokenDate1, state.tokenState
                                            [token1]);
            NUnit.Framework.Assert.IsTrue("missing token 2", state.tokenState.Contains(token2
                                                                                       ));
            NUnit.Framework.Assert.AreEqual("incorrect token 2 date", tokenDate2, state.tokenState
                                            [token2]);
            NUnit.Framework.Assert.AreEqual("incorrect master key count", 1, state.tokenMasterKeyState
                                            .Count);
            NUnit.Framework.Assert.IsTrue("missing master key 1", state.tokenMasterKeyState.Contains
                                              (key1));
            // store some more keys and tokens, remove the previous key and one
            // of the tokens, and renew a previous token
            DelegationKey key2 = new DelegationKey(3, 4, Sharpen.Runtime.GetBytesForString("keyData2"
                                                                                           ));
            DelegationKey key3 = new DelegationKey(5, 6, Sharpen.Runtime.GetBytesForString("keyData3"
                                                                                           ));
            MRDelegationTokenIdentifier token3 = new MRDelegationTokenIdentifier(new Text("tokenOwner3"
                                                                                          ), new Text("tokenRenewer3"), new Text("tokenUser3"));

            token3.SetSequenceNumber(12345679);
            long tokenDate3 = 87654321L;

            store.RemoveToken(token1);
            store.StoreTokenMasterKey(key2);
            long newTokenDate2 = 975318642L;

            store.UpdateToken(token2, newTokenDate2);
            store.RemoveTokenMasterKey(key1);
            store.StoreTokenMasterKey(key3);
            store.StoreToken(token3, tokenDate3);
            store.Close();
            // verify the new keys and tokens are recovered, the removed key and
            // token are no longer present, and the renewed token has the updated
            // expiration date
            store = CreateAndStartStore();
            state = store.LoadState();
            NUnit.Framework.Assert.AreEqual("incorrect loaded token count", 2, state.tokenState
                                            .Count);
            NUnit.Framework.Assert.IsFalse("token 1 not removed", state.tokenState.Contains(token1
                                                                                            ));
            NUnit.Framework.Assert.IsTrue("missing token 2", state.tokenState.Contains(token2
                                                                                       ));
            NUnit.Framework.Assert.AreEqual("incorrect token 2 date", newTokenDate2, state.tokenState
                                            [token2]);
            NUnit.Framework.Assert.IsTrue("missing token 3", state.tokenState.Contains(token3
                                                                                       ));
            NUnit.Framework.Assert.AreEqual("incorrect token 3 date", tokenDate3, state.tokenState
                                            [token3]);
            NUnit.Framework.Assert.AreEqual("incorrect master key count", 2, state.tokenMasterKeyState
                                            .Count);
            NUnit.Framework.Assert.IsFalse("master key 1 not removed", state.tokenMasterKeyState
                                           .Contains(key1));
            NUnit.Framework.Assert.IsTrue("missing master key 2", state.tokenMasterKeyState.Contains
                                              (key2));
            NUnit.Framework.Assert.IsTrue("missing master key 3", state.tokenMasterKeyState.Contains
                                              (key3));
            store.Close();
        }
예제 #6
0
        /// <exception cref="System.IO.IOException"/>
        private void TestTokenStore(string stateStoreUri)
        {
            conf.Set(JHAdminConfig.MrHsFsStateStoreUri, stateStoreUri);
            HistoryServerStateStoreService store = CreateAndStartStore();

            HistoryServerStateStoreService.HistoryServerState state = store.LoadState();
            NUnit.Framework.Assert.IsTrue("token state not empty", state.tokenState.IsEmpty()
                                          );
            NUnit.Framework.Assert.IsTrue("key state not empty", state.tokenMasterKeyState.IsEmpty
                                              ());
            DelegationKey key1 = new DelegationKey(1, 2, Sharpen.Runtime.GetBytesForString("keyData1"
                                                                                           ));
            MRDelegationTokenIdentifier token1 = new MRDelegationTokenIdentifier(new Text("tokenOwner1"
                                                                                          ), new Text("tokenRenewer1"), new Text("tokenUser1"));

            token1.SetSequenceNumber(1);
            long tokenDate1 = 1L;
            MRDelegationTokenIdentifier token2 = new MRDelegationTokenIdentifier(new Text("tokenOwner2"
                                                                                          ), new Text("tokenRenewer2"), new Text("tokenUser2"));

            token2.SetSequenceNumber(12345678);
            long tokenDate2 = 87654321L;

            store.StoreTokenMasterKey(key1);
            try
            {
                store.StoreTokenMasterKey(key1);
                NUnit.Framework.Assert.Fail("redundant store of key undetected");
            }
            catch (IOException)
            {
            }
            // expected
            store.StoreToken(token1, tokenDate1);
            store.StoreToken(token2, tokenDate2);
            try
            {
                store.StoreToken(token1, tokenDate1);
                NUnit.Framework.Assert.Fail("redundant store of token undetected");
            }
            catch (IOException)
            {
            }
            // expected
            store.Close();
            store = CreateAndStartStore();
            state = store.LoadState();
            NUnit.Framework.Assert.AreEqual("incorrect loaded token count", 2, state.tokenState
                                            .Count);
            NUnit.Framework.Assert.IsTrue("missing token 1", state.tokenState.Contains(token1
                                                                                       ));
            NUnit.Framework.Assert.AreEqual("incorrect token 1 date", tokenDate1, state.tokenState
                                            [token1]);
            NUnit.Framework.Assert.IsTrue("missing token 2", state.tokenState.Contains(token2
                                                                                       ));
            NUnit.Framework.Assert.AreEqual("incorrect token 2 date", tokenDate2, state.tokenState
                                            [token2]);
            NUnit.Framework.Assert.AreEqual("incorrect master key count", 1, state.tokenMasterKeyState
                                            .Count);
            NUnit.Framework.Assert.IsTrue("missing master key 1", state.tokenMasterKeyState.Contains
                                              (key1));
            DelegationKey key2 = new DelegationKey(3, 4, Sharpen.Runtime.GetBytesForString("keyData2"
                                                                                           ));
            DelegationKey key3 = new DelegationKey(5, 6, Sharpen.Runtime.GetBytesForString("keyData3"
                                                                                           ));
            MRDelegationTokenIdentifier token3 = new MRDelegationTokenIdentifier(new Text("tokenOwner3"
                                                                                          ), new Text("tokenRenewer3"), new Text("tokenUser3"));

            token3.SetSequenceNumber(12345679);
            long tokenDate3 = 87654321L;

            store.RemoveToken(token1);
            store.StoreTokenMasterKey(key2);
            long newTokenDate2 = 975318642L;

            store.UpdateToken(token2, newTokenDate2);
            store.RemoveTokenMasterKey(key1);
            store.StoreTokenMasterKey(key3);
            store.StoreToken(token3, tokenDate3);
            store.Close();
            store = CreateAndStartStore();
            state = store.LoadState();
            NUnit.Framework.Assert.AreEqual("incorrect loaded token count", 2, state.tokenState
                                            .Count);
            NUnit.Framework.Assert.IsFalse("token 1 not removed", state.tokenState.Contains(token1
                                                                                            ));
            NUnit.Framework.Assert.IsTrue("missing token 2", state.tokenState.Contains(token2
                                                                                       ));
            NUnit.Framework.Assert.AreEqual("incorrect token 2 date", newTokenDate2, state.tokenState
                                            [token2]);
            NUnit.Framework.Assert.IsTrue("missing token 3", state.tokenState.Contains(token3
                                                                                       ));
            NUnit.Framework.Assert.AreEqual("incorrect token 3 date", tokenDate3, state.tokenState
                                            [token3]);
            NUnit.Framework.Assert.AreEqual("incorrect master key count", 2, state.tokenMasterKeyState
                                            .Count);
            NUnit.Framework.Assert.IsFalse("master key 1 not removed", state.tokenMasterKeyState
                                           .Contains(key1));
            NUnit.Framework.Assert.IsTrue("missing master key 2", state.tokenMasterKeyState.Contains
                                              (key2));
            NUnit.Framework.Assert.IsTrue("missing master key 3", state.tokenMasterKeyState.Contains
                                              (key3));
        }
 public JHSDelegationTokenSecretManagerForTest(HistoryServerStateStoreService store
                                               )
     : base(10000, 10000, 10000, 10000, store)
 {
 }