/// <exception cref="System.IO.IOException"/> private int LoadTokens(HistoryServerStateStoreService.HistoryServerState state) { FileStatus[] stats = fs.ListStatus(tokenStatePath); int numTokens = 0; foreach (FileStatus stat in stats) { string name = stat.GetPath().GetName(); if (name.StartsWith(TokenBucketDirPrefix)) { numTokens += LoadTokensFromBucket(state, stat.GetPath()); } else { if (name.Equals(TokenKeysDirName)) { // key loading is done elsewhere continue; } else { Log.Warn("Skipping unexpected file in history server token state: " + stat.GetPath ()); } } } return(numTokens); }
// don't close the filesystem as it's part of the filesystem cache // and other clients may still be using it /// <exception cref="System.IO.IOException"/> public override HistoryServerStateStoreService.HistoryServerState LoadState() { Log.Info("Loading history server state from " + rootStatePath); HistoryServerStateStoreService.HistoryServerState state = new HistoryServerStateStoreService.HistoryServerState (); LoadTokenState(state); return(state); }
/// <exception cref="System.IO.IOException"/> public override HistoryServerStateStoreService.HistoryServerState LoadState() { HistoryServerStateStoreService.HistoryServerState result = new HistoryServerStateStoreService.HistoryServerState (); result.tokenState.PutAll(state.tokenState); Sharpen.Collections.AddAll(result.tokenMasterKeyState, state.tokenMasterKeyState); return(result); }
/// <exception cref="System.IO.IOException"/> private void LoadTokenState(HistoryServerStateStoreService.HistoryServerState state ) { int numKeys = LoadKeys(state); int numTokens = LoadTokens(state); Log.Info("Loaded " + numKeys + " master keys and " + numTokens + " tokens from " + tokenStatePath); }
/// <exception cref="System.IO.IOException"/> private int LoadTokensFromBucket(HistoryServerStateStoreService.HistoryServerState state, Path bucket) { string numStr = Sharpen.Runtime.Substring(bucket.GetName(), TokenBucketDirPrefix. Length); int bucketId = System.Convert.ToInt32(numStr); int numTokens = 0; FileStatus[] tokenStats = fs.ListStatus(bucket); ICollection <string> loadedTokens = new HashSet <string>(tokenStats.Length); foreach (FileStatus stat in tokenStats) { string name = stat.GetPath().GetName(); if (name.StartsWith(TokenFilePrefix)) { LoadTokenFromBucket(bucketId, state, stat.GetPath(), stat.GetLen()); loadedTokens.AddItem(name); ++numTokens; } else { if (name.StartsWith(UpdateTmpFilePrefix)) { string tokenName = Sharpen.Runtime.Substring(name, UpdateTmpFilePrefix.Length); if (loadedTokens.Contains(tokenName)) { // already have the token, update may be partial so ignore it fs.Delete(stat.GetPath(), false); } else { // token is missing, so try to parse the update temp file LoadTokenFromBucket(bucketId, state, stat.GetPath(), stat.GetLen()); fs.Rename(stat.GetPath(), new Path(stat.GetPath().GetParent(), tokenName)); loadedTokens.AddItem(tokenName); ++numTokens; } } else { if (name.StartsWith(TmpFilePrefix)) { // cleanup incomplete temp files fs.Delete(stat.GetPath(), false); } else { Log.Warn("Skipping unexpected file in history server token bucket: " + stat.GetPath ()); } } } } return(numTokens); }
/// <exception cref="System.IO.IOException"/> public override HistoryServerStateStoreService.HistoryServerState LoadState() { HistoryServerStateStoreService.HistoryServerState state = new HistoryServerStateStoreService.HistoryServerState (); int numKeys = LoadTokenMasterKeys(state); Log.Info("Recovered " + numKeys + " token master keys"); int numTokens = LoadTokens(state); Log.Info("Recovered " + numTokens + " tokens"); return state; }
/// <exception cref="System.IO.IOException"/> public virtual void Recover(HistoryServerStateStoreService.HistoryServerState state ) { Log.Info("Recovering " + GetType().Name); foreach (DelegationKey key in state.tokenMasterKeyState) { AddKey(key); } foreach (KeyValuePair <MRDelegationTokenIdentifier, long> entry in state.tokenState) { AddPersistedDelegationToken(entry.Key, entry.Value); } }
/// <exception cref="System.IO.IOException"/> private void LoadTokenFromBucket(int bucketId, HistoryServerStateStoreService.HistoryServerState state, Path tokenFile, long numTokenFileBytes) { MRDelegationTokenIdentifier token = LoadToken(state, tokenFile, numTokenFileBytes ); int tokenBucketId = GetBucketId(token); if (tokenBucketId != bucketId) { throw new IOException("Token " + tokenFile + " should be in bucket " + tokenBucketId + ", found in bucket " + bucketId); } }
public virtual void TestUpdatedTokenRecovery() { IOException intentionalErr = new IOException("intentional error"); FileSystem fs = FileSystem.GetLocal(conf); FileSystem spyfs = Org.Mockito.Mockito.Spy(fs); // make the update token process fail halfway through where we're left // with just the temporary update file and no token file ArgumentMatcher <Path> updateTmpMatcher = new _ArgumentMatcher_196(); Org.Mockito.Mockito.DoThrow(intentionalErr).When(spyfs).Rename(Matchers.ArgThat(updateTmpMatcher ), Matchers.IsA <Path>()); conf.Set(JHAdminConfig.MrHsFsStateStoreUri, testDir.GetAbsoluteFile().ToURI().ToString ()); HistoryServerStateStoreService store = new _HistoryServerFileSystemStateStoreService_211 (spyfs); store.Init(conf); store.Start(); MRDelegationTokenIdentifier token1 = new MRDelegationTokenIdentifier(new Text("tokenOwner1" ), new Text("tokenRenewer1"), new Text("tokenUser1")); token1.SetSequenceNumber(1); long tokenDate1 = 1L; store.StoreToken(token1, tokenDate1); long newTokenDate1 = 975318642L; try { store.UpdateToken(token1, newTokenDate1); NUnit.Framework.Assert.Fail("intentional error not thrown"); } catch (IOException e) { NUnit.Framework.Assert.AreEqual(intentionalErr, e); } store.Close(); // verify the update file is seen and parsed upon recovery when // original token file is missing store = CreateAndStartStore(); HistoryServerStateStoreService.HistoryServerState state = store.LoadState(); NUnit.Framework.Assert.AreEqual("incorrect loaded token count", 1, state.tokenState .Count); NUnit.Framework.Assert.IsTrue("missing token 1", state.tokenState.Contains(token1 )); NUnit.Framework.Assert.AreEqual("incorrect token 1 date", newTokenDate1, state.tokenState [token1]); store.Close(); }
/// <exception cref="System.IO.IOException"/> private void LoadTokenMasterKey(HistoryServerStateStoreService.HistoryServerState state, byte[] data) { DelegationKey key = new DelegationKey(); DataInputStream @in = new DataInputStream(new ByteArrayInputStream(data)); try { key.ReadFields(@in); } finally { IOUtils.Cleanup(Log, @in); } state.tokenMasterKeyState.AddItem(key); }
/// <exception cref="System.IO.IOException"/> private void LoadToken(HistoryServerStateStoreService.HistoryServerState state, byte [] data) { MRDelegationTokenIdentifier tokenId = new MRDelegationTokenIdentifier(); long renewDate; DataInputStream @in = new DataInputStream(new ByteArrayInputStream(data)); try { tokenId.ReadFields(@in); renewDate = @in.ReadLong(); } finally { IOUtils.Cleanup(Log, @in); } state.tokenState[tokenId] = renewDate; }
/// <exception cref="System.IO.IOException"/> private int LoadTokenMasterKeys(HistoryServerStateStoreService.HistoryServerState state) { int numKeys = 0; LeveldbIterator iter = null; try { iter = new LeveldbIterator(db); iter.Seek(JniDBFactory.Bytes(TokenMasterKeyKeyPrefix)); while (iter.HasNext()) { KeyValuePair<byte[], byte[]> entry = iter.Next(); string key = JniDBFactory.AsString(entry.Key); if (!key.StartsWith(TokenMasterKeyKeyPrefix)) { break; } if (Log.IsDebugEnabled()) { Log.Debug("Loading master key from " + key); } try { LoadTokenMasterKey(state, entry.Value); } catch (IOException e) { throw new IOException("Error loading token master key from " + key, e); } ++numKeys; } } catch (DBException e) { throw new IOException(e); } finally { if (iter != null) { iter.Close(); } } return numKeys; }
/// <exception cref="System.IO.IOException"/> private int LoadKeys(HistoryServerStateStoreService.HistoryServerState state) { FileStatus[] stats = fs.ListStatus(tokenKeysStatePath); int numKeys = 0; foreach (FileStatus stat in stats) { string name = stat.GetPath().GetName(); if (name.StartsWith(TokenMasterKeyFilePrefix)) { LoadTokenMasterKey(state, stat.GetPath(), stat.GetLen()); ++numKeys; } else { Log.Warn("Skipping unexpected file in history server token state: " + stat.GetPath ()); } } return(numKeys); }
/// <exception cref="System.IO.IOException"/> private MRDelegationTokenIdentifier LoadToken(HistoryServerStateStoreService.HistoryServerState state, Path tokenFile, long numTokenFileBytes) { MRDelegationTokenIdentifier tokenId = new MRDelegationTokenIdentifier(); long renewDate; byte[] tokenData = ReadFile(tokenFile, numTokenFileBytes); DataInputStream @in = new DataInputStream(new ByteArrayInputStream(tokenData)); try { tokenId.ReadFields(@in); renewDate = @in.ReadLong(); } finally { IOUtils.Cleanup(Log, @in); } state.tokenState[tokenId] = renewDate; return(tokenId); }
// utility class to start and stop secret manager as part of service // framework and implement state recovery for secret manager on startup /// <exception cref="System.Exception"/> protected override void ServiceStart() { bool recoveryEnabled = this.GetConfig().GetBoolean(JHAdminConfig.MrHsRecoveryEnable , JHAdminConfig.DefaultMrHsRecoveryEnable); if (recoveryEnabled) { System.Diagnostics.Debug.Assert(this._enclosing.stateStore.IsInState(Service.STATE .Started)); HistoryServerStateStoreService.HistoryServerState state = this._enclosing.stateStore .LoadState(); this._enclosing.jhsDTSecretManager.Recover(state); } try { this._enclosing.jhsDTSecretManager.StartThreads(); } catch (IOException io) { JobHistoryServer.Log.Error("Error while starting the Secret Manager threads", io); throw; } base.ServiceStart(); }
/// <exception cref="System.IO.IOException"/> protected internal override void StartStorage() { state = new HistoryServerStateStoreService.HistoryServerState(); }
/// <exception cref="System.IO.IOException"/> private void TestTokenStore(string stateStoreUri) { conf.Set(JHAdminConfig.MrHsFsStateStoreUri, stateStoreUri); HistoryServerStateStoreService store = CreateAndStartStore(); HistoryServerStateStoreService.HistoryServerState state = store.LoadState(); NUnit.Framework.Assert.IsTrue("token state not empty", state.tokenState.IsEmpty() ); NUnit.Framework.Assert.IsTrue("key state not empty", state.tokenMasterKeyState.IsEmpty ()); DelegationKey key1 = new DelegationKey(1, 2, Sharpen.Runtime.GetBytesForString("keyData1" )); MRDelegationTokenIdentifier token1 = new MRDelegationTokenIdentifier(new Text("tokenOwner1" ), new Text("tokenRenewer1"), new Text("tokenUser1")); token1.SetSequenceNumber(1); long tokenDate1 = 1L; MRDelegationTokenIdentifier token2 = new MRDelegationTokenIdentifier(new Text("tokenOwner2" ), new Text("tokenRenewer2"), new Text("tokenUser2")); token2.SetSequenceNumber(12345678); long tokenDate2 = 87654321L; store.StoreTokenMasterKey(key1); try { store.StoreTokenMasterKey(key1); NUnit.Framework.Assert.Fail("redundant store of key undetected"); } catch (IOException) { } // expected store.StoreToken(token1, tokenDate1); store.StoreToken(token2, tokenDate2); try { store.StoreToken(token1, tokenDate1); NUnit.Framework.Assert.Fail("redundant store of token undetected"); } catch (IOException) { } // expected store.Close(); store = CreateAndStartStore(); state = store.LoadState(); NUnit.Framework.Assert.AreEqual("incorrect loaded token count", 2, state.tokenState .Count); NUnit.Framework.Assert.IsTrue("missing token 1", state.tokenState.Contains(token1 )); NUnit.Framework.Assert.AreEqual("incorrect token 1 date", tokenDate1, state.tokenState [token1]); NUnit.Framework.Assert.IsTrue("missing token 2", state.tokenState.Contains(token2 )); NUnit.Framework.Assert.AreEqual("incorrect token 2 date", tokenDate2, state.tokenState [token2]); NUnit.Framework.Assert.AreEqual("incorrect master key count", 1, state.tokenMasterKeyState .Count); NUnit.Framework.Assert.IsTrue("missing master key 1", state.tokenMasterKeyState.Contains (key1)); DelegationKey key2 = new DelegationKey(3, 4, Sharpen.Runtime.GetBytesForString("keyData2" )); DelegationKey key3 = new DelegationKey(5, 6, Sharpen.Runtime.GetBytesForString("keyData3" )); MRDelegationTokenIdentifier token3 = new MRDelegationTokenIdentifier(new Text("tokenOwner3" ), new Text("tokenRenewer3"), new Text("tokenUser3")); token3.SetSequenceNumber(12345679); long tokenDate3 = 87654321L; store.RemoveToken(token1); store.StoreTokenMasterKey(key2); long newTokenDate2 = 975318642L; store.UpdateToken(token2, newTokenDate2); store.RemoveTokenMasterKey(key1); store.StoreTokenMasterKey(key3); store.StoreToken(token3, tokenDate3); store.Close(); store = CreateAndStartStore(); state = store.LoadState(); NUnit.Framework.Assert.AreEqual("incorrect loaded token count", 2, state.tokenState .Count); NUnit.Framework.Assert.IsFalse("token 1 not removed", state.tokenState.Contains(token1 )); NUnit.Framework.Assert.IsTrue("missing token 2", state.tokenState.Contains(token2 )); NUnit.Framework.Assert.AreEqual("incorrect token 2 date", newTokenDate2, state.tokenState [token2]); NUnit.Framework.Assert.IsTrue("missing token 3", state.tokenState.Contains(token3 )); NUnit.Framework.Assert.AreEqual("incorrect token 3 date", tokenDate3, state.tokenState [token3]); NUnit.Framework.Assert.AreEqual("incorrect master key count", 2, state.tokenMasterKeyState .Count); NUnit.Framework.Assert.IsFalse("master key 1 not removed", state.tokenMasterKeyState .Contains(key1)); NUnit.Framework.Assert.IsTrue("missing master key 2", state.tokenMasterKeyState.Contains (key2)); NUnit.Framework.Assert.IsTrue("missing master key 3", state.tokenMasterKeyState.Contains (key3)); }
public virtual void TestTokenStore() { HistoryServerStateStoreService store = CreateAndStartStore(); // verify initially the store is empty HistoryServerStateStoreService.HistoryServerState state = store.LoadState(); NUnit.Framework.Assert.IsTrue("token state not empty", state.tokenState.IsEmpty() ); NUnit.Framework.Assert.IsTrue("key state not empty", state.tokenMasterKeyState.IsEmpty ()); // store a key and some tokens DelegationKey key1 = new DelegationKey(1, 2, Sharpen.Runtime.GetBytesForString("keyData1" )); MRDelegationTokenIdentifier token1 = new MRDelegationTokenIdentifier(new Text("tokenOwner1" ), new Text("tokenRenewer1"), new Text("tokenUser1")); token1.SetSequenceNumber(1); long tokenDate1 = 1L; MRDelegationTokenIdentifier token2 = new MRDelegationTokenIdentifier(new Text("tokenOwner2" ), new Text("tokenRenewer2"), new Text("tokenUser2")); token2.SetSequenceNumber(12345678); long tokenDate2 = 87654321L; store.StoreTokenMasterKey(key1); store.StoreToken(token1, tokenDate1); store.StoreToken(token2, tokenDate2); store.Close(); // verify the key and tokens can be recovered store = CreateAndStartStore(); state = store.LoadState(); NUnit.Framework.Assert.AreEqual("incorrect loaded token count", 2, state.tokenState .Count); NUnit.Framework.Assert.IsTrue("missing token 1", state.tokenState.Contains(token1 )); NUnit.Framework.Assert.AreEqual("incorrect token 1 date", tokenDate1, state.tokenState [token1]); NUnit.Framework.Assert.IsTrue("missing token 2", state.tokenState.Contains(token2 )); NUnit.Framework.Assert.AreEqual("incorrect token 2 date", tokenDate2, state.tokenState [token2]); NUnit.Framework.Assert.AreEqual("incorrect master key count", 1, state.tokenMasterKeyState .Count); NUnit.Framework.Assert.IsTrue("missing master key 1", state.tokenMasterKeyState.Contains (key1)); // store some more keys and tokens, remove the previous key and one // of the tokens, and renew a previous token DelegationKey key2 = new DelegationKey(3, 4, Sharpen.Runtime.GetBytesForString("keyData2" )); DelegationKey key3 = new DelegationKey(5, 6, Sharpen.Runtime.GetBytesForString("keyData3" )); MRDelegationTokenIdentifier token3 = new MRDelegationTokenIdentifier(new Text("tokenOwner3" ), new Text("tokenRenewer3"), new Text("tokenUser3")); token3.SetSequenceNumber(12345679); long tokenDate3 = 87654321L; store.RemoveToken(token1); store.StoreTokenMasterKey(key2); long newTokenDate2 = 975318642L; store.UpdateToken(token2, newTokenDate2); store.RemoveTokenMasterKey(key1); store.StoreTokenMasterKey(key3); store.StoreToken(token3, tokenDate3); store.Close(); // verify the new keys and tokens are recovered, the removed key and // token are no longer present, and the renewed token has the updated // expiration date store = CreateAndStartStore(); state = store.LoadState(); NUnit.Framework.Assert.AreEqual("incorrect loaded token count", 2, state.tokenState .Count); NUnit.Framework.Assert.IsFalse("token 1 not removed", state.tokenState.Contains(token1 )); NUnit.Framework.Assert.IsTrue("missing token 2", state.tokenState.Contains(token2 )); NUnit.Framework.Assert.AreEqual("incorrect token 2 date", newTokenDate2, state.tokenState [token2]); NUnit.Framework.Assert.IsTrue("missing token 3", state.tokenState.Contains(token3 )); NUnit.Framework.Assert.AreEqual("incorrect token 3 date", tokenDate3, state.tokenState [token3]); NUnit.Framework.Assert.AreEqual("incorrect master key count", 2, state.tokenMasterKeyState .Count); NUnit.Framework.Assert.IsFalse("master key 1 not removed", state.tokenMasterKeyState .Contains(key1)); NUnit.Framework.Assert.IsTrue("missing master key 2", state.tokenMasterKeyState.Contains (key2)); NUnit.Framework.Assert.IsTrue("missing master key 3", state.tokenMasterKeyState.Contains (key3)); store.Close(); }
/// <exception cref="System.IO.IOException"/> protected internal override void CloseStorage() { state = null; }