/// <exception cref="System.IO.IOException"/>
private int LoadTokens(HistoryServerStateStoreService.HistoryServerState state)
{
FileStatus[] stats = fs.ListStatus(tokenStatePath);
int numTokens = 0;
foreach (FileStatus stat in stats)
{
string name = stat.GetPath().GetName();
if (name.StartsWith(TokenBucketDirPrefix))
{
numTokens += LoadTokensFromBucket(state, stat.GetPath());
}
else
{
if (name.Equals(TokenKeysDirName))
{
// key loading is done elsewhere
continue;
}
else
{
Log.Warn("Skipping unexpected file in history server token state: " + stat.GetPath
());
}
}
}
return(numTokens);
}
// don't close the filesystem as it's part of the filesystem cache
// and other clients may still be using it
/// <exception cref="System.IO.IOException"/>
public override HistoryServerStateStoreService.HistoryServerState LoadState()
{
Log.Info("Loading history server state from " + rootStatePath);
HistoryServerStateStoreService.HistoryServerState state = new HistoryServerStateStoreService.HistoryServerState
();
LoadTokenState(state);
return(state);
}
/// <exception cref="System.IO.IOException"/>
public override HistoryServerStateStoreService.HistoryServerState LoadState()
{
HistoryServerStateStoreService.HistoryServerState result = new HistoryServerStateStoreService.HistoryServerState
();
result.tokenState.PutAll(state.tokenState);
Sharpen.Collections.AddAll(result.tokenMasterKeyState, state.tokenMasterKeyState);
return(result);
}
/// <exception cref="System.IO.IOException"/>
private void LoadTokenState(HistoryServerStateStoreService.HistoryServerState state
)
{
int numKeys = LoadKeys(state);
int numTokens = LoadTokens(state);
Log.Info("Loaded " + numKeys + " master keys and " + numTokens + " tokens from "
+ tokenStatePath);
}
/// <exception cref="System.IO.IOException"/>
private int LoadTokensFromBucket(HistoryServerStateStoreService.HistoryServerState
state, Path bucket)
{
string numStr = Sharpen.Runtime.Substring(bucket.GetName(), TokenBucketDirPrefix.
Length);
int bucketId = System.Convert.ToInt32(numStr);
int numTokens = 0;
FileStatus[] tokenStats = fs.ListStatus(bucket);
ICollection <string> loadedTokens = new HashSet <string>(tokenStats.Length);
foreach (FileStatus stat in tokenStats)
{
string name = stat.GetPath().GetName();
if (name.StartsWith(TokenFilePrefix))
{
LoadTokenFromBucket(bucketId, state, stat.GetPath(), stat.GetLen());
loadedTokens.AddItem(name);
++numTokens;
}
else
{
if (name.StartsWith(UpdateTmpFilePrefix))
{
string tokenName = Sharpen.Runtime.Substring(name, UpdateTmpFilePrefix.Length);
if (loadedTokens.Contains(tokenName))
{
// already have the token, update may be partial so ignore it
fs.Delete(stat.GetPath(), false);
}
else
{
// token is missing, so try to parse the update temp file
LoadTokenFromBucket(bucketId, state, stat.GetPath(), stat.GetLen());
fs.Rename(stat.GetPath(), new Path(stat.GetPath().GetParent(), tokenName));
loadedTokens.AddItem(tokenName);
++numTokens;
}
}
else
{
if (name.StartsWith(TmpFilePrefix))
{
// cleanup incomplete temp files
fs.Delete(stat.GetPath(), false);
}
else
{
Log.Warn("Skipping unexpected file in history server token bucket: " + stat.GetPath
());
}
}
}
}
return(numTokens);
}
/// <exception cref="System.IO.IOException"/>
public override HistoryServerStateStoreService.HistoryServerState LoadState()
{
HistoryServerStateStoreService.HistoryServerState state = new HistoryServerStateStoreService.HistoryServerState
();
int numKeys = LoadTokenMasterKeys(state);
Log.Info("Recovered " + numKeys + " token master keys");
int numTokens = LoadTokens(state);
Log.Info("Recovered " + numTokens + " tokens");
return state;
}
/// <exception cref="System.IO.IOException"/>
public virtual void Recover(HistoryServerStateStoreService.HistoryServerState state
)
{
Log.Info("Recovering " + GetType().Name);
foreach (DelegationKey key in state.tokenMasterKeyState)
{
AddKey(key);
}
foreach (KeyValuePair <MRDelegationTokenIdentifier, long> entry in state.tokenState)
{
AddPersistedDelegationToken(entry.Key, entry.Value);
}
}
/// <exception cref="System.IO.IOException"/>
private void LoadTokenFromBucket(int bucketId, HistoryServerStateStoreService.HistoryServerState
state, Path tokenFile, long numTokenFileBytes)
{
MRDelegationTokenIdentifier token = LoadToken(state, tokenFile, numTokenFileBytes
);
int tokenBucketId = GetBucketId(token);
if (tokenBucketId != bucketId)
{
throw new IOException("Token " + tokenFile + " should be in bucket " + tokenBucketId
+ ", found in bucket " + bucketId);
}
}
public virtual void TestUpdatedTokenRecovery()
{
IOException intentionalErr = new IOException("intentional error");
FileSystem fs = FileSystem.GetLocal(conf);
FileSystem spyfs = Org.Mockito.Mockito.Spy(fs);
// make the update token process fail halfway through where we're left
// with just the temporary update file and no token file
ArgumentMatcher <Path> updateTmpMatcher = new _ArgumentMatcher_196();
Org.Mockito.Mockito.DoThrow(intentionalErr).When(spyfs).Rename(Matchers.ArgThat(updateTmpMatcher
), Matchers.IsA <Path>());
conf.Set(JHAdminConfig.MrHsFsStateStoreUri, testDir.GetAbsoluteFile().ToURI().ToString
());
HistoryServerStateStoreService store = new _HistoryServerFileSystemStateStoreService_211
(spyfs);
store.Init(conf);
store.Start();
MRDelegationTokenIdentifier token1 = new MRDelegationTokenIdentifier(new Text("tokenOwner1"
), new Text("tokenRenewer1"), new Text("tokenUser1"));
token1.SetSequenceNumber(1);
long tokenDate1 = 1L;
store.StoreToken(token1, tokenDate1);
long newTokenDate1 = 975318642L;
try
{
store.UpdateToken(token1, newTokenDate1);
NUnit.Framework.Assert.Fail("intentional error not thrown");
}
catch (IOException e)
{
NUnit.Framework.Assert.AreEqual(intentionalErr, e);
}
store.Close();
// verify the update file is seen and parsed upon recovery when
// original token file is missing
store = CreateAndStartStore();
HistoryServerStateStoreService.HistoryServerState state = store.LoadState();
NUnit.Framework.Assert.AreEqual("incorrect loaded token count", 1, state.tokenState
.Count);
NUnit.Framework.Assert.IsTrue("missing token 1", state.tokenState.Contains(token1
));
NUnit.Framework.Assert.AreEqual("incorrect token 1 date", newTokenDate1, state.tokenState
[token1]);
store.Close();
}
/// <exception cref="System.IO.IOException"/>
private void LoadTokenMasterKey(HistoryServerStateStoreService.HistoryServerState
state, byte[] data)
{
DelegationKey key = new DelegationKey();
DataInputStream @in = new DataInputStream(new ByteArrayInputStream(data));
try
{
key.ReadFields(@in);
}
finally
{
IOUtils.Cleanup(Log, @in);
}
state.tokenMasterKeyState.AddItem(key);
}
/// <exception cref="System.IO.IOException"/>
private void LoadToken(HistoryServerStateStoreService.HistoryServerState state, byte
[] data)
{
MRDelegationTokenIdentifier tokenId = new MRDelegationTokenIdentifier();
long renewDate;
DataInputStream @in = new DataInputStream(new ByteArrayInputStream(data));
try
{
tokenId.ReadFields(@in);
renewDate = @in.ReadLong();
}
finally
{
IOUtils.Cleanup(Log, @in);
}
state.tokenState[tokenId] = renewDate;
}
/// <exception cref="System.IO.IOException"/>
private int LoadTokenMasterKeys(HistoryServerStateStoreService.HistoryServerState
state)
{
int numKeys = 0;
LeveldbIterator iter = null;
try
{
iter = new LeveldbIterator(db);
iter.Seek(JniDBFactory.Bytes(TokenMasterKeyKeyPrefix));
while (iter.HasNext())
{
KeyValuePair<byte[], byte[]> entry = iter.Next();
string key = JniDBFactory.AsString(entry.Key);
if (!key.StartsWith(TokenMasterKeyKeyPrefix))
{
break;
}
if (Log.IsDebugEnabled())
{
Log.Debug("Loading master key from " + key);
}
try
{
LoadTokenMasterKey(state, entry.Value);
}
catch (IOException e)
{
throw new IOException("Error loading token master key from " + key, e);
}
++numKeys;
}
}
catch (DBException e)
{
throw new IOException(e);
}
finally
{
if (iter != null)
{
iter.Close();
}
}
return numKeys;
}
/// <exception cref="System.IO.IOException"/>
private int LoadKeys(HistoryServerStateStoreService.HistoryServerState state)
{
FileStatus[] stats = fs.ListStatus(tokenKeysStatePath);
int numKeys = 0;
foreach (FileStatus stat in stats)
{
string name = stat.GetPath().GetName();
if (name.StartsWith(TokenMasterKeyFilePrefix))
{
LoadTokenMasterKey(state, stat.GetPath(), stat.GetLen());
++numKeys;
}
else
{
Log.Warn("Skipping unexpected file in history server token state: " + stat.GetPath
());
}
}
return(numKeys);
}
/// <exception cref="System.IO.IOException"/>
private MRDelegationTokenIdentifier LoadToken(HistoryServerStateStoreService.HistoryServerState
state, Path tokenFile, long numTokenFileBytes)
{
MRDelegationTokenIdentifier tokenId = new MRDelegationTokenIdentifier();
long renewDate;
byte[] tokenData = ReadFile(tokenFile, numTokenFileBytes);
DataInputStream @in = new DataInputStream(new ByteArrayInputStream(tokenData));
try
{
tokenId.ReadFields(@in);
renewDate = @in.ReadLong();
}
finally
{
IOUtils.Cleanup(Log, @in);
}
state.tokenState[tokenId] = renewDate;
return(tokenId);
}
// utility class to start and stop secret manager as part of service
// framework and implement state recovery for secret manager on startup
/// <exception cref="System.Exception"/>
protected override void ServiceStart()
{
bool recoveryEnabled = this.GetConfig().GetBoolean(JHAdminConfig.MrHsRecoveryEnable
, JHAdminConfig.DefaultMrHsRecoveryEnable);
if (recoveryEnabled)
{
System.Diagnostics.Debug.Assert(this._enclosing.stateStore.IsInState(Service.STATE
.Started));
HistoryServerStateStoreService.HistoryServerState state = this._enclosing.stateStore
.LoadState();
this._enclosing.jhsDTSecretManager.Recover(state);
}
try
{
this._enclosing.jhsDTSecretManager.StartThreads();
}
catch (IOException io)
{
JobHistoryServer.Log.Error("Error while starting the Secret Manager threads", io);
throw;
}
base.ServiceStart();
}
/// <exception cref="System.IO.IOException"/>
protected internal override void StartStorage()
{
state = new HistoryServerStateStoreService.HistoryServerState();
}
/// <exception cref="System.IO.IOException"/>
private void TestTokenStore(string stateStoreUri)
{
conf.Set(JHAdminConfig.MrHsFsStateStoreUri, stateStoreUri);
HistoryServerStateStoreService store = CreateAndStartStore();
HistoryServerStateStoreService.HistoryServerState state = store.LoadState();
NUnit.Framework.Assert.IsTrue("token state not empty", state.tokenState.IsEmpty()
);
NUnit.Framework.Assert.IsTrue("key state not empty", state.tokenMasterKeyState.IsEmpty
());
DelegationKey key1 = new DelegationKey(1, 2, Sharpen.Runtime.GetBytesForString("keyData1"
));
MRDelegationTokenIdentifier token1 = new MRDelegationTokenIdentifier(new Text("tokenOwner1"
), new Text("tokenRenewer1"), new Text("tokenUser1"));
token1.SetSequenceNumber(1);
long tokenDate1 = 1L;
MRDelegationTokenIdentifier token2 = new MRDelegationTokenIdentifier(new Text("tokenOwner2"
), new Text("tokenRenewer2"), new Text("tokenUser2"));
token2.SetSequenceNumber(12345678);
long tokenDate2 = 87654321L;
store.StoreTokenMasterKey(key1);
try
{
store.StoreTokenMasterKey(key1);
NUnit.Framework.Assert.Fail("redundant store of key undetected");
}
catch (IOException)
{
}
// expected
store.StoreToken(token1, tokenDate1);
store.StoreToken(token2, tokenDate2);
try
{
store.StoreToken(token1, tokenDate1);
NUnit.Framework.Assert.Fail("redundant store of token undetected");
}
catch (IOException)
{
}
// expected
store.Close();
store = CreateAndStartStore();
state = store.LoadState();
NUnit.Framework.Assert.AreEqual("incorrect loaded token count", 2, state.tokenState
.Count);
NUnit.Framework.Assert.IsTrue("missing token 1", state.tokenState.Contains(token1
));
NUnit.Framework.Assert.AreEqual("incorrect token 1 date", tokenDate1, state.tokenState
[token1]);
NUnit.Framework.Assert.IsTrue("missing token 2", state.tokenState.Contains(token2
));
NUnit.Framework.Assert.AreEqual("incorrect token 2 date", tokenDate2, state.tokenState
[token2]);
NUnit.Framework.Assert.AreEqual("incorrect master key count", 1, state.tokenMasterKeyState
.Count);
NUnit.Framework.Assert.IsTrue("missing master key 1", state.tokenMasterKeyState.Contains
(key1));
DelegationKey key2 = new DelegationKey(3, 4, Sharpen.Runtime.GetBytesForString("keyData2"
));
DelegationKey key3 = new DelegationKey(5, 6, Sharpen.Runtime.GetBytesForString("keyData3"
));
MRDelegationTokenIdentifier token3 = new MRDelegationTokenIdentifier(new Text("tokenOwner3"
), new Text("tokenRenewer3"), new Text("tokenUser3"));
token3.SetSequenceNumber(12345679);
long tokenDate3 = 87654321L;
store.RemoveToken(token1);
store.StoreTokenMasterKey(key2);
long newTokenDate2 = 975318642L;
store.UpdateToken(token2, newTokenDate2);
store.RemoveTokenMasterKey(key1);
store.StoreTokenMasterKey(key3);
store.StoreToken(token3, tokenDate3);
store.Close();
store = CreateAndStartStore();
state = store.LoadState();
NUnit.Framework.Assert.AreEqual("incorrect loaded token count", 2, state.tokenState
.Count);
NUnit.Framework.Assert.IsFalse("token 1 not removed", state.tokenState.Contains(token1
));
NUnit.Framework.Assert.IsTrue("missing token 2", state.tokenState.Contains(token2
));
NUnit.Framework.Assert.AreEqual("incorrect token 2 date", newTokenDate2, state.tokenState
[token2]);
NUnit.Framework.Assert.IsTrue("missing token 3", state.tokenState.Contains(token3
));
NUnit.Framework.Assert.AreEqual("incorrect token 3 date", tokenDate3, state.tokenState
[token3]);
NUnit.Framework.Assert.AreEqual("incorrect master key count", 2, state.tokenMasterKeyState
.Count);
NUnit.Framework.Assert.IsFalse("master key 1 not removed", state.tokenMasterKeyState
.Contains(key1));
NUnit.Framework.Assert.IsTrue("missing master key 2", state.tokenMasterKeyState.Contains
(key2));
NUnit.Framework.Assert.IsTrue("missing master key 3", state.tokenMasterKeyState.Contains
(key3));
}
public virtual void TestTokenStore()
{
HistoryServerStateStoreService store = CreateAndStartStore();
// verify initially the store is empty
HistoryServerStateStoreService.HistoryServerState state = store.LoadState();
NUnit.Framework.Assert.IsTrue("token state not empty", state.tokenState.IsEmpty()
);
NUnit.Framework.Assert.IsTrue("key state not empty", state.tokenMasterKeyState.IsEmpty
());
// store a key and some tokens
DelegationKey key1 = new DelegationKey(1, 2, Sharpen.Runtime.GetBytesForString("keyData1"
));
MRDelegationTokenIdentifier token1 = new MRDelegationTokenIdentifier(new Text("tokenOwner1"
), new Text("tokenRenewer1"), new Text("tokenUser1"));
token1.SetSequenceNumber(1);
long tokenDate1 = 1L;
MRDelegationTokenIdentifier token2 = new MRDelegationTokenIdentifier(new Text("tokenOwner2"
), new Text("tokenRenewer2"), new Text("tokenUser2"));
token2.SetSequenceNumber(12345678);
long tokenDate2 = 87654321L;
store.StoreTokenMasterKey(key1);
store.StoreToken(token1, tokenDate1);
store.StoreToken(token2, tokenDate2);
store.Close();
// verify the key and tokens can be recovered
store = CreateAndStartStore();
state = store.LoadState();
NUnit.Framework.Assert.AreEqual("incorrect loaded token count", 2, state.tokenState
.Count);
NUnit.Framework.Assert.IsTrue("missing token 1", state.tokenState.Contains(token1
));
NUnit.Framework.Assert.AreEqual("incorrect token 1 date", tokenDate1, state.tokenState
[token1]);
NUnit.Framework.Assert.IsTrue("missing token 2", state.tokenState.Contains(token2
));
NUnit.Framework.Assert.AreEqual("incorrect token 2 date", tokenDate2, state.tokenState
[token2]);
NUnit.Framework.Assert.AreEqual("incorrect master key count", 1, state.tokenMasterKeyState
.Count);
NUnit.Framework.Assert.IsTrue("missing master key 1", state.tokenMasterKeyState.Contains
(key1));
// store some more keys and tokens, remove the previous key and one
// of the tokens, and renew a previous token
DelegationKey key2 = new DelegationKey(3, 4, Sharpen.Runtime.GetBytesForString("keyData2"
));
DelegationKey key3 = new DelegationKey(5, 6, Sharpen.Runtime.GetBytesForString("keyData3"
));
MRDelegationTokenIdentifier token3 = new MRDelegationTokenIdentifier(new Text("tokenOwner3"
), new Text("tokenRenewer3"), new Text("tokenUser3"));
token3.SetSequenceNumber(12345679);
long tokenDate3 = 87654321L;
store.RemoveToken(token1);
store.StoreTokenMasterKey(key2);
long newTokenDate2 = 975318642L;
store.UpdateToken(token2, newTokenDate2);
store.RemoveTokenMasterKey(key1);
store.StoreTokenMasterKey(key3);
store.StoreToken(token3, tokenDate3);
store.Close();
// verify the new keys and tokens are recovered, the removed key and
// token are no longer present, and the renewed token has the updated
// expiration date
store = CreateAndStartStore();
state = store.LoadState();
NUnit.Framework.Assert.AreEqual("incorrect loaded token count", 2, state.tokenState
.Count);
NUnit.Framework.Assert.IsFalse("token 1 not removed", state.tokenState.Contains(token1
));
NUnit.Framework.Assert.IsTrue("missing token 2", state.tokenState.Contains(token2
));
NUnit.Framework.Assert.AreEqual("incorrect token 2 date", newTokenDate2, state.tokenState
[token2]);
NUnit.Framework.Assert.IsTrue("missing token 3", state.tokenState.Contains(token3
));
NUnit.Framework.Assert.AreEqual("incorrect token 3 date", tokenDate3, state.tokenState
[token3]);
NUnit.Framework.Assert.AreEqual("incorrect master key count", 2, state.tokenMasterKeyState
.Count);
NUnit.Framework.Assert.IsFalse("master key 1 not removed", state.tokenMasterKeyState
.Contains(key1));
NUnit.Framework.Assert.IsTrue("missing master key 2", state.tokenMasterKeyState.Contains
(key2));
NUnit.Framework.Assert.IsTrue("missing master key 3", state.tokenMasterKeyState.Contains
(key3));
store.Close();
}
/// <exception cref="System.IO.IOException"/>
protected internal override void CloseStorage()
{
state = null;
}
