/// <exception cref="System.IO.IOException"/>
public virtual GetDelegationTokenResponse GetDelegationToken(GetDelegationTokenRequest
request)
{
UserGroupInformation ugi = UserGroupInformation.GetCurrentUser();
// Verify that the connection is kerberos authenticated
if (!this.IsAllowedDelegationTokenOp())
{
throw new IOException("Delegation Token can be issued only with kerberos authentication"
);
}
GetDelegationTokenResponse response = this.recordFactory.NewRecordInstance <GetDelegationTokenResponse
>();
string user = ugi.GetUserName();
Text owner = new Text(user);
Text realUser = null;
if (ugi.GetRealUser() != null)
{
realUser = new Text(ugi.GetRealUser().GetUserName());
}
MRDelegationTokenIdentifier tokenIdentifier = new MRDelegationTokenIdentifier(owner
, new Text(request.GetRenewer()), realUser);
Org.Apache.Hadoop.Security.Token.Token <MRDelegationTokenIdentifier> realJHSToken =
new Org.Apache.Hadoop.Security.Token.Token <MRDelegationTokenIdentifier>(tokenIdentifier
, this._enclosing.jhsDTSecretManager);
Org.Apache.Hadoop.Yarn.Api.Records.Token mrDToken = Org.Apache.Hadoop.Yarn.Api.Records.Token
.NewInstance(realJHSToken.GetIdentifier(), realJHSToken.GetKind().ToString(), realJHSToken
.GetPassword(), realJHSToken.GetService().ToString());
response.SetDelegationToken(mrDToken);
return(response);
}
/// <exception cref="System.IO.IOException"/>
public override void StoreToken(MRDelegationTokenIdentifier tokenId, long renewDate
)
{
if (Log.IsDebugEnabled())
{
Log.Debug("Storing token " + tokenId.GetSequenceNumber());
}
ByteArrayOutputStream memStream = new ByteArrayOutputStream();
DataOutputStream dataStream = new DataOutputStream(memStream);
try
{
tokenId.Write(dataStream);
dataStream.WriteLong(renewDate);
dataStream.Close();
dataStream = null;
}
finally
{
IOUtils.Cleanup(Log, dataStream);
}
string dbKey = GetTokenDatabaseKey(tokenId);
try
{
db.Put(JniDBFactory.Bytes(dbKey), memStream.ToByteArray());
}
catch (DBException e)
{
throw new IOException(e);
}
}
/// <exception cref="System.IO.IOException"/>
public override void UpdateToken(MRDelegationTokenIdentifier tokenId, long renewDate
)
{
if (Log.IsDebugEnabled())
{
Log.Debug("Updating token " + tokenId.GetSequenceNumber());
}
// Files cannot be atomically replaced, therefore we write a temporary
// update file, remove the original token file, then rename the update
// file to the token file. During recovery either the token file will be
// used or if that is missing and an update file is present then the
// update file is used.
Path tokenPath = GetTokenPath(tokenId);
Path tmp = new Path(tokenPath.GetParent(), UpdateTmpFilePrefix + tokenPath.GetName
());
WriteFile(tmp, BuildTokenData(tokenId, renewDate));
try
{
DeleteFile(tokenPath);
}
catch (IOException e)
{
fs.Delete(tmp, false);
throw;
}
if (!fs.Rename(tmp, tokenPath))
{
throw new IOException("Could not rename " + tmp + " to " + tokenPath);
}
}
/// <exception cref="System.IO.IOException"/>
public override void RemoveToken(MRDelegationTokenIdentifier tokenId)
{
if (Log.IsDebugEnabled())
{
Log.Debug("Removing token " + tokenId.GetSequenceNumber());
}
DeleteFile(GetTokenPath(tokenId));
}
/// <exception cref="System.IO.IOException"/>
public override void UpdateToken(MRDelegationTokenIdentifier tokenId, long renewDate
)
{
if (!state.tokenState.Contains(tokenId))
{
throw new IOException("token " + tokenId + " not in store");
}
state.tokenState[tokenId] = renewDate;
}
/// <exception cref="System.IO.IOException"/>
public override void StoreToken(MRDelegationTokenIdentifier tokenId, long renewDate
)
{
if (state.tokenState.Contains(tokenId))
{
throw new IOException("token " + tokenId + " was stored twice");
}
state.tokenState[tokenId] = renewDate;
}
/// <exception cref="System.IO.IOException"/>
public override void RemoveToken(MRDelegationTokenIdentifier tokenId)
{
string dbKey = GetTokenDatabaseKey(tokenId);
try
{
db.Delete(JniDBFactory.Bytes(dbKey));
}
catch (DBException e)
{
throw new IOException(e);
}
}
/// <exception cref="System.IO.IOException"/>
private void LoadTokenFromBucket(int bucketId, HistoryServerStateStoreService.HistoryServerState
state, Path tokenFile, long numTokenFileBytes)
{
MRDelegationTokenIdentifier token = LoadToken(state, tokenFile, numTokenFileBytes
);
int tokenBucketId = GetBucketId(token);
if (tokenBucketId != bucketId)
{
throw new IOException("Token " + tokenFile + " should be in bucket " + tokenBucketId
+ ", found in bucket " + bucketId);
}
}
public virtual void TestUpdatedTokenRecovery()
{
IOException intentionalErr = new IOException("intentional error");
FileSystem fs = FileSystem.GetLocal(conf);
FileSystem spyfs = Org.Mockito.Mockito.Spy(fs);
// make the update token process fail halfway through where we're left
// with just the temporary update file and no token file
ArgumentMatcher <Path> updateTmpMatcher = new _ArgumentMatcher_196();
Org.Mockito.Mockito.DoThrow(intentionalErr).When(spyfs).Rename(Matchers.ArgThat(updateTmpMatcher
), Matchers.IsA <Path>());
conf.Set(JHAdminConfig.MrHsFsStateStoreUri, testDir.GetAbsoluteFile().ToURI().ToString
());
HistoryServerStateStoreService store = new _HistoryServerFileSystemStateStoreService_211
(spyfs);
store.Init(conf);
store.Start();
MRDelegationTokenIdentifier token1 = new MRDelegationTokenIdentifier(new Text("tokenOwner1"
), new Text("tokenRenewer1"), new Text("tokenUser1"));
token1.SetSequenceNumber(1);
long tokenDate1 = 1L;
store.StoreToken(token1, tokenDate1);
long newTokenDate1 = 975318642L;
try
{
store.UpdateToken(token1, newTokenDate1);
NUnit.Framework.Assert.Fail("intentional error not thrown");
}
catch (IOException e)
{
NUnit.Framework.Assert.AreEqual(intentionalErr, e);
}
store.Close();
// verify the update file is seen and parsed upon recovery when
// original token file is missing
store = CreateAndStartStore();
HistoryServerStateStoreService.HistoryServerState state = store.LoadState();
NUnit.Framework.Assert.AreEqual("incorrect loaded token count", 1, state.tokenState
.Count);
NUnit.Framework.Assert.IsTrue("missing token 1", state.tokenState.Contains(token1
));
NUnit.Framework.Assert.AreEqual("incorrect token 1 date", newTokenDate1, state.tokenState
[token1]);
store.Close();
}
/// <exception cref="System.IO.IOException"/>
public override void StoreToken(MRDelegationTokenIdentifier tokenId, long renewDate
)
{
if (Log.IsDebugEnabled())
{
Log.Debug("Storing token " + tokenId.GetSequenceNumber());
}
Path tokenPath = GetTokenPath(tokenId);
if (fs.Exists(tokenPath))
{
throw new IOException(tokenPath + " already exists");
}
CreateNewFile(tokenPath, BuildTokenData(tokenId, renewDate));
}
/// <exception cref="System.IO.IOException"/>
private void LoadToken(HistoryServerStateStoreService.HistoryServerState state, byte
[] data)
{
MRDelegationTokenIdentifier tokenId = new MRDelegationTokenIdentifier();
long renewDate;
DataInputStream @in = new DataInputStream(new ByteArrayInputStream(data));
try
{
tokenId.ReadFields(@in);
renewDate = @in.ReadLong();
}
finally
{
IOUtils.Cleanup(Log, @in);
}
state.tokenState[tokenId] = renewDate;
}
/// <exception cref="System.IO.IOException"/>
private byte[] BuildTokenData(MRDelegationTokenIdentifier tokenId, long renewDate
)
{
ByteArrayOutputStream memStream = new ByteArrayOutputStream();
DataOutputStream dataStream = new DataOutputStream(memStream);
try
{
tokenId.Write(dataStream);
dataStream.WriteLong(renewDate);
dataStream.Close();
dataStream = null;
}
finally
{
IOUtils.Cleanup(Log, dataStream);
}
return(memStream.ToByteArray());
}
/// <exception cref="System.IO.IOException"/>
private MRDelegationTokenIdentifier LoadToken(HistoryServerStateStoreService.HistoryServerState
state, Path tokenFile, long numTokenFileBytes)
{
MRDelegationTokenIdentifier tokenId = new MRDelegationTokenIdentifier();
long renewDate;
byte[] tokenData = ReadFile(tokenFile, numTokenFileBytes);
DataInputStream @in = new DataInputStream(new ByteArrayInputStream(tokenData));
try
{
tokenId.ReadFields(@in);
renewDate = @in.ReadLong();
}
finally
{
IOUtils.Cleanup(Log, @in);
}
state.tokenState[tokenId] = renewDate;
return(tokenId);
}
/// <exception cref="System.IO.IOException"/>
private void TestTokenStore(string stateStoreUri)
{
conf.Set(JHAdminConfig.MrHsFsStateStoreUri, stateStoreUri);
HistoryServerStateStoreService store = CreateAndStartStore();
HistoryServerStateStoreService.HistoryServerState state = store.LoadState();
NUnit.Framework.Assert.IsTrue("token state not empty", state.tokenState.IsEmpty()
);
NUnit.Framework.Assert.IsTrue("key state not empty", state.tokenMasterKeyState.IsEmpty
());
DelegationKey key1 = new DelegationKey(1, 2, Sharpen.Runtime.GetBytesForString("keyData1"
));
MRDelegationTokenIdentifier token1 = new MRDelegationTokenIdentifier(new Text("tokenOwner1"
), new Text("tokenRenewer1"), new Text("tokenUser1"));
token1.SetSequenceNumber(1);
long tokenDate1 = 1L;
MRDelegationTokenIdentifier token2 = new MRDelegationTokenIdentifier(new Text("tokenOwner2"
), new Text("tokenRenewer2"), new Text("tokenUser2"));
token2.SetSequenceNumber(12345678);
long tokenDate2 = 87654321L;
store.StoreTokenMasterKey(key1);
try
{
store.StoreTokenMasterKey(key1);
NUnit.Framework.Assert.Fail("redundant store of key undetected");
}
catch (IOException)
{
}
// expected
store.StoreToken(token1, tokenDate1);
store.StoreToken(token2, tokenDate2);
try
{
store.StoreToken(token1, tokenDate1);
NUnit.Framework.Assert.Fail("redundant store of token undetected");
}
catch (IOException)
{
}
// expected
store.Close();
store = CreateAndStartStore();
state = store.LoadState();
NUnit.Framework.Assert.AreEqual("incorrect loaded token count", 2, state.tokenState
.Count);
NUnit.Framework.Assert.IsTrue("missing token 1", state.tokenState.Contains(token1
));
NUnit.Framework.Assert.AreEqual("incorrect token 1 date", tokenDate1, state.tokenState
[token1]);
NUnit.Framework.Assert.IsTrue("missing token 2", state.tokenState.Contains(token2
));
NUnit.Framework.Assert.AreEqual("incorrect token 2 date", tokenDate2, state.tokenState
[token2]);
NUnit.Framework.Assert.AreEqual("incorrect master key count", 1, state.tokenMasterKeyState
.Count);
NUnit.Framework.Assert.IsTrue("missing master key 1", state.tokenMasterKeyState.Contains
(key1));
DelegationKey key2 = new DelegationKey(3, 4, Sharpen.Runtime.GetBytesForString("keyData2"
));
DelegationKey key3 = new DelegationKey(5, 6, Sharpen.Runtime.GetBytesForString("keyData3"
));
MRDelegationTokenIdentifier token3 = new MRDelegationTokenIdentifier(new Text("tokenOwner3"
), new Text("tokenRenewer3"), new Text("tokenUser3"));
token3.SetSequenceNumber(12345679);
long tokenDate3 = 87654321L;
store.RemoveToken(token1);
store.StoreTokenMasterKey(key2);
long newTokenDate2 = 975318642L;
store.UpdateToken(token2, newTokenDate2);
store.RemoveTokenMasterKey(key1);
store.StoreTokenMasterKey(key3);
store.StoreToken(token3, tokenDate3);
store.Close();
store = CreateAndStartStore();
state = store.LoadState();
NUnit.Framework.Assert.AreEqual("incorrect loaded token count", 2, state.tokenState
.Count);
NUnit.Framework.Assert.IsFalse("token 1 not removed", state.tokenState.Contains(token1
));
NUnit.Framework.Assert.IsTrue("missing token 2", state.tokenState.Contains(token2
));
NUnit.Framework.Assert.AreEqual("incorrect token 2 date", newTokenDate2, state.tokenState
[token2]);
NUnit.Framework.Assert.IsTrue("missing token 3", state.tokenState.Contains(token3
));
NUnit.Framework.Assert.AreEqual("incorrect token 3 date", tokenDate3, state.tokenState
[token3]);
NUnit.Framework.Assert.AreEqual("incorrect master key count", 2, state.tokenMasterKeyState
.Count);
NUnit.Framework.Assert.IsFalse("master key 1 not removed", state.tokenMasterKeyState
.Contains(key1));
NUnit.Framework.Assert.IsTrue("missing master key 2", state.tokenMasterKeyState.Contains
(key2));
NUnit.Framework.Assert.IsTrue("missing master key 3", state.tokenMasterKeyState.Contains
(key3));
}
// Do nothing
/// <exception cref="System.IO.IOException"/>
public override void UpdateToken(MRDelegationTokenIdentifier tokenId, long renewDate
)
{
}
private string GetTokenDatabaseKey(MRDelegationTokenIdentifier tokenId)
{
return TokenStateKeyPrefix + tokenId.GetSequenceNumber();
}
private Path GetTokenPath(MRDelegationTokenIdentifier tokenId)
{
Path bucketPath = GetTokenBucketPath(GetBucketId(tokenId));
return(new Path(bucketPath, TokenFilePrefix + tokenId.GetSequenceNumber()));
}
private static int GetBucketId(MRDelegationTokenIdentifier tokenId)
{
return(tokenId.GetSequenceNumber() % NumTokenBuckets);
}
/// <exception cref="System.IO.IOException"/>
public override void RemoveToken(MRDelegationTokenIdentifier tokenId)
{
Sharpen.Collections.Remove(state.tokenState, tokenId);
}
/// <summary>Blocking method to remove a delegation token from the state storage.</summary> /// <remarks> /// Blocking method to remove a delegation token from the state storage. /// Implementations must not return from this method until the token has been /// removed from the state store. /// </remarks> /// <param name="tokenId">the token to remove</param> /// <exception cref="System.IO.IOException"/> public abstract void RemoveToken(MRDelegationTokenIdentifier tokenId);
/// <summary>
/// Blocking method to store a delegation token along with the current token
/// sequence number to the state storage.
/// </summary>
/// <remarks>
/// Blocking method to store a delegation token along with the current token
/// sequence number to the state storage.
/// Implementations must not return from this method until the token has been
/// committed to the state store.
/// </remarks>
/// <param name="tokenId">the token to store</param>
/// <param name="renewDate">the token renewal deadline</param>
/// <exception cref="System.IO.IOException"/>
public abstract void StoreToken(MRDelegationTokenIdentifier tokenId, long renewDate
);
// Do nothing
/// <exception cref="System.IO.IOException"/>
public override void RemoveToken(MRDelegationTokenIdentifier tokenId)
{
}
public virtual void TestTokenStore()
{
HistoryServerStateStoreService store = CreateAndStartStore();
// verify initially the store is empty
HistoryServerStateStoreService.HistoryServerState state = store.LoadState();
NUnit.Framework.Assert.IsTrue("token state not empty", state.tokenState.IsEmpty()
);
NUnit.Framework.Assert.IsTrue("key state not empty", state.tokenMasterKeyState.IsEmpty
());
// store a key and some tokens
DelegationKey key1 = new DelegationKey(1, 2, Sharpen.Runtime.GetBytesForString("keyData1"
));
MRDelegationTokenIdentifier token1 = new MRDelegationTokenIdentifier(new Text("tokenOwner1"
), new Text("tokenRenewer1"), new Text("tokenUser1"));
token1.SetSequenceNumber(1);
long tokenDate1 = 1L;
MRDelegationTokenIdentifier token2 = new MRDelegationTokenIdentifier(new Text("tokenOwner2"
), new Text("tokenRenewer2"), new Text("tokenUser2"));
token2.SetSequenceNumber(12345678);
long tokenDate2 = 87654321L;
store.StoreTokenMasterKey(key1);
store.StoreToken(token1, tokenDate1);
store.StoreToken(token2, tokenDate2);
store.Close();
// verify the key and tokens can be recovered
store = CreateAndStartStore();
state = store.LoadState();
NUnit.Framework.Assert.AreEqual("incorrect loaded token count", 2, state.tokenState
.Count);
NUnit.Framework.Assert.IsTrue("missing token 1", state.tokenState.Contains(token1
));
NUnit.Framework.Assert.AreEqual("incorrect token 1 date", tokenDate1, state.tokenState
[token1]);
NUnit.Framework.Assert.IsTrue("missing token 2", state.tokenState.Contains(token2
));
NUnit.Framework.Assert.AreEqual("incorrect token 2 date", tokenDate2, state.tokenState
[token2]);
NUnit.Framework.Assert.AreEqual("incorrect master key count", 1, state.tokenMasterKeyState
.Count);
NUnit.Framework.Assert.IsTrue("missing master key 1", state.tokenMasterKeyState.Contains
(key1));
// store some more keys and tokens, remove the previous key and one
// of the tokens, and renew a previous token
DelegationKey key2 = new DelegationKey(3, 4, Sharpen.Runtime.GetBytesForString("keyData2"
));
DelegationKey key3 = new DelegationKey(5, 6, Sharpen.Runtime.GetBytesForString("keyData3"
));
MRDelegationTokenIdentifier token3 = new MRDelegationTokenIdentifier(new Text("tokenOwner3"
), new Text("tokenRenewer3"), new Text("tokenUser3"));
token3.SetSequenceNumber(12345679);
long tokenDate3 = 87654321L;
store.RemoveToken(token1);
store.StoreTokenMasterKey(key2);
long newTokenDate2 = 975318642L;
store.UpdateToken(token2, newTokenDate2);
store.RemoveTokenMasterKey(key1);
store.StoreTokenMasterKey(key3);
store.StoreToken(token3, tokenDate3);
store.Close();
// verify the new keys and tokens are recovered, the removed key and
// token are no longer present, and the renewed token has the updated
// expiration date
store = CreateAndStartStore();
state = store.LoadState();
NUnit.Framework.Assert.AreEqual("incorrect loaded token count", 2, state.tokenState
.Count);
NUnit.Framework.Assert.IsFalse("token 1 not removed", state.tokenState.Contains(token1
));
NUnit.Framework.Assert.IsTrue("missing token 2", state.tokenState.Contains(token2
));
NUnit.Framework.Assert.AreEqual("incorrect token 2 date", newTokenDate2, state.tokenState
[token2]);
NUnit.Framework.Assert.IsTrue("missing token 3", state.tokenState.Contains(token3
));
NUnit.Framework.Assert.AreEqual("incorrect token 3 date", tokenDate3, state.tokenState
[token3]);
NUnit.Framework.Assert.AreEqual("incorrect master key count", 2, state.tokenMasterKeyState
.Count);
NUnit.Framework.Assert.IsFalse("master key 1 not removed", state.tokenMasterKeyState
.Contains(key1));
NUnit.Framework.Assert.IsTrue("missing master key 2", state.tokenMasterKeyState.Contains
(key2));
NUnit.Framework.Assert.IsTrue("missing master key 3", state.tokenMasterKeyState.Contains
(key3));
store.Close();
}
public virtual void TestRecovery()
{
Configuration conf = new Configuration();
HistoryServerStateStoreService store = new HistoryServerMemStateStoreService();
store.Init(conf);
store.Start();
TestJHSDelegationTokenSecretManager.JHSDelegationTokenSecretManagerForTest mgr =
new TestJHSDelegationTokenSecretManager.JHSDelegationTokenSecretManagerForTest(store
);
mgr.StartThreads();
MRDelegationTokenIdentifier tokenId1 = new MRDelegationTokenIdentifier(new Text("tokenOwner"
), new Text("tokenRenewer"), new Text("tokenUser"));
Org.Apache.Hadoop.Security.Token.Token <MRDelegationTokenIdentifier> token1 = new
Org.Apache.Hadoop.Security.Token.Token <MRDelegationTokenIdentifier>(tokenId1, mgr
);
MRDelegationTokenIdentifier tokenId2 = new MRDelegationTokenIdentifier(new Text("tokenOwner"
), new Text("tokenRenewer"), new Text("tokenUser"));
Org.Apache.Hadoop.Security.Token.Token <MRDelegationTokenIdentifier> token2 = new
Org.Apache.Hadoop.Security.Token.Token <MRDelegationTokenIdentifier>(tokenId2, mgr
);
DelegationKey[] keys = mgr.GetAllKeys();
long tokenRenewDate1 = mgr.GetAllTokens()[tokenId1].GetRenewDate();
long tokenRenewDate2 = mgr.GetAllTokens()[tokenId2].GetRenewDate();
mgr.StopThreads();
mgr = new TestJHSDelegationTokenSecretManager.JHSDelegationTokenSecretManagerForTest
(store);
mgr.Recover(store.LoadState());
IList <DelegationKey> recoveredKeys = Arrays.AsList(mgr.GetAllKeys());
foreach (DelegationKey key in keys)
{
NUnit.Framework.Assert.IsTrue("key missing after recovery", recoveredKeys.Contains
(key));
}
NUnit.Framework.Assert.IsTrue("token1 missing", mgr.GetAllTokens().Contains(tokenId1
));
NUnit.Framework.Assert.AreEqual("token1 renew date", tokenRenewDate1, mgr.GetAllTokens
()[tokenId1].GetRenewDate());
NUnit.Framework.Assert.IsTrue("token2 missing", mgr.GetAllTokens().Contains(tokenId2
));
NUnit.Framework.Assert.AreEqual("token2 renew date", tokenRenewDate2, mgr.GetAllTokens
()[tokenId2].GetRenewDate());
mgr.StartThreads();
mgr.VerifyToken(tokenId1, token1.GetPassword());
mgr.VerifyToken(tokenId2, token2.GetPassword());
MRDelegationTokenIdentifier tokenId3 = new MRDelegationTokenIdentifier(new Text("tokenOwner"
), new Text("tokenRenewer"), new Text("tokenUser"));
Org.Apache.Hadoop.Security.Token.Token <MRDelegationTokenIdentifier> token3 = new
Org.Apache.Hadoop.Security.Token.Token <MRDelegationTokenIdentifier>(tokenId3, mgr
);
NUnit.Framework.Assert.AreEqual("sequence number restore", tokenId2.GetSequenceNumber
() + 1, tokenId3.GetSequenceNumber());
mgr.CancelToken(token1, "tokenOwner");
// Testing with full principal name
MRDelegationTokenIdentifier tokenIdFull = new MRDelegationTokenIdentifier(new Text
("tokenOwner/localhost@LOCALHOST"), new Text("tokenRenewer"), new Text("tokenUser"
));
KerberosName.SetRules("RULE:[1:$1]\nRULE:[2:$1]");
Org.Apache.Hadoop.Security.Token.Token <MRDelegationTokenIdentifier> tokenFull = new
Org.Apache.Hadoop.Security.Token.Token <MRDelegationTokenIdentifier>(tokenIdFull,
mgr);
// Negative test
try
{
mgr.CancelToken(tokenFull, "tokenOwner");
}
catch (AccessControlException ace)
{
NUnit.Framework.Assert.IsTrue(ace.Message.Contains("is not authorized to cancel the token"
));
}
// Succeed to cancel with full principal
mgr.CancelToken(tokenFull, tokenIdFull.GetOwner().ToString());
long tokenRenewDate3 = mgr.GetAllTokens()[tokenId3].GetRenewDate();
mgr.StopThreads();
mgr = new TestJHSDelegationTokenSecretManager.JHSDelegationTokenSecretManagerForTest
(store);
mgr.Recover(store.LoadState());
NUnit.Framework.Assert.IsFalse("token1 should be missing", mgr.GetAllTokens().Contains
(tokenId1));
NUnit.Framework.Assert.IsTrue("token2 missing", mgr.GetAllTokens().Contains(tokenId2
));
NUnit.Framework.Assert.AreEqual("token2 renew date", tokenRenewDate2, mgr.GetAllTokens
()[tokenId2].GetRenewDate());
NUnit.Framework.Assert.IsTrue("token3 missing", mgr.GetAllTokens().Contains(tokenId3
));
NUnit.Framework.Assert.AreEqual("token3 renew date", tokenRenewDate3, mgr.GetAllTokens
()[tokenId3].GetRenewDate());
mgr.StartThreads();
mgr.VerifyToken(tokenId2, token2.GetPassword());
mgr.VerifyToken(tokenId3, token3.GetPassword());
mgr.StopThreads();
}
