/// <summary>
/// This method is intended to be used for recovering persisted delegation
/// tokens
/// This method must be called before this secret manager is activated (before
/// startThreads() is called)
/// </summary>
/// <param name="identifier">identifier read from persistent storage</param>
/// <param name="renewDate">token renew time</param>
/// <exception cref="System.IO.IOException"/>
public virtual void AddPersistedDelegationToken(TokenIdent identifier, long renewDate
)
{
lock (this)
{
if (running)
{
// a safety check
throw new IOException("Can't add persisted delegation token to a running SecretManager."
);
}
int keyId = identifier.GetMasterKeyId();
DelegationKey dKey = allKeys[keyId];
if (dKey == null)
{
Log.Warn("No KEY found for persisted identifier " + identifier.ToString());
return;
}
byte[] password = CreatePassword(identifier.GetBytes(), dKey.GetKey());
if (identifier.GetSequenceNumber() > GetDelegationTokenSeqNum())
{
SetDelegationTokenSeqNum(identifier.GetSequenceNumber());
}
if (GetTokenInfo(identifier) == null)
{
currentTokens[identifier] = new AbstractDelegationTokenSecretManager.DelegationTokenInformation
(renewDate, password, GetTrackingIdIfEnabled(identifier));
}
else
{
throw new IOException("Same delegation token being added twice.");
}
}
}
protected internal override void RemoveStoredMasterKey(DelegationKey key)
{
string nodeRemovePath = GetNodePath(ZkDtsmMasterKeyRoot, DelegationKeyPrefix + key
.GetKeyId());
if (Log.IsDebugEnabled())
{
Log.Debug("Removing ZKDTSMDelegationKey_" + key.GetKeyId());
}
try
{
if (zkClient.CheckExists().ForPath(nodeRemovePath) != null)
{
while (zkClient.CheckExists().ForPath(nodeRemovePath) != null)
{
zkClient.Delete().Guaranteed().ForPath(nodeRemovePath);
}
}
else
{
Log.Debug("Attempted to delete a non-existing znode " + nodeRemovePath);
}
}
catch (Exception)
{
Log.Debug(nodeRemovePath + " znode could not be removed!!");
}
}
/// <exception cref="System.IO.IOException"/>
private DelegationKey GetKeyFromZK(int keyId)
{
string nodePath = GetNodePath(ZkDtsmMasterKeyRoot, DelegationKeyPrefix + keyId);
try
{
byte[] data = zkClient.GetData().ForPath(nodePath);
if ((data == null) || (data.Length == 0))
{
return(null);
}
ByteArrayInputStream bin = new ByteArrayInputStream(data);
DataInputStream din = new DataInputStream(bin);
DelegationKey key = new DelegationKey();
key.ReadFields(din);
return(key);
}
catch (KeeperException.NoNodeException)
{
Log.Error("No node in path [" + nodePath + "]");
}
catch (Exception ex)
{
throw new IOException(ex);
}
return(null);
}
public virtual void TestDelegationKeyEqualAndHash()
{
DelegationKey key1 = new DelegationKey(1111, 2222, Runtime.GetBytesForString
("keyBytes"));
DelegationKey key2 = new DelegationKey(1111, 2222, Runtime.GetBytesForString
("keyBytes"));
DelegationKey key3 = new DelegationKey(3333, 2222, Runtime.GetBytesForString
("keyBytes"));
Assert.Equal(key1, key2);
NUnit.Framework.Assert.IsFalse(key2.Equals(key3));
}
/// <exception cref="System.IO.IOException"/>
private void ProcessKeyAddOrUpdate(byte[] data)
{
ByteArrayInputStream bin = new ByteArrayInputStream(data);
DataInputStream din = new DataInputStream(bin);
DelegationKey key = new DelegationKey();
key.ReadFields(din);
lock (this)
{
allKeys[key.GetKeyId()] = key;
}
}
/// <summary>Renew a delegation token.</summary>
/// <param name="token">the token to renew</param>
/// <param name="renewer">the full principal name of the user doing the renewal</param>
/// <returns>the new expiration time</returns>
/// <exception cref="Org.Apache.Hadoop.Security.Token.SecretManager.InvalidToken">if the token is invalid
/// </exception>
/// <exception cref="Org.Apache.Hadoop.Security.AccessControlException">if the user can't renew token
/// </exception>
/// <exception cref="System.IO.IOException"/>
public virtual long RenewToken(Org.Apache.Hadoop.Security.Token.Token <TokenIdent>
token, string renewer)
{
lock (this)
{
ByteArrayInputStream buf = new ByteArrayInputStream(token.GetIdentifier());
DataInputStream @in = new DataInputStream(buf);
TokenIdent id = CreateIdentifier();
id.ReadFields(@in);
Log.Info("Token renewal for identifier: " + id + "; total currentTokens " + currentTokens
.Count);
long now = Time.Now();
if (id.GetMaxDate() < now)
{
throw new SecretManager.InvalidToken(renewer + " tried to renew an expired token"
);
}
if ((id.GetRenewer() == null) || (id.GetRenewer().ToString().IsEmpty()))
{
throw new AccessControlException(renewer + " tried to renew a token without a renewer"
);
}
if (!id.GetRenewer().ToString().Equals(renewer))
{
throw new AccessControlException(renewer + " tries to renew a token with renewer "
+ id.GetRenewer());
}
DelegationKey key = GetDelegationKey(id.GetMasterKeyId());
if (key == null)
{
throw new SecretManager.InvalidToken("Unable to find master key for keyId=" + id.
GetMasterKeyId() + " from cache. Failed to renew an unexpired token" + " with sequenceNumber="
+ id.GetSequenceNumber());
}
byte[] password = CreatePassword(token.GetIdentifier(), key.GetKey());
if (!Arrays.Equals(password, token.GetPassword()))
{
throw new AccessControlException(renewer + " is trying to renew a token with wrong password"
);
}
long renewTime = Math.Min(id.GetMaxDate(), now + tokenRenewInterval);
string trackingId = GetTrackingIdIfEnabled(id);
AbstractDelegationTokenSecretManager.DelegationTokenInformation info = new AbstractDelegationTokenSecretManager.DelegationTokenInformation
(renewTime, password, trackingId);
if (GetTokenInfo(id) == null)
{
throw new SecretManager.InvalidToken("Renewal request for unknown token");
}
UpdateToken(id, info);
return(renewTime);
}
}
/// <summary>
/// Add a previously used master key to cache (when NN restarts),
/// should be called before activate().
/// </summary>
/// <exception cref="System.IO.IOException"/>
public virtual void AddKey(DelegationKey key)
{
lock (this)
{
if (running)
{
// a safety check
throw new IOException("Can't add delegation key to a running SecretManager.");
}
if (key.GetKeyId() > GetCurrentKeyId())
{
SetCurrentKeyId(key.GetKeyId());
}
allKeys[key.GetKeyId()] = key;
}
}
/// <exception cref="System.IO.IOException"/>
private void AddOrUpdateDelegationKey(DelegationKey key, bool isUpdate)
{
string nodeCreatePath = GetNodePath(ZkDtsmMasterKeyRoot, DelegationKeyPrefix + key
.GetKeyId());
ByteArrayOutputStream os = new ByteArrayOutputStream();
DataOutputStream fsOut = new DataOutputStream(os);
if (Log.IsDebugEnabled())
{
Log.Debug("Storing ZKDTSMDelegationKey_" + key.GetKeyId());
}
key.Write(fsOut);
try
{
if (zkClient.CheckExists().ForPath(nodeCreatePath) != null)
{
zkClient.SetData().ForPath(nodeCreatePath, os.ToByteArray()).SetVersion(-1);
if (!isUpdate)
{
Log.Debug("Key with path [" + nodeCreatePath + "] already exists.. Updating !!");
}
}
else
{
zkClient.Create().WithMode(CreateMode.Persistent).ForPath(nodeCreatePath, os.ToByteArray
());
if (isUpdate)
{
Log.Debug("Updating non existent Key path [" + nodeCreatePath + "].. Adding new !!"
);
}
}
}
catch (KeeperException.NodeExistsException)
{
Log.Debug(nodeCreatePath + " znode already exists !!");
}
catch (Exception ex)
{
throw new IOException(ex);
}
finally
{
os.Close();
}
}
public virtual void TestParallelDelegationTokenCreation()
{
TestDelegationToken.TestDelegationTokenSecretManager dtSecretManager = new TestDelegationToken.TestDelegationTokenSecretManager
(2000, 24 * 60 * 60 * 1000, 7 * 24 * 60 * 60 * 1000, 2000);
try
{
dtSecretManager.StartThreads();
int numThreads = 100;
int numTokensPerThread = 100;
Thread[] issuers = new Thread[numThreads];
for (int i = 0; i < numThreads; i++)
{
issuers[i] = new Daemon(new _T1720540651(this));
issuers[i].Start();
}
for (int i_1 = 0; i_1 < numThreads; i_1++)
{
issuers[i_1].Join();
}
IDictionary <TestDelegationToken.TestDelegationTokenIdentifier, AbstractDelegationTokenSecretManager.DelegationTokenInformation
> tokenCache = dtSecretManager.GetAllTokens();
Assert.Equal(numTokensPerThread * numThreads, tokenCache.Count
);
IEnumerator <TestDelegationToken.TestDelegationTokenIdentifier> iter = tokenCache.
Keys.GetEnumerator();
while (iter.HasNext())
{
TestDelegationToken.TestDelegationTokenIdentifier id = iter.Next();
AbstractDelegationTokenSecretManager.DelegationTokenInformation info = tokenCache
[id];
Assert.True(info != null);
DelegationKey key = dtSecretManager.GetKey(id);
Assert.True(key != null);
byte[] storedPassword = dtSecretManager.RetrievePassword(id);
byte[] password = dtSecretManager.CreatePassword(id, key);
Assert.True(Arrays.Equals(password, storedPassword));
//verify by secret manager api
dtSecretManager.VerifyToken(id, password);
}
}
finally
{
dtSecretManager.StopThreads();
}
}
/// <summary>
/// Update the current master key
/// This is called once by startThreads before tokenRemoverThread is created,
/// and only by tokenRemoverThread afterwards.
/// </summary>
/// <exception cref="System.IO.IOException"/>
private void UpdateCurrentKey()
{
Log.Info("Updating the current master key for generating delegation tokens");
/* Create a new currentKey with an estimated expiry date. */
int newCurrentId;
lock (this)
{
newCurrentId = IncrementCurrentKeyId();
}
DelegationKey newKey = new DelegationKey(newCurrentId, Runtime.CurrentTimeMillis(
) + keyUpdateInterval + tokenMaxLifetime, GenerateSecret());
//Log must be invoked outside the lock on 'this'
LogUpdateMasterKey(newKey);
lock (this)
{
currentKey = newKey;
StoreDelegationKey(currentKey);
}
}
protected internal override DelegationKey GetDelegationKey(int keyId)
{
// First check if its I already have this key
DelegationKey key = allKeys[keyId];
// Then query ZK
if (key == null)
{
try
{
key = GetKeyFromZK(keyId);
if (key != null)
{
allKeys[keyId] = key;
}
}
catch (IOException e)
{
Log.Error("Error retrieving key [" + keyId + "] from ZK", e);
}
}
return(key);
}
// RM
/// <exception cref="System.IO.IOException"/>
protected internal virtual void StoreNewMasterKey(DelegationKey key)
{
return;
}
public virtual byte[] CreatePassword(TestDelegationToken.TestDelegationTokenIdentifier
t, DelegationKey key)
{
return(SecretManager.CreatePassword(t.GetBytes(), key.GetKey()));
}
/// <exception cref="System.IO.IOException"/>
protected internal override void StoreDelegationKey(DelegationKey key)
{
AddOrUpdateDelegationKey(key, false);
}
/// <summary>
/// For subclasses externalizing the storage, for example Zookeeper
/// based implementations
/// </summary>
/// <exception cref="System.IO.IOException"/>
protected internal virtual void UpdateDelegationKey(DelegationKey key)
{
allKeys[key.GetKeyId()] = key;
}
/// <summary>
/// For subclasses externalizing the storage, for example Zookeeper
/// based implementations
/// </summary>
/// <exception cref="System.IO.IOException"/>
protected internal virtual void StoreDelegationKey(DelegationKey key)
{
allKeys[key.GetKeyId()] = key;
StoreNewMasterKey(key);
}
/// <exception cref="System.IO.IOException"/>
protected internal override void StoreNewMasterKey(DelegationKey key)
{
isStoreNewMasterKeyCalled = true;
base.StoreNewMasterKey(key);
}
// HDFS
/// <exception cref="System.IO.IOException"/>
protected internal virtual void LogUpdateMasterKey(DelegationKey key)
{
return;
}
/// <exception cref="System.IO.IOException"/>
protected internal override void UpdateDelegationKey(DelegationKey key)
{
AddOrUpdateDelegationKey(key, true);
}
protected internal override void RemoveStoredMasterKey(DelegationKey key)
{
isRemoveStoredMasterKeyCalled = true;
NUnit.Framework.Assert.IsFalse(key.Equals(allKeys[currentId]));
}
// RM
protected internal virtual void RemoveStoredMasterKey(DelegationKey key)
{
return;
}