/// <summary> /// This method is intended to be used for recovering persisted delegation /// tokens /// This method must be called before this secret manager is activated (before /// startThreads() is called) /// </summary> /// <param name="identifier">identifier read from persistent storage</param> /// <param name="renewDate">token renew time</param> /// <exception cref="System.IO.IOException"/> public virtual void AddPersistedDelegationToken(TokenIdent identifier, long renewDate ) { lock (this) { if (running) { // a safety check throw new IOException("Can't add persisted delegation token to a running SecretManager." ); } int keyId = identifier.GetMasterKeyId(); DelegationKey dKey = allKeys[keyId]; if (dKey == null) { Log.Warn("No KEY found for persisted identifier " + identifier.ToString()); return; } byte[] password = CreatePassword(identifier.GetBytes(), dKey.GetKey()); if (identifier.GetSequenceNumber() > GetDelegationTokenSeqNum()) { SetDelegationTokenSeqNum(identifier.GetSequenceNumber()); } if (GetTokenInfo(identifier) == null) { currentTokens[identifier] = new AbstractDelegationTokenSecretManager.DelegationTokenInformation (renewDate, password, GetTrackingIdIfEnabled(identifier)); } else { throw new IOException("Same delegation token being added twice."); } } }
protected internal override void RemoveStoredMasterKey(DelegationKey key) { string nodeRemovePath = GetNodePath(ZkDtsmMasterKeyRoot, DelegationKeyPrefix + key .GetKeyId()); if (Log.IsDebugEnabled()) { Log.Debug("Removing ZKDTSMDelegationKey_" + key.GetKeyId()); } try { if (zkClient.CheckExists().ForPath(nodeRemovePath) != null) { while (zkClient.CheckExists().ForPath(nodeRemovePath) != null) { zkClient.Delete().Guaranteed().ForPath(nodeRemovePath); } } else { Log.Debug("Attempted to delete a non-existing znode " + nodeRemovePath); } } catch (Exception) { Log.Debug(nodeRemovePath + " znode could not be removed!!"); } }
/// <exception cref="System.IO.IOException"/> private DelegationKey GetKeyFromZK(int keyId) { string nodePath = GetNodePath(ZkDtsmMasterKeyRoot, DelegationKeyPrefix + keyId); try { byte[] data = zkClient.GetData().ForPath(nodePath); if ((data == null) || (data.Length == 0)) { return(null); } ByteArrayInputStream bin = new ByteArrayInputStream(data); DataInputStream din = new DataInputStream(bin); DelegationKey key = new DelegationKey(); key.ReadFields(din); return(key); } catch (KeeperException.NoNodeException) { Log.Error("No node in path [" + nodePath + "]"); } catch (Exception ex) { throw new IOException(ex); } return(null); }
public virtual void TestDelegationKeyEqualAndHash() { DelegationKey key1 = new DelegationKey(1111, 2222, Runtime.GetBytesForString ("keyBytes")); DelegationKey key2 = new DelegationKey(1111, 2222, Runtime.GetBytesForString ("keyBytes")); DelegationKey key3 = new DelegationKey(3333, 2222, Runtime.GetBytesForString ("keyBytes")); Assert.Equal(key1, key2); NUnit.Framework.Assert.IsFalse(key2.Equals(key3)); }
/// <exception cref="System.IO.IOException"/> private void ProcessKeyAddOrUpdate(byte[] data) { ByteArrayInputStream bin = new ByteArrayInputStream(data); DataInputStream din = new DataInputStream(bin); DelegationKey key = new DelegationKey(); key.ReadFields(din); lock (this) { allKeys[key.GetKeyId()] = key; } }
/// <summary>Renew a delegation token.</summary> /// <param name="token">the token to renew</param> /// <param name="renewer">the full principal name of the user doing the renewal</param> /// <returns>the new expiration time</returns> /// <exception cref="Org.Apache.Hadoop.Security.Token.SecretManager.InvalidToken">if the token is invalid /// </exception> /// <exception cref="Org.Apache.Hadoop.Security.AccessControlException">if the user can't renew token /// </exception> /// <exception cref="System.IO.IOException"/> public virtual long RenewToken(Org.Apache.Hadoop.Security.Token.Token <TokenIdent> token, string renewer) { lock (this) { ByteArrayInputStream buf = new ByteArrayInputStream(token.GetIdentifier()); DataInputStream @in = new DataInputStream(buf); TokenIdent id = CreateIdentifier(); id.ReadFields(@in); Log.Info("Token renewal for identifier: " + id + "; total currentTokens " + currentTokens .Count); long now = Time.Now(); if (id.GetMaxDate() < now) { throw new SecretManager.InvalidToken(renewer + " tried to renew an expired token" ); } if ((id.GetRenewer() == null) || (id.GetRenewer().ToString().IsEmpty())) { throw new AccessControlException(renewer + " tried to renew a token without a renewer" ); } if (!id.GetRenewer().ToString().Equals(renewer)) { throw new AccessControlException(renewer + " tries to renew a token with renewer " + id.GetRenewer()); } DelegationKey key = GetDelegationKey(id.GetMasterKeyId()); if (key == null) { throw new SecretManager.InvalidToken("Unable to find master key for keyId=" + id. GetMasterKeyId() + " from cache. Failed to renew an unexpired token" + " with sequenceNumber=" + id.GetSequenceNumber()); } byte[] password = CreatePassword(token.GetIdentifier(), key.GetKey()); if (!Arrays.Equals(password, token.GetPassword())) { throw new AccessControlException(renewer + " is trying to renew a token with wrong password" ); } long renewTime = Math.Min(id.GetMaxDate(), now + tokenRenewInterval); string trackingId = GetTrackingIdIfEnabled(id); AbstractDelegationTokenSecretManager.DelegationTokenInformation info = new AbstractDelegationTokenSecretManager.DelegationTokenInformation (renewTime, password, trackingId); if (GetTokenInfo(id) == null) { throw new SecretManager.InvalidToken("Renewal request for unknown token"); } UpdateToken(id, info); return(renewTime); } }
/// <summary> /// Add a previously used master key to cache (when NN restarts), /// should be called before activate(). /// </summary> /// <exception cref="System.IO.IOException"/> public virtual void AddKey(DelegationKey key) { lock (this) { if (running) { // a safety check throw new IOException("Can't add delegation key to a running SecretManager."); } if (key.GetKeyId() > GetCurrentKeyId()) { SetCurrentKeyId(key.GetKeyId()); } allKeys[key.GetKeyId()] = key; } }
/// <exception cref="System.IO.IOException"/> private void AddOrUpdateDelegationKey(DelegationKey key, bool isUpdate) { string nodeCreatePath = GetNodePath(ZkDtsmMasterKeyRoot, DelegationKeyPrefix + key .GetKeyId()); ByteArrayOutputStream os = new ByteArrayOutputStream(); DataOutputStream fsOut = new DataOutputStream(os); if (Log.IsDebugEnabled()) { Log.Debug("Storing ZKDTSMDelegationKey_" + key.GetKeyId()); } key.Write(fsOut); try { if (zkClient.CheckExists().ForPath(nodeCreatePath) != null) { zkClient.SetData().ForPath(nodeCreatePath, os.ToByteArray()).SetVersion(-1); if (!isUpdate) { Log.Debug("Key with path [" + nodeCreatePath + "] already exists.. Updating !!"); } } else { zkClient.Create().WithMode(CreateMode.Persistent).ForPath(nodeCreatePath, os.ToByteArray ()); if (isUpdate) { Log.Debug("Updating non existent Key path [" + nodeCreatePath + "].. Adding new !!" ); } } } catch (KeeperException.NodeExistsException) { Log.Debug(nodeCreatePath + " znode already exists !!"); } catch (Exception ex) { throw new IOException(ex); } finally { os.Close(); } }
public virtual void TestParallelDelegationTokenCreation() { TestDelegationToken.TestDelegationTokenSecretManager dtSecretManager = new TestDelegationToken.TestDelegationTokenSecretManager (2000, 24 * 60 * 60 * 1000, 7 * 24 * 60 * 60 * 1000, 2000); try { dtSecretManager.StartThreads(); int numThreads = 100; int numTokensPerThread = 100; Thread[] issuers = new Thread[numThreads]; for (int i = 0; i < numThreads; i++) { issuers[i] = new Daemon(new _T1720540651(this)); issuers[i].Start(); } for (int i_1 = 0; i_1 < numThreads; i_1++) { issuers[i_1].Join(); } IDictionary <TestDelegationToken.TestDelegationTokenIdentifier, AbstractDelegationTokenSecretManager.DelegationTokenInformation > tokenCache = dtSecretManager.GetAllTokens(); Assert.Equal(numTokensPerThread * numThreads, tokenCache.Count ); IEnumerator <TestDelegationToken.TestDelegationTokenIdentifier> iter = tokenCache. Keys.GetEnumerator(); while (iter.HasNext()) { TestDelegationToken.TestDelegationTokenIdentifier id = iter.Next(); AbstractDelegationTokenSecretManager.DelegationTokenInformation info = tokenCache [id]; Assert.True(info != null); DelegationKey key = dtSecretManager.GetKey(id); Assert.True(key != null); byte[] storedPassword = dtSecretManager.RetrievePassword(id); byte[] password = dtSecretManager.CreatePassword(id, key); Assert.True(Arrays.Equals(password, storedPassword)); //verify by secret manager api dtSecretManager.VerifyToken(id, password); } } finally { dtSecretManager.StopThreads(); } }
/// <summary> /// Update the current master key /// This is called once by startThreads before tokenRemoverThread is created, /// and only by tokenRemoverThread afterwards. /// </summary> /// <exception cref="System.IO.IOException"/> private void UpdateCurrentKey() { Log.Info("Updating the current master key for generating delegation tokens"); /* Create a new currentKey with an estimated expiry date. */ int newCurrentId; lock (this) { newCurrentId = IncrementCurrentKeyId(); } DelegationKey newKey = new DelegationKey(newCurrentId, Runtime.CurrentTimeMillis( ) + keyUpdateInterval + tokenMaxLifetime, GenerateSecret()); //Log must be invoked outside the lock on 'this' LogUpdateMasterKey(newKey); lock (this) { currentKey = newKey; StoreDelegationKey(currentKey); } }
protected internal override DelegationKey GetDelegationKey(int keyId) { // First check if its I already have this key DelegationKey key = allKeys[keyId]; // Then query ZK if (key == null) { try { key = GetKeyFromZK(keyId); if (key != null) { allKeys[keyId] = key; } } catch (IOException e) { Log.Error("Error retrieving key [" + keyId + "] from ZK", e); } } return(key); }
// RM /// <exception cref="System.IO.IOException"/> protected internal virtual void StoreNewMasterKey(DelegationKey key) { return; }
public virtual byte[] CreatePassword(TestDelegationToken.TestDelegationTokenIdentifier t, DelegationKey key) { return(SecretManager.CreatePassword(t.GetBytes(), key.GetKey())); }
/// <exception cref="System.IO.IOException"/> protected internal override void StoreDelegationKey(DelegationKey key) { AddOrUpdateDelegationKey(key, false); }
/// <summary> /// For subclasses externalizing the storage, for example Zookeeper /// based implementations /// </summary> /// <exception cref="System.IO.IOException"/> protected internal virtual void UpdateDelegationKey(DelegationKey key) { allKeys[key.GetKeyId()] = key; }
/// <summary> /// For subclasses externalizing the storage, for example Zookeeper /// based implementations /// </summary> /// <exception cref="System.IO.IOException"/> protected internal virtual void StoreDelegationKey(DelegationKey key) { allKeys[key.GetKeyId()] = key; StoreNewMasterKey(key); }
/// <exception cref="System.IO.IOException"/> protected internal override void StoreNewMasterKey(DelegationKey key) { isStoreNewMasterKeyCalled = true; base.StoreNewMasterKey(key); }
// HDFS /// <exception cref="System.IO.IOException"/> protected internal virtual void LogUpdateMasterKey(DelegationKey key) { return; }
/// <exception cref="System.IO.IOException"/> protected internal override void UpdateDelegationKey(DelegationKey key) { AddOrUpdateDelegationKey(key, true); }
protected internal override void RemoveStoredMasterKey(DelegationKey key) { isRemoveStoredMasterKeyCalled = true; NUnit.Framework.Assert.IsFalse(key.Equals(allKeys[currentId])); }
// RM protected internal virtual void RemoveStoredMasterKey(DelegationKey key) { return; }