public virtual void TestKeyMaterial()
{
byte[] key1 = new byte[] { 1, 2, 3, 4 };
KeyProvider.KeyVersion obj = new KeyProvider.KeyVersion("key1", "key1@1", key1);
Assert.Equal("key1@1", obj.GetVersionName());
Assert.AssertArrayEquals(new byte[] { 1, 2, 3, 4 }, obj.GetMaterial());
}
public virtual void TestDeleteKey()
{
KeyProvider.KeyVersion mockKey = Org.Mockito.Mockito.Mock <KeyProvider.KeyVersion>
();
KeyProvider mockProv = Org.Mockito.Mockito.Mock <KeyProvider>();
Org.Mockito.Mockito.When(mockProv.GetCurrentKey(Org.Mockito.Mockito.Eq("k1"))).ThenReturn
(mockKey);
Org.Mockito.Mockito.When(mockProv.GetKeyVersion(Org.Mockito.Mockito.Eq("k1@0"))).
ThenReturn(mockKey);
Org.Mockito.Mockito.When(mockProv.GetMetadata(Org.Mockito.Mockito.Eq("k1"))).ThenReturn
(new KMSClientProvider.KMSMetadata("c", 0, "l", null, new DateTime(), 1));
Org.Mockito.Mockito.When(mockProv.GetConf()).ThenReturn(new Configuration());
KeyProvider cache = new CachingKeyProvider(mockProv, 100, 100);
Assert.Equal(mockKey, cache.GetCurrentKey("k1"));
Org.Mockito.Mockito.Verify(mockProv, Org.Mockito.Mockito.Times(1)).GetCurrentKey(
Org.Mockito.Mockito.Eq("k1"));
Assert.Equal(mockKey, cache.GetKeyVersion("k1@0"));
Org.Mockito.Mockito.Verify(mockProv, Org.Mockito.Mockito.Times(1)).GetKeyVersion(
Org.Mockito.Mockito.Eq("k1@0"));
cache.DeleteKey("k1");
// asserting the cache is purged
Assert.Equal(mockKey, cache.GetCurrentKey("k1"));
Org.Mockito.Mockito.Verify(mockProv, Org.Mockito.Mockito.Times(2)).GetCurrentKey(
Org.Mockito.Mockito.Eq("k1"));
Assert.Equal(mockKey, cache.GetKeyVersion("k1@0"));
Org.Mockito.Mockito.Verify(mockProv, Org.Mockito.Mockito.Times(2)).GetKeyVersion(
Org.Mockito.Mockito.Eq("k1@0"));
}
public virtual void TestKeyVersion()
{
KeyProvider.KeyVersion mockKey = Org.Mockito.Mockito.Mock <KeyProvider.KeyVersion>
();
KeyProvider mockProv = Org.Mockito.Mockito.Mock <KeyProvider>();
Org.Mockito.Mockito.When(mockProv.GetKeyVersion(Org.Mockito.Mockito.Eq("k1@0"))).
ThenReturn(mockKey);
Org.Mockito.Mockito.When(mockProv.GetKeyVersion(Org.Mockito.Mockito.Eq("k2@0"))).
ThenReturn(null);
Org.Mockito.Mockito.When(mockProv.GetConf()).ThenReturn(new Configuration());
KeyProvider cache = new CachingKeyProvider(mockProv, 100, 100);
// asserting caching
Assert.Equal(mockKey, cache.GetKeyVersion("k1@0"));
Org.Mockito.Mockito.Verify(mockProv, Org.Mockito.Mockito.Times(1)).GetKeyVersion(
Org.Mockito.Mockito.Eq("k1@0"));
Assert.Equal(mockKey, cache.GetKeyVersion("k1@0"));
Org.Mockito.Mockito.Verify(mockProv, Org.Mockito.Mockito.Times(1)).GetKeyVersion(
Org.Mockito.Mockito.Eq("k1@0"));
Thread.Sleep(200);
Assert.Equal(mockKey, cache.GetKeyVersion("k1@0"));
Org.Mockito.Mockito.Verify(mockProv, Org.Mockito.Mockito.Times(2)).GetKeyVersion(
Org.Mockito.Mockito.Eq("k1@0"));
// asserting no caching when key is not known
cache = new CachingKeyProvider(mockProv, 100, 100);
Assert.Equal(null, cache.GetKeyVersion("k2@0"));
Org.Mockito.Mockito.Verify(mockProv, Org.Mockito.Mockito.Times(1)).GetKeyVersion(
Org.Mockito.Mockito.Eq("k2@0"));
Assert.Equal(null, cache.GetKeyVersion("k2@0"));
Org.Mockito.Mockito.Verify(mockProv, Org.Mockito.Mockito.Times(2)).GetKeyVersion(
Org.Mockito.Mockito.Eq("k2@0"));
}
/// <exception cref="System.IO.IOException"/>
public override IList <KeyProvider.KeyVersion> GetKeyVersions(string name)
{
readLock.Lock();
try
{
IList <KeyProvider.KeyVersion> list = new AList <KeyProvider.KeyVersion>();
KeyProvider.Metadata km = GetMetadata(name);
if (km != null)
{
int latestVersion = km.GetVersions();
KeyProvider.KeyVersion v = null;
string versionName = null;
for (int i = 0; i < latestVersion; i++)
{
versionName = BuildVersionName(name, i);
v = GetKeyVersion(versionName);
if (v != null)
{
list.AddItem(v);
}
}
}
return(list);
}
finally
{
readLock.Unlock();
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="GeneralSecurityException"/>
public virtual KeyProvider.KeyVersion DecryptEncryptedKey(KeyProviderCryptoExtension.EncryptedKeyVersion
encryptedKeyVersion)
{
// Fetch the encryption key material
string encryptionKeyVersionName = encryptedKeyVersion.GetEncryptionKeyVersionName
();
KeyProvider.KeyVersion encryptionKey = keyProvider.GetKeyVersion(encryptionKeyVersionName
);
Preconditions.CheckNotNull(encryptionKey, "KeyVersion name '%s' does not exist",
encryptionKeyVersionName);
Preconditions.CheckArgument(encryptedKeyVersion.GetEncryptedKeyVersion().GetVersionName
().Equals(KeyProviderCryptoExtension.Eek), "encryptedKey version name must be '%s', is '%s'"
, KeyProviderCryptoExtension.Eek, encryptedKeyVersion.GetEncryptedKeyVersion().GetVersionName
());
// Encryption key IV is determined from encrypted key's IV
byte[] encryptionIV = KeyProviderCryptoExtension.EncryptedKeyVersion.DeriveIV(encryptedKeyVersion
.GetEncryptedKeyIv());
CryptoCodec cc = CryptoCodec.GetInstance(keyProvider.GetConf());
Decryptor decryptor = cc.CreateDecryptor();
decryptor.Init(encryptionKey.GetMaterial(), encryptionIV);
KeyProvider.KeyVersion encryptedKV = encryptedKeyVersion.GetEncryptedKeyVersion();
int keyLen = encryptedKV.GetMaterial().Length;
ByteBuffer bbIn = ByteBuffer.AllocateDirect(keyLen);
ByteBuffer bbOut = ByteBuffer.AllocateDirect(keyLen);
bbIn.Put(encryptedKV.GetMaterial());
bbIn.Flip();
decryptor.Decrypt(bbIn, bbOut);
bbOut.Flip();
byte[] decryptedKey = new byte[keyLen];
bbOut.Get(decryptedKey);
return(new KeyProvider.KeyVersion(encryptionKey.GetName(), Ek, decryptedKey));
}
/// <exception cref="NoSuchAlgorithmException"/>
/// <exception cref="System.IO.IOException"/>
public override KeyProvider.KeyVersion RollNewVersion(string name)
{
KeyProvider.KeyVersion key = GetKeyProvider().RollNewVersion(name);
GetExtension().currentKeyCache.Invalidate(name);
GetExtension().keyMetadataCache.Invalidate(name);
return(key);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="GeneralSecurityException"/>
public virtual KeyProviderCryptoExtension.EncryptedKeyVersion GenerateEncryptedKey
(string encryptionKeyName)
{
// Fetch the encryption key
KeyProvider.KeyVersion encryptionKey = keyProvider.GetCurrentKey(encryptionKeyName
);
Preconditions.CheckNotNull(encryptionKey, "No KeyVersion exists for key '%s' ", encryptionKeyName
);
// Generate random bytes for new key and IV
CryptoCodec cc = CryptoCodec.GetInstance(keyProvider.GetConf());
byte[] newKey = new byte[encryptionKey.GetMaterial().Length];
cc.GenerateSecureRandom(newKey);
byte[] iv = new byte[cc.GetCipherSuite().GetAlgorithmBlockSize()];
cc.GenerateSecureRandom(iv);
// Encryption key IV is derived from new key's IV
byte[] encryptionIV = KeyProviderCryptoExtension.EncryptedKeyVersion.DeriveIV(iv);
Encryptor encryptor = cc.CreateEncryptor();
encryptor.Init(encryptionKey.GetMaterial(), encryptionIV);
int keyLen = newKey.Length;
ByteBuffer bbIn = ByteBuffer.AllocateDirect(keyLen);
ByteBuffer bbOut = ByteBuffer.AllocateDirect(keyLen);
bbIn.Put(newKey);
bbIn.Flip();
encryptor.Encrypt(bbIn, bbOut);
bbOut.Flip();
byte[] encryptedKey = new byte[keyLen];
bbOut.Get(encryptedKey);
return(new KeyProviderCryptoExtension.EncryptedKeyVersion(encryptionKeyName, encryptionKey
.GetVersionName(), iv, new KeyProvider.KeyVersion(encryptionKey.GetName(), Eek,
encryptedKey)));
}
/// <summary>Create a new EncryptedKeyVersion.</summary>
/// <param name="keyName">
/// Name of the encryption key used to
/// encrypt the encrypted key.
/// </param>
/// <param name="encryptionKeyVersionName">
/// Version name of the encryption key used
/// to encrypt the encrypted key.
/// </param>
/// <param name="encryptedKeyIv">
/// Initialization vector of the encrypted
/// key. The IV of the encryption key used to
/// encrypt the encrypted key is derived from
/// this IV.
/// </param>
/// <param name="encryptedKeyVersion">The encrypted encryption key version.</param>
protected internal EncryptedKeyVersion(string keyName, string encryptionKeyVersionName
, byte[] encryptedKeyIv, KeyProvider.KeyVersion encryptedKeyVersion)
{
this.encryptionKeyName = keyName;
this.encryptionKeyVersionName = encryptionKeyVersionName;
this.encryptedKeyIv = encryptedKeyIv;
this.encryptedKeyVersion = encryptedKeyVersion;
}
/// <summary>
/// Factory method to create a new EncryptedKeyVersion that can then be
/// passed into
/// <see cref="#decryptEncryptedKey"/>
/// . Note that the fields of the
/// returned EncryptedKeyVersion will only partially be populated; it is not
/// necessarily suitable for operations besides decryption.
/// </summary>
/// <param name="keyName">
/// Key name of the encryption key use to encrypt the
/// encrypted key.
/// </param>
/// <param name="encryptionKeyVersionName">
/// Version name of the encryption key used
/// to encrypt the encrypted key.
/// </param>
/// <param name="encryptedKeyIv">
/// Initialization vector of the encrypted
/// key. The IV of the encryption key used to
/// encrypt the encrypted key is derived from
/// this IV.
/// </param>
/// <param name="encryptedKeyMaterial">Key material of the encrypted key.</param>
/// <returns>EncryptedKeyVersion suitable for decryption.</returns>
public static KeyProviderCryptoExtension.EncryptedKeyVersion CreateForDecryption(
string keyName, string encryptionKeyVersionName, byte[] encryptedKeyIv, byte[] encryptedKeyMaterial
)
{
KeyProvider.KeyVersion encryptedKeyVersion = new KeyProvider.KeyVersion(null, Eek
, encryptedKeyMaterial);
return(new KeyProviderCryptoExtension.EncryptedKeyVersion(keyName, encryptionKeyVersionName
, encryptedKeyIv, encryptedKeyVersion));
}
public static void Setup()
{
conf = new Configuration();
kp = new UserProvider.Factory().CreateProvider(new URI("user:///"), conf);
kpExt = KeyProviderCryptoExtension.CreateKeyProviderCryptoExtension(kp);
options = new KeyProvider.Options(conf);
options.SetCipher(Cipher);
options.SetBitLength(128);
encryptionKey = kp.CreateKey(EncryptionKeyName, SecureRandom.GetSeed(16), options
);
}
public virtual void TestJksProviderWithKeytoolKeys()
{
Configuration conf = new Configuration();
string keystoreDirAbsolutePath = conf.GetResource("hdfs7067.keystore").AbsolutePath;
string ourUrl = JavaKeyStoreProvider.SchemeName + "://file@/" + keystoreDirAbsolutePath;
conf.Set(KeyProviderFactory.KeyProviderPath, ourUrl);
KeyProvider provider = KeyProviderFactory.GetProviders(conf)[0];
// Sanity check that we are using the right keystore
KeyProvider.KeyVersion keyVersion = provider.GetKeyVersion("testkey5@0");
try
{
KeyProvider.KeyVersion keyVersionWrongKeyNameFormat = provider.GetKeyVersion("testkey2"
);
NUnit.Framework.Assert.Fail("should have thrown an exception");
}
catch (IOException e)
{
// No version in key path testkey2/
GenericTestUtils.AssertExceptionContains("No version in key path", e);
}
try
{
KeyProvider.KeyVersion keyVersionCurrentKeyNotWrongKeyNameFormat = provider.GetCurrentKey
("testkey5@0");
NUnit.Framework.Assert.Fail("should have thrown an exception getting testkey5@0");
}
catch (IOException e)
{
// javax.crypto.spec.SecretKeySpec cannot be cast to
// org.apache.hadoop.crypto.key.JavaKeyStoreProvider$KeyMetadata
GenericTestUtils.AssertExceptionContains("other non-Hadoop method", e);
}
try
{
KeyProvider.KeyVersion keyVersionCurrentKeyNotReally = provider.GetCurrentKey("testkey2"
);
NUnit.Framework.Assert.Fail("should have thrown an exception getting testkey2");
}
catch (IOException e)
{
// javax.crypto.spec.SecretKeySpec cannot be cast to
// org.apache.hadoop.crypto.key.JavaKeyStoreProvider$KeyMetadata
GenericTestUtils.AssertExceptionContains("other non-Hadoop method", e);
}
}
public virtual void TestGenerateEncryptedKey()
{
// Generate a new EEK and check it
KeyProviderCryptoExtension.EncryptedKeyVersion ek1 = kpExt.GenerateEncryptedKey(encryptionKey
.GetName());
Assert.Equal("Version name of EEK should be EEK", KeyProviderCryptoExtension
.Eek, ek1.GetEncryptedKeyVersion().GetVersionName());
Assert.Equal("Name of EEK should be encryption key name", EncryptionKeyName
, ek1.GetEncryptionKeyName());
NUnit.Framework.Assert.IsNotNull("Expected encrypted key material", ek1.GetEncryptedKeyVersion
().GetMaterial());
Assert.Equal("Length of encryption key material and EEK material should "
+ "be the same", encryptionKey.GetMaterial().Length, ek1.GetEncryptedKeyVersion
().GetMaterial().Length);
// Decrypt EEK into an EK and check it
KeyProvider.KeyVersion k1 = kpExt.DecryptEncryptedKey(ek1);
Assert.Equal(KeyProviderCryptoExtension.Ek, k1.GetVersionName(
));
Assert.Equal(encryptionKey.GetMaterial().Length, k1.GetMaterial
().Length);
if (Arrays.Equals(k1.GetMaterial(), encryptionKey.GetMaterial()))
{
NUnit.Framework.Assert.Fail("Encrypted key material should not equal encryption key material"
);
}
if (Arrays.Equals(ek1.GetEncryptedKeyVersion().GetMaterial(), encryptionKey.GetMaterial
()))
{
NUnit.Framework.Assert.Fail("Encrypted key material should not equal decrypted key material"
);
}
// Decrypt it again and it should be the same
KeyProvider.KeyVersion k1a = kpExt.DecryptEncryptedKey(ek1);
Assert.AssertArrayEquals(k1.GetMaterial(), k1a.GetMaterial());
// Generate another EEK and make sure it's different from the first
KeyProviderCryptoExtension.EncryptedKeyVersion ek2 = kpExt.GenerateEncryptedKey(encryptionKey
.GetName());
KeyProvider.KeyVersion k2 = kpExt.DecryptEncryptedKey(ek2);
if (Arrays.Equals(k1.GetMaterial(), k2.GetMaterial()))
{
NUnit.Framework.Assert.Fail("Generated EEKs should have different material!");
}
if (Arrays.Equals(ek1.GetEncryptedKeyIv(), ek2.GetEncryptedKeyIv()))
{
NUnit.Framework.Assert.Fail("Generated EEKs should have different IVs!");
}
}
/// <exception cref="System.IO.IOException"/>
public override IList <KeyProvider.KeyVersion> GetKeyVersions(string name)
{
lock (this)
{
IList <KeyProvider.KeyVersion> list = new AList <KeyProvider.KeyVersion>();
KeyProvider.Metadata km = GetMetadata(name);
if (km != null)
{
int latestVersion = km.GetVersions();
for (int i = 0; i < latestVersion; i++)
{
KeyProvider.KeyVersion v = GetKeyVersion(BuildVersionName(name, i));
if (v != null)
{
list.AddItem(v);
}
}
}
return(list);
}
}
public virtual void TestRollNewVersion()
{
KeyProvider.KeyVersion mockKey = Org.Mockito.Mockito.Mock <KeyProvider.KeyVersion>
();
KeyProvider mockProv = Org.Mockito.Mockito.Mock <KeyProvider>();
Org.Mockito.Mockito.When(mockProv.GetCurrentKey(Org.Mockito.Mockito.Eq("k1"))).ThenReturn
(mockKey);
Org.Mockito.Mockito.When(mockProv.GetConf()).ThenReturn(new Configuration());
KeyProvider cache = new CachingKeyProvider(mockProv, 100, 100);
Assert.Equal(mockKey, cache.GetCurrentKey("k1"));
Org.Mockito.Mockito.Verify(mockProv, Org.Mockito.Mockito.Times(1)).GetCurrentKey(
Org.Mockito.Mockito.Eq("k1"));
cache.RollNewVersion("k1");
// asserting the cache is purged
Assert.Equal(mockKey, cache.GetCurrentKey("k1"));
Org.Mockito.Mockito.Verify(mockProv, Org.Mockito.Mockito.Times(2)).GetCurrentKey(
Org.Mockito.Mockito.Eq("k1"));
cache.RollNewVersion("k1", new byte[0]);
Assert.Equal(mockKey, cache.GetCurrentKey("k1"));
Org.Mockito.Mockito.Verify(mockProv, Org.Mockito.Mockito.Times(3)).GetCurrentKey(
Org.Mockito.Mockito.Eq("k1"));
}
public virtual void TestEncryptDecrypt()
{
// Get an EEK
KeyProviderCryptoExtension.EncryptedKeyVersion eek = kpExt.GenerateEncryptedKey(encryptionKey
.GetName());
byte[] encryptedKeyIv = eek.GetEncryptedKeyIv();
byte[] encryptedKeyMaterial = eek.GetEncryptedKeyVersion().GetMaterial();
// Decrypt it manually
Cipher cipher = Cipher.GetInstance("AES/CTR/NoPadding");
cipher.Init(Cipher.DecryptMode, new SecretKeySpec(encryptionKey.GetMaterial
(), "AES"), new IvParameterSpec(KeyProviderCryptoExtension.EncryptedKeyVersion.DeriveIV
(encryptedKeyIv)));
byte[] manualMaterial = cipher.DoFinal(encryptedKeyMaterial);
// Test the createForDecryption factory method
KeyProviderCryptoExtension.EncryptedKeyVersion eek2 = KeyProviderCryptoExtension.EncryptedKeyVersion
.CreateForDecryption(eek.GetEncryptionKeyName(), eek.GetEncryptionKeyVersionName
(), eek.GetEncryptedKeyIv(), eek.GetEncryptedKeyVersion().GetMaterial());
// Decrypt it with the API
KeyProvider.KeyVersion decryptedKey = kpExt.DecryptEncryptedKey(eek2);
byte[] apiMaterial = decryptedKey.GetMaterial();
Assert.AssertArrayEquals("Wrong key material from decryptEncryptedKey", manualMaterial
, apiMaterial);
}
