/// <exception cref="System.IO.IOException"/>
public CryptoFSDataOutputStream(FSDataOutputStream @out, CryptoCodec codec, byte[]
key, byte[] iv)
: base(new CryptoOutputStream(@out, codec, key, iv, @out.GetPos()), null, @out.GetPos
())
{
this.fsOut = @out;
}
/// <summary>Wraps a given FSDataInputStream with a CryptoInputStream.</summary>
/// <remarks>
/// Wraps a given FSDataInputStream with a CryptoInputStream. The size of the
/// data buffer required for the stream is specified by the
/// "mapreduce.job.encrypted-intermediate-data.buffer.kb" Job configuration
/// variable.
/// </remarks>
/// <param name="conf"/>
/// <param name="in"/>
/// <returns>FSDataInputStream</returns>
/// <exception cref="System.IO.IOException"/>
public static FSDataInputStream WrapIfNecessary(Configuration conf, FSDataInputStream
@in)
{
if (IsEncryptedSpillEnabled(conf))
{
CryptoCodec cryptoCodec = CryptoCodec.GetInstance(conf);
int bufferSize = GetBufferSize(conf);
// Not going to be used... but still has to be read...
// Since the O/P stream always writes it..
IOUtils.ReadFully(@in, new byte[8], 0, 8);
byte[] iv = new byte[cryptoCodec.GetCipherSuite().GetAlgorithmBlockSize()];
IOUtils.ReadFully(@in, iv, 0, cryptoCodec.GetCipherSuite().GetAlgorithmBlockSize(
));
if (Log.IsDebugEnabled())
{
Log.Debug("IV read from Stream [" + Base64.EncodeBase64URLSafeString(iv) + "]");
}
return(new CryptoFSDataInputStream(@in, cryptoCodec, bufferSize, GetEncryptionKey
(), iv));
}
else
{
return(@in);
}
}
/// <summary>Wraps a given InputStream with a CryptoInputStream.</summary>
/// <remarks>
/// Wraps a given InputStream with a CryptoInputStream. The size of the data
/// buffer required for the stream is specified by the
/// "mapreduce.job.encrypted-intermediate-data.buffer.kb" Job configuration
/// variable.
/// If the value of 'length' is > -1, The InputStream is additionally
/// wrapped in a LimitInputStream. CryptoStreams are late buffering in nature.
/// This means they will always try to read ahead if they can. The
/// LimitInputStream will ensure that the CryptoStream does not read past the
/// provided length from the given Input Stream.
/// </remarks>
/// <param name="conf"/>
/// <param name="in"/>
/// <param name="length"/>
/// <returns>InputStream</returns>
/// <exception cref="System.IO.IOException"/>
public static InputStream WrapIfNecessary(Configuration conf, InputStream @in, long
length)
{
if (IsEncryptedSpillEnabled(conf))
{
int bufferSize = GetBufferSize(conf);
if (length > -1)
{
@in = new LimitInputStream(@in, length);
}
byte[] offsetArray = new byte[8];
IOUtils.ReadFully(@in, offsetArray, 0, 8);
long offset = ByteBuffer.Wrap(offsetArray).GetLong();
CryptoCodec cryptoCodec = CryptoCodec.GetInstance(conf);
byte[] iv = new byte[cryptoCodec.GetCipherSuite().GetAlgorithmBlockSize()];
IOUtils.ReadFully(@in, iv, 0, cryptoCodec.GetCipherSuite().GetAlgorithmBlockSize(
));
if (Log.IsDebugEnabled())
{
Log.Debug("IV read from [" + Base64.EncodeBase64URLSafeString(iv) + "]");
}
return(new CryptoInputStream(@in, cryptoCodec, bufferSize, GetEncryptionKey(), iv
, offset + CryptoPadding(conf)));
}
else
{
return(@in);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="GeneralSecurityException"/>
public virtual KeyProvider.KeyVersion DecryptEncryptedKey(KeyProviderCryptoExtension.EncryptedKeyVersion
encryptedKeyVersion)
{
// Fetch the encryption key material
string encryptionKeyVersionName = encryptedKeyVersion.GetEncryptionKeyVersionName
();
KeyProvider.KeyVersion encryptionKey = keyProvider.GetKeyVersion(encryptionKeyVersionName
);
Preconditions.CheckNotNull(encryptionKey, "KeyVersion name '%s' does not exist",
encryptionKeyVersionName);
Preconditions.CheckArgument(encryptedKeyVersion.GetEncryptedKeyVersion().GetVersionName
().Equals(KeyProviderCryptoExtension.Eek), "encryptedKey version name must be '%s', is '%s'"
, KeyProviderCryptoExtension.Eek, encryptedKeyVersion.GetEncryptedKeyVersion().GetVersionName
());
// Encryption key IV is determined from encrypted key's IV
byte[] encryptionIV = KeyProviderCryptoExtension.EncryptedKeyVersion.DeriveIV(encryptedKeyVersion
.GetEncryptedKeyIv());
CryptoCodec cc = CryptoCodec.GetInstance(keyProvider.GetConf());
Decryptor decryptor = cc.CreateDecryptor();
decryptor.Init(encryptionKey.GetMaterial(), encryptionIV);
KeyProvider.KeyVersion encryptedKV = encryptedKeyVersion.GetEncryptedKeyVersion();
int keyLen = encryptedKV.GetMaterial().Length;
ByteBuffer bbIn = ByteBuffer.AllocateDirect(keyLen);
ByteBuffer bbOut = ByteBuffer.AllocateDirect(keyLen);
bbIn.Put(encryptedKV.GetMaterial());
bbIn.Flip();
decryptor.Decrypt(bbIn, bbOut);
bbOut.Flip();
byte[] decryptedKey = new byte[keyLen];
bbOut.Get(decryptedKey);
return(new KeyProvider.KeyVersion(encryptionKey.GetName(), Ek, decryptedKey));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="GeneralSecurityException"/>
public virtual KeyProviderCryptoExtension.EncryptedKeyVersion GenerateEncryptedKey
(string encryptionKeyName)
{
// Fetch the encryption key
KeyProvider.KeyVersion encryptionKey = keyProvider.GetCurrentKey(encryptionKeyName
);
Preconditions.CheckNotNull(encryptionKey, "No KeyVersion exists for key '%s' ", encryptionKeyName
);
// Generate random bytes for new key and IV
CryptoCodec cc = CryptoCodec.GetInstance(keyProvider.GetConf());
byte[] newKey = new byte[encryptionKey.GetMaterial().Length];
cc.GenerateSecureRandom(newKey);
byte[] iv = new byte[cc.GetCipherSuite().GetAlgorithmBlockSize()];
cc.GenerateSecureRandom(iv);
// Encryption key IV is derived from new key's IV
byte[] encryptionIV = KeyProviderCryptoExtension.EncryptedKeyVersion.DeriveIV(iv);
Encryptor encryptor = cc.CreateEncryptor();
encryptor.Init(encryptionKey.GetMaterial(), encryptionIV);
int keyLen = newKey.Length;
ByteBuffer bbIn = ByteBuffer.AllocateDirect(keyLen);
ByteBuffer bbOut = ByteBuffer.AllocateDirect(keyLen);
bbIn.Put(newKey);
bbIn.Flip();
encryptor.Encrypt(bbIn, bbOut);
bbOut.Flip();
byte[] encryptedKey = new byte[keyLen];
bbOut.Get(encryptedKey);
return(new KeyProviderCryptoExtension.EncryptedKeyVersion(encryptionKeyName, encryptionKey
.GetVersionName(), iv, new KeyProvider.KeyVersion(encryptionKey.GetName(), Eek,
encryptedKey)));
}
public static void Init()
{
Configuration conf = new HdfsConfiguration();
dfsCluster = new MiniDFSCluster.Builder(conf).Build();
dfsCluster.WaitClusterUp();
fs = dfsCluster.GetFileSystem();
codec = CryptoCodec.GetInstance(conf);
}
/// <summary>This method creates and initializes an IV (Initialization Vector)</summary>
/// <param name="conf"/>
/// <returns>byte[]</returns>
/// <exception cref="System.IO.IOException"/>
public static byte[] CreateIV(Configuration conf)
{
CryptoCodec cryptoCodec = CryptoCodec.GetInstance(conf);
if (IsEncryptedSpillEnabled(conf))
{
byte[] iv = new byte[cryptoCodec.GetCipherSuite().GetAlgorithmBlockSize()];
cryptoCodec.GenerateSecureRandom(iv);
return(iv);
}
else
{
return(null);
}
}
/// <summary>Negotiate a cipher option which server supports.</summary>
/// <param name="conf">the configuration</param>
/// <param name="options">the cipher options which client supports</param>
/// <returns>CipherOption negotiated cipher option</returns>
/// <exception cref="System.IO.IOException"/>
public static CipherOption NegotiateCipherOption(Configuration conf, IList <CipherOption
> options)
{
// Negotiate cipher suites if configured. Currently, the only supported
// cipher suite is AES/CTR/NoPadding, but the protocol allows multiple
// values for future expansion.
string cipherSuites = conf.Get(DFSConfigKeys.DfsEncryptDataTransferCipherSuitesKey
);
if (cipherSuites == null || cipherSuites.IsEmpty())
{
return(null);
}
if (!cipherSuites.Equals(CipherSuite.AesCtrNopadding.GetName()))
{
throw new IOException(string.Format("Invalid cipher suite, %s=%s", DFSConfigKeys.
DfsEncryptDataTransferCipherSuitesKey, cipherSuites));
}
if (options != null)
{
foreach (CipherOption option in options)
{
CipherSuite suite = option.GetCipherSuite();
if (suite == CipherSuite.AesCtrNopadding)
{
int keyLen = conf.GetInt(DFSConfigKeys.DfsEncryptDataTransferCipherKeyBitlengthKey
, DFSConfigKeys.DfsEncryptDataTransferCipherKeyBitlengthDefault) / 8;
CryptoCodec codec = CryptoCodec.GetInstance(conf, suite);
byte[] inKey = new byte[keyLen];
byte[] inIv = new byte[suite.GetAlgorithmBlockSize()];
byte[] outKey = new byte[keyLen];
byte[] outIv = new byte[suite.GetAlgorithmBlockSize()];
codec.GenerateSecureRandom(inKey);
codec.GenerateSecureRandom(inIv);
codec.GenerateSecureRandom(outKey);
codec.GenerateSecureRandom(outIv);
return(new CipherOption(suite, inKey, inIv, outKey, outIv));
}
}
}
return(null);
}
/// <summary>Wraps a given FSDataOutputStream with a CryptoOutputStream.</summary>
/// <remarks>
/// Wraps a given FSDataOutputStream with a CryptoOutputStream. The size of the
/// data buffer required for the stream is specified by the
/// "mapreduce.job.encrypted-intermediate-data.buffer.kb" Job configuration
/// variable.
/// </remarks>
/// <param name="conf"/>
/// <param name="out"/>
/// <returns>FSDataOutputStream</returns>
/// <exception cref="System.IO.IOException"/>
public static FSDataOutputStream WrapIfNecessary(Configuration conf, FSDataOutputStream
@out)
{
if (IsEncryptedSpillEnabled(conf))
{
@out.Write(((byte[])ByteBuffer.Allocate(8).PutLong(@out.GetPos()).Array()));
byte[] iv = CreateIV(conf);
@out.Write(iv);
if (Log.IsDebugEnabled())
{
Log.Debug("IV written to Stream [" + Base64.EncodeBase64URLSafeString(iv) + "]");
}
return(new CryptoFSDataOutputStream(@out, CryptoCodec.GetInstance(conf), GetBufferSize
(conf), GetEncryptionKey(), iv));
}
else
{
return(@out);
}
}
/// <summary>
/// Create IOStreamPair of
/// <see cref="Org.Apache.Hadoop.Crypto.CryptoInputStream"/>
/// and
/// <see cref="Org.Apache.Hadoop.Crypto.CryptoOutputStream"/>
/// </summary>
/// <param name="conf">the configuration</param>
/// <param name="cipherOption">negotiated cipher option</param>
/// <param name="out">underlying output stream</param>
/// <param name="in">underlying input stream</param>
/// <param name="isServer">is server side</param>
/// <returns>IOStreamPair the stream pair</returns>
/// <exception cref="System.IO.IOException">for any error</exception>
public static IOStreamPair CreateStreamPair(Configuration conf, CipherOption cipherOption
, OutputStream @out, InputStream @in, bool isServer)
{
if (Log.IsDebugEnabled())
{
Log.Debug("Creating IOStreamPair of CryptoInputStream and " + "CryptoOutputStream."
);
}
CryptoCodec codec = CryptoCodec.GetInstance(conf, cipherOption.GetCipherSuite());
byte[] inKey = cipherOption.GetInKey();
byte[] inIv = cipherOption.GetInIv();
byte[] outKey = cipherOption.GetOutKey();
byte[] outIv = cipherOption.GetOutIv();
InputStream cIn = new CryptoInputStream(@in, codec, isServer ? inKey : outKey, isServer
? inIv : outIv);
OutputStream cOut = new CryptoOutputStream(@out, codec, isServer ? outKey : inKey
, isServer ? outIv : inIv);
return(new IOStreamPair(cIn, cOut));
}
/// <exception cref="System.IO.IOException"/>
public CryptoFSDataInputStream(FSDataInputStream @in, CryptoCodec codec, int bufferSize
, byte[] key, byte[] iv)
: base(new CryptoInputStream(@in, codec, bufferSize, key, iv))
{
}
public static int CryptoPadding(Configuration conf)
{
// Sizeof(IV) + long(start-offset)
return(IsEncryptedSpillEnabled(conf) ? CryptoCodec.GetInstance(conf).GetCipherSuite
().GetAlgorithmBlockSize() + 8 : 0);
}
