/// <exception cref="System.IO.IOException"/> public CryptoFSDataOutputStream(FSDataOutputStream @out, CryptoCodec codec, byte[] key, byte[] iv) : base(new CryptoOutputStream(@out, codec, key, iv, @out.GetPos()), null, @out.GetPos ()) { this.fsOut = @out; }
/// <summary>Wraps a given FSDataInputStream with a CryptoInputStream.</summary> /// <remarks> /// Wraps a given FSDataInputStream with a CryptoInputStream. The size of the /// data buffer required for the stream is specified by the /// "mapreduce.job.encrypted-intermediate-data.buffer.kb" Job configuration /// variable. /// </remarks> /// <param name="conf"/> /// <param name="in"/> /// <returns>FSDataInputStream</returns> /// <exception cref="System.IO.IOException"/> public static FSDataInputStream WrapIfNecessary(Configuration conf, FSDataInputStream @in) { if (IsEncryptedSpillEnabled(conf)) { CryptoCodec cryptoCodec = CryptoCodec.GetInstance(conf); int bufferSize = GetBufferSize(conf); // Not going to be used... but still has to be read... // Since the O/P stream always writes it.. IOUtils.ReadFully(@in, new byte[8], 0, 8); byte[] iv = new byte[cryptoCodec.GetCipherSuite().GetAlgorithmBlockSize()]; IOUtils.ReadFully(@in, iv, 0, cryptoCodec.GetCipherSuite().GetAlgorithmBlockSize( )); if (Log.IsDebugEnabled()) { Log.Debug("IV read from Stream [" + Base64.EncodeBase64URLSafeString(iv) + "]"); } return(new CryptoFSDataInputStream(@in, cryptoCodec, bufferSize, GetEncryptionKey (), iv)); } else { return(@in); } }
/// <summary>Wraps a given InputStream with a CryptoInputStream.</summary> /// <remarks> /// Wraps a given InputStream with a CryptoInputStream. The size of the data /// buffer required for the stream is specified by the /// "mapreduce.job.encrypted-intermediate-data.buffer.kb" Job configuration /// variable. /// If the value of 'length' is > -1, The InputStream is additionally /// wrapped in a LimitInputStream. CryptoStreams are late buffering in nature. /// This means they will always try to read ahead if they can. The /// LimitInputStream will ensure that the CryptoStream does not read past the /// provided length from the given Input Stream. /// </remarks> /// <param name="conf"/> /// <param name="in"/> /// <param name="length"/> /// <returns>InputStream</returns> /// <exception cref="System.IO.IOException"/> public static InputStream WrapIfNecessary(Configuration conf, InputStream @in, long length) { if (IsEncryptedSpillEnabled(conf)) { int bufferSize = GetBufferSize(conf); if (length > -1) { @in = new LimitInputStream(@in, length); } byte[] offsetArray = new byte[8]; IOUtils.ReadFully(@in, offsetArray, 0, 8); long offset = ByteBuffer.Wrap(offsetArray).GetLong(); CryptoCodec cryptoCodec = CryptoCodec.GetInstance(conf); byte[] iv = new byte[cryptoCodec.GetCipherSuite().GetAlgorithmBlockSize()]; IOUtils.ReadFully(@in, iv, 0, cryptoCodec.GetCipherSuite().GetAlgorithmBlockSize( )); if (Log.IsDebugEnabled()) { Log.Debug("IV read from [" + Base64.EncodeBase64URLSafeString(iv) + "]"); } return(new CryptoInputStream(@in, cryptoCodec, bufferSize, GetEncryptionKey(), iv , offset + CryptoPadding(conf))); } else { return(@in); } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="GeneralSecurityException"/> public virtual KeyProvider.KeyVersion DecryptEncryptedKey(KeyProviderCryptoExtension.EncryptedKeyVersion encryptedKeyVersion) { // Fetch the encryption key material string encryptionKeyVersionName = encryptedKeyVersion.GetEncryptionKeyVersionName (); KeyProvider.KeyVersion encryptionKey = keyProvider.GetKeyVersion(encryptionKeyVersionName ); Preconditions.CheckNotNull(encryptionKey, "KeyVersion name '%s' does not exist", encryptionKeyVersionName); Preconditions.CheckArgument(encryptedKeyVersion.GetEncryptedKeyVersion().GetVersionName ().Equals(KeyProviderCryptoExtension.Eek), "encryptedKey version name must be '%s', is '%s'" , KeyProviderCryptoExtension.Eek, encryptedKeyVersion.GetEncryptedKeyVersion().GetVersionName ()); // Encryption key IV is determined from encrypted key's IV byte[] encryptionIV = KeyProviderCryptoExtension.EncryptedKeyVersion.DeriveIV(encryptedKeyVersion .GetEncryptedKeyIv()); CryptoCodec cc = CryptoCodec.GetInstance(keyProvider.GetConf()); Decryptor decryptor = cc.CreateDecryptor(); decryptor.Init(encryptionKey.GetMaterial(), encryptionIV); KeyProvider.KeyVersion encryptedKV = encryptedKeyVersion.GetEncryptedKeyVersion(); int keyLen = encryptedKV.GetMaterial().Length; ByteBuffer bbIn = ByteBuffer.AllocateDirect(keyLen); ByteBuffer bbOut = ByteBuffer.AllocateDirect(keyLen); bbIn.Put(encryptedKV.GetMaterial()); bbIn.Flip(); decryptor.Decrypt(bbIn, bbOut); bbOut.Flip(); byte[] decryptedKey = new byte[keyLen]; bbOut.Get(decryptedKey); return(new KeyProvider.KeyVersion(encryptionKey.GetName(), Ek, decryptedKey)); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="GeneralSecurityException"/> public virtual KeyProviderCryptoExtension.EncryptedKeyVersion GenerateEncryptedKey (string encryptionKeyName) { // Fetch the encryption key KeyProvider.KeyVersion encryptionKey = keyProvider.GetCurrentKey(encryptionKeyName ); Preconditions.CheckNotNull(encryptionKey, "No KeyVersion exists for key '%s' ", encryptionKeyName ); // Generate random bytes for new key and IV CryptoCodec cc = CryptoCodec.GetInstance(keyProvider.GetConf()); byte[] newKey = new byte[encryptionKey.GetMaterial().Length]; cc.GenerateSecureRandom(newKey); byte[] iv = new byte[cc.GetCipherSuite().GetAlgorithmBlockSize()]; cc.GenerateSecureRandom(iv); // Encryption key IV is derived from new key's IV byte[] encryptionIV = KeyProviderCryptoExtension.EncryptedKeyVersion.DeriveIV(iv); Encryptor encryptor = cc.CreateEncryptor(); encryptor.Init(encryptionKey.GetMaterial(), encryptionIV); int keyLen = newKey.Length; ByteBuffer bbIn = ByteBuffer.AllocateDirect(keyLen); ByteBuffer bbOut = ByteBuffer.AllocateDirect(keyLen); bbIn.Put(newKey); bbIn.Flip(); encryptor.Encrypt(bbIn, bbOut); bbOut.Flip(); byte[] encryptedKey = new byte[keyLen]; bbOut.Get(encryptedKey); return(new KeyProviderCryptoExtension.EncryptedKeyVersion(encryptionKeyName, encryptionKey .GetVersionName(), iv, new KeyProvider.KeyVersion(encryptionKey.GetName(), Eek, encryptedKey))); }
public static void Init() { Configuration conf = new HdfsConfiguration(); dfsCluster = new MiniDFSCluster.Builder(conf).Build(); dfsCluster.WaitClusterUp(); fs = dfsCluster.GetFileSystem(); codec = CryptoCodec.GetInstance(conf); }
/// <summary>This method creates and initializes an IV (Initialization Vector)</summary> /// <param name="conf"/> /// <returns>byte[]</returns> /// <exception cref="System.IO.IOException"/> public static byte[] CreateIV(Configuration conf) { CryptoCodec cryptoCodec = CryptoCodec.GetInstance(conf); if (IsEncryptedSpillEnabled(conf)) { byte[] iv = new byte[cryptoCodec.GetCipherSuite().GetAlgorithmBlockSize()]; cryptoCodec.GenerateSecureRandom(iv); return(iv); } else { return(null); } }
/// <summary>Negotiate a cipher option which server supports.</summary> /// <param name="conf">the configuration</param> /// <param name="options">the cipher options which client supports</param> /// <returns>CipherOption negotiated cipher option</returns> /// <exception cref="System.IO.IOException"/> public static CipherOption NegotiateCipherOption(Configuration conf, IList <CipherOption > options) { // Negotiate cipher suites if configured. Currently, the only supported // cipher suite is AES/CTR/NoPadding, but the protocol allows multiple // values for future expansion. string cipherSuites = conf.Get(DFSConfigKeys.DfsEncryptDataTransferCipherSuitesKey ); if (cipherSuites == null || cipherSuites.IsEmpty()) { return(null); } if (!cipherSuites.Equals(CipherSuite.AesCtrNopadding.GetName())) { throw new IOException(string.Format("Invalid cipher suite, %s=%s", DFSConfigKeys. DfsEncryptDataTransferCipherSuitesKey, cipherSuites)); } if (options != null) { foreach (CipherOption option in options) { CipherSuite suite = option.GetCipherSuite(); if (suite == CipherSuite.AesCtrNopadding) { int keyLen = conf.GetInt(DFSConfigKeys.DfsEncryptDataTransferCipherKeyBitlengthKey , DFSConfigKeys.DfsEncryptDataTransferCipherKeyBitlengthDefault) / 8; CryptoCodec codec = CryptoCodec.GetInstance(conf, suite); byte[] inKey = new byte[keyLen]; byte[] inIv = new byte[suite.GetAlgorithmBlockSize()]; byte[] outKey = new byte[keyLen]; byte[] outIv = new byte[suite.GetAlgorithmBlockSize()]; codec.GenerateSecureRandom(inKey); codec.GenerateSecureRandom(inIv); codec.GenerateSecureRandom(outKey); codec.GenerateSecureRandom(outIv); return(new CipherOption(suite, inKey, inIv, outKey, outIv)); } } } return(null); }
/// <summary>Wraps a given FSDataOutputStream with a CryptoOutputStream.</summary> /// <remarks> /// Wraps a given FSDataOutputStream with a CryptoOutputStream. The size of the /// data buffer required for the stream is specified by the /// "mapreduce.job.encrypted-intermediate-data.buffer.kb" Job configuration /// variable. /// </remarks> /// <param name="conf"/> /// <param name="out"/> /// <returns>FSDataOutputStream</returns> /// <exception cref="System.IO.IOException"/> public static FSDataOutputStream WrapIfNecessary(Configuration conf, FSDataOutputStream @out) { if (IsEncryptedSpillEnabled(conf)) { @out.Write(((byte[])ByteBuffer.Allocate(8).PutLong(@out.GetPos()).Array())); byte[] iv = CreateIV(conf); @out.Write(iv); if (Log.IsDebugEnabled()) { Log.Debug("IV written to Stream [" + Base64.EncodeBase64URLSafeString(iv) + "]"); } return(new CryptoFSDataOutputStream(@out, CryptoCodec.GetInstance(conf), GetBufferSize (conf), GetEncryptionKey(), iv)); } else { return(@out); } }
/// <summary> /// Create IOStreamPair of /// <see cref="Org.Apache.Hadoop.Crypto.CryptoInputStream"/> /// and /// <see cref="Org.Apache.Hadoop.Crypto.CryptoOutputStream"/> /// </summary> /// <param name="conf">the configuration</param> /// <param name="cipherOption">negotiated cipher option</param> /// <param name="out">underlying output stream</param> /// <param name="in">underlying input stream</param> /// <param name="isServer">is server side</param> /// <returns>IOStreamPair the stream pair</returns> /// <exception cref="System.IO.IOException">for any error</exception> public static IOStreamPair CreateStreamPair(Configuration conf, CipherOption cipherOption , OutputStream @out, InputStream @in, bool isServer) { if (Log.IsDebugEnabled()) { Log.Debug("Creating IOStreamPair of CryptoInputStream and " + "CryptoOutputStream." ); } CryptoCodec codec = CryptoCodec.GetInstance(conf, cipherOption.GetCipherSuite()); byte[] inKey = cipherOption.GetInKey(); byte[] inIv = cipherOption.GetInIv(); byte[] outKey = cipherOption.GetOutKey(); byte[] outIv = cipherOption.GetOutIv(); InputStream cIn = new CryptoInputStream(@in, codec, isServer ? inKey : outKey, isServer ? inIv : outIv); OutputStream cOut = new CryptoOutputStream(@out, codec, isServer ? outKey : inKey , isServer ? outIv : inIv); return(new IOStreamPair(cIn, cOut)); }
/// <exception cref="System.IO.IOException"/> public CryptoFSDataInputStream(FSDataInputStream @in, CryptoCodec codec, int bufferSize , byte[] key, byte[] iv) : base(new CryptoInputStream(@in, codec, bufferSize, key, iv)) { }
public static int CryptoPadding(Configuration conf) { // Sizeof(IV) + long(start-offset) return(IsEncryptedSpillEnabled(conf) ? CryptoCodec.GetInstance(conf).GetCipherSuite ().GetAlgorithmBlockSize() + 8 : 0); }