/// <inheritdoc/>
public void Write(IList <IColumn> columns)
{
List <DataColumn> parquetColumns = CreateParquetColumns(columns);
List <DataField> parquetFields = parquetColumns.Select(p => p.Field).ToList();
Schema schema = new Schema(parquetFields);
using (var parquetWriter = new ParquetWriter(schema, FileStream))
{
// TODO - Write is called many times; one for each rowgroup in the file. We do not need to compile
// and write metadata many times. Refactor to write metadata only once.
CryptoMetadata metadata = CompileMetadata(columns, FileEncryptionSettings);
if (!metadata.IsEmpty())
{
parquetWriter.CustomMetadata = new Dictionary <string, string>
{
[nameof(CryptoMetadata)] = JsonConvert.SerializeObject(
value: metadata,
settings: new JsonSerializerSettings()
{
NullValueHandling = NullValueHandling.Ignore,
Converters = { new StringEnumConverter() },
Formatting = Formatting.Indented
})
};
}
// create a new row group in the file
using (ParquetRowGroupWriter groupWriter = parquetWriter.CreateRowGroup())
{
parquetColumns.ForEach(groupWriter.WriteColumn);
}
}
}
/// <summary>
/// Compiles a <see cref="CryptoMetadata"/> object from <see cref="List{T}"/>s of <see cref="IColumn"/>s and <see cref="FileEncryptionSettings"/>.
/// </summary>
/// <param name="columns">The <see cref="IColumn"/>s on which to compile metadata.</param>
/// <param name="encryptionSettings">The <see cref="FileEncryptionSettings"/> on which to compile metadata.</param>
/// <returns></returns>
public static CryptoMetadata CompileMetadata(IList <IColumn> columns, IList <FileEncryptionSettings> encryptionSettings)
{
if (columns.Count != encryptionSettings.Count)
{
throw new ArgumentException($"{nameof(columns)}.Count does not equal {nameof(encryptionSettings)}.Count");
}
CryptoMetadata cryptoMetadata = new CryptoMetadata();
for (int i = 0; i < columns.Count; i++)
{
FileEncryptionSettings settings = encryptionSettings[i];
if (settings.EncryptionType != EncryptionType.Plaintext)
{
ColumnEncryptionMetadata columnEncryptionInformation = new ColumnEncryptionMetadata()
{
ColumnName = columns[i].Name,
DataEncryptionKeyName = settings.DataEncryptionKey.Name,
ColumnIndex = encryptionSettings.IndexOf(settings),
EncryptionAlgorithm = DataEncryptionKeyAlgorithm.AEAD_AES_256_CBC_HMAC_SHA256,
EncryptionType = settings.EncryptionType,
Serializer = settings.GetSerializer()
};
DataEncryptionKeyMetadata columnKeyInformation = new DataEncryptionKeyMetadata()
{
KeyEncryptionKeyName = settings.DataEncryptionKey.KeyEncryptionKey.Name,
EncryptedDataEncryptionKey = settings.DataEncryptionKey.EncryptedValue.ToHexString(),
Name = settings.DataEncryptionKey.Name
};
KeyEncryptionKeyMetadata columnMasterKeyInformation = new KeyEncryptionKeyMetadata()
{
KeyPath = settings.DataEncryptionKey.KeyEncryptionKey.Path,
KeyProvider = settings.DataEncryptionKey.KeyEncryptionKey.KeyStoreProvider.ProviderName,
Name = settings.DataEncryptionKey.KeyEncryptionKey.Name
};
cryptoMetadata.ColumnEncryptionInformation.Add(columnEncryptionInformation);
cryptoMetadata.DataEncryptionKeyInformation.Add(columnKeyInformation);
cryptoMetadata.KeyEncryptionKeyInformation.Add(columnMasterKeyInformation);
}
}
return(cryptoMetadata);
}