public override async Task <FlowGuiConfig> HandleSensitiveData(FlowGuiConfig guiConfig)
{
var functions = guiConfig?.Process?.Functions;
if (functions != null && functions.Count > 0)
{
foreach (var func in functions.Where(f => f.Type == FunctionTypeName))
{
var code = func.Properties?.Code;
if (code != null && !KeyVaultUri.IsSecretUri(code))
{
var secretName = $"{guiConfig.Name}-azurefunc";
var secretUri = await KeyVaultClient.SaveSecretAsync(
keyvaultName : RuntimeKeyVaultName.Value,
secretName : secretName,
secretValue : code,
hashSuffix : true);
func.Properties.Code = secretUri;
}
}
}
return(guiConfig);
}
public override async Task <FlowGuiConfig> HandleSensitiveData(FlowGuiConfig guiConfig)
{
var functions = guiConfig?.Process?.Functions;
if (functions != null && functions.Count > 0)
{
foreach (var func in functions.Where(f => f.Type == "jarUDF" || f.Type == "jarUDAF"))
{
var path = func.Properties?.Path;
if (path != null && !KeyVaultUri.IsSecretUri(path))
{
var secretName = $"{guiConfig.Name}-jarpath";
var secretUri = await KeyVaultClient.SaveSecretAsync(
keyvaultName : RuntimeKeyVaultName.Value,
secretName : secretName,
secretValue : path,
sparkType : Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string sparkType)?sparkType : null,
hashSuffix : true);
func.Properties.Path = secretUri;
}
}
}
return(guiConfig);
}
public override async Task <FlowGuiConfig> HandleSensitiveData(FlowGuiConfig guiConfig)
{
var referenceData = guiConfig?.Input?.ReferenceData;
if (referenceData != null && referenceData.Length > 0)
{
foreach (var rd in referenceData)
{
var path = rd.Properties?.Path;
if (path != null && !KeyVaultUri.IsSecretUri(path))
{
var secretName = $"{guiConfig.Name}-referencedata";
var secretUri = await KeyVaultClient.SaveSecretAsync(
keyvaultName : RuntimeKeyVaultName.Value,
secretName : secretName,
secretValue : path,
sparkType : Configuration[Constants.ConfigSettingName_SparkType],
hashSuffix : true);
rd.Properties.Path = secretUri;
}
}
}
return(guiConfig);
}
public override async Task <FlowGuiConfig> HandleSensitiveData(FlowGuiConfig guiConfig)
{
var outputsData = guiConfig?.Outputs;
if (outputsData != null && outputsData.Length > 0)
{
foreach (var rd in outputsData)
{
var connStr = rd.Properties?.ConnectionString;
if (connStr != null && !KeyVaultUri.IsSecretUri(connStr))
{
var secretName = $"{guiConfig.Name}-output";
var secretUri = await KeyVaultClient.SaveSecretAsync(
keyvaultName : RuntimeKeyVaultName.Value,
secretName : secretName,
secretValue : connStr,
hashSuffix : true);
rd.Properties.ConnectionString = secretUri;
}
}
}
return(guiConfig);
}
public override async Task <FlowGuiConfig> HandleSensitiveData(FlowGuiConfig guiConfig)
{
var functions = guiConfig?.Process?.Functions;
if (functions != null && functions.Count > 0)
{
foreach (var func in functions.Where(f => f.Type == FunctionTypeName))
{
var code = func.Properties?.Code;
if (!string.IsNullOrEmpty(code) && !KeyVaultUri.IsSecretUri(code))
{
var secretName = $"{guiConfig.Name}-azurefunc";
var secretUri = await KeyVaultClient.SaveSecretAsync(
keyvaultName : RuntimeKeyVaultName.Value,
secretName : secretName,
secretValue : code,
sparkType : Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string sparkType)?sparkType : null,
hashSuffix : true);
func.Properties.Code = secretUri;
}
}
}
return(guiConfig);
}
public override async Task <FlowGuiConfig> HandleSensitiveData(FlowGuiConfig guiConfig)
{
var runtimeKeyVaultName = Configuration[Constants.ConfigSettingName_RuntimeKeyVaultName];
var sparkType = Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string value) ? value : null;
Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");
// Replace Input Event Hub Connection String
var eventHubConnectionString = guiConfig?.Input?.Properties?.InputEventhubConnection;
if (!string.IsNullOrEmpty(eventHubConnectionString) && !KeyVaultUri.IsSecretUri(eventHubConnectionString))
{
// create new secret
var secretName = $"{guiConfig.Name}-input-eventhubconnectionstring";
var secretId = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, eventHubConnectionString, sparkType);
guiConfig.Input.Properties.InputEventhubConnection = secretId;
}
// Replace Input Event Hub SubscriptionId
var inputSubscriptionId = guiConfig?.Input?.Properties?.InputSubscriptionId;
if (!string.IsNullOrEmpty(inputSubscriptionId) && !KeyVaultUri.IsSecretUri(inputSubscriptionId))
{
var secretName = $"{guiConfig.Name}-input-inputsubscriptionid";
var secretId = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, inputSubscriptionId, sparkType);
guiConfig.Input.Properties.InputSubscriptionId = secretId;
}
// Replace Input Event Hub ResourceGroup
var inputResourceGroup = guiConfig?.Input?.Properties?.InputResourceGroup;
if (!string.IsNullOrEmpty(inputResourceGroup) && !KeyVaultUri.IsSecretUri(inputResourceGroup))
{
var secretName = $"{guiConfig.Name}-input-inputResourceGroup";
var secretId = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, inputResourceGroup, sparkType);
guiConfig.Input.Properties.InputResourceGroup = secretId;
}
// Replace Info Databricks Token
var infoDatabricksToken = guiConfig?.DatabricksToken;
if (!string.IsNullOrEmpty(infoDatabricksToken) && !KeyVaultUri.IsSecretUri(infoDatabricksToken))
{
var secretName = $"{guiConfig.Name}-info-databricksToken";
var secretId = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, infoDatabricksToken, sparkType);
guiConfig.DatabricksToken = secretId;
}
return(guiConfig);
}
public override async Task <FlowGuiConfig> HandleSensitiveData(FlowGuiConfig guiConfig)
{
var runtimeKeyVaultName = Configuration[Constants.ConfigSettingName_RuntimeKeyVaultName];
Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");
// Replace Input Event Hub Connection String
var eventHubConnectionString = guiConfig?.Input?.Properties?.InputEventhubConnection;
if (eventHubConnectionString != null && !KeyVaultUri.IsSecretUri(eventHubConnectionString))
{
//TODO: create new secret
var secretName = $"{guiConfig.Name}-input-eventhubconnectionstring";
var secretId = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, eventHubConnectionString);
guiConfig.Input.Properties.InputEventhubConnection = secretId;
}
// Replace Input Event Hub SubscriptionId
var inputSubscriptionId = guiConfig?.Input?.Properties?.InputSubscriptionId;
if (!string.IsNullOrEmpty(inputSubscriptionId) && !KeyVaultUri.IsSecretUri(inputSubscriptionId))
{
var secretName = $"{guiConfig.Name}-input-inputsubscriptionid";
var secretId = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, inputSubscriptionId);
guiConfig.Input.Properties.InputSubscriptionId = secretId;
}
// Replace Input Event Hub ResourceGroup
var inputResourceGroup = guiConfig?.Input?.Properties?.InputResourceGroup;
if (!string.IsNullOrEmpty(inputResourceGroup) && !KeyVaultUri.IsSecretUri(inputResourceGroup))
{
var secretName = $"{guiConfig.Name}-input-inputResourceGroup";
var secretId = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, inputResourceGroup);
guiConfig.Input.Properties.InputResourceGroup = secretId;
}
return(guiConfig);
}
private FlowEventHubOutputSpec ProcessOutputMetric(FlowGuiOutput uiOutput)
{
if (uiOutput != null && uiOutput.Properties != null)
{
var sparkKeyVaultName = Configuration[Constants.ConfigSettingName_RuntimeKeyVaultName];
var metricsEhConnectionStringKey = Configuration[Constants.ConfigSettingName_MetricEventHubConnectionKey];
FlowEventHubOutputSpec eventhubOutput = new FlowEventHubOutputSpec()
{
ConnectionStringRef = KeyVaultUri.ComposeUri(sparkKeyVaultName, metricsEhConnectionStringKey),
CompressionType = "none",
Format = "json"
};
return(eventhubOutput);
}
else
{
return(null);
}
}
public override async Task <FlowGuiConfig> HandleSensitiveData(FlowGuiConfig guiConfig)
{
if (guiConfig?.Input?.Mode == Constants.InputMode_Batching)
{
var runtimeKeyVaultName = Configuration[Constants.ConfigSettingName_RuntimeKeyVaultName];
Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");
var sparkType = Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string value) ? value : null;
for (int i = 0; i < guiConfig?.Input?.Batch?.Length; i++)
{
// Replace Input Path
var input = guiConfig?.Input?.Batch[i];
var inputConnection = input.Properties.Connection;
if (!string.IsNullOrEmpty(inputConnection) && !KeyVaultUri.IsSecretUri(inputConnection))
{
var secretName = $"{guiConfig.Name}-input-{i}-inputConnection";
var secretId = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, inputConnection, sparkType).ConfigureAwait(false);
input.Properties.Connection = secretId;
var accountName = ConfigHelper.ParseBlobAccountName(inputConnection);
await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, $"{Constants.AccountSecretPrefix}{accountName}", ConfigHelper.ParseBlobAccountKey(inputConnection), sparkType, false);
}
var inputPath = input.Properties.Path;
if (!string.IsNullOrEmpty(inputPath) && !KeyVaultUri.IsSecretUri(inputPath))
{
var secretName = $"{guiConfig.Name}-input-{i}-inputPath";
var secretId = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, inputPath, Configuration[Constants.ConfigSettingName_SparkType]).ConfigureAwait(false);
input.Properties.Path = secretId;
}
}
}
return(guiConfig);
}
public PSEncryptionConfigAttributes(KeyVaultProperties keyVaultProperties)
{
if (keyVaultProperties != null)
{
KeyName = keyVaultProperties?.KeyName;
KeyVaultUri = keyVaultProperties?.KeyVaultUri;
KeyVersion = keyVaultProperties?.KeyVersion;
if (KeyVersion == null)
{
KeyVersion = "";
}
if (KeyVaultUri != null)
{
KeyVaultUri = KeyVaultUri.EndsWith("/") ? KeyVaultUri.Substring(0, KeyVaultUri.Length - 1) : KeyVaultUri;
}
UserAssignedIdentity = keyVaultProperties?.Identity?.UserAssignedIdentity;
}
}
