public override async Task <FlowGuiConfig> HandleSensitiveData(FlowGuiConfig guiConfig)
{
var functions = guiConfig?.Process?.Functions;
if (functions != null && functions.Count > 0)
{
foreach (var func in functions.Where(f => f.Type == FunctionTypeName))
{
var code = func.Properties?.Code;
if (code != null && !KeyVaultUri.IsSecretUri(code))
{
var secretName = $"{guiConfig.Name}-azurefunc";
var secretUri = await KeyVaultClient.SaveSecretAsync(
keyvaultName : RuntimeKeyVaultName.Value,
secretName : secretName,
secretValue : code,
hashSuffix : true);
func.Properties.Code = secretUri;
}
}
}
return(guiConfig);
}
public override async Task <FlowGuiConfig> HandleSensitiveData(FlowGuiConfig guiConfig)
{
var functions = guiConfig?.Process?.Functions;
if (functions != null && functions.Count > 0)
{
foreach (var func in functions.Where(f => f.Type == "jarUDF" || f.Type == "jarUDAF"))
{
var path = func.Properties?.Path;
if (path != null && !KeyVaultUri.IsSecretUri(path))
{
var secretName = $"{guiConfig.Name}-jarpath";
var secretUri = await KeyVaultClient.SaveSecretAsync(
keyvaultName : RuntimeKeyVaultName.Value,
secretName : secretName,
secretValue : path,
hashSuffix : true);
func.Properties.Path = secretUri;
}
}
}
return(guiConfig);
}
public override async Task <FlowGuiConfig> HandleSensitiveData(FlowGuiConfig guiConfig)
{
var referenceData = guiConfig?.Input?.ReferenceData;
if (referenceData != null && referenceData.Length > 0)
{
foreach (var rd in referenceData)
{
var path = rd.Properties?.Path;
if (path != null && !KeyVaultUri.IsSecretUri(path))
{
var secretName = $"{guiConfig.Name}-referencedata";
var secretUri = await KeyVaultClient.SaveSecretAsync(
keyvaultName : RuntimeKeyVaultName.Value,
secretName : secretName,
secretValue : path,
sparkType : Configuration[Constants.ConfigSettingName_SparkType],
hashSuffix : true);
rd.Properties.Path = secretUri;
}
}
}
return(guiConfig);
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var jobs = flowToDeploy.GetJobs();
if (jobs == null || !jobs.Where(j => j.JobConfigs.Any()).Any())
{
return("no jobs, skipped");
}
var config = flowToDeploy.Config;
var rulesCode = flowToDeploy.GetAttachment <RulesCode>(PrepareTransformFile.AttachmentName_CodeGenObject);
Ensure.NotNull(rulesCode, "rulesCode");
var runtimeConfigBaseFolder = flowToDeploy.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder);
Ensure.NotNull(runtimeConfigBaseFolder, "runtimeConfigBaseFolder");
var filePath = ResourcePathUtil.Combine(runtimeConfigBaseFolder, $"{config.Name}-combined.txt");
var transformFilePath = await RuntimeStorage.SaveFile(filePath, rulesCode.Code);
var transformFileSecret = flowToDeploy.GetTokenString(PrepareTransformFile.TokenName_TransformFile);
await KeyVaultClient.SaveSecretAsync(transformFileSecret, transformFilePath);
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var config = flowToDeploy.Config;
var guiConfig = config?.GetGuiConfig();
if (guiConfig == null)
{
return("no gui input, skipped.");
}
var schema = guiConfig.Input?.Properties?.InputSchemaFile;
Ensure.NotNull(schema, "guiConfig.input.properties.inputschemafile");
var runtimeConfigBaseFolder = flowToDeploy.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder);
Ensure.NotNull(runtimeConfigBaseFolder, "runtimeConfigBaseFolder");
var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName);
Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");
var filePath = ResourcePathUtil.Combine(runtimeConfigBaseFolder, "inputschema.json");
var schemaFile = await RuntimeStorage.SaveFile(filePath, schema);
var secretName = $"{config.Name}-inputschemafile";
var schemaFileSecret = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, schemaFile, Configuration[Constants.ConfigSettingName_SparkType]);
flowToDeploy.SetStringToken(TokenName_InputSchemaFilePath, schemaFileSecret);
return("done");
}
public override async Task <FlowGuiConfig> HandleSensitiveData(FlowGuiConfig guiConfig)
{
var outputsData = guiConfig?.Outputs;
if (outputsData != null && outputsData.Length > 0)
{
foreach (var rd in outputsData)
{
var connStr = rd.Properties?.ConnectionString;
if (connStr != null && !KeyVaultUri.IsSecretUri(connStr))
{
var secretName = $"{guiConfig.Name}-output";
var secretUri = await KeyVaultClient.SaveSecretAsync(
keyvaultName : RuntimeKeyVaultName.Value,
secretName : secretName,
secretValue : connStr,
hashSuffix : true);
rd.Properties.ConnectionString = secretUri;
}
}
}
return(guiConfig);
}
private async Task <FlowBlobOutputSpec> ProcessOutputBlob(string configName, FlowGuiOutput uiOutput)
{
if (uiOutput != null && uiOutput.Properties != null)
{
var sparkKeyVaultName = Configuration[Constants.ConfigSettingName_RuntimeKeyVaultName];
string connectionString = await KeyVaultClient.GetSecretFromKeyVaultAsync(sparkKeyVaultName, uiOutput.Properties.ConnectionString);
var blobPath = $"wasbs://{uiOutput.Properties.ContainerName}{ParseBlobAccountName(connectionString)}.blob.core.windows.net/{uiOutput.Properties.BlobPrefix}/%1$tY/%1$tm/%1$td/%1$tH/${{quarterBucket}}/${{minuteBucket}}";
var secretId = $"{configName}-output";
var blobPathSecret = await KeyVaultClient.SaveSecretAsync(sparkKeyVaultName, secretId, blobPath, true);
FlowBlobOutputSpec blobOutput = new FlowBlobOutputSpec()
{
CompressionType = uiOutput.Properties.CompressionType,
Format = uiOutput.Properties.Format,
Groups = new BlobOutputGroups()
{
Main = new BlobOutputMain()
{
Folder = blobPathSecret
}
}
};
return(blobOutput);
}
else
{
return(null);
}
}
private async Task <FlowSqlOutputSpec> ProcessOutputSql(string configName, FlowGuiOutput uiOutput)
{
if (uiOutput != null && uiOutput.Properties != null)
{
var sparkKeyVaultName = Configuration[Constants.ConfigSettingName_RuntimeKeyVaultName];
string connectionString = await KeyVaultClient.ResolveSecretUriAsync(uiOutput.Properties.ConnectionString).ConfigureAwait(false);
var database = GetValueFromJdbcConnection(connectionString, "database");
var user = GetValueFromJdbcConnection(connectionString, "user");
var pwd = GetValueFromJdbcConnection(connectionString, "password");
var url = GetUrlFromJdbcConnection(connectionString);
var sparkType = Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string value) ? value : null;
// Save password and url in keyvault
var pwdSecretId = $"{configName}-outSqlPassword";
var pwdRef = await KeyVaultClient.SaveSecretAsync(sparkKeyVaultName, pwdSecretId, pwd, sparkType, true).ConfigureAwait(false);
var urlSecretId = $"{configName}-outSqlUrl";
var urlRef = await KeyVaultClient.SaveSecretAsync(sparkKeyVaultName, urlSecretId, url, sparkType, true).ConfigureAwait(false);
FlowSqlOutputSpec sqlOutput = new FlowSqlOutputSpec()
{
ConnectionStringRef = uiOutput.Properties.ConnectionString,
TableName = uiOutput.Properties.TableName,
WriteMode = uiOutput.Properties.WriteMode,
UseBulkInsert = uiOutput.Properties.UseBulkInsert,
DatabaseName = database,
User = user,
Password = pwdRef,
Url = urlRef
};
return(sqlOutput);
}
else
{
return(null);
}
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var config = flowToDeploy.Config;
var guiConfig = config?.GetGuiConfig();
if (guiConfig == null)
{
return("no gui input, skipped.");
}
string queries = string.Join("\n", guiConfig.Process?.Queries);
string ruleDefinitions = RuleDefinitionGenerator.GenerateRuleDefinitions(guiConfig.Rules, config.Name);
RulesCode rulesCode = CodeGen.GenerateCode(queries, ruleDefinitions, config.Name);
Ensure.NotNull(rulesCode, "rulesCode");
// Save the rulesCode object for downstream processing
flowToDeploy.SetAttachment(AttachmentName_CodeGenObject, rulesCode);
var runtimeConfigBaseFolder = flowToDeploy.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder);
Ensure.NotNull(runtimeConfigBaseFolder, "runtimeConfigBaseFolder");
var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName);
Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");
var filePath = ResourcePathUtil.Combine(runtimeConfigBaseFolder, $"{config.Name}-combined.txt");
var transformFilePath = await RuntimeStorage.SaveFile(filePath, rulesCode.Code);
var secretName = $"{config.Name}-transform";
var transformFileSecret = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, transformFilePath);
flowToDeploy.SetStringToken(TokenName_TransformFile, transformFileSecret);
return("done");
}
private async Task <FlowBlobOutputSpec> ProcessOutputBlob(string inputMode, string configName, FlowGuiOutput uiOutput)
{
if (uiOutput != null && uiOutput.Properties != null)
{
var sparkKeyVaultName = Configuration[Constants.ConfigSettingName_RuntimeKeyVaultName];
string connectionString = await KeyVaultClient.ResolveSecretUriAsync(uiOutput.Properties.ConnectionString).ConfigureAwait(false);
var accountName = ConfigHelper.ParseBlobAccountName(connectionString);
var timeFormat = ConfigHelper.GetBlobPartitionFormat(inputMode, uiOutput.Properties.BlobPartitionFormat);
var blobPath = $"wasbs://{uiOutput.Properties.ContainerName}@{accountName}.blob.core.windows.net/{uiOutput.Properties.BlobPrefix}/{timeFormat}";
var secretId = $"{configName}-output";
var sparkType = Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string value) ? value : null;
var blobPathSecret = await KeyVaultClient.SaveSecretAsync(sparkKeyVaultName, secretId, blobPath, sparkType, true);
await KeyVaultClient.SaveSecretAsync(sparkKeyVaultName, $"{Constants.AccountSecretPrefix}{accountName}", ConfigHelper.ParseBlobAccountKey(connectionString), sparkType, false);
FlowBlobOutputSpec blobOutput = new FlowBlobOutputSpec()
{
CompressionType = uiOutput.Properties.CompressionType,
Format = uiOutput.Properties.Format,
Groups = new BlobOutputGroups()
{
Main = new BlobOutputMain()
{
Folder = blobPathSecret
}
}
};
return(blobOutput);
}
else
{
return(null);
}
}
