public override async Task <FlowGuiConfig> HandleSensitiveData(FlowGuiConfig guiConfig) { var functions = guiConfig?.Process?.Functions; if (functions != null && functions.Count > 0) { foreach (var func in functions.Where(f => f.Type == FunctionTypeName)) { var code = func.Properties?.Code; if (code != null && !KeyVaultUri.IsSecretUri(code)) { var secretName = $"{guiConfig.Name}-azurefunc"; var secretUri = await KeyVaultClient.SaveSecretAsync( keyvaultName : RuntimeKeyVaultName.Value, secretName : secretName, secretValue : code, hashSuffix : true); func.Properties.Code = secretUri; } } } return(guiConfig); }
public override async Task <FlowGuiConfig> HandleSensitiveData(FlowGuiConfig guiConfig) { var functions = guiConfig?.Process?.Functions; if (functions != null && functions.Count > 0) { foreach (var func in functions.Where(f => f.Type == "jarUDF" || f.Type == "jarUDAF")) { var path = func.Properties?.Path; if (path != null && !KeyVaultUri.IsSecretUri(path)) { var secretName = $"{guiConfig.Name}-jarpath"; var secretUri = await KeyVaultClient.SaveSecretAsync( keyvaultName : RuntimeKeyVaultName.Value, secretName : secretName, secretValue : path, hashSuffix : true); func.Properties.Path = secretUri; } } } return(guiConfig); }
public override async Task <FlowGuiConfig> HandleSensitiveData(FlowGuiConfig guiConfig) { var referenceData = guiConfig?.Input?.ReferenceData; if (referenceData != null && referenceData.Length > 0) { foreach (var rd in referenceData) { var path = rd.Properties?.Path; if (path != null && !KeyVaultUri.IsSecretUri(path)) { var secretName = $"{guiConfig.Name}-referencedata"; var secretUri = await KeyVaultClient.SaveSecretAsync( keyvaultName : RuntimeKeyVaultName.Value, secretName : secretName, secretValue : path, sparkType : Configuration[Constants.ConfigSettingName_SparkType], hashSuffix : true); rd.Properties.Path = secretUri; } } } return(guiConfig); }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var jobs = flowToDeploy.GetJobs(); if (jobs == null || !jobs.Where(j => j.JobConfigs.Any()).Any()) { return("no jobs, skipped"); } var config = flowToDeploy.Config; var rulesCode = flowToDeploy.GetAttachment <RulesCode>(PrepareTransformFile.AttachmentName_CodeGenObject); Ensure.NotNull(rulesCode, "rulesCode"); var runtimeConfigBaseFolder = flowToDeploy.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder); Ensure.NotNull(runtimeConfigBaseFolder, "runtimeConfigBaseFolder"); var filePath = ResourcePathUtil.Combine(runtimeConfigBaseFolder, $"{config.Name}-combined.txt"); var transformFilePath = await RuntimeStorage.SaveFile(filePath, rulesCode.Code); var transformFileSecret = flowToDeploy.GetTokenString(PrepareTransformFile.TokenName_TransformFile); await KeyVaultClient.SaveSecretAsync(transformFileSecret, transformFilePath); return("done"); }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var config = flowToDeploy.Config; var guiConfig = config?.GetGuiConfig(); if (guiConfig == null) { return("no gui input, skipped."); } var schema = guiConfig.Input?.Properties?.InputSchemaFile; Ensure.NotNull(schema, "guiConfig.input.properties.inputschemafile"); var runtimeConfigBaseFolder = flowToDeploy.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder); Ensure.NotNull(runtimeConfigBaseFolder, "runtimeConfigBaseFolder"); var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName); Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName"); var filePath = ResourcePathUtil.Combine(runtimeConfigBaseFolder, "inputschema.json"); var schemaFile = await RuntimeStorage.SaveFile(filePath, schema); var secretName = $"{config.Name}-inputschemafile"; var schemaFileSecret = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, schemaFile, Configuration[Constants.ConfigSettingName_SparkType]); flowToDeploy.SetStringToken(TokenName_InputSchemaFilePath, schemaFileSecret); return("done"); }
public override async Task <FlowGuiConfig> HandleSensitiveData(FlowGuiConfig guiConfig) { var outputsData = guiConfig?.Outputs; if (outputsData != null && outputsData.Length > 0) { foreach (var rd in outputsData) { var connStr = rd.Properties?.ConnectionString; if (connStr != null && !KeyVaultUri.IsSecretUri(connStr)) { var secretName = $"{guiConfig.Name}-output"; var secretUri = await KeyVaultClient.SaveSecretAsync( keyvaultName : RuntimeKeyVaultName.Value, secretName : secretName, secretValue : connStr, hashSuffix : true); rd.Properties.ConnectionString = secretUri; } } } return(guiConfig); }
private async Task <FlowBlobOutputSpec> ProcessOutputBlob(string configName, FlowGuiOutput uiOutput) { if (uiOutput != null && uiOutput.Properties != null) { var sparkKeyVaultName = Configuration[Constants.ConfigSettingName_RuntimeKeyVaultName]; string connectionString = await KeyVaultClient.GetSecretFromKeyVaultAsync(sparkKeyVaultName, uiOutput.Properties.ConnectionString); var blobPath = $"wasbs://{uiOutput.Properties.ContainerName}{ParseBlobAccountName(connectionString)}.blob.core.windows.net/{uiOutput.Properties.BlobPrefix}/%1$tY/%1$tm/%1$td/%1$tH/${{quarterBucket}}/${{minuteBucket}}"; var secretId = $"{configName}-output"; var blobPathSecret = await KeyVaultClient.SaveSecretAsync(sparkKeyVaultName, secretId, blobPath, true); FlowBlobOutputSpec blobOutput = new FlowBlobOutputSpec() { CompressionType = uiOutput.Properties.CompressionType, Format = uiOutput.Properties.Format, Groups = new BlobOutputGroups() { Main = new BlobOutputMain() { Folder = blobPathSecret } } }; return(blobOutput); } else { return(null); } }
private async Task <FlowSqlOutputSpec> ProcessOutputSql(string configName, FlowGuiOutput uiOutput) { if (uiOutput != null && uiOutput.Properties != null) { var sparkKeyVaultName = Configuration[Constants.ConfigSettingName_RuntimeKeyVaultName]; string connectionString = await KeyVaultClient.ResolveSecretUriAsync(uiOutput.Properties.ConnectionString).ConfigureAwait(false); var database = GetValueFromJdbcConnection(connectionString, "database"); var user = GetValueFromJdbcConnection(connectionString, "user"); var pwd = GetValueFromJdbcConnection(connectionString, "password"); var url = GetUrlFromJdbcConnection(connectionString); var sparkType = Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string value) ? value : null; // Save password and url in keyvault var pwdSecretId = $"{configName}-outSqlPassword"; var pwdRef = await KeyVaultClient.SaveSecretAsync(sparkKeyVaultName, pwdSecretId, pwd, sparkType, true).ConfigureAwait(false); var urlSecretId = $"{configName}-outSqlUrl"; var urlRef = await KeyVaultClient.SaveSecretAsync(sparkKeyVaultName, urlSecretId, url, sparkType, true).ConfigureAwait(false); FlowSqlOutputSpec sqlOutput = new FlowSqlOutputSpec() { ConnectionStringRef = uiOutput.Properties.ConnectionString, TableName = uiOutput.Properties.TableName, WriteMode = uiOutput.Properties.WriteMode, UseBulkInsert = uiOutput.Properties.UseBulkInsert, DatabaseName = database, User = user, Password = pwdRef, Url = urlRef }; return(sqlOutput); } else { return(null); } }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var config = flowToDeploy.Config; var guiConfig = config?.GetGuiConfig(); if (guiConfig == null) { return("no gui input, skipped."); } string queries = string.Join("\n", guiConfig.Process?.Queries); string ruleDefinitions = RuleDefinitionGenerator.GenerateRuleDefinitions(guiConfig.Rules, config.Name); RulesCode rulesCode = CodeGen.GenerateCode(queries, ruleDefinitions, config.Name); Ensure.NotNull(rulesCode, "rulesCode"); // Save the rulesCode object for downstream processing flowToDeploy.SetAttachment(AttachmentName_CodeGenObject, rulesCode); var runtimeConfigBaseFolder = flowToDeploy.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder); Ensure.NotNull(runtimeConfigBaseFolder, "runtimeConfigBaseFolder"); var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName); Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName"); var filePath = ResourcePathUtil.Combine(runtimeConfigBaseFolder, $"{config.Name}-combined.txt"); var transformFilePath = await RuntimeStorage.SaveFile(filePath, rulesCode.Code); var secretName = $"{config.Name}-transform"; var transformFileSecret = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, transformFilePath); flowToDeploy.SetStringToken(TokenName_TransformFile, transformFileSecret); return("done"); }
private async Task <FlowBlobOutputSpec> ProcessOutputBlob(string inputMode, string configName, FlowGuiOutput uiOutput) { if (uiOutput != null && uiOutput.Properties != null) { var sparkKeyVaultName = Configuration[Constants.ConfigSettingName_RuntimeKeyVaultName]; string connectionString = await KeyVaultClient.ResolveSecretUriAsync(uiOutput.Properties.ConnectionString).ConfigureAwait(false); var accountName = ConfigHelper.ParseBlobAccountName(connectionString); var timeFormat = ConfigHelper.GetBlobPartitionFormat(inputMode, uiOutput.Properties.BlobPartitionFormat); var blobPath = $"wasbs://{uiOutput.Properties.ContainerName}@{accountName}.blob.core.windows.net/{uiOutput.Properties.BlobPrefix}/{timeFormat}"; var secretId = $"{configName}-output"; var sparkType = Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string value) ? value : null; var blobPathSecret = await KeyVaultClient.SaveSecretAsync(sparkKeyVaultName, secretId, blobPath, sparkType, true); await KeyVaultClient.SaveSecretAsync(sparkKeyVaultName, $"{Constants.AccountSecretPrefix}{accountName}", ConfigHelper.ParseBlobAccountKey(connectionString), sparkType, false); FlowBlobOutputSpec blobOutput = new FlowBlobOutputSpec() { CompressionType = uiOutput.Properties.CompressionType, Format = uiOutput.Properties.Format, Groups = new BlobOutputGroups() { Main = new BlobOutputMain() { Folder = blobPathSecret } } }; return(blobOutput); } else { return(null); } }