public async Task <ApiResult> SaveFlow([FromBody] JObject config)
{
try
{
RolesCheck.EnsureWriter(Request, _isLocal);
Ensure.NotNull(config, "config");
var inputJson = JsonConfig.From(config.ToString());
var result = await _flowOperation.SaveFlowConfig(FlowGuiConfig.From(inputJson));
Ensure.IsSuccessResult(result);
if (result.Properties != null && result.Properties.TryGetValue(FlowOperation.FlowConfigPropertyName, out object flowConfig))
{
return(ApiResult.CreateSuccess(JToken.FromObject(flowConfig)));
}
else
{
return(ApiResult.CreateSuccess(JToken.FromObject(string.Empty)));
}
}
catch (Exception e)
{
_logger.LogError(e, e.Message);
return(ApiResult.CreateError(e.Message));
}
}
public override async Task <FlowGuiConfig> HandleSensitiveData(FlowGuiConfig guiConfig)
{
var functions = guiConfig?.Process?.Functions;
if (functions != null && functions.Count > 0)
{
foreach (var func in functions.Where(f => f.Type == FunctionTypeName))
{
var code = func.Properties?.Code;
if (!string.IsNullOrEmpty(code) && !KeyVaultUri.IsSecretUri(code))
{
var secretName = $"{guiConfig.Name}-azurefunc";
var secretUri = await KeyVaultClient.SaveSecretAsync(
keyvaultName : RuntimeKeyVaultName.Value,
secretName : secretName,
secretValue : code,
sparkType : Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string sparkType)?sparkType : null,
hashSuffix : true);
func.Properties.Code = secretUri;
}
}
}
return(guiConfig);
}
public override async Task <FlowGuiConfig> HandleSensitiveData(FlowGuiConfig guiConfig)
{
var functions = guiConfig?.Process?.Functions;
if (functions != null && functions.Count > 0)
{
foreach (var func in functions.Where(f => f.Type == FunctionTypeName))
{
var code = func.Properties?.Code;
if (code != null && !KeyVaultUri.IsSecretUri(code))
{
var secretName = $"{guiConfig.Name}-azurefunc";
var secretUri = await KeyVaultClient.SaveSecretAsync(
keyvaultName : RuntimeKeyVaultName.Value,
secretName : secretName,
secretValue : code,
hashSuffix : true);
func.Properties.Code = secretUri;
}
}
}
return(guiConfig);
}
public override async Task <FlowGuiConfig> HandleSensitiveData(FlowGuiConfig guiConfig)
{
var functions = guiConfig?.Process?.Functions;
if (functions != null && functions.Count > 0)
{
foreach (var func in functions.Where(f => f.Type == "jarUDF" || f.Type == "jarUDAF"))
{
var path = func.Properties?.Path;
if (path != null && !KeyVaultUri.IsSecretUri(path))
{
var secretName = $"{guiConfig.Name}-jarpath";
var secretUri = await KeyVaultClient.SaveSecretAsync(
keyvaultName : RuntimeKeyVaultName.Value,
secretName : secretName,
secretValue : path,
sparkType : Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string sparkType)?sparkType : null,
hashSuffix : true);
func.Properties.Path = secretUri;
}
}
}
return(guiConfig);
}
public override async Task <FlowGuiConfig> HandleSensitiveData(FlowGuiConfig guiConfig)
{
var referenceData = guiConfig?.Input?.ReferenceData;
if (referenceData != null && referenceData.Length > 0)
{
foreach (var rd in referenceData)
{
var path = rd.Properties?.Path;
if (path != null && !KeyVaultUri.IsSecretUri(path))
{
var secretName = $"{guiConfig.Name}-referencedata";
var secretUri = await KeyVaultClient.SaveSecretAsync(
keyvaultName : RuntimeKeyVaultName.Value,
secretName : secretName,
secretValue : path,
sparkType : Configuration[Constants.ConfigSettingName_SparkType],
hashSuffix : true);
rd.Properties.Path = secretUri;
}
}
}
return(guiConfig);
}
public override async Task <FlowGuiConfig> HandleSensitiveData(FlowGuiConfig guiConfig)
{
var outputsData = guiConfig?.Outputs;
if (outputsData != null && outputsData.Length > 0)
{
foreach (var rd in outputsData)
{
var connStr = rd.Properties?.ConnectionString;
if (connStr != null && !KeyVaultUri.IsSecretUri(connStr))
{
var secretName = $"{guiConfig.Name}-output";
var secretUri = await KeyVaultClient.SaveSecretAsync(
keyvaultName : RuntimeKeyVaultName.Value,
secretName : secretName,
secretValue : connStr,
hashSuffix : true);
rd.Properties.ConnectionString = secretUri;
}
}
}
return(guiConfig);
}
public async Task TestEndToEnd()
{
string flowName = "localconfiggentest";
// Point these two settings to your filesystem locations
InitialConfiguration.Set(LocalSparkClient.ConfigSettingName_SparkHomeFolder, @"<LOCAL SPARK HOME FOLDER>");
InitialConfiguration.Set(Constants.ConfigSettingName_LocalRoot, @"<LOCAL ROOT>");
var input = await File.ReadAllTextAsync(@"Resource\guiInputLocal.json");
var inputJson = JsonConfig.From(input);
var result = await this.FlowOperation.SaveFlowConfig(FlowGuiConfig.From(inputJson));
Assert.IsTrue(result.IsSuccess, result.Message);
var result2 = await this.RuntimeConfigGeneration.GenerateRuntimeConfigs(flowName);
var runtimeConfigFolderUri = result2.Properties?.GetValueOrDefault(PrepareJobConfigVariables.ResultPropertyName_RuntimeConfigFolder, null);
_runtimeConfigFolder = new System.Uri(runtimeConfigFolderUri.ToString()).AbsolutePath;
var jobResult1 = await JobOperation.StartJob(flowName);
Assert.IsTrue(jobResult1.IsSuccess);
// Wait for few seconds for the job to do some work
System.Threading.Thread.Sleep(5000);
var jobResult2 = await JobOperation.StopJob(flowName);
Assert.IsTrue(jobResult2.IsSuccess);
}
public override async Task <FlowGuiConfig> HandleSensitiveData(FlowGuiConfig guiConfig)
{
var runtimeKeyVaultName = Configuration[Constants.ConfigSettingName_RuntimeKeyVaultName];
var sparkType = Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string value) ? value : null;
Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");
// Replace Input Event Hub Connection String
var eventHubConnectionString = guiConfig?.Input?.Properties?.InputEventhubConnection;
if (!string.IsNullOrEmpty(eventHubConnectionString) && !KeyVaultUri.IsSecretUri(eventHubConnectionString))
{
// create new secret
var secretName = $"{guiConfig.Name}-input-eventhubconnectionstring";
var secretId = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, eventHubConnectionString, sparkType);
guiConfig.Input.Properties.InputEventhubConnection = secretId;
}
// Replace Input Event Hub SubscriptionId
var inputSubscriptionId = guiConfig?.Input?.Properties?.InputSubscriptionId;
if (!string.IsNullOrEmpty(inputSubscriptionId) && !KeyVaultUri.IsSecretUri(inputSubscriptionId))
{
var secretName = $"{guiConfig.Name}-input-inputsubscriptionid";
var secretId = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, inputSubscriptionId, sparkType);
guiConfig.Input.Properties.InputSubscriptionId = secretId;
}
// Replace Input Event Hub ResourceGroup
var inputResourceGroup = guiConfig?.Input?.Properties?.InputResourceGroup;
if (!string.IsNullOrEmpty(inputResourceGroup) && !KeyVaultUri.IsSecretUri(inputResourceGroup))
{
var secretName = $"{guiConfig.Name}-input-inputResourceGroup";
var secretId = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, inputResourceGroup, sparkType);
guiConfig.Input.Properties.InputResourceGroup = secretId;
}
// Replace Info Databricks Token
var infoDatabricksToken = guiConfig?.DatabricksToken;
if (!string.IsNullOrEmpty(infoDatabricksToken) && !KeyVaultUri.IsSecretUri(infoDatabricksToken))
{
var secretName = $"{guiConfig.Name}-info-databricksToken";
var secretId = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, infoDatabricksToken, sparkType);
guiConfig.DatabricksToken = secretId;
}
return(guiConfig);
}
public void TestDeserializedFlowGuiConfig()
{
var jsonString = File.ReadAllText(@"Resource\guiInput.json");
var json = JsonConfig.From(jsonString);
var instance = FlowGuiConfig.From(json);
Assert.AreEqual(expected: "configgentest", actual: instance.Name);
Assert.AreEqual(expected: "configgentest", actual: instance.DisplayName);
Assert.AreEqual(expected: "iothub", actual: instance.Input.InputType);
Assert.AreEqual(expected: "8000", actual: instance.Process.JobConfig.JobExecutorMemory);
}
/// <summary>
/// Save a flow with the gui input
/// </summary>
/// <param name="flowConfig"></param>
/// <returns></returns>
public async Task <Result> SaveFlowConfig(FlowGuiConfig flowConfig)
{
var config = await ConfigBuilder.Build(flowConfig);
var existingFlow = await FlowData.GetByName(config.Name);
Result result = null;
if (existingFlow != null)
{
existingFlow.Gui = config.Gui;
existingFlow.CommonProcessor.Template = config.CommonProcessor.Template;
existingFlow.CommonProcessor.SparkJobTemplateRef = config.CommonProcessor.SparkJobTemplateRef;
existingFlow.DisplayName = config.DisplayName;
config = existingFlow;
result = await FlowData.UpdateGuiForFlow(config.Name, config.Gui);
if (result.IsSuccess)
{
result = await FlowData.UpdateCommonProcessorForFlow(config.Name, config.CommonProcessor);
}
}
else
{
result = await this.FlowData.Upsert(config);
}
if (result.IsSuccess)
{
// pass the generated config back with the result
var properties = new Dictionary <string, object>()
{
{ FlowConfigPropertyName, config }
};
if (result.Properties != null)
{
result.Properties.AppendDictionary(properties);
}
else
{
result.Properties = properties;
}
}
return(result);
}
public override async Task <FlowGuiConfig> HandleSensitiveData(FlowGuiConfig guiConfig)
{
var runtimeKeyVaultName = Configuration[Constants.ConfigSettingName_RuntimeKeyVaultName];
Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");
// Replace Input Event Hub Connection String
var eventHubConnectionString = guiConfig?.Input?.Properties?.InputEventhubConnection;
if (eventHubConnectionString != null && !KeyVaultUri.IsSecretUri(eventHubConnectionString))
{
//TODO: create new secret
var secretName = $"{guiConfig.Name}-input-eventhubconnectionstring";
var secretId = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, eventHubConnectionString);
guiConfig.Input.Properties.InputEventhubConnection = secretId;
}
// Replace Input Event Hub SubscriptionId
var inputSubscriptionId = guiConfig?.Input?.Properties?.InputSubscriptionId;
if (!string.IsNullOrEmpty(inputSubscriptionId) && !KeyVaultUri.IsSecretUri(inputSubscriptionId))
{
var secretName = $"{guiConfig.Name}-input-inputsubscriptionid";
var secretId = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, inputSubscriptionId);
guiConfig.Input.Properties.InputSubscriptionId = secretId;
}
// Replace Input Event Hub ResourceGroup
var inputResourceGroup = guiConfig?.Input?.Properties?.InputResourceGroup;
if (!string.IsNullOrEmpty(inputResourceGroup) && !KeyVaultUri.IsSecretUri(inputResourceGroup))
{
var secretName = $"{guiConfig.Name}-input-inputResourceGroup";
var secretId = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, inputResourceGroup);
guiConfig.Input.Properties.InputResourceGroup = secretId;
}
return(guiConfig);
}
public async Task TestConfigSaved()
{
string flowName = "localconfiggentest";
var input = await File.ReadAllTextAsync(@"Resource\guiInput.json");
var inputJson = JsonConfig.From(input);
var result = await this.FlowOperation.SaveFlowConfig(FlowGuiConfig.From(inputJson));
Assert.IsTrue(result.IsSuccess, result.Message);
var actualJson = await this.FlowOperation.GetFlowByName(flowName);
var expectedJson = FlowConfig.From(await File.ReadAllTextAsync(@"Resource\flowSaved.json"));
foreach (var match in EntityConfig.Match(expectedJson, actualJson))
{
Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"path:{match.Item1}");
}
}
public async Task <FlowConfig> Build(FlowGuiConfig inputGuiConfig)
{
var defaultConfig = await FlowData.GetFlowConfigByInputType(inputGuiConfig.Input.InputType);
var config = defaultConfig ?? FlowConfig.From(JsonConfig.CreateEmpty());
// Set the flowName. This will be the case for new flow creation
if (string.IsNullOrWhiteSpace(inputGuiConfig.Name))
{
string flowName = GenerateValidFlowName(inputGuiConfig.DisplayName);
config.Name = flowName;
inputGuiConfig.Name = flowName;
}
var guiConfig = await HandleSensitiveData(inputGuiConfig);
config.Name = guiConfig?.Name ?? config.Name;
config.DisplayName = guiConfig?.DisplayName ?? config.DisplayName;
config.Gui = guiConfig.ToJson()._jt;
return(FlowConfig.From(config.ToString()));
}
public async Task TestConfigSaved()
{
await CommonData.Add(FlowDataManager.CommonDataName_DefaultFlowConfig, @"Resource\flowDefault.json");
var input = await File.ReadAllTextAsync(@"Resource\guiInput.json");
var inputJson = JsonConfig.From(input);
var result = await this.FlowOperation.SaveFlowConfig(FlowGuiConfig.From(inputJson));
Assert.IsTrue(result.IsSuccess, result.Message);
var actualJson = await this.FlowOperation.GetFlowByName("configgentest");
var expectedJson = FlowConfig.From(await File.ReadAllTextAsync(@"Resource\flowSaved.json"));
foreach (var match in EntityConfig.Match(expectedJson, actualJson))
{
Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"path:{match.Item1}");
}
Cleanup();
}
public override async Task <FlowGuiConfig> HandleSensitiveData(FlowGuiConfig guiConfig)
{
if (guiConfig?.Input?.Mode == Constants.InputMode_Batching)
{
var runtimeKeyVaultName = Configuration[Constants.ConfigSettingName_RuntimeKeyVaultName];
Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");
var sparkType = Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string value) ? value : null;
for (int i = 0; i < guiConfig?.Input?.Batch?.Length; i++)
{
// Replace Input Path
var input = guiConfig?.Input?.Batch[i];
var inputConnection = input.Properties.Connection;
if (!string.IsNullOrEmpty(inputConnection) && !KeyVaultUri.IsSecretUri(inputConnection))
{
var secretName = $"{guiConfig.Name}-input-{i}-inputConnection";
var secretId = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, inputConnection, sparkType).ConfigureAwait(false);
input.Properties.Connection = secretId;
var accountName = ConfigHelper.ParseBlobAccountName(inputConnection);
await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, $"{Constants.AccountSecretPrefix}{accountName}", ConfigHelper.ParseBlobAccountKey(inputConnection), sparkType, false);
}
var inputPath = input.Properties.Path;
if (!string.IsNullOrEmpty(inputPath) && !KeyVaultUri.IsSecretUri(inputPath))
{
var secretName = $"{guiConfig.Name}-input-{i}-inputPath";
var secretId = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, inputPath, Configuration[Constants.ConfigSettingName_SparkType]).ConfigureAwait(false);
input.Properties.Path = secretId;
}
}
}
return(guiConfig);
}
private async Task <FlowGuiConfig> HandleSensitiveData(FlowGuiConfig inputConfig)
{
await Processors.ChainTask(p => p.HandleSensitiveData(inputConfig));
return(inputConfig);
}
public virtual Task <FlowGuiConfig> HandleSensitiveData(FlowGuiConfig guiConfig)
{
return(Task.FromResult(guiConfig));
}