public async Task <ApiResult> SaveFlow([FromBody] JObject config)
        {
            try
            {
                RolesCheck.EnsureWriter(Request, _isLocal);
                Ensure.NotNull(config, "config");

                var inputJson = JsonConfig.From(config.ToString());
                var result    = await _flowOperation.SaveFlowConfig(FlowGuiConfig.From(inputJson));

                Ensure.IsSuccessResult(result);

                if (result.Properties != null && result.Properties.TryGetValue(FlowOperation.FlowConfigPropertyName, out object flowConfig))
                {
                    return(ApiResult.CreateSuccess(JToken.FromObject(flowConfig)));
                }
                else
                {
                    return(ApiResult.CreateSuccess(JToken.FromObject(string.Empty)));
                }
            }
            catch (Exception e)
            {
                _logger.LogError(e, e.Message);
                return(ApiResult.CreateError(e.Message));
            }
        }
        public override async Task <FlowGuiConfig> HandleSensitiveData(FlowGuiConfig guiConfig)
        {
            var functions = guiConfig?.Process?.Functions;

            if (functions != null && functions.Count > 0)
            {
                foreach (var func in functions.Where(f => f.Type == FunctionTypeName))
                {
                    var code = func.Properties?.Code;
                    if (!string.IsNullOrEmpty(code) && !KeyVaultUri.IsSecretUri(code))
                    {
                        var secretName = $"{guiConfig.Name}-azurefunc";
                        var secretUri  = await KeyVaultClient.SaveSecretAsync(
                            keyvaultName : RuntimeKeyVaultName.Value,
                            secretName : secretName,
                            secretValue : code,
                            sparkType : Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string sparkType)?sparkType : null,
                            hashSuffix : true);

                        func.Properties.Code = secretUri;
                    }
                }
            }

            return(guiConfig);
        }
        public override async Task <FlowGuiConfig> HandleSensitiveData(FlowGuiConfig guiConfig)
        {
            var functions = guiConfig?.Process?.Functions;

            if (functions != null && functions.Count > 0)
            {
                foreach (var func in functions.Where(f => f.Type == FunctionTypeName))
                {
                    var code = func.Properties?.Code;
                    if (code != null && !KeyVaultUri.IsSecretUri(code))
                    {
                        var secretName = $"{guiConfig.Name}-azurefunc";
                        var secretUri  = await KeyVaultClient.SaveSecretAsync(
                            keyvaultName : RuntimeKeyVaultName.Value,
                            secretName : secretName,
                            secretValue : code,
                            hashSuffix : true);

                        func.Properties.Code = secretUri;
                    }
                }
            }

            return(guiConfig);
        }
Esempio n. 4
0
        public override async Task <FlowGuiConfig> HandleSensitiveData(FlowGuiConfig guiConfig)
        {
            var functions = guiConfig?.Process?.Functions;

            if (functions != null && functions.Count > 0)
            {
                foreach (var func in functions.Where(f => f.Type == "jarUDF" || f.Type == "jarUDAF"))
                {
                    var path = func.Properties?.Path;
                    if (path != null && !KeyVaultUri.IsSecretUri(path))
                    {
                        var secretName = $"{guiConfig.Name}-jarpath";
                        var secretUri  = await KeyVaultClient.SaveSecretAsync(
                            keyvaultName : RuntimeKeyVaultName.Value,
                            secretName : secretName,
                            secretValue : path,
                            sparkType : Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string sparkType)?sparkType : null,
                            hashSuffix : true);

                        func.Properties.Path = secretUri;
                    }
                }
            }

            return(guiConfig);
        }
Esempio n. 5
0
        public override async Task <FlowGuiConfig> HandleSensitiveData(FlowGuiConfig guiConfig)
        {
            var referenceData = guiConfig?.Input?.ReferenceData;

            if (referenceData != null && referenceData.Length > 0)
            {
                foreach (var rd in referenceData)
                {
                    var path = rd.Properties?.Path;
                    if (path != null && !KeyVaultUri.IsSecretUri(path))
                    {
                        var secretName = $"{guiConfig.Name}-referencedata";
                        var secretUri  = await KeyVaultClient.SaveSecretAsync(
                            keyvaultName : RuntimeKeyVaultName.Value,
                            secretName : secretName,
                            secretValue : path,
                            sparkType : Configuration[Constants.ConfigSettingName_SparkType],
                            hashSuffix : true);

                        rd.Properties.Path = secretUri;
                    }
                }
            }

            return(guiConfig);
        }
Esempio n. 6
0
        public override async Task <FlowGuiConfig> HandleSensitiveData(FlowGuiConfig guiConfig)
        {
            var outputsData = guiConfig?.Outputs;

            if (outputsData != null && outputsData.Length > 0)
            {
                foreach (var rd in outputsData)
                {
                    var connStr = rd.Properties?.ConnectionString;
                    if (connStr != null && !KeyVaultUri.IsSecretUri(connStr))
                    {
                        var secretName = $"{guiConfig.Name}-output";
                        var secretUri  = await KeyVaultClient.SaveSecretAsync(
                            keyvaultName : RuntimeKeyVaultName.Value,
                            secretName : secretName,
                            secretValue : connStr,
                            hashSuffix : true);

                        rd.Properties.ConnectionString = secretUri;
                    }
                }
            }

            return(guiConfig);
        }
Esempio n. 7
0
        public async Task TestEndToEnd()
        {
            string flowName = "localconfiggentest";

            // Point these two settings to your filesystem locations
            InitialConfiguration.Set(LocalSparkClient.ConfigSettingName_SparkHomeFolder, @"<LOCAL SPARK HOME FOLDER>");
            InitialConfiguration.Set(Constants.ConfigSettingName_LocalRoot, @"<LOCAL ROOT>");

            var input = await File.ReadAllTextAsync(@"Resource\guiInputLocal.json");

            var inputJson = JsonConfig.From(input);
            var result    = await this.FlowOperation.SaveFlowConfig(FlowGuiConfig.From(inputJson));

            Assert.IsTrue(result.IsSuccess, result.Message);

            var result2 = await this.RuntimeConfigGeneration.GenerateRuntimeConfigs(flowName);

            var runtimeConfigFolderUri = result2.Properties?.GetValueOrDefault(PrepareJobConfigVariables.ResultPropertyName_RuntimeConfigFolder, null);

            _runtimeConfigFolder = new System.Uri(runtimeConfigFolderUri.ToString()).AbsolutePath;

            var jobResult1 = await JobOperation.StartJob(flowName);

            Assert.IsTrue(jobResult1.IsSuccess);
            // Wait for few seconds for the job to do some work
            System.Threading.Thread.Sleep(5000);
            var jobResult2 = await JobOperation.StopJob(flowName);

            Assert.IsTrue(jobResult2.IsSuccess);
        }
        public override async Task <FlowGuiConfig> HandleSensitiveData(FlowGuiConfig guiConfig)
        {
            var runtimeKeyVaultName = Configuration[Constants.ConfigSettingName_RuntimeKeyVaultName];
            var sparkType           = Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string value) ? value : null;

            Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");

            // Replace Input Event Hub Connection String
            var eventHubConnectionString = guiConfig?.Input?.Properties?.InputEventhubConnection;

            if (!string.IsNullOrEmpty(eventHubConnectionString) && !KeyVaultUri.IsSecretUri(eventHubConnectionString))
            {
                // create new secret
                var secretName = $"{guiConfig.Name}-input-eventhubconnectionstring";
                var secretId   = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, eventHubConnectionString, sparkType);

                guiConfig.Input.Properties.InputEventhubConnection = secretId;
            }

            // Replace Input Event Hub SubscriptionId
            var inputSubscriptionId = guiConfig?.Input?.Properties?.InputSubscriptionId;

            if (!string.IsNullOrEmpty(inputSubscriptionId) && !KeyVaultUri.IsSecretUri(inputSubscriptionId))
            {
                var secretName = $"{guiConfig.Name}-input-inputsubscriptionid";
                var secretId   = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, inputSubscriptionId, sparkType);

                guiConfig.Input.Properties.InputSubscriptionId = secretId;
            }

            // Replace Input Event Hub ResourceGroup
            var inputResourceGroup = guiConfig?.Input?.Properties?.InputResourceGroup;

            if (!string.IsNullOrEmpty(inputResourceGroup) && !KeyVaultUri.IsSecretUri(inputResourceGroup))
            {
                var secretName = $"{guiConfig.Name}-input-inputResourceGroup";
                var secretId   = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, inputResourceGroup, sparkType);

                guiConfig.Input.Properties.InputResourceGroup = secretId;
            }

            // Replace Info Databricks Token
            var infoDatabricksToken = guiConfig?.DatabricksToken;

            if (!string.IsNullOrEmpty(infoDatabricksToken) && !KeyVaultUri.IsSecretUri(infoDatabricksToken))
            {
                var secretName = $"{guiConfig.Name}-info-databricksToken";
                var secretId   = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, infoDatabricksToken, sparkType);

                guiConfig.DatabricksToken = secretId;
            }

            return(guiConfig);
        }
        public void TestDeserializedFlowGuiConfig()
        {
            var jsonString = File.ReadAllText(@"Resource\guiInput.json");
            var json       = JsonConfig.From(jsonString);
            var instance   = FlowGuiConfig.From(json);

            Assert.AreEqual(expected: "configgentest", actual: instance.Name);
            Assert.AreEqual(expected: "configgentest", actual: instance.DisplayName);
            Assert.AreEqual(expected: "iothub", actual: instance.Input.InputType);
            Assert.AreEqual(expected: "8000", actual: instance.Process.JobConfig.JobExecutorMemory);
        }
Esempio n. 10
0
        /// <summary>
        /// Save a flow with the gui input
        /// </summary>
        /// <param name="flowConfig"></param>
        /// <returns></returns>
        public async Task <Result> SaveFlowConfig(FlowGuiConfig flowConfig)
        {
            var config = await ConfigBuilder.Build(flowConfig);

            var existingFlow = await FlowData.GetByName(config.Name);

            Result result = null;

            if (existingFlow != null)
            {
                existingFlow.Gui = config.Gui;
                existingFlow.CommonProcessor.Template            = config.CommonProcessor.Template;
                existingFlow.CommonProcessor.SparkJobTemplateRef = config.CommonProcessor.SparkJobTemplateRef;
                existingFlow.DisplayName = config.DisplayName;
                config = existingFlow;
                result = await FlowData.UpdateGuiForFlow(config.Name, config.Gui);

                if (result.IsSuccess)
                {
                    result = await FlowData.UpdateCommonProcessorForFlow(config.Name, config.CommonProcessor);
                }
            }
            else
            {
                result = await this.FlowData.Upsert(config);
            }

            if (result.IsSuccess)
            {
                // pass the generated config back with the result
                var properties = new Dictionary <string, object>()
                {
                    { FlowConfigPropertyName, config }
                };

                if (result.Properties != null)
                {
                    result.Properties.AppendDictionary(properties);
                }
                else
                {
                    result.Properties = properties;
                }
            }
            return(result);
        }
Esempio n. 11
0
        public override async Task <FlowGuiConfig> HandleSensitiveData(FlowGuiConfig guiConfig)
        {
            var runtimeKeyVaultName = Configuration[Constants.ConfigSettingName_RuntimeKeyVaultName];

            Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");

            // Replace Input Event Hub Connection String
            var eventHubConnectionString = guiConfig?.Input?.Properties?.InputEventhubConnection;

            if (eventHubConnectionString != null && !KeyVaultUri.IsSecretUri(eventHubConnectionString))
            {
                //TODO: create new secret
                var secretName = $"{guiConfig.Name}-input-eventhubconnectionstring";
                var secretId   = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, eventHubConnectionString);

                guiConfig.Input.Properties.InputEventhubConnection = secretId;
            }

            // Replace Input Event Hub SubscriptionId
            var inputSubscriptionId = guiConfig?.Input?.Properties?.InputSubscriptionId;

            if (!string.IsNullOrEmpty(inputSubscriptionId) && !KeyVaultUri.IsSecretUri(inputSubscriptionId))
            {
                var secretName = $"{guiConfig.Name}-input-inputsubscriptionid";
                var secretId   = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, inputSubscriptionId);

                guiConfig.Input.Properties.InputSubscriptionId = secretId;
            }

            // Replace Input Event Hub ResourceGroup
            var inputResourceGroup = guiConfig?.Input?.Properties?.InputResourceGroup;

            if (!string.IsNullOrEmpty(inputResourceGroup) && !KeyVaultUri.IsSecretUri(inputResourceGroup))
            {
                var secretName = $"{guiConfig.Name}-input-inputResourceGroup";
                var secretId   = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, inputResourceGroup);

                guiConfig.Input.Properties.InputResourceGroup = secretId;
            }

            return(guiConfig);
        }
Esempio n. 12
0
        public async Task TestConfigSaved()
        {
            string flowName = "localconfiggentest";

            var input = await File.ReadAllTextAsync(@"Resource\guiInput.json");

            var inputJson = JsonConfig.From(input);
            var result    = await this.FlowOperation.SaveFlowConfig(FlowGuiConfig.From(inputJson));

            Assert.IsTrue(result.IsSuccess, result.Message);

            var actualJson = await this.FlowOperation.GetFlowByName(flowName);

            var expectedJson = FlowConfig.From(await File.ReadAllTextAsync(@"Resource\flowSaved.json"));

            foreach (var match in EntityConfig.Match(expectedJson, actualJson))
            {
                Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"path:{match.Item1}");
            }
        }
        public async Task <FlowConfig> Build(FlowGuiConfig inputGuiConfig)
        {
            var defaultConfig = await FlowData.GetFlowConfigByInputType(inputGuiConfig.Input.InputType);

            var config = defaultConfig ?? FlowConfig.From(JsonConfig.CreateEmpty());

            // Set the flowName. This will be the case for new flow creation
            if (string.IsNullOrWhiteSpace(inputGuiConfig.Name))
            {
                string flowName = GenerateValidFlowName(inputGuiConfig.DisplayName);
                config.Name         = flowName;
                inputGuiConfig.Name = flowName;
            }
            var guiConfig = await HandleSensitiveData(inputGuiConfig);

            config.Name = guiConfig?.Name ?? config.Name;

            config.DisplayName = guiConfig?.DisplayName ?? config.DisplayName;
            config.Gui         = guiConfig.ToJson()._jt;

            return(FlowConfig.From(config.ToString()));
        }
Esempio n. 14
0
        public async Task TestConfigSaved()
        {
            await CommonData.Add(FlowDataManager.CommonDataName_DefaultFlowConfig, @"Resource\flowDefault.json");

            var input = await File.ReadAllTextAsync(@"Resource\guiInput.json");

            var inputJson = JsonConfig.From(input);
            var result    = await this.FlowOperation.SaveFlowConfig(FlowGuiConfig.From(inputJson));

            Assert.IsTrue(result.IsSuccess, result.Message);

            var actualJson = await this.FlowOperation.GetFlowByName("configgentest");

            var expectedJson = FlowConfig.From(await File.ReadAllTextAsync(@"Resource\flowSaved.json"));

            foreach (var match in EntityConfig.Match(expectedJson, actualJson))
            {
                Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"path:{match.Item1}");
            }

            Cleanup();
        }
        public override async Task <FlowGuiConfig> HandleSensitiveData(FlowGuiConfig guiConfig)
        {
            if (guiConfig?.Input?.Mode == Constants.InputMode_Batching)
            {
                var runtimeKeyVaultName = Configuration[Constants.ConfigSettingName_RuntimeKeyVaultName];
                Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");

                var sparkType = Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string value) ? value : null;

                for (int i = 0; i < guiConfig?.Input?.Batch?.Length; i++)
                {
                    // Replace Input Path
                    var input           = guiConfig?.Input?.Batch[i];
                    var inputConnection = input.Properties.Connection;
                    if (!string.IsNullOrEmpty(inputConnection) && !KeyVaultUri.IsSecretUri(inputConnection))
                    {
                        var secretName = $"{guiConfig.Name}-input-{i}-inputConnection";
                        var secretId   = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, inputConnection, sparkType).ConfigureAwait(false);

                        input.Properties.Connection = secretId;

                        var accountName = ConfigHelper.ParseBlobAccountName(inputConnection);
                        await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, $"{Constants.AccountSecretPrefix}{accountName}", ConfigHelper.ParseBlobAccountKey(inputConnection), sparkType, false);
                    }

                    var inputPath = input.Properties.Path;
                    if (!string.IsNullOrEmpty(inputPath) && !KeyVaultUri.IsSecretUri(inputPath))
                    {
                        var secretName = $"{guiConfig.Name}-input-{i}-inputPath";
                        var secretId   = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, inputPath, Configuration[Constants.ConfigSettingName_SparkType]).ConfigureAwait(false);

                        input.Properties.Path = secretId;
                    }
                }
            }

            return(guiConfig);
        }
        private async Task <FlowGuiConfig> HandleSensitiveData(FlowGuiConfig inputConfig)
        {
            await Processors.ChainTask(p => p.HandleSensitiveData(inputConfig));

            return(inputConfig);
        }
Esempio n. 17
0
 public virtual Task <FlowGuiConfig> HandleSensitiveData(FlowGuiConfig guiConfig)
 {
     return(Task.FromResult(guiConfig));
 }