Esempio n. 1
0
        public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
        {
            var flowConfig = flowToDeploy.Config;

            if (flowConfig.GetGuiConfig()?.Input?.Mode != Constants.InputMode_Batching)
            {
                return("done");
            }

            // set the default job config
            var defaultJobConfig = JsonConfig.From(flowConfig.CommonProcessor?.Template);

            Ensure.NotNull(defaultJobConfig, "defaultJobConfig");
            flowToDeploy.SetAttachment(TokenName__DefaultJobConfig, defaultJobConfig);

            // Deploy job configs
            var jobsToDeploy = flowToDeploy?.GetJobs();

            if (jobsToDeploy != null)
            {
                foreach (var job in jobsToDeploy)
                {
                    await GenerateJobConfigContent(job, job.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder), defaultJobConfig).ConfigureAwait(false);
                }

                return("done");
            }
            else
            {
                await Task.Yield();

                return("no jobs, skipped");
            }
        }
        public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
        {
            var flowConfig = flowToDeploy.Config;

            // set the default job config
            var defaultJobConfig = JsonConfig.From(flowConfig.CommonProcessor?.Template);

            Ensure.NotNull(defaultJobConfig, "defaultJobConfig");
            flowToDeploy.SetAttachment(ParameterObjectName_DefaultJobConfig, defaultJobConfig);

            // Deploy job configs
            var jobsToDeploy = flowToDeploy?.GetJobs();

            if (jobsToDeploy != null)
            {
                foreach (var job in jobsToDeploy)
                {
                    GenerateJobConfigContent(job, job.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder), defaultJobConfig);
                }

                return("done");
            }
            else
            {
                await Task.Yield();

                return("no jobs, skipped");
            }
        }
        public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
        {
            if (flowToDeploy.Config.GetGuiConfig().Input?.Mode == Constants.InputMode_Batching)
            {
                var inputConfig = flowToDeploy.Config.GetGuiConfig();

                var inputBatching = inputConfig.Input.Batch ?? Array.Empty <FlowGuiInputBatchInput>();
                var specsTasks    = inputBatching.Select(async rd =>
                {
                    var connectionString = await KeyVaultClient.ResolveSecretUriAsync(rd.Properties.Connection).ConfigureAwait(false);
                    var inputPath        = await KeyVaultClient.ResolveSecretUriAsync(rd.Properties.Path).ConfigureAwait(false);

                    return(new InputBatchingSpec()
                    {
                        Name = ConfigHelper.ParseBlobAccountName(connectionString),
                        Path = rd.Properties.Path,
                        Format = rd.Properties.FormatType,
                        CompressionType = rd.Properties.CompressionType,
                        ProcessStartTime = "",
                        ProcessEndTime = "",
                        PartitionIncrement = GetPartitionIncrement(inputPath).ToString(CultureInfo.InvariantCulture),
                    });
                }).ToArray();

                var specs = await Task.WhenAll(specsTasks).ConfigureAwait(false);

                flowToDeploy.SetAttachment(TokenName_InputBatching, specs);
            }

            return("done");
        }
        /// <summary>
        /// Generate and set the info for the transform file which will be used to generate JobConfig
        /// </summary>
        /// <returns></returns>
        public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
        {
            var config    = flowToDeploy.Config;
            var guiConfig = config?.GetGuiConfig();

            if (guiConfig == null)
            {
                return("no gui input, skipped.");
            }
            string queries = string.Join("\n", guiConfig.Process?.Queries);

            string    ruleDefinitions = RuleDefinitionGenerator.GenerateRuleDefinitions(guiConfig.Rules, config.Name);
            RulesCode rulesCode       = CodeGen.GenerateCode(queries, ruleDefinitions, config.Name);

            Ensure.NotNull(rulesCode, "rulesCode");

            // Save the rulesCode object for downstream processing
            flowToDeploy.SetAttachment(AttachmentName_CodeGenObject, rulesCode);

            var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName);

            Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");

            var secretName = $"{config.Name}-transform";

            Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string sparkType);
            var uriPrefix           = KeyVaultClient.GetUriPrefix(sparkType);
            var transformFileSecret = SecretUriParser.ComposeUri(runtimeKeyVaultName, secretName, uriPrefix);

            flowToDeploy.SetStringToken(TokenName_TransformFile, transformFileSecret);

            await Task.CompletedTask;

            return("done");
        }
Esempio n. 5
0
        public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
        {
            var flowConfig = flowToDeploy.Config;

            var defaultConfig = await FlowData.GetFlowConfigByInputType(flowConfig.GetGuiConfig().Input.InputType, flowToDeploy.Tokens);

            if (defaultConfig == null)
            {
                return("defaultConfig is null, skipped");
            }

            var newConfig = flowConfig.RebaseOn(defaultConfig);

            flowToDeploy.UpdateFlowConfig(newConfig);
            flowToDeploy.SetAttachment(AttachementName_DefaultFlowConfig, defaultConfig);

            return("done");
        }
        public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
        {
            // Get spark job config template with the reference from flow config
            var jobConfigTemplateName = flowToDeploy.Config?.GetSparkJobConfigTemplateName();
            var defaultSparkJobConfig = await this.SparkJobTemplateData.GetByName(jobConfigTemplateName);

            Ensure.NotNull(defaultSparkJobConfig, "defaultSparkJobConfig");

            var sparkJobUpsertions = flowToDeploy.GetJobs()?.Select(job => DeploySingleJob(job, defaultSparkJobConfig));

            if (sparkJobUpsertions == null)
            {
                return("no jobs are defined, skipped");
            }

            // Ensure all creations are done successfully
            var upsertedJobNames = await Task.WhenAll(sparkJobUpsertions);

            flowToDeploy.SetAttachment(AttachmentName_SparkJobNames, upsertedJobNames);

            return("done");
        }
        public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
        {
            var config    = flowToDeploy.Config;
            var guiConfig = config?.GetGuiConfig();

            if (guiConfig == null)
            {
                return("no gui input, skipped.");
            }
            string queries = string.Join("\n", guiConfig.Process?.Queries);

            string    ruleDefinitions = RuleDefinitionGenerator.GenerateRuleDefinitions(guiConfig.Rules, config.Name);
            RulesCode rulesCode       = CodeGen.GenerateCode(queries, ruleDefinitions, config.Name);

            Ensure.NotNull(rulesCode, "rulesCode");

            // Save the rulesCode object for downstream processing
            flowToDeploy.SetAttachment(AttachmentName_CodeGenObject, rulesCode);

            var runtimeConfigBaseFolder = flowToDeploy.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder);

            Ensure.NotNull(runtimeConfigBaseFolder, "runtimeConfigBaseFolder");

            var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName);

            Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");

            var filePath          = ResourcePathUtil.Combine(runtimeConfigBaseFolder, $"{config.Name}-combined.txt");
            var transformFilePath = await RuntimeStorage.SaveFile(filePath, rulesCode.Code);

            var secretName          = $"{config.Name}-transform";
            var transformFileSecret = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, transformFilePath);

            flowToDeploy.SetStringToken(TokenName_TransformFile, transformFileSecret);

            return("done");
        }