public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var flowConfig = flowToDeploy.Config; if (flowConfig.GetGuiConfig()?.Input?.Mode != Constants.InputMode_Batching) { return("done"); } // set the default job config var defaultJobConfig = JsonConfig.From(flowConfig.CommonProcessor?.Template); Ensure.NotNull(defaultJobConfig, "defaultJobConfig"); flowToDeploy.SetAttachment(TokenName__DefaultJobConfig, defaultJobConfig); // Deploy job configs var jobsToDeploy = flowToDeploy?.GetJobs(); if (jobsToDeploy != null) { foreach (var job in jobsToDeploy) { await GenerateJobConfigContent(job, job.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder), defaultJobConfig).ConfigureAwait(false); } return("done"); } else { await Task.Yield(); return("no jobs, skipped"); } }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var flowConfig = flowToDeploy.Config; // set the default job config var defaultJobConfig = JsonConfig.From(flowConfig.CommonProcessor?.Template); Ensure.NotNull(defaultJobConfig, "defaultJobConfig"); flowToDeploy.SetAttachment(ParameterObjectName_DefaultJobConfig, defaultJobConfig); // Deploy job configs var jobsToDeploy = flowToDeploy?.GetJobs(); if (jobsToDeploy != null) { foreach (var job in jobsToDeploy) { GenerateJobConfigContent(job, job.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder), defaultJobConfig); } return("done"); } else { await Task.Yield(); return("no jobs, skipped"); } }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { if (flowToDeploy.Config.GetGuiConfig().Input?.Mode == Constants.InputMode_Batching) { var inputConfig = flowToDeploy.Config.GetGuiConfig(); var inputBatching = inputConfig.Input.Batch ?? Array.Empty <FlowGuiInputBatchInput>(); var specsTasks = inputBatching.Select(async rd => { var connectionString = await KeyVaultClient.ResolveSecretUriAsync(rd.Properties.Connection).ConfigureAwait(false); var inputPath = await KeyVaultClient.ResolveSecretUriAsync(rd.Properties.Path).ConfigureAwait(false); return(new InputBatchingSpec() { Name = ConfigHelper.ParseBlobAccountName(connectionString), Path = rd.Properties.Path, Format = rd.Properties.FormatType, CompressionType = rd.Properties.CompressionType, ProcessStartTime = "", ProcessEndTime = "", PartitionIncrement = GetPartitionIncrement(inputPath).ToString(CultureInfo.InvariantCulture), }); }).ToArray(); var specs = await Task.WhenAll(specsTasks).ConfigureAwait(false); flowToDeploy.SetAttachment(TokenName_InputBatching, specs); } return("done"); }
/// <summary> /// Generate and set the info for the transform file which will be used to generate JobConfig /// </summary> /// <returns></returns> public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var config = flowToDeploy.Config; var guiConfig = config?.GetGuiConfig(); if (guiConfig == null) { return("no gui input, skipped."); } string queries = string.Join("\n", guiConfig.Process?.Queries); string ruleDefinitions = RuleDefinitionGenerator.GenerateRuleDefinitions(guiConfig.Rules, config.Name); RulesCode rulesCode = CodeGen.GenerateCode(queries, ruleDefinitions, config.Name); Ensure.NotNull(rulesCode, "rulesCode"); // Save the rulesCode object for downstream processing flowToDeploy.SetAttachment(AttachmentName_CodeGenObject, rulesCode); var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName); Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName"); var secretName = $"{config.Name}-transform"; Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string sparkType); var uriPrefix = KeyVaultClient.GetUriPrefix(sparkType); var transformFileSecret = SecretUriParser.ComposeUri(runtimeKeyVaultName, secretName, uriPrefix); flowToDeploy.SetStringToken(TokenName_TransformFile, transformFileSecret); await Task.CompletedTask; return("done"); }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var flowConfig = flowToDeploy.Config; var defaultConfig = await FlowData.GetFlowConfigByInputType(flowConfig.GetGuiConfig().Input.InputType, flowToDeploy.Tokens); if (defaultConfig == null) { return("defaultConfig is null, skipped"); } var newConfig = flowConfig.RebaseOn(defaultConfig); flowToDeploy.UpdateFlowConfig(newConfig); flowToDeploy.SetAttachment(AttachementName_DefaultFlowConfig, defaultConfig); return("done"); }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { // Get spark job config template with the reference from flow config var jobConfigTemplateName = flowToDeploy.Config?.GetSparkJobConfigTemplateName(); var defaultSparkJobConfig = await this.SparkJobTemplateData.GetByName(jobConfigTemplateName); Ensure.NotNull(defaultSparkJobConfig, "defaultSparkJobConfig"); var sparkJobUpsertions = flowToDeploy.GetJobs()?.Select(job => DeploySingleJob(job, defaultSparkJobConfig)); if (sparkJobUpsertions == null) { return("no jobs are defined, skipped"); } // Ensure all creations are done successfully var upsertedJobNames = await Task.WhenAll(sparkJobUpsertions); flowToDeploy.SetAttachment(AttachmentName_SparkJobNames, upsertedJobNames); return("done"); }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var config = flowToDeploy.Config; var guiConfig = config?.GetGuiConfig(); if (guiConfig == null) { return("no gui input, skipped."); } string queries = string.Join("\n", guiConfig.Process?.Queries); string ruleDefinitions = RuleDefinitionGenerator.GenerateRuleDefinitions(guiConfig.Rules, config.Name); RulesCode rulesCode = CodeGen.GenerateCode(queries, ruleDefinitions, config.Name); Ensure.NotNull(rulesCode, "rulesCode"); // Save the rulesCode object for downstream processing flowToDeploy.SetAttachment(AttachmentName_CodeGenObject, rulesCode); var runtimeConfigBaseFolder = flowToDeploy.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder); Ensure.NotNull(runtimeConfigBaseFolder, "runtimeConfigBaseFolder"); var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName); Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName"); var filePath = ResourcePathUtil.Combine(runtimeConfigBaseFolder, $"{config.Name}-combined.txt"); var transformFilePath = await RuntimeStorage.SaveFile(filePath, rulesCode.Code); var secretName = $"{config.Name}-transform"; var transformFileSecret = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, transformFilePath); flowToDeploy.SetStringToken(TokenName_TransformFile, transformFileSecret); return("done"); }