public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var flowConfig = flowToDeploy.Config;
if (flowConfig.GetGuiConfig()?.Input?.Mode != Constants.InputMode_Batching)
{
return("done");
}
// set the default job config
var defaultJobConfig = JsonConfig.From(flowConfig.CommonProcessor?.Template);
Ensure.NotNull(defaultJobConfig, "defaultJobConfig");
flowToDeploy.SetAttachment(TokenName__DefaultJobConfig, defaultJobConfig);
// Deploy job configs
var jobsToDeploy = flowToDeploy?.GetJobs();
if (jobsToDeploy != null)
{
foreach (var job in jobsToDeploy)
{
await GenerateJobConfigContent(job, job.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder), defaultJobConfig).ConfigureAwait(false);
}
return("done");
}
else
{
await Task.Yield();
return("no jobs, skipped");
}
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var flowConfig = flowToDeploy.Config;
// set the default job config
var defaultJobConfig = JsonConfig.From(flowConfig.CommonProcessor?.Template);
Ensure.NotNull(defaultJobConfig, "defaultJobConfig");
flowToDeploy.SetAttachment(ParameterObjectName_DefaultJobConfig, defaultJobConfig);
// Deploy job configs
var jobsToDeploy = flowToDeploy?.GetJobs();
if (jobsToDeploy != null)
{
foreach (var job in jobsToDeploy)
{
GenerateJobConfigContent(job, job.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder), defaultJobConfig);
}
return("done");
}
else
{
await Task.Yield();
return("no jobs, skipped");
}
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
if (flowToDeploy.Config.GetGuiConfig().Input?.Mode == Constants.InputMode_Batching)
{
var inputConfig = flowToDeploy.Config.GetGuiConfig();
var inputBatching = inputConfig.Input.Batch ?? Array.Empty <FlowGuiInputBatchInput>();
var specsTasks = inputBatching.Select(async rd =>
{
var connectionString = await KeyVaultClient.ResolveSecretUriAsync(rd.Properties.Connection).ConfigureAwait(false);
var inputPath = await KeyVaultClient.ResolveSecretUriAsync(rd.Properties.Path).ConfigureAwait(false);
return(new InputBatchingSpec()
{
Name = ConfigHelper.ParseBlobAccountName(connectionString),
Path = rd.Properties.Path,
Format = rd.Properties.FormatType,
CompressionType = rd.Properties.CompressionType,
ProcessStartTime = "",
ProcessEndTime = "",
PartitionIncrement = GetPartitionIncrement(inputPath).ToString(CultureInfo.InvariantCulture),
});
}).ToArray();
var specs = await Task.WhenAll(specsTasks).ConfigureAwait(false);
flowToDeploy.SetAttachment(TokenName_InputBatching, specs);
}
return("done");
}
/// <summary>
/// Generate and set the info for the transform file which will be used to generate JobConfig
/// </summary>
/// <returns></returns>
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var config = flowToDeploy.Config;
var guiConfig = config?.GetGuiConfig();
if (guiConfig == null)
{
return("no gui input, skipped.");
}
string queries = string.Join("\n", guiConfig.Process?.Queries);
string ruleDefinitions = RuleDefinitionGenerator.GenerateRuleDefinitions(guiConfig.Rules, config.Name);
RulesCode rulesCode = CodeGen.GenerateCode(queries, ruleDefinitions, config.Name);
Ensure.NotNull(rulesCode, "rulesCode");
// Save the rulesCode object for downstream processing
flowToDeploy.SetAttachment(AttachmentName_CodeGenObject, rulesCode);
var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName);
Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");
var secretName = $"{config.Name}-transform";
Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string sparkType);
var uriPrefix = KeyVaultClient.GetUriPrefix(sparkType);
var transformFileSecret = SecretUriParser.ComposeUri(runtimeKeyVaultName, secretName, uriPrefix);
flowToDeploy.SetStringToken(TokenName_TransformFile, transformFileSecret);
await Task.CompletedTask;
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var flowConfig = flowToDeploy.Config;
var defaultConfig = await FlowData.GetFlowConfigByInputType(flowConfig.GetGuiConfig().Input.InputType, flowToDeploy.Tokens);
if (defaultConfig == null)
{
return("defaultConfig is null, skipped");
}
var newConfig = flowConfig.RebaseOn(defaultConfig);
flowToDeploy.UpdateFlowConfig(newConfig);
flowToDeploy.SetAttachment(AttachementName_DefaultFlowConfig, defaultConfig);
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
// Get spark job config template with the reference from flow config
var jobConfigTemplateName = flowToDeploy.Config?.GetSparkJobConfigTemplateName();
var defaultSparkJobConfig = await this.SparkJobTemplateData.GetByName(jobConfigTemplateName);
Ensure.NotNull(defaultSparkJobConfig, "defaultSparkJobConfig");
var sparkJobUpsertions = flowToDeploy.GetJobs()?.Select(job => DeploySingleJob(job, defaultSparkJobConfig));
if (sparkJobUpsertions == null)
{
return("no jobs are defined, skipped");
}
// Ensure all creations are done successfully
var upsertedJobNames = await Task.WhenAll(sparkJobUpsertions);
flowToDeploy.SetAttachment(AttachmentName_SparkJobNames, upsertedJobNames);
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var config = flowToDeploy.Config;
var guiConfig = config?.GetGuiConfig();
if (guiConfig == null)
{
return("no gui input, skipped.");
}
string queries = string.Join("\n", guiConfig.Process?.Queries);
string ruleDefinitions = RuleDefinitionGenerator.GenerateRuleDefinitions(guiConfig.Rules, config.Name);
RulesCode rulesCode = CodeGen.GenerateCode(queries, ruleDefinitions, config.Name);
Ensure.NotNull(rulesCode, "rulesCode");
// Save the rulesCode object for downstream processing
flowToDeploy.SetAttachment(AttachmentName_CodeGenObject, rulesCode);
var runtimeConfigBaseFolder = flowToDeploy.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder);
Ensure.NotNull(runtimeConfigBaseFolder, "runtimeConfigBaseFolder");
var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName);
Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");
var filePath = ResourcePathUtil.Combine(runtimeConfigBaseFolder, $"{config.Name}-combined.txt");
var transformFilePath = await RuntimeStorage.SaveFile(filePath, rulesCode.Code);
var secretName = $"{config.Name}-transform";
var transformFileSecret = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, transformFilePath);
flowToDeploy.SetStringToken(TokenName_TransformFile, transformFileSecret);
return("done");
}