public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var config = flowToDeploy.Config;
var guiConfig = config?.GetGuiConfig();
if (guiConfig == null)
{
return("no gui input, skipped.");
}
var rulesCode = flowToDeploy.GetAttachment <RulesCode>(PrepareTransformFile.AttachmentName_CodeGenObject);
Ensure.NotNull(rulesCode, "rulesCode");
Ensure.NotNull(rulesCode.MetricsRoot, "rulesCode.MetricsRoot");
Ensure.NotNull(rulesCode.MetricsRoot.metrics, "rulesCode.MetricsRoot.metrics");
var stateTables = rulesCode.AccumlationTables.Select(t =>
{
return(new StateTableSpec()
{
Name = t.Key,
Schema = t.Value,
Location = ConstructStateTablePath(guiConfig.Name, t.Key)
});
}).ToArray();
flowToDeploy.SetObjectToken(TokenName_StateTables, stateTables);
await Task.Yield();
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var config = flowToDeploy.Config;
var guiConfig = config?.GetGuiConfig();
if (guiConfig == null)
{
return("no gui input, skipped.");
}
var rulesCode = flowToDeploy.GetAttachment <RulesCode>(GenerateTransformFile.AttachmentName_CodeGenObject);
Ensure.NotNull(rulesCode, "rulesCode");
Ensure.NotNull(rulesCode.MetricsRoot, "rulesCode.MetricsRoot");
Ensure.NotNull(rulesCode.MetricsRoot.metrics, "rulesCode.MetricsRoot.metrics");
var timewindows = rulesCode.TimeWindows.Select(t =>
{
return(new TimeWindowSpec()
{
Name = t.Key,
WindowDuration = t.Value
});
}).ToArray();
flowToDeploy.SetObjectToken(TokenName_TimeWindows, timewindows);
await Task.Yield();
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var jobs = flowToDeploy.GetJobs();
if (jobs == null || !jobs.Where(j => j.JobConfigs.Any()).Any())
{
return("no jobs, skipped");
}
var config = flowToDeploy.Config;
var rulesCode = flowToDeploy.GetAttachment <RulesCode>(PrepareTransformFile.AttachmentName_CodeGenObject);
Ensure.NotNull(rulesCode, "rulesCode");
var runtimeConfigBaseFolder = flowToDeploy.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder);
Ensure.NotNull(runtimeConfigBaseFolder, "runtimeConfigBaseFolder");
var filePath = ResourcePathUtil.Combine(runtimeConfigBaseFolder, $"{config.Name}-combined.txt");
var transformFilePath = await RuntimeStorage.SaveFile(filePath, rulesCode.Code);
var transformFileSecret = flowToDeploy.GetTokenString(PrepareTransformFile.TokenName_TransformFile);
await KeyVaultClient.SaveSecretAsync(transformFileSecret, transformFilePath);
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var config = flowToDeploy.Config;
var guiConfig = config.GetGuiConfig();
if (guiConfig == null)
{
return("no gui input, skipped");
}
// Get code gen data from earlier step
var rulesCode = flowToDeploy.GetAttachment <RulesCode>(PrepareTransformFile.AttachmentName_CodeGenObject);
Ensure.NotNull(rulesCode, "rulesCode");
Ensure.NotNull(rulesCode.MetricsRoot, "rulesCode.MetricsRoot");
Ensure.NotNull(rulesCode.MetricsRoot.metrics, "rulesCode.MetricsRoot.metrics");
// Set the metrics interval to the batch interval
var intervalInSeconds = guiConfig?.Input?.Properties?.WindowDuration;
Ensure.NotNull(intervalInSeconds, "input.properties.windowDuration");
foreach (var s in rulesCode.MetricsRoot.metrics.sources)
{
s.input.pollingInterval = Convert.ToInt32(intervalInSeconds) * 1000;
}
string metricsRaw = JsonConvert.SerializeObject(rulesCode.MetricsRoot.metrics);
string metrics = metricsRaw.Replace("_FLOW_", config.Name);
var newMetrics = JsonConvert.DeserializeObject <MetricsConfig>(metrics);
var defaultConfig = flowToDeploy.GetAttachment <FlowConfig>(MergeDefaultConfig.AttachementName_DefaultFlowConfig);
var defaultMetrics = defaultConfig?.Metrics;
newMetrics.Sources = MergeLists(defaultMetrics?.Sources, newMetrics?.Sources);
newMetrics.Widgets = MergeLists(defaultMetrics?.Widgets, newMetrics?.Widgets);
await FlowData.UpdateMetricsForFlow(config.Name, newMetrics);
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var config = flowToDeploy.Config;
var guiConfig = config.GetGuiConfig();
if (guiConfig == null)
{
return("no gui input, skipped");
}
var rulesCode = flowToDeploy.GetAttachment <RulesCode>(GenerateTransformFile.AttachmentName_CodeGenObject);
Ensure.NotNull(rulesCode, "rulesCode");
Ensure.NotNull(rulesCode.MetricsRoot, "rulesCode.MetricsRoot");
Ensure.NotNull(rulesCode.MetricsRoot.metrics, "rulesCode.MetricsRoot.metrics");
var outputs = await ProcessOutputs(guiConfig.Outputs, rulesCode, config.Name);
flowToDeploy.SetObjectToken(TokenName_Outputs, outputs);
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
// Update jobs name in the flow config
var jobNames = flowToDeploy.GetAttachment <string[]>(DeploySparkJob.AttachmentName_SparkJobNames);
if (jobNames == null)
{
flowToDeploy.Result = new SuccessResult($"no jobs are deployed", flowToDeploy.ResultProperties);
return("no jobs, skipped");
}
var result = await this.FlowData.UpdateJobNamesForFlow(flowToDeploy.Name, jobNames);
if (!result.IsSuccess)
{
throw new ConfigGenerationException(result.Message);
}
flowToDeploy.Result = new SuccessResult($"Deployed jobs:[{string.Join(",", jobNames)}]", flowToDeploy.ResultProperties);
return("done");
}