public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var config = flowToDeploy.Config; var guiConfig = config?.GetGuiConfig(); if (guiConfig == null) { return("no gui input, skipped."); } var rulesCode = flowToDeploy.GetAttachment <RulesCode>(PrepareTransformFile.AttachmentName_CodeGenObject); Ensure.NotNull(rulesCode, "rulesCode"); Ensure.NotNull(rulesCode.MetricsRoot, "rulesCode.MetricsRoot"); Ensure.NotNull(rulesCode.MetricsRoot.metrics, "rulesCode.MetricsRoot.metrics"); var stateTables = rulesCode.AccumlationTables.Select(t => { return(new StateTableSpec() { Name = t.Key, Schema = t.Value, Location = ConstructStateTablePath(guiConfig.Name, t.Key) }); }).ToArray(); flowToDeploy.SetObjectToken(TokenName_StateTables, stateTables); await Task.Yield(); return("done"); }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var config = flowToDeploy.Config; var guiConfig = config?.GetGuiConfig(); if (guiConfig == null) { return("no gui input, skipped."); } var rulesCode = flowToDeploy.GetAttachment <RulesCode>(GenerateTransformFile.AttachmentName_CodeGenObject); Ensure.NotNull(rulesCode, "rulesCode"); Ensure.NotNull(rulesCode.MetricsRoot, "rulesCode.MetricsRoot"); Ensure.NotNull(rulesCode.MetricsRoot.metrics, "rulesCode.MetricsRoot.metrics"); var timewindows = rulesCode.TimeWindows.Select(t => { return(new TimeWindowSpec() { Name = t.Key, WindowDuration = t.Value }); }).ToArray(); flowToDeploy.SetObjectToken(TokenName_TimeWindows, timewindows); await Task.Yield(); return("done"); }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var jobs = flowToDeploy.GetJobs(); if (jobs == null || !jobs.Where(j => j.JobConfigs.Any()).Any()) { return("no jobs, skipped"); } var config = flowToDeploy.Config; var rulesCode = flowToDeploy.GetAttachment <RulesCode>(PrepareTransformFile.AttachmentName_CodeGenObject); Ensure.NotNull(rulesCode, "rulesCode"); var runtimeConfigBaseFolder = flowToDeploy.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder); Ensure.NotNull(runtimeConfigBaseFolder, "runtimeConfigBaseFolder"); var filePath = ResourcePathUtil.Combine(runtimeConfigBaseFolder, $"{config.Name}-combined.txt"); var transformFilePath = await RuntimeStorage.SaveFile(filePath, rulesCode.Code); var transformFileSecret = flowToDeploy.GetTokenString(PrepareTransformFile.TokenName_TransformFile); await KeyVaultClient.SaveSecretAsync(transformFileSecret, transformFilePath); return("done"); }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var config = flowToDeploy.Config; var guiConfig = config.GetGuiConfig(); if (guiConfig == null) { return("no gui input, skipped"); } // Get code gen data from earlier step var rulesCode = flowToDeploy.GetAttachment <RulesCode>(PrepareTransformFile.AttachmentName_CodeGenObject); Ensure.NotNull(rulesCode, "rulesCode"); Ensure.NotNull(rulesCode.MetricsRoot, "rulesCode.MetricsRoot"); Ensure.NotNull(rulesCode.MetricsRoot.metrics, "rulesCode.MetricsRoot.metrics"); // Set the metrics interval to the batch interval var intervalInSeconds = guiConfig?.Input?.Properties?.WindowDuration; Ensure.NotNull(intervalInSeconds, "input.properties.windowDuration"); foreach (var s in rulesCode.MetricsRoot.metrics.sources) { s.input.pollingInterval = Convert.ToInt32(intervalInSeconds) * 1000; } string metricsRaw = JsonConvert.SerializeObject(rulesCode.MetricsRoot.metrics); string metrics = metricsRaw.Replace("_FLOW_", config.Name); var newMetrics = JsonConvert.DeserializeObject <MetricsConfig>(metrics); var defaultConfig = flowToDeploy.GetAttachment <FlowConfig>(MergeDefaultConfig.AttachementName_DefaultFlowConfig); var defaultMetrics = defaultConfig?.Metrics; newMetrics.Sources = MergeLists(defaultMetrics?.Sources, newMetrics?.Sources); newMetrics.Widgets = MergeLists(defaultMetrics?.Widgets, newMetrics?.Widgets); await FlowData.UpdateMetricsForFlow(config.Name, newMetrics); return("done"); }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var config = flowToDeploy.Config; var guiConfig = config.GetGuiConfig(); if (guiConfig == null) { return("no gui input, skipped"); } var rulesCode = flowToDeploy.GetAttachment <RulesCode>(GenerateTransformFile.AttachmentName_CodeGenObject); Ensure.NotNull(rulesCode, "rulesCode"); Ensure.NotNull(rulesCode.MetricsRoot, "rulesCode.MetricsRoot"); Ensure.NotNull(rulesCode.MetricsRoot.metrics, "rulesCode.MetricsRoot.metrics"); var outputs = await ProcessOutputs(guiConfig.Outputs, rulesCode, config.Name); flowToDeploy.SetObjectToken(TokenName_Outputs, outputs); return("done"); }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { // Update jobs name in the flow config var jobNames = flowToDeploy.GetAttachment <string[]>(DeploySparkJob.AttachmentName_SparkJobNames); if (jobNames == null) { flowToDeploy.Result = new SuccessResult($"no jobs are deployed", flowToDeploy.ResultProperties); return("no jobs, skipped"); } var result = await this.FlowData.UpdateJobNamesForFlow(flowToDeploy.Name, jobNames); if (!result.IsSuccess) { throw new ConfigGenerationException(result.Message); } flowToDeploy.Result = new SuccessResult($"Deployed jobs:[{string.Join(",", jobNames)}]", flowToDeploy.ResultProperties); return("done"); }