public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var flowConfig = flowToDeploy.Config;
// set the default job config
var defaultJobConfig = JsonConfig.From(flowConfig.CommonProcessor?.Template);
Ensure.NotNull(defaultJobConfig, "defaultJobConfig");
flowToDeploy.SetAttachment(ParameterObjectName_DefaultJobConfig, defaultJobConfig);
// Deploy job configs
var jobsToDeploy = flowToDeploy?.GetJobs();
if (jobsToDeploy != null)
{
foreach (var job in jobsToDeploy)
{
GenerateJobConfigContent(job, job.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder), defaultJobConfig);
}
return("done");
}
else
{
await Task.Yield();
return("no jobs, skipped");
}
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
// Deploy job configs
var jobs = flowToDeploy.GetJobs();
var deploymentTasks = jobs?.Select(async job => {
var content = job.GetTokenString(GenerateJobConfig.TokenName_JobConfigContent);
var filePath = job.GetTokenString(GenerateJobConfig.TokenName_JobConfigFilePath);
if (content != null && filePath != null)
{
job.SparkJobConfigFilePath = await this.JobData.SaveFile(filePath, content);
}
else
{
job.SparkJobConfigFilePath = null;
}
return(job.SparkJobConfigFilePath);
});
// Ensure all jobs configs are written successfully
if (deploymentTasks == null)
{
return("no jobs, skipped");
}
await Task.WhenAll(deploymentTasks);
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
// get a flattener
var flattener = await this.ConfigFlatteners.GetDefault();
if (flattener == null)
{
return("no flattern config, skipped");
}
// flatten each job config
var jobs = flowToDeploy.GetJobs();
if (jobs == null)
{
return("no jobs, skipped");
}
foreach (var job in jobs)
{
foreach (var jc in job.JobConfigs)
{
var jsonContent = job.Tokens.Resolve(jc.Content);
if (jsonContent != null)
{
var json = JsonConfig.From(jsonContent);
jc.Content = flattener.Flatten(json);
}
}
}
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var flowConfig = flowToDeploy.Config;
if (flowConfig.GetGuiConfig()?.Input?.Mode != Constants.InputMode_Batching)
{
return("done");
}
// set the default job config
var defaultJobConfig = JsonConfig.From(flowConfig.CommonProcessor?.Template);
Ensure.NotNull(defaultJobConfig, "defaultJobConfig");
flowToDeploy.SetAttachment(TokenName__DefaultJobConfig, defaultJobConfig);
// Deploy job configs
var jobsToDeploy = flowToDeploy?.GetJobs();
if (jobsToDeploy != null)
{
foreach (var job in jobsToDeploy)
{
await GenerateJobConfigContent(job, job.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder), defaultJobConfig).ConfigureAwait(false);
}
return("done");
}
else
{
await Task.Yield();
return("no jobs, skipped");
}
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var jobs = flowToDeploy.GetJobs();
if (jobs == null || !jobs.Where(j => j.JobConfigs.Any()).Any())
{
return("no jobs, skipped");
}
var config = flowToDeploy.Config;
var rulesCode = flowToDeploy.GetAttachment <RulesCode>(PrepareTransformFile.AttachmentName_CodeGenObject);
Ensure.NotNull(rulesCode, "rulesCode");
var runtimeConfigBaseFolder = flowToDeploy.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder);
Ensure.NotNull(runtimeConfigBaseFolder, "runtimeConfigBaseFolder");
var filePath = ResourcePathUtil.Combine(runtimeConfigBaseFolder, $"{config.Name}-combined.txt");
var transformFilePath = await RuntimeStorage.SaveFile(filePath, rulesCode.Code);
var transformFileSecret = flowToDeploy.GetTokenString(PrepareTransformFile.TokenName_TransformFile);
await KeyVaultClient.SaveSecretAsync(transformFileSecret, transformFilePath);
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var guiConfig = flowToDeploy.Config.GetGuiConfig();
if (guiConfig?.Input?.Mode == Constants.InputMode_Batching)
{
var jobConfigs = flowToDeploy.GetJobs().First().JobConfigs;
var reConfigs = jobConfigs.Where(j => !j.IsOneTime).ToList();
var otConfigs = jobConfigs.Where(j => j.IsOneTime).ToList();
Assert.AreEqual(expected: 2, actual: reConfigs.Count);
Assert.AreEqual(expected: 3, actual: otConfigs.Count);
var startTimeConfig = (DateTime)guiConfig.BatchList[0].Properties.StartTime;
var curTime = startTimeConfig.AddDays(1);
var normalizedStartTime = startTimeConfig.Add(-startTimeConfig.TimeOfDay);
TimeSpan interval = new TimeSpan(1, 0, 0, 0);
var expected = normalizedStartTime;
foreach (var c in reConfigs)
{
var actualStart = DateTime.Parse(c.ProcessStartTime).ToUniversalTime();
var actualEnd = DateTime.Parse(c.ProcessEndTime).ToUniversalTime();
var window = (actualEnd - actualStart).TotalSeconds;
Assert.AreEqual(expected: expected.AddDays(-2), actual: actualStart, message: $"StartTime:{actualStart}");
Assert.AreEqual(expected: expected.AddDays(1).AddSeconds(-1), actual: actualEnd, message: $"EndTime:{actualEnd}");
Assert.AreEqual(expected: 259199, actual: window, message: $"Window:{window}");
Assert.IsTrue(actualStart < curTime);
expected = expected.Add(interval);
}
var lastScheduleTime = DateTime.Parse(reConfigs.Last().ProcessStartTime).ToUniversalTime();
Assert.IsTrue(lastScheduleTime < curTime);
expected = normalizedStartTime;
foreach (var c in otConfigs)
{
var actualStart = DateTime.Parse(c.ProcessStartTime).ToUniversalTime();
var actualEnd = DateTime.Parse(c.ProcessEndTime).ToUniversalTime();
var window = (actualEnd - actualStart).TotalSeconds;
Assert.AreEqual(expected: expected, actual: actualStart, message: $"StartTime:{actualStart}");
Assert.AreEqual(expected: expected.AddDays(1).AddSeconds(-1), actual: actualEnd, message: $"EndTime:{actualEnd}");
Assert.AreEqual(expected: 86399, actual: window, message: $"Window:{window}");
expected = expected.Add(interval);
}
lastScheduleTime = DateTime.Parse(otConfigs.Last().ProcessStartTime).ToUniversalTime();
Assert.IsTrue(lastScheduleTime > curTime);
}
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var expectedConfigContent = await File.ReadAllTextAsync(@"Resource\jobConfig.json");
var expectedJson = JsonConfig.From(expectedConfigContent);
var actualContentContent = flowToDeploy.GetJobs().First().GetTokenString(GenerateJobConfig.TokenName_JobConfigContent);
var actualJson = JsonConfig.From(actualContentContent);
foreach (var match in JsonConfig.Match(expectedJson, actualJson))
{
Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"path:{match.Item1}");
}
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var jobs = flowToDeploy.GetJobs();
if (jobs == null || !jobs.Where(j => j.JobConfigs.Any()).Any())
{
return("no jobs, skipped");
}
var config = flowToDeploy.Config;
var guiConfig = config?.GetGuiConfig();
if (guiConfig == null)
{
return("no gui input, skipped.");
}
var projectColumns = guiConfig.Input?.Properties?.NormalizationSnippet?.Trim('\t', ' ', '\r', '\n');
//TODO: make the hardcoded "Raw.*" configurable?
var finalProjections = string.IsNullOrEmpty(projectColumns) ? "Raw.*" : projectColumns;
var runtimeConfigBaseFolder = flowToDeploy.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder);
Ensure.NotNull(runtimeConfigBaseFolder, "runtimeConfigBaseFolder");
var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName);
Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");
var filePath = ResourcePathUtil.Combine(runtimeConfigBaseFolder, "projection.txt");
var savedFile = await RuntimeStorage.SaveFile(filePath, finalProjections);
var tokenValue = flowToDeploy.GetTokenString(PrepareProjectionFile.TokenName_ProjectionFiles);
var projectionFileSecret = JArray.Parse(tokenValue).FirstOrDefault()?.Value <string>();
if (!string.IsNullOrEmpty(projectionFileSecret))
{
await KeyVaultClient.SaveSecretAsync(projectionFileSecret, savedFile);
}
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
// Get spark job config template with the reference from flow config
var jobConfigTemplateName = flowToDeploy.Config?.GetSparkJobConfigTemplateName();
var defaultSparkJobConfig = await this.SparkJobTemplateData.GetByName(jobConfigTemplateName);
Ensure.NotNull(defaultSparkJobConfig, "defaultSparkJobConfig");
var sparkJobUpsertions = flowToDeploy.GetJobs()?.Select(job => DeploySingleJob(job, defaultSparkJobConfig));
if (sparkJobUpsertions == null)
{
return("no jobs are defined, skipped");
}
// Ensure all creations are done successfully
var upsertedJobNames = await Task.WhenAll(sparkJobUpsertions);
flowToDeploy.SetAttachment(AttachmentName_SparkJobNames, upsertedJobNames);
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var jobs = flowToDeploy.GetJobs();
if (jobs == null || !jobs.Where(j => j.JobConfigs.Any()).Any())
{
return("no jobs, skipped");
}
var config = flowToDeploy.Config;
var guiConfig = config?.GetGuiConfig();
if (guiConfig == null)
{
return("no gui input, skipped.");
}
var schema = guiConfig.Input?.Properties?.InputSchemaFile;
Ensure.NotNull(schema, "guiConfig.input.properties.inputschemafile");
var runtimeConfigBaseFolder = flowToDeploy.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder);
Ensure.NotNull(runtimeConfigBaseFolder, "runtimeConfigBaseFolder");
var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName);
Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");
var filePath = ResourcePathUtil.Combine(runtimeConfigBaseFolder, "inputschema.json");
var schemaFile = await RuntimeStorage.SaveFile(filePath, schema);
var secretName = $"{config.Name}-inputschemafile";
var schemaFileSecret = flowToDeploy.GetTokenString(PrepareSchemaFile.TokenName_InputSchemaFilePath);
await KeyVaultClient.SaveSecretAsync(schemaFileSecret, schemaFile);
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var flowConfig = flowToDeploy.Config;
// get a flattener
var flattener = await this.ConfigFlatteners.GetDefault();
if (flattener == null)
{
return("no flattern config, skipped");
}
// flatten each job config
var jobs = flowToDeploy.GetJobs();
if (jobs == null)
{
return("no jobs, skipped");
}
foreach (var job in jobs)
{
var jsonContent = job.GetTokenString(GenerateJobConfig.TokenName_JobConfigContent);
var destFolder = job.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder);
if (jsonContent != null)
{
var json = JsonConfig.From(jsonContent);
job.SetStringToken(GenerateJobConfig.TokenName_JobConfigContent, flattener.Flatten(json));
var destinationPath = ResourcePathUtil.Combine(destFolder, job.Name + ".conf");
job.SetStringToken(GenerateJobConfig.TokenName_JobConfigFilePath, destinationPath);
}
}
return("done");
}
