/// <summary>
/// Schedule a batch job
/// </summary>
/// <returns></returns>
private JobConfig ScheduleSingleJob(JobDeploymentSession job, string destFolder, JsonConfig defaultJobConfig, bool isOneTime, TimeSpan interval, TimeSpan window, DateTime processTime, DateTime scheduledTime, string prefix = "")
{
var ps_s = processTime;
var ps_e = processTime.Add(interval).AddMilliseconds(-1); //ENDTIME
var pe_s = ps_s.Add(-window);
var pe_e = ps_e.Add(-window); // STARTTIME
var dateString = ConvertDateToString(scheduledTime);
var suffix = prefix + $"-{Regex.Replace(dateString, "[^0-9]", "")}";
var jobName = job.Name + suffix;
job.SetStringToken("name", jobName);
Ensure.NotNull(defaultJobConfig, "defaultJobConfig");
var processStartTime = ConvertDateToString(pe_e);
var processEndTime = ConvertDateToString(ps_e);
destFolder = GetJobConfigFilePath(isOneTime, dateString, destFolder);
var jc = new JobConfig
{
Content = GetBatchConfigContent(job, defaultJobConfig.ToString(), processStartTime, processEndTime),
FilePath = ResourcePathUtil.Combine(destFolder, job.Name + ".conf"),
Name = jobName,
SparkJobName = job.SparkJobName + suffix,
ProcessStartTime = processStartTime,
ProcessEndTime = processEndTime,
ProcessingTime = dateString,
IsOneTime = isOneTime
};
return(jc);
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var config = flowToDeploy.Config;
var guiConfig = config?.GetGuiConfig();
if (guiConfig == null)
{
return("no gui input, skipped.");
}
var projectColumns = guiConfig.Input?.Properties?.NormalizationSnippet?.Trim('\t', ' ', '\r', '\n');
//TODO: make the hardcoded "Raw.*" configurable?
var finalProjections = string.IsNullOrEmpty(projectColumns) ? "Raw.*" : projectColumns;
var runtimeConfigBaseFolder = flowToDeploy.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder);
Ensure.NotNull(runtimeConfigBaseFolder, "runtimeConfigBaseFolder");
var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName);
Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");
var filePath = ResourcePathUtil.Combine(runtimeConfigBaseFolder, "projection.txt");
var savedFile = await RuntimeStorage.SaveFile(filePath, finalProjections);
var secretName = $"{config.Name}-projectionfile";
var savedSecretId = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, savedFile, Configuration[Constants.ConfigSettingName_SparkType]);
flowToDeploy.SetObjectToken(TokenName_ProjectionFiles, new string[] { savedSecretId });
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var config = flowToDeploy.Config;
var guiConfig = config?.GetGuiConfig();
if (guiConfig == null)
{
return("no gui input, skipped.");
}
var schema = guiConfig.Input?.Properties?.InputSchemaFile;
Ensure.NotNull(schema, "guiConfig.input.properties.inputschemafile");
var runtimeConfigBaseFolder = flowToDeploy.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder);
Ensure.NotNull(runtimeConfigBaseFolder, "runtimeConfigBaseFolder");
var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName);
Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");
var filePath = ResourcePathUtil.Combine(runtimeConfigBaseFolder, "inputschema.json");
var schemaFile = await RuntimeStorage.SaveFile(filePath, schema);
var secretName = $"{config.Name}-inputschemafile";
var schemaFileSecret = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, schemaFile, Configuration[Constants.ConfigSettingName_SparkType]);
flowToDeploy.SetStringToken(TokenName_InputSchemaFilePath, schemaFileSecret);
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var jobs = flowToDeploy.GetJobs();
if (jobs == null || !jobs.Where(j => j.JobConfigs.Any()).Any())
{
return("no jobs, skipped");
}
var config = flowToDeploy.Config;
var rulesCode = flowToDeploy.GetAttachment <RulesCode>(PrepareTransformFile.AttachmentName_CodeGenObject);
Ensure.NotNull(rulesCode, "rulesCode");
var runtimeConfigBaseFolder = flowToDeploy.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder);
Ensure.NotNull(runtimeConfigBaseFolder, "runtimeConfigBaseFolder");
var filePath = ResourcePathUtil.Combine(runtimeConfigBaseFolder, $"{config.Name}-combined.txt");
var transformFilePath = await RuntimeStorage.SaveFile(filePath, rulesCode.Code);
var transformFileSecret = flowToDeploy.GetTokenString(PrepareTransformFile.TokenName_TransformFile);
await KeyVaultClient.SaveSecretAsync(transformFileSecret, transformFilePath);
return("done");
}
public async Task TestEndToEndGeneration()
{
var flowName = "localconfiggentest";
var testingConfig = await File.ReadAllTextAsync(@"Resource\flowSaved.json");
await DesignTimeStorage.SaveByName(flowName, testingConfig, FlowDataManager.DataCollectionName);
// generate runtime configs
var result = await this.RuntimeConfigGeneration.GenerateRuntimeConfigs(flowName);
var runtimeConfigFolderUri = result.Properties?.GetValueOrDefault(PrepareJobConfigVariables.ResultPropertyName_RuntimeConfigFolder, null);
_runtimeConfigFolder = new System.Uri(runtimeConfigFolderUri.ToString()).AbsolutePath;
Assert.IsTrue(result.IsSuccess);
// verify output schema file is expected
var expectedSchema = JsonConfig.From(await File.ReadAllTextAsync(@"Resource\schema.json"));
var p = ResourcePathUtil.Combine(_runtimeConfigFolder, "inputschema.json");
var pp = new System.Uri(p).AbsolutePath;
var actualSchema = JsonConfig.From(File.ReadAllText(Path.Combine(_runtimeConfigFolder, "inputschema.json")));
foreach (var match in JsonConfig.Match(expectedSchema, actualSchema))
{
Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"path:{match.Item1}");
}
// verify output projection file is expected
var expectedProjection = await File.ReadAllTextAsync(@"Resource\projection.txt");
var actualProjection = File.ReadAllText(Path.Combine(_runtimeConfigFolder, "projection.txt"));
Assert.AreEqual(expected: expectedProjection, actual: actualProjection);
// verify transform file is exepcted
var expectedTransform = await File.ReadAllTextAsync(@"Resource\localconfiggentest-combined.txt");
var actualTransform = File.ReadAllText(Path.Combine(_runtimeConfigFolder, "localconfiggentest-combined.txt"));
Assert.AreEqual(expected: expectedTransform, actual: actualTransform);
// verify output configuration
var actualConf = PropertiesDictionary.From(File.ReadAllText(Path.Combine(_runtimeConfigFolder, $"{flowName}.conf")));
Assert.IsTrue(actualConf.Count == 74, $"Actual entries in .conf not as expected. Expected=65, Got={actualConf.Count}");
// verify metrics
var expectedConfig = JsonConfig.From(await File.ReadAllTextAsync(@"Resource\flowStarted.json"));
var actualConfig = JsonConfig.From(await this.DesignTimeStorage.GetByName(flowName, FlowDataManager.DataCollectionName));
foreach (var match in JsonConfig.Match(expectedConfig, actualConfig))
{
Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"path:{match.Item1}");
}
}
public async Task EndToEndGenerationCustom()
{
var flowName = "customconfiggentest";
var testingConfig = await File.ReadAllTextAsync(@"Resource\customflow.json");
await DesignTimeStorage.SaveByName(flowName, testingConfig, FlowDataManager.DataCollectionName);
await CommonData.Add("defaultFlowConfig", @"Resource\customflow.json");
await CommonData.Add("flattener", @"Resource\customFlattenerConfig.json");
await CommonData.Add("defaultJobTemplate", @"Resource\sparkJobTemplate.json");
var result = await this.RuntimeConfigGeneration.GenerateRuntimeConfigs(flowName);
var runtimeConfigFolder = result.Properties?.GetValueOrDefault(PrepareJobConfigVariables.ResultPropertyName_RuntimeConfigFolder, null);
Assert.IsTrue(result.IsSuccess);
Assert.AreEqual(expected: 2, actual: RuntimeStorage.Cache.Count);
var jobConfigDestinationFolder = runtimeConfigFolder?.ToString().Split("Generation_").First();
// Verify output configuration is expected
var actualConf = PropertiesDictionary.From(this.RuntimeStorage.Cache[ResourcePathUtil.Combine(runtimeConfigFolder.ToString(), "customconfiggentest1.conf")]);
var expectedConf = PropertiesDictionary.From(await File.ReadAllTextAsync(@"Resource\customJobConfig1.conf"));
var matches = PropertiesDictionary.Match(expectedConf, actualConf).ToList();
foreach (var match in matches)
{
Console.WriteLine($"prop:{match.Item1 ?? "null"}, expected:<{match.Item2 ?? "null"}>, actual:<{match.Item3 ?? "null"}>");
}
foreach (var match in matches)
{
Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"property:{match.Item1}");
}
//verify second job conf is generated as expected
flowName = "customconfiggentest2";
actualConf = PropertiesDictionary.From(this.RuntimeStorage.Cache[ResourcePathUtil.Combine(runtimeConfigFolder.ToString(), "customconfiggentest2.conf")]);
expectedConf = PropertiesDictionary.From(await File.ReadAllTextAsync(@"Resource\customJobConfig2.conf"));
matches = PropertiesDictionary.Match(expectedConf, actualConf).ToList();
foreach (var match in matches)
{
Console.WriteLine($"prop:{match.Item1 ?? "null"}, expected:<{match.Item2 ?? "null"}>, actual:<{match.Item3 ?? "null"}>");
}
foreach (var match in matches)
{
Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"property:{match.Item1}");
}
Cleanup();
}
public async Task TestBatchSchedule()
{
var flowName = "configgenbatchtest";
var testingConfig = await File.ReadAllTextAsync(@"Resource\batchFlow.json");
var current = DateTime.UtcNow;
var startTimeConfig = current.AddDays(-1);
var endTimeConfig = current.AddDays(1);
var normalizedStartTime = startTimeConfig.Add(-startTimeConfig.TimeOfDay);
testingConfig = testingConfig.Replace("${startTime}", startTimeConfig.ToString("o"));
testingConfig = testingConfig.Replace("${endTime}", endTimeConfig.ToString("o"));
await DesignTimeStorage.SaveByName(flowName, testingConfig, FlowDataManager.DataCollectionName);
await CommonData.Add("defaultJobTemplate", @"Resource\batchSparkJobTemplate.json");
await CommonData.Add(ConfigFlattenerManager.DefaultConfigName, @"Resource\flattenerConfig.json");
await CommonData.Add(FlowDataManager.CommonDataName_DefaultFlowConfig, @"Resource\batchFlowDefault.json");
var result = await this.RuntimeConfigGeneration.GenerateRuntimeConfigs(flowName);
var runtimeConfigFolder = result.Properties?.GetValueOrDefault(PrepareJobConfigVariables.ResultPropertyName_RuntimeConfigFolder, null);
Assert.IsTrue(result.IsSuccess);
Assert.AreEqual(expected: 8, actual: RuntimeStorage.Cache.Count);
var processTime = Regex.Replace(normalizedStartTime.ToString("s"), "[^0-9]", "");
var actualConf = PropertiesDictionary.From(this.RuntimeStorage.Cache[ResourcePathUtil.Combine(runtimeConfigFolder.ToString() + $@"/Recurring/{processTime}", $"{flowName}.conf")]);
var conf = await File.ReadAllTextAsync(@"Resource\batchJobConfig.conf");
conf = conf.Replace("${processTime}", processTime);
conf = conf.Replace("${startTime}", normalizedStartTime.AddDays(-2).ToString("o"));
conf = conf.Replace("${endTime}", normalizedStartTime.AddDays(1).AddSeconds(-1).ToString("o"));
var expectedConf = PropertiesDictionary.From(conf);
var matches = PropertiesDictionary.Match(expectedConf, actualConf).ToList();
foreach (var match in matches)
{
Console.WriteLine($"prop:{match.Item1 ?? "null"}, expected:<{match.Item2 ?? "null"}>, actual:<{match.Item3 ?? "null"}>");
}
foreach (var match in matches)
{
Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"property:{match.Item1}");
}
Cleanup();
}
/// <summary>
/// Figure out the destination folder of runtime configs
/// Resolve the destination folder from flow config
/// Calculate the version of generation
/// Finalize the destination folder with the configured one and the version
/// </summary>
/// <param name="baseFolderPath">the base folder path to determine the final destination folder for job configs</param>
/// <param name="version">the version to enclose in the folder path</param>
/// <returns></returns>
public string FigureOutDestinationFolder(string baseFolderPath, string version)
{
if (baseFolderPath == null)
{
return(null);
}
var folderName = "Generation_" + version;
return(ResourcePathUtil.Combine(baseFolderPath, folderName));
}
/// <summary>
/// Get the job config path based on the job type
/// </summary>
/// <returns></returns>
private static string GetJobConfigFilePath(bool isOneTime, string partitionName, string baseFolder)
{
var oneTimeFolderName = "";
if (isOneTime)
{
oneTimeFolderName = $"OneTime/{Regex.Replace(partitionName, "[^0-9]", "")}";
}
else
{
oneTimeFolderName = $"Recurring/{Regex.Replace(partitionName, "[^0-9]", "")}";
}
return(ResourcePathUtil.Combine(baseFolder, oneTimeFolderName));
}
private void GenerateJobConfigContent(JobDeploymentSession job, string destFolder, JsonConfig defaultJobConfig)
{
Ensure.NotNull(destFolder, "destFolder");
Ensure.NotNull(defaultJobConfig, "defaultJobConfig");
// replace config with tokens from job and the flow
var newJobConfig = job.Tokens.Resolve(defaultJobConfig);
Ensure.NotNull(newJobConfig, "newJobConfig");
job.SetJsonToken(TokenName_JobConfigContent, newJobConfig);
var destinationPath = ResourcePathUtil.Combine(destFolder, job.Name + ".json");
job.SetStringToken(TokenName_JobConfigFilePath, destinationPath);
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var jobs = flowToDeploy.GetJobs();
if (jobs == null || !jobs.Where(j => j.JobConfigs.Any()).Any())
{
return("no jobs, skipped");
}
var config = flowToDeploy.Config;
var guiConfig = config?.GetGuiConfig();
if (guiConfig == null)
{
return("no gui input, skipped.");
}
var projectColumns = guiConfig.Input?.Properties?.NormalizationSnippet?.Trim('\t', ' ', '\r', '\n');
//TODO: make the hardcoded "Raw.*" configurable?
var finalProjections = string.IsNullOrEmpty(projectColumns) ? "Raw.*" : projectColumns;
var runtimeConfigBaseFolder = flowToDeploy.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder);
Ensure.NotNull(runtimeConfigBaseFolder, "runtimeConfigBaseFolder");
var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName);
Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");
var filePath = ResourcePathUtil.Combine(runtimeConfigBaseFolder, "projection.txt");
var savedFile = await RuntimeStorage.SaveFile(filePath, finalProjections);
var tokenValue = flowToDeploy.GetTokenString(PrepareProjectionFile.TokenName_ProjectionFiles);
var projectionFileSecret = JArray.Parse(tokenValue).FirstOrDefault()?.Value <string>();
if (!string.IsNullOrEmpty(projectionFileSecret))
{
await KeyVaultClient.SaveSecretAsync(projectionFileSecret, savedFile);
}
return("done");
}
private void GenerateJobConfigContent(JobDeploymentSession job, string destFolder, JsonConfig defaultJobConfig)
{
Ensure.NotNull(destFolder, "destFolder");
Ensure.NotNull(defaultJobConfig, "defaultJobConfig");
// replace config with tokens from job and the flow
var newJobConfig = job.Tokens.Resolve(defaultJobConfig);
Ensure.NotNull(newJobConfig, "newJobConfig");
job.SetJsonToken(TokenName_JobConfigContent, newJobConfig);
var jc = new JobConfig
{
Content = newJobConfig.ToString(),
FilePath = ResourcePathUtil.Combine(destFolder, job.Name + ".conf"),
SparkJobName = job.SparkJobName,
};
job.JobConfigs.Add(jc);
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var config = flowToDeploy.Config;
var guiConfig = config?.GetGuiConfig();
if (guiConfig == null)
{
return("no gui input, skipped.");
}
string queries = string.Join("\n", guiConfig.Process?.Queries);
string ruleDefinitions = RuleDefinitionGenerator.GenerateRuleDefinitions(guiConfig.Rules, config.Name);
RulesCode rulesCode = CodeGen.GenerateCode(queries, ruleDefinitions, config.Name);
Ensure.NotNull(rulesCode, "rulesCode");
// Save the rulesCode object for downstream processing
flowToDeploy.SetAttachment(AttachmentName_CodeGenObject, rulesCode);
var runtimeConfigBaseFolder = flowToDeploy.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder);
Ensure.NotNull(runtimeConfigBaseFolder, "runtimeConfigBaseFolder");
var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName);
Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");
var filePath = ResourcePathUtil.Combine(runtimeConfigBaseFolder, $"{config.Name}-combined.txt");
var transformFilePath = await RuntimeStorage.SaveFile(filePath, rulesCode.Code);
var secretName = $"{config.Name}-transform";
var transformFileSecret = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, transformFilePath);
flowToDeploy.SetStringToken(TokenName_TransformFile, transformFileSecret);
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var flowConfig = flowToDeploy.Config;
// get a flattener
var flattener = await this.ConfigFlatteners.GetDefault();
if (flattener == null)
{
return("no flattern config, skipped");
}
// flatten each job config
var jobs = flowToDeploy.GetJobs();
if (jobs == null)
{
return("no jobs, skipped");
}
foreach (var job in jobs)
{
var jsonContent = job.GetTokenString(GenerateJobConfig.TokenName_JobConfigContent);
var destFolder = job.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder);
if (jsonContent != null)
{
var json = JsonConfig.From(jsonContent);
job.SetStringToken(GenerateJobConfig.TokenName_JobConfigContent, flattener.Flatten(json));
var destinationPath = ResourcePathUtil.Combine(destFolder, job.Name + ".conf");
job.SetStringToken(GenerateJobConfig.TokenName_JobConfigFilePath, destinationPath);
}
}
return("done");
}
public async Task DatabricksTestEndToEndGeneration()
{
var flowName = "dbconfiggentest";
var testingConfig = await File.ReadAllTextAsync(@"Resource\databricksFlowSaved.json");
await DesignTimeStorage.SaveByName(flowName, testingConfig, FlowDataManager.DataCollectionName);
await CommonData.Add("defaultJobTemplate", @"Resource\sparkJobTemplate.json");
await CommonData.Add(ConfigFlattenerManager.DefaultConfigName, @"Resource\flattenerConfig.json");
await CommonData.Add(FlowDataManager.CommonDataName_DefaultFlowConfig, @"Resource\flowDefault.json");
var result = await this.RuntimeConfigGeneration.GenerateRuntimeConfigs(flowName);
var runtimeConfigFolder = result.Properties?.GetValueOrDefault(PrepareJobConfigVariables.ResultPropertyName_RuntimeConfigFolder, null);
Assert.IsTrue(result.IsSuccess);
Assert.AreEqual(expected: 4, actual: RuntimeStorage.Cache.Count);
// verify output schema file is expected
var expectedSchema = JsonConfig.From(await File.ReadAllTextAsync(@"Resource\schema.json"));
var actualSchema = JsonConfig.From(RuntimeStorage.Cache[ResourcePathUtil.Combine(runtimeConfigFolder.ToString(), "inputschema.json")]);
foreach (var match in JsonConfig.Match(expectedSchema, actualSchema))
{
Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"path:{match.Item1}");
}
// verify output projection file is expected
var expectedProjection = await File.ReadAllTextAsync(@"Resource\projection.txt");
var actualProjection = RuntimeStorage.Cache[ResourcePathUtil.Combine(runtimeConfigFolder.ToString(), "projection.txt")];
Assert.AreEqual(expected: expectedProjection, actual: actualProjection);
// verify transform file is exepcted
var expectedTransform = await File.ReadAllTextAsync(@"Resource\configgentest-combined.txt");
var actualTransform = RuntimeStorage.Cache[ResourcePathUtil.Combine(runtimeConfigFolder.ToString(), "dbconfiggentest-combined.txt")];
Assert.AreEqual(expected: expectedTransform, actual: actualTransform);
// Verify output configuration is expected
var actualConf = PropertiesDictionary.From(this.RuntimeStorage.Cache[ResourcePathUtil.Combine(runtimeConfigFolder.ToString(), $"{flowName}.conf")]);
var expectedConf = PropertiesDictionary.From(await File.ReadAllTextAsync(@"Resource\databricksJobConfig.conf"));
var matches = PropertiesDictionary.Match(expectedConf, actualConf).ToList();
foreach (var match in matches)
{
Console.WriteLine($"prop:{match.Item1 ?? "null"}, expected:<{match.Item2 ?? "null"}>, actual:<{match.Item3 ?? "null"}>");
}
foreach (var match in matches)
{
Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"property:{match.Item1}");
}
// Verify metrics
var expectedConfig = JsonConfig.From(await File.ReadAllTextAsync(@"Resource\databricksFlowStarted.json"));
var actualConfig = JsonConfig.From(await this.DesignTimeStorage.GetByName(flowName, FlowDataManager.DataCollectionName));
foreach (var match in JsonConfig.Match(expectedConfig, actualConfig))
{
Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"path:{match.Item1}");
}
Cleanup();
}